From ab6dbabb1791c46e6e65cba20f0fc1960c7875c7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 14 Jan 2019 15:59:06 +0100
Subject: [PATCH 001/540] Fix cylindrical bug and BZ bug in feltor_hpc

---
 src/feltor/feltor.cu        |  1 +
 src/feltor/feltor_hpc.cu    | 12 ++++++------
 src/feltor/window_params.js |  2 +-
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 4d2654cf7..59f6308e7 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -158,6 +158,7 @@ int main( int argc, char* argv[])
         }
         feltor.update_quantities();
     }
+    q.display( std::cout );
     double energy0 = q.energy, mass0 = q.mass, E0 = energy0, M0 = mass0;
     /////////////////////////set up transfer for glfw
     dg::DVec dvisual( grid.size(), 0.);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 8cf292f85..44382f129 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -19,7 +19,7 @@ using DVec = dg::MDVec;
 using DMatrix = dg::MDMatrix;
 using IDMatrix = dg::MIDMatrix;
 using IHMatrix = dg::MIHMatrix;
-using Geometry = dg::CartesianMPIGrid3d;
+using Geometry = dg::CylindricalMPIGrid3d;
 #define MPI_OUT if(rank==0)
 #else //FELTOR_MPI
 using HVec = dg::HVec;
@@ -27,7 +27,7 @@ using DVec = dg::DVec;
 using DMatrix = dg::DMatrix;
 using IDMatrix = dg::IDMatrix;
 using IHMatrix = dg::IHMatrix;
-using Geometry = dg::CartesianGrid3d;
+using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #endif //FELTOR_MPI
 
@@ -232,10 +232,10 @@ int main( int argc, char* argv[])
         dg::geo::BFieldZ fieldZ(mag);
         dg::geo::BFieldP fieldP(mag);
 
-        HVec vecR = dg::pullback( fieldR, grid_out);
-        HVec vecZ = dg::pullback( fieldZ, grid_out);
-        HVec vecP = dg::pullback( fieldP, grid_out);
-        HVec psip = dg::pullback( mag.psip(), grid_out);
+        HVec vecR = dg::pullback( fieldR, grid);
+        HVec vecZ = dg::pullback( fieldZ, grid);
+        HVec vecP = dg::pullback( fieldP, grid);
+        HVec psip = dg::pullback( mag.psip(), grid);
         std::map<std::string, const HVec*> v3d{
             {"BR", &vecR}, {"BZ", &vecZ}, {"BP", &vecP},
             {"Psip", &psip}, {"Nprof", &profile },
diff --git a/src/feltor/window_params.js b/src/feltor/window_params.js
index a072aedfe..a1051f677 100644
--- a/src/feltor/window_params.js
+++ b/src/feltor/window_params.js
@@ -1,7 +1,7 @@
 {
 
     "rows": 6,      //# of rows
-    "reduction": 1, //# of cols = Nz/reduction + 1
+    "reduction": 4, //# of cols = Nz/reduction + 1
     "width": 200,   //box width (in pixel)
     "height": 200   //box height(in pixel)
 }
-- 
GitLab


From ab36b1de1a3df9c39901e489d6a7141d33efe0ab Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 15 Jan 2019 17:16:17 +0100
Subject: [PATCH 002/540] Fix bug in createEPhi (contravariant component)

---
 inc/geometries/magnetic_field.h | 17 ++++++++++++-----
 src/feltor/feltor.cuh           |  4 +++-
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 041f1c0c5..440c8394c 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -523,26 +523,30 @@ struct BHatP: public aCylindricalFunctor<BHatP>
 };
 
 /**
- * @brief Magnetic unit vector field (BHatR, BHatZ, BHatP)
- *
+ * @brief Contravariant components of the magnetic unit vector field
+ * in cylindrical coordinates.
  * @param mag the tokamak magnetic field
  * @return the tuple BHatR, BHatZ, BHatP constructed from mag
  */
 inline CylindricalVectorLvl0 createBHat( const TokamakMagneticField& mag){
     return CylindricalVectorLvl0( BHatR(mag), BHatZ(mag), BHatP(mag));
 }
+
 /**
- * @brief Unit vector field (0, 0, 1)
- * @return the tuple dg::geo::Constant(0), dg::geo::Constant(0), dg::geo::Constant(1)
+ * @brief Contravariant components of the unit vector field (0, 0, 1/R)
+ * in cylindrical coordinates.
+ * @return the tuple dg::geo::Constant(0), dg::geo::Constant(0), \f$ 1/R \f$
+ * @note This is equivalent to inserting a toroidal magnetic field into the \c dg::geo::createBHat function.
  */
 inline CylindricalVectorLvl0 createEPhi( ){
-    return CylindricalVectorLvl0( Constant(0), Constant(0), Constant(1));
+    return CylindricalVectorLvl0( Constant(0), Constant(0), [](double x, double y){ return 1./x;});
 }
 /**
  * @brief Approximate curvature vector field (CurvatureNablaBR, CurvatureNablaBZ, Constant(0))
  *
  * @param mag the tokamak magnetic field
  * @return the tuple CurvatureNablaBR, CurvatureNablaBZ, dg::geo::Constant(0) constructed from mag
+ * @note The contravariant components in cylindrical coordinates
  */
 inline CylindricalVectorLvl0 createCurvatureNablaB( const TokamakMagneticField& mag){
     return CylindricalVectorLvl0( CurvatureNablaBR(mag), CurvatureNablaBZ(mag), Constant(0));
@@ -552,6 +556,7 @@ inline CylindricalVectorLvl0 createCurvatureNablaB( const TokamakMagneticField&
  *
  * @param mag the tokamak magnetic field
  * @return the tuple CurvatureKappaR, CurvatureKappaZ, dg::geo::Constant(0) constructed from mag
+ * @note The contravariant components in cylindrical coordinates
  */
 inline CylindricalVectorLvl0 createCurvatureKappa( const TokamakMagneticField& mag){
     return CylindricalVectorLvl0( CurvatureKappaR(mag), CurvatureKappaZ(mag), Constant(0));
@@ -561,6 +566,7 @@ inline CylindricalVectorLvl0 createCurvatureKappa( const TokamakMagneticField& m
  *
  * @param mag the tokamak magnetic field
  * @return the tuple TrueCurvatureKappaR, TrueCurvatureKappaZ, TrueCurvatureKappaP constructed from mag
+ * @note The contravariant components in cylindrical coordinates
  */
 inline CylindricalVectorLvl0 createTrueCurvatureKappa( const TokamakMagneticField& mag){
     return CylindricalVectorLvl0( TrueCurvatureKappaR(mag), TrueCurvatureKappaZ(mag), TrueCurvatureKappaP(mag));
@@ -570,6 +576,7 @@ inline CylindricalVectorLvl0 createTrueCurvatureKappa( const TokamakMagneticFiel
  *
  * @param mag the tokamak magnetic field
  * @return the tuple TrueCurvatureNablaBR, TrueCurvatureNablaBZ, TrueCurvatureNablaBP constructed from mag
+ * @note The contravariant components in cylindrical coordinates
  */
 inline CylindricalVectorLvl0 createTrueCurvatureNablaB( const TokamakMagneticField& mag){
     return CylindricalVectorLvl0( TrueCurvatureNablaBR(mag), TrueCurvatureNablaBZ(mag), TrueCurvatureNablaBP(mag));
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 3ddfe4cac..045481fbf 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -477,7 +477,9 @@ void Explicit<Grid, IMatrix, Matrix, container>::construct_invert(
         m_multi_invgammaP[u].elliptic().set_chi( hh);
         m_multi_invgammaN[u].construct(  m_multigrid.grid(u),
             p.bcxN, p.bcyN, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered);
-        m_multi_invgammaN[u].elliptic().set_chi( hh); m_multi_induction[u].construct(  m_multigrid.grid(u), p.bcxA, p.bcyA, dg::PER, -1., dg::centered);
+        m_multi_invgammaN[u].elliptic().set_chi( hh);
+        m_multi_induction[u].construct(  m_multigrid.grid(u),
+            p.bcxA, p.bcyA, dg::PER, -1., dg::centered);
         m_multi_induction[u].elliptic().set_chi( hh);
     }
 }
-- 
GitLab


From e5425f15173eb0b4925c7e03e8cacf44ce735fc1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 16 Jan 2019 15:37:57 +0100
Subject: [PATCH 003/540] Begin to work on damping terms

and improve W viscosity
---
 src/feltor/feltor.cu    |  4 ++--
 src/feltor/feltor.cuh   | 45 ++++++++++++++++++++++++-----------------
 src/feltor/feltor.tex   | 38 ++++++++++++++++++----------------
 src/feltor/parameters.h | 10 ++++++---
 4 files changed, 56 insertions(+), 41 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 59f6308e7..c22d98364 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -67,8 +67,8 @@ int main( int argc, char* argv[])
         //then shift tanh
         p.rho_source-3.*p.alpha, p.alpha, -1.), grid);
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
-    if( p.omega_source != 0)
-        feltor.set_source( p.omega_source, profile, source_damping);
+    if( p.omega_source != 0 || p.omega_damping != 0)
+        feltor.set_source_and_sink( profile, p.omega_source, source_damping);
 
     dg::HVec profile_damping = dg::pullback( dg::geo::TanhDamping(
         mag.psip(), -3.*p.alpha, p.alpha, -1), grid);
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 045481fbf..81e61d6d1 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -164,8 +164,8 @@ struct ComputeLogN{
 struct ComputeSource{
     DG_DEVICE
     void operator()( double& result, double tilde_n, double profne,
-        double source, double omega_source) const{
-        double temp = omega_source*source*(profne - tilde_n);
+        double source, double omega_s, double damping, double omega_d) const{
+        double temp = (omega_s*source + omega_d*damping)*(profne - tilde_n);
         result = temp;
     }
 };
@@ -247,6 +247,7 @@ struct Quantities
     //resisitive and diffusive terms
     double Dres = 0, Dpar[4] = {0,0,0,0}, Dperp[4] = {0,0,0,0};
     double aligned = 0; //alignment parameter
+    double source[2] = {0,0}; //source terms
     void display( std::ostream& os = std::cout ) const
     {
         os << "Quantities: \n"
@@ -258,6 +259,7 @@ struct Quantities
            << "    Dres: "<<Dres<<"\n"
            << "    Dpar: ["<<Dpar[0]<<", "<<Dpar[1]<<", "<<Dpar[2]<<", "<<Dpar[3]<<"]\n"
            << "   Dperp: ["<<Dperp[0]<<", "<<Dperp[1]<<", "<<Dperp[2]<<", "<<Dperp[3]<<"]\n"
+           << " Sources: ["<<source[0]<<", "<<source[1]<<"]\n"
            << " aligned: "<<aligned<<"\n";
     }
 };
@@ -308,12 +310,14 @@ struct Explicit
     }
 
     //source strength, profile - 1 and damping
-    void set_source( double omega_source, container profile, container damping)
+    void set_source_and_sink( container profile, double omega_source,
+        container source, double omega_damping, container damping)
     {
-        m_omega_source = omega_source;
         m_profne = profile;
-        m_source = damping;
-
+        m_omega_source = omega_source;
+        m_source = source;
+        m_omega_damping = omega_damping;
+        m_damping = damping;
     }
   private:
     void compute_phi( double t, const std::array<container,2>& y);
@@ -339,6 +343,7 @@ struct Explicit
         dg::geo::TokamakMagneticField);
 
     container m_UE2;
+    std::array<container,2> m_sn;
     container m_temp0, m_temp1, m_temp2;//helper variables
 #ifdef DG_MANUFACTURED
     container m_R, m_Z, m_P; //coordinates
@@ -348,7 +353,7 @@ struct Explicit
     std::array<container,3> m_curv, m_curvKappa, m_b;
     container m_divCurvKappa;
     container m_binv, m_divb;
-    container m_source, m_profne;
+    container m_source, m_damping, m_profne;
     container m_vol3d;
 
     container m_apar, m_dxA, m_dyA, m_dzA;
@@ -374,7 +379,7 @@ struct Explicit
 
     const feltor::Parameters m_p;
     Quantities m_q;
-    double m_omega_source =0.;
+    double m_omega_source = 0., m_omega_damping = 0.;
 
 };
 
@@ -507,7 +512,7 @@ Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g,
 {
     //--------------------------init vectors to 0-----------------//
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
-    m_UE2 = m_temp2 = m_temp1 = m_temp0;
+    m_UE2 = m_sn[0] = m_sn[1] = m_temp2 = m_temp1 = m_temp0;
     m_phi[0] = m_phi[1] = m_temp0;
     m_dxPhi = m_dyPhi = m_dzPhi = m_fields[0] = m_fields[1] = m_logn = m_phi;
     m_dxN = m_dyN = m_dzN = m_dsN = m_dxU = m_dyU = m_dzU = m_dsU = m_phi;
@@ -816,6 +821,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_dissipation(
         // Z*(tau (1+lnN )+psi + 0.5 mu U^2)
         dg::blas1::subroutine( routines::ComputeDiss(m_p.mu[i], m_p.tau[i]),
                 m_temp2, m_logn[i], m_phi[i], fields[1][i]);
+        m_q.source[i] = z[i]*dg::blas2::dot( m_temp2, m_vol3d, m_sn[i]);
         // perp dissipation for N: nu_perp Delta_p N or -nu_perp Delta_p**2 N
         if( m_p.perp_diff == "viscous")
         {
@@ -862,7 +868,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_dissipation(
     dg::blas1::pointwiseDot(1., fields[0][0], fields[1][1],
         -1., fields[0][0], fields[1][0], 0., m_temp0);
     m_q.Dres = -m_p.c*dg::blas2::dot(m_temp0, m_vol3d, m_temp0);
-    m_q.ediff = m_q.Dres
+    m_q.ediff = m_q.Dres + m_q.source[0] + m_q.source[1]
         + m_q.Dpar[0]+m_q.Dperp[0]+m_q.Dpar[1]+m_q.Dperp[1]
         + m_q.Dpar[2]+m_q.Dperp[2]+m_q.Dpar[3]+m_q.Dperp[3];
 }
@@ -915,16 +921,17 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
     compute_parallel( t, y, m_fields, yp);
 #endif
 
-    //Add particle source to dtNe
-    if( m_omega_source != 0)
+    //Add particle source to dtNe and dtNi
+    if( m_omega_source != 0 || m_omega_damping != 0)
     {
-        dg::blas1::subroutine( routines::ComputeSource(),
-            m_temp1, y[0][0], m_profne, m_source, m_omega_source);
-        dg::blas1::axpby( 1., m_temp1, 1.0, yp[0][0]);
-        //add FLR correction to dtNi
-        dg::blas1::axpby( 1., m_temp1, 1.0, yp[1][1]);
-        dg::blas2::gemv( 0.5*m_p.tau[1]*m_p.mu[1],
-            m_lapperpN, m_temp1, 1.0, yp[1][1]);
+        dg::blas1::subroutine( routines::ComputeSource(), m_sn[0], y[0][0],
+            m_profne, m_source, m_omega_source, m_damping, m_omega_damping);
+        //compute FLR correction
+        dg::blas2::gemv( m_lapperpN, m_sn[0], m_temp0);
+        dg::blas1::axpby( 1., m_sn[0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_sn[1]);
+
+        dg::blas1::axpby( 1., m_sn[0], 1.0, yp[0][0]);
+        dg::blas1::axpby( 1., m_sn[1], 1.0, yp[0][1]);
     }
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( yp[0][0], dg::plus_equals(), manufactured::SNe{
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 4a163f56b..6ac92d2b4 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -433,16 +433,14 @@ For numerical stabilisation we choose:
 \Lambda_{u_e,\parallel} &= \nu_\parallel \Delta_\parallel u_e &
 \Lambda_{U_i,\parallel} &= \nu_\parallel \Delta_\parallel U_i 
 \end{align}
-Similarly, for the perpendicular dissipation we apply viscous terms.
+Similarly, for the perpendicular dissipation we apply viscous or hyperviscous terms.
 \begin{align}\label{eq:perpdiffNT}
- \Lambda_{n_e,\perp} &=  \nu_\perp \Delta_\perp n_e &
- \Lambda_{N_i,\perp} &=  \nu_\perp \Delta_\perp N_i  & \\
- \Lambda_{u_e,\perp} &=  \nu_\perp \Delta_\perp (u_e - eA_\parallel/m_e)  &
- \Lambda_{U_i,\perp} &=  \nu_\perp \Delta_\perp (U_i + eA_\parallel/m_i)
+ \Lambda_{n_e,\perp} &=  \nu_\perp \Delta_\perp n_e \text{ or } -\nu_\perp \Delta_\perp^2 n_e&
+ \Lambda_{N_i,\perp} &=  \nu_\perp \Delta_\perp N_i \text{ or } -\nu_\perp \Delta_\perp^2 N_i & \\
+ \Lambda_{u_e,\perp} &=  \nu_\perp \Delta_\perp u_e \text{ or } -\nu_\perp \Delta_\perp^2 u_e &
+ \Lambda_{U_i,\perp} &=  \nu_\perp \Delta_\perp U_i \text{ or } -\nu_\perp \Delta_\perp^2 U_i
 \end{align}
 Here the mass diffusion coefficient coincides with the viscous coefficient, hence we fixed the Schmidt number \(\mathit{Sc}_\parallel:= \frac{\nu_U}{\nu_N}\) to unity.
-Note that we let the perpendicular diffusion act on $A_\parallel$ in addition to $U$ for
-numerical reasons.
 
 \subsection{Boundary and Initial conditions}
 We define the simulation box as
@@ -529,17 +527,20 @@ wavelength $k_\psi$ aligned with the magnetic flux surfaces.
 \tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta(-\psi_p)H(Z-Z_X)
 \end{align}
 
-\subsection{Particle sources} \label{sec:sources}
+\subsection{Particle sinks/sources} \label{sec:sources}
 The idea for the source terms $S$ is to fix the profile $n_{prof}$ in the
-core of our domain, where our model does not apply.
-We thus define a particle source for electrons as
+core and the wall shadow (the corners) of our domain, where our model does not apply.
+We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
   S_{n_e}(R,Z,\varphi, t) &= \omega_s
-    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho_{\max} -\rho(R,Z)) H(Z-Z_X) \\
+    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho_{s} -\rho(R,Z)) H(Z-Z_X) \nonumber\\
+                            &+ \omega_d
+    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho(R,Z) - \rho_{d}) \\
     \rho(R,Z) &:= \frac{\psi_p(R_0,0)- \psi_p(R,Z) }{\psi_p(R_0,0)},
 \end{align}
-with $0 < \rho_{\max}<1$
-where $\omega_s$ is the source strength parameter. This will result in exponential adaption of the core
+with $0 < \rho_{s}<1$ and $\rho_{d}>1$
+where $\omega_s$ is the source strength parameter and $\omega_d$ the damping parameter.
+This will result in exponential adaption of the core and wall
 density profile of the form $n_e \propto n_{prof}+(n_{prof}-n_{e,0})e^{-\omega_st}$.
 For ions we use
 \begin{align}
@@ -634,7 +635,7 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
         - \left(2\tau + {\mu}U^2\right) \mathcal K_\kappa (U)
         -2\tau U\mathcal K_\kappa(\ln N)
         - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e)
-        + \nu_\perp\Delta_\perp W + \nu_\parallel \Delta_\parallel U ,
+        + \nu_\perp\Delta_\perp U + \nu_\parallel \Delta_\parallel U ,
         \label{eq:EgyrofluidU} \\
         W&:= \left( U + \frac{\beta}{\mu}A_\parallel\right)
     \end{align}
@@ -776,9 +777,11 @@ posY       & float &0.0    & - & blob Z-position in units of $a$ \\
 sigma\_z    & float &0.25   & - & variance in units of $R_0$  \\
 k\_psi     & float &0    & - & zonal mode wave number  \\
 nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} \\
-source  & float & 0     & - & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
+source  & float & 0     & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
+damping  & float & 0     & 0 & profile damping rate $\omega_d$ in Eq.~\eqref{eq:electron_source} \\
 alpha     & float  & 0.02 & - & Width $\alpha$ of Heaviside profile in Eq.~\eqref{eq:heaviside_profile} \\
-rho\_source & float  & 0.2   & 0.2 & Source region boundary $0<\rho_{\max}<1$ in Eq.~\eqref{eq:electron_source}  \\
+rho\_source & float  & 0.2   & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source}  \\
+rho\_damping & float  & 1.1   & 1.1 & Damping region boundary $0<\rho_{d}<1$ in Eq.~\eqref{eq:electron_source}  \\
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
@@ -821,7 +824,8 @@ y\_XYZ           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi
 z\_XYZ           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 Psip             & Dataset & 3 (z,y,x) & Flux function $\psi_p(R,Z)$ \\
 Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ \\
-Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta(\rho_{\max} - \rho(R,Z)) H(Z-Z_X)$\\
+Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta(\rho_{s} - \rho(R,Z)) H(Z-Z_X)$\\
+Damping          & Dataset & 3 (z,y,x) & Damping  profile $\Theta(\rho(R,Z) - \rho_{d} )$\\
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 3bc82def8..82c017af9 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -27,7 +27,7 @@ struct Parameters
     std::array<double,2> mu; // mu[0] = mu_e, m[1] = mu_i
     std::array<double,2> tau; // tau[0] = -1, tau[1] = tau_i
     double alpha, beta;
-    double rho_source;
+    double rho_source, rho_damping;
 
     double nu_perp, nu_parallel;
     double c;
@@ -38,7 +38,7 @@ struct Parameters
     double sigma_z;
     double k_psi;
 
-    double omega_source;
+    double omega_source, omega_damping;
     double nprofamp;
     double boxscaleRm, boxscaleRp;
     double boxscaleZm, boxscaleZp;
@@ -87,9 +87,11 @@ struct Parameters
         k_psi       = js["k_psi"].asDouble();
 
         nprofamp  = js["nprofileamp"].asDouble();
-        omega_source = js["source"].asDouble();
+        omega_source  = js.get("source", 0.).asDouble();
+        omega_damping = js.get("damping",0.).asDouble();
         alpha        = js.get("alpha", 0.02).asDouble();
         rho_source   = js.get("rho_source", 0.2).asDouble();
+        rho_damping  = js.get("rho_damping", 1.1).asDouble();
 
         bcxN = dg::str2bc(js["bc"]["density"][0].asString());
         bcyN = dg::str2bc(js["bc"]["density"][1].asString());
@@ -135,7 +137,9 @@ struct Parameters
             <<"    init Phi:     "<<initphi<<"\n";
         os << "Profile parameters are: \n"
             <<"     omega_source:                 "<<omega_source<<"\n"
+            <<"     omega_damping:                "<<omega_damping<<"\n"
             <<"     rho_source:                   "<<rho_source<<"\n"
+            <<"     rho_damping:                  "<<rho_damping<<"\n"
             <<"     alpha:                        "<<alpha<<"\n"
             <<"     density profile amplitude:    "<<nprofamp<<"\n"
             <<"     boxscale R+:                  "<<boxscaleRp<<"\n"
-- 
GitLab


From 07fa39e765c5083867a1b713df4aa1ac104e1e19 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 18 Jan 2019 14:36:56 +0100
Subject: [PATCH 004/540] Fix various bugs for feltor3d

- boundary conditions of Apar same as U
- correction of Delta W with Delta Apar in explicit part
- include source terms in invariants
- fix fieldaligned init bug due to unfavorable summation
- correct Delta2 in writeup
- fix source profile init bug
- no damping in U (pb fixed by using more z planes)
---
 inc/geometries/fieldaligned.h |   8 ++-
 src/feltor/feltor.cu          |  34 ++++++++--
 src/feltor/feltor.cuh         | 118 ++++++++++++++++++----------------
 src/feltor/feltor.tex         |  38 +++++------
 src/feltor/feltor_hpc.cu      |  29 +++++++--
 src/feltor/input/default.json |   3 +-
 src/feltor/parameters.h       |  16 ++---
 7 files changed, 147 insertions(+), 99 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 5f3af66f4..426af82c9 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -300,6 +300,7 @@ struct Fieldaligned
      * @param p0 The index of the plane to start
      *
      * @return Returns an instance of container
+     * @attention It is recommended to use  \c mx>1 and \c my>1 when this function is used, else there might occur some unfavourable summation effects due to the repeated use of transformations especially for low perpendicular resolution.
      */
     template< class BinaryOp>
     container evaluate( const BinaryOp& binary, unsigned p0=0) const
@@ -313,8 +314,8 @@ struct Fieldaligned
      * The algorithm does the equivalent of the following:
      *  - Evaluate the given \c BinaryOp on a 2d plane
      *  - Apply the plus and minus transformation each \f$ r N_z\f$ times where \f$ N_z\f$ is the number of planes in the global 3d grid and \f$ r\f$ is the number of rounds.
-     *  - Scale the transformations with \f$ u ( \pm (iN_z + j)h_z) \f$, where \c u is the given \c UnarayOp, \c i is the round index and \c j is the plane index.
-     *  - Sum all transformations with the same plane index \c j , where the minus transformations get the inverted index \f$ N_z - j\f$.
+     *  - Scale the transformations with \f$ u ( \pm (iN_z + j)h_z) \f$, where \c u is the given \c UnarayOp, \c i in [0..r] is the round index and \c j in [0..Nz] is the plane index.
+     *  - %Sum all transformations with the same plane index \c j , where the minus transformations get the inverted index \f$ N_z - j\f$.
      *  - Shift the index by \f$ p_0\f$
      *  .
      * @tparam BinaryOp Binary Functor
@@ -323,7 +324,8 @@ struct Fieldaligned
      * @param unary Functor to evaluate in z
      * @param p0 The index of the plane to start
      * @param rounds The number of rounds \c r to follow a fieldline; can be zero, then the fieldlines are only followed within the current box ( no periodicity)
-     * @note g is evaluated such that p0 corresponds to z=0, p0+1 corresponds to z=hz, p0-1 to z=-hz, ...
+     * @note \c unary is evaluated such that \c p0 corresponds to z=0, p0+1 corresponds to z=hz, p0-1 to z=-hz, ...
+     * @attention It is recommended to use  \c mx>1 and \c my>1 when this function is used, else there might occur some unfavourable summation effects due to the repeated use of transformations especially for low perpendicular resolution.
      *
      * @return Returns an instance of container
      */
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index c22d98364..e57119c96 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -67,14 +67,15 @@ int main( int argc, char* argv[])
         //then shift tanh
         p.rho_source-3.*p.alpha, p.alpha, -1.), grid);
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
-    if( p.omega_source != 0 || p.omega_damping != 0)
-        feltor.set_source_and_sink( profile, p.omega_source, source_damping);
 
     dg::HVec profile_damping = dg::pullback( dg::geo::TanhDamping(
         mag.psip(), -3.*p.alpha, p.alpha, -1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
+    if( p.omega_source != 0 )
+        feltor.set_source( profile, p.omega_source, source_damping);
+
     //Now perturbation
     dg::HVec ntilde = dg::evaluate(dg::zero,grid);
     if( p.initne == "blob" || p.initne == "straight blob")
@@ -84,9 +85,21 @@ int main( int argc, char* argv[])
         if( p.symmetric)
             ntilde = dg::pullback( init0, grid);
         else if( p.initne == "blob")//rounds =3 ->2*3-1
-            ntilde = feltor.fieldalignedn( init0, gaussianZ, (unsigned)p.Nz/2, 3);
+        {
+            dg::geo::Fieldaligned<dg::CylindricalGrid3d, dg::IHMatrix,
+                dg::HVec> fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+            //evaluate should always be used with mx,my > 1
+            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 3);
+        }
         else if( p.initne == "straight blob")//rounds =1 ->2*1-1
-            ntilde = feltor.fieldalignedn( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+        {
+            dg::geo::Fieldaligned<dg::CylindricalGrid3d, dg::IHMatrix,
+                dg::HVec> fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+            //evaluate should always be used with mx,my > 1
+            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+        }
     }
     else if( p.initne == "turbulence")
     {
@@ -95,7 +108,13 @@ int main( int argc, char* argv[])
         if( p.symmetric)
             ntilde = dg::pullback( init0, grid);
         else
-            ntilde = feltor.fieldalignedn( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+        {
+            dg::geo::Fieldaligned<dg::CylindricalGrid3d, dg::IHMatrix,
+                dg::HVec> fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+            //evaluate should always be used with mx,my > 1
+            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+        }
         dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
     }
     else if( p.initne == "zonal")
@@ -198,6 +217,11 @@ int main( int argc, char* argv[])
                 dg::blas2::gemv( laplacianM, *pair.second, dvisual);
                 dg::assign( dvisual, hvisual);
             }
+            else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
+            {
+                dg::assign( *pair.second, hvisual);
+                dg::blas1::axpby( 1., hvisual, -1., profile, hvisual);
+            }
             else
                 dg::assign( *pair.second, hvisual);
             dg::blas2::gemv( equi, hvisual, visual);
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 81e61d6d1..a4d16d052 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -146,13 +146,13 @@ struct ComputePsi{
     }
 };
 struct ComputeDiss{
-    ComputeDiss( double mu, double tau):m_mu(mu), m_tau(tau){}
+    ComputeDiss( double z, double mu, double tau):m_z(z), m_mu(mu), m_tau(tau){}
     DG_DEVICE
     void operator()( double& energy, double logN, double phi, double U) const{
-        energy = m_tau*(1.+logN) + phi + 0.5*m_mu*U*U;
+        energy = m_z*(m_tau*(1.+logN) + phi + 0.5*m_mu*U*U);
     }
     private:
-    double m_mu, m_tau;
+    double m_z, m_mu, m_tau;
 };
 struct ComputeLogN{
     DG_DEVICE
@@ -164,8 +164,8 @@ struct ComputeLogN{
 struct ComputeSource{
     DG_DEVICE
     void operator()( double& result, double tilde_n, double profne,
-        double source, double omega_s, double damping, double omega_d) const{
-        double temp = (omega_s*source + omega_d*damping)*(profne - tilde_n);
+        double source, double omega_source) const{
+        double temp = omega_source*source*(profne - tilde_n);
         result = temp;
     }
 };
@@ -282,11 +282,6 @@ struct Explicit
     //Given n_e-1 initialize N_i-1 such that phi=0
     void initializeni( const container& ne, container& ni);
 
-    template<class ...Params>
-    container fieldalignedn( Params&&... ps){
-        return m_ds_N.fieldaligned().evaluate( std::forward<Params>(ps)...);
-    }
-
     void operator()( double t,
         const std::array<std::array<container,2>,2>& y,
         std::array<std::array<container,2>,2>& yp);
@@ -308,16 +303,16 @@ struct Explicit
     const std::array<std::array<container,2>,2>& fields() const{
         return m_fields;
     }
+    const std::array<container,2>& sources() const{
+        return m_s;
+    }
 
-    //source strength, profile - 1 and damping
-    void set_source_and_sink( container profile, double omega_source,
-        container source, double omega_damping, container damping)
+    //source strength, profile - 1
+    void set_source( container profile, double omega_source, container source)
     {
         m_profne = profile;
         m_omega_source = omega_source;
         m_source = source;
-        m_omega_damping = omega_damping;
-        m_damping = damping;
     }
   private:
     void compute_phi( double t, const std::array<container,2>& y);
@@ -343,7 +338,6 @@ struct Explicit
         dg::geo::TokamakMagneticField);
 
     container m_UE2;
-    std::array<container,2> m_sn;
     container m_temp0, m_temp1, m_temp2;//helper variables
 #ifdef DG_MANUFACTURED
     container m_R, m_Z, m_P; //coordinates
@@ -353,19 +347,19 @@ struct Explicit
     std::array<container,3> m_curv, m_curvKappa, m_b;
     container m_divCurvKappa;
     container m_binv, m_divb;
-    container m_source, m_damping, m_profne;
+    container m_source, m_profne;
     container m_vol3d;
 
     container m_apar, m_dxA, m_dyA, m_dzA;
     std::array<container,2> m_phi, m_dxPhi, m_dyPhi, m_dzPhi;
     std::array<container,2> m_logn, m_dxN, m_dyN, m_dzN, m_dsN;
-    std::array<container,2> m_dxU, m_dyU, m_dzU, m_dsU;
+    std::array<container,2> m_dxU, m_dyU, m_dzU, m_dsU, m_s;
     std::array<std::array<container,2>,2> m_fields;
 
     std::vector<container> m_multi_chi;
 
     //matrices and solvers
-    Matrix m_dx_N, m_dx_U, m_dx_P, m_dx_A, m_dy_N, m_dy_U, m_dy_P, m_dy_A, m_dz;
+    Matrix m_dx_N, m_dx_U, m_dx_P, m_dy_N, m_dy_U, m_dy_P, m_dz;
     dg::geo::DS<Geometry, IMatrix, Matrix, container> m_ds_P, m_ds_N, m_ds_U;
     dg::Elliptic3d< Geometry, Matrix, container> m_lapperpN, m_lapperpU;
     std::vector<dg::Elliptic3d< Geometry, Matrix, container> > m_multi_pol;
@@ -379,7 +373,7 @@ struct Explicit
 
     const feltor::Parameters m_p;
     Quantities m_q;
-    double m_omega_source = 0., m_omega_damping = 0.;
+    double m_omega_source = 0.;
 
 };
 
@@ -484,7 +478,7 @@ void Explicit<Grid, IMatrix, Matrix, container>::construct_invert(
             p.bcxN, p.bcyN, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered);
         m_multi_invgammaN[u].elliptic().set_chi( hh);
         m_multi_induction[u].construct(  m_multigrid.grid(u),
-            p.bcxA, p.bcyA, dg::PER, -1., dg::centered);
+            p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
         m_multi_induction[u].elliptic().set_chi( hh);
     }
 }
@@ -499,11 +493,9 @@ Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g,
     m_dx_N( dg::create::dx( g, p.bcxN) ),
     m_dx_U( dg::create::dx( g, p.bcxU) ),
     m_dx_P( dg::create::dx( g, p.bcxP) ),
-    m_dx_A( dg::create::dx( g, p.bcxA) ),
     m_dy_N( dg::create::dy( g, p.bcyN) ),
     m_dy_U( dg::create::dy( g, p.bcyU) ),
     m_dy_P( dg::create::dy( g, p.bcyP) ),
-    m_dy_A( dg::create::dy( g, p.bcyA) ),
     m_dz( dg::create::dz( g, dg::PER) ),
     m_multigrid( g, p.stages),
     m_old_phi( 2, dg::evaluate( dg::zero, g)),
@@ -512,9 +504,10 @@ Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g,
 {
     //--------------------------init vectors to 0-----------------//
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
-    m_UE2 = m_sn[0] = m_sn[1] = m_temp2 = m_temp1 = m_temp0;
+    m_UE2 = m_temp2 = m_temp1 = m_temp0;
     m_phi[0] = m_phi[1] = m_temp0;
     m_dxPhi = m_dyPhi = m_dzPhi = m_fields[0] = m_fields[1] = m_logn = m_phi;
+    m_s = m_phi;
     m_dxN = m_dyN = m_dzN = m_dsN = m_dxU = m_dyU = m_dzU = m_dsU = m_phi;
     m_apar = m_dxA = m_dyA = m_dzA = m_phi[0];
     //--------------------------Construct-------------------------//
@@ -677,8 +670,8 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_apar(
     if(  number[0] == m_multigrid.max_iter())
         throw dg::Fail( m_p.eps_pol);
     //----------Compute Derivatives----------------------------//
-    dg::blas2::symv( m_dx_A, m_apar, m_dxA);
-    dg::blas2::symv( m_dy_A, m_apar, m_dyA);
+    dg::blas2::symv( m_dx_U, m_apar, m_dxA);
+    dg::blas2::symv( m_dy_U, m_apar, m_dyA);
     if(!m_p.symmetric) dg::blas2::symv( m_dz, m_apar, m_dzA);
 
     //----------Compute Velocities-----------------------------//
@@ -734,6 +727,24 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_perp(
             );
         }
     }
+    // Add correction to perpendicular viscosity in W
+    if( m_p.beta != 0)
+    {
+        if( m_p.perp_diff == "viscous")
+        {
+            dg::blas2::gemv(  m_lapperpU, m_apar, m_temp0); //!minus
+        }
+        else
+        {
+            dg::blas2::gemv( m_lapperpU, m_apar, m_temp1);
+            dg::blas2::gemv( m_lapperpU, m_temp1, m_temp0); //!plus
+        }
+        for( unsigned i=0; i<2; i++)
+        {
+            //-nu_perp*beta/mu Delta_perp A_par
+            dg::blas1::axpby( m_p.nu_perp*m_p.beta/m_p.mu[i], m_temp0, 1., yp[1][i]);
+        }
+    }
 }
 
 template<class Geometry, class IMatrix, class Matrix, class container>
@@ -813,56 +824,55 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_dissipation(
     m_q.aligned = dg::blas2::dot( m_logn[0], m_vol3d, m_dsN[0]);
     /////////////////DISSIPATION TERMS//////////////////////////////
     m_q.diff = m_p.nu_parallel*dg::blas1::dot( m_vol3d, m_dsN[0]);
+    m_q.diff += dg::blas1::dot( m_vol3d, m_s[0]); //particle sources
     // energy dissipation through diffusion
     double z[2] = {-1.0,1.0};
     for( unsigned i=0; i<2;i++)
     {
         //Compute dissipation for N
         // Z*(tau (1+lnN )+psi + 0.5 mu U^2)
-        dg::blas1::subroutine( routines::ComputeDiss(m_p.mu[i], m_p.tau[i]),
+        dg::blas1::subroutine( routines::ComputeDiss(z[i], m_p.mu[i], m_p.tau[i]),
                 m_temp2, m_logn[i], m_phi[i], fields[1][i]);
-        m_q.source[i] = z[i]*dg::blas2::dot( m_temp2, m_vol3d, m_sn[i]);
+        // Dissipation through sink/source terms
+        m_q.source[i] = dg::blas2::dot( m_temp2, m_vol3d, m_s[i]);
+        // parallel dissipation for N: nu_parallel *(Delta_s N)
+        m_q.Dpar[i] = m_p.nu_parallel*dg::blas2::dot(
+                        m_temp2, m_vol3d, m_dsN[i]);
         // perp dissipation for N: nu_perp Delta_p N or -nu_perp Delta_p**2 N
         if( m_p.perp_diff == "viscous")
         {
             dg::blas1::transform( fields[0][i], m_temp1, dg::PLUS<double>(-1));
-            dg::blas2::gemv( m_lapperpN, m_temp1, m_temp0);
+            dg::blas2::gemv( m_lapperpN, m_temp1, m_temp0); //!minus
         }
         else
         {
             dg::blas1::transform( fields[0][i], m_temp0, dg::PLUS<double>(-1));
             dg::blas2::gemv( m_lapperpN, m_temp0, m_temp1);
-            dg::blas2::gemv( m_lapperpN, m_temp1, m_temp0);
+            dg::blas2::gemv( m_lapperpN, m_temp1, m_temp0); //!plus
         }
-        //minus in Laplacian!
         if( i==0)
             m_q.diff += -m_p.nu_perp*dg::blas1::dot( m_vol3d, m_temp0);
-        m_q.Dperp[i] = -z[i]*m_p.nu_perp*dg::blas2::dot(
+        m_q.Dperp[i] = -m_p.nu_perp*dg::blas2::dot(
                         m_temp2, m_vol3d, m_temp0);
-        // parallel dissipation for N: nu_parallel *(Delta_s N)
-        m_q.Dpar[i] = z[i]*m_p.nu_parallel*dg::blas2::dot(
-                        m_temp2, m_vol3d, m_dsN[i]);
-        //Compute parallel dissipation for U
-        //Z*mu*N*U nu_parallel *( Delta_s U)
+        //Compute dissipation for U
+        //Z*mu*N*U
         dg::blas1::pointwiseDot( z[i]*m_p.mu[i], fields[0][i], fields[1][i],
                 0, m_temp2);
-        // perp dissipation for U: nu_perp Delta_p W or -nu_perp Delta_p**2 W
+        // parallel dissipation for U: nu_parallel *(Delta_s U)
+        m_q.Dpar[i+2] = m_p.nu_parallel*dg::blas2::dot(
+            m_temp2, m_vol3d, m_dsU[i]);
+        // perp dissipation for U: nu_perp Delta_p U or -nu_perp Delta_p**2 U
         if( m_p.perp_diff == "viscous")
         {
-            dg::blas1::axpby( m_p.beta/m_p.mu[i], m_apar, 1., fields[1][i], m_temp1);
-            dg::blas2::gemv( m_lapperpU, m_temp1, m_temp0);
+            dg::blas2::gemv( m_lapperpU, fields[1][i], m_temp0); //!minus
         }
         else
         {
-            dg::blas1::axpby( m_p.beta/m_p.mu[i], m_apar, 1., fields[1][i], m_temp0);
-            dg::blas2::gemv( m_lapperpU, m_temp0, m_temp1);
-            dg::blas2::gemv( m_lapperpU, m_temp1, m_temp0);
+            dg::blas2::gemv( m_lapperpU, fields[1][i], m_temp1);
+            dg::blas2::gemv( m_lapperpU, m_temp1, m_temp0); //!plus
         }
         m_q.Dperp[i+2] = -m_p.nu_perp *dg::blas2::dot(
             m_temp2, m_vol3d, m_temp0);
-        // parallel dissipation for U: nu_parallel *(Delta_s U)
-        m_q.Dpar[i+2] = m_p.nu_parallel*dg::blas2::dot(
-            m_temp2, m_vol3d, m_dsU[i]);
     }
     // resistive energy (quadratic current): -C (n_e (U_i-u_e))**2
     dg::blas1::pointwiseDot(1., fields[0][0], fields[1][1],
@@ -921,17 +931,17 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
     compute_parallel( t, y, m_fields, yp);
 #endif
 
-    //Add particle source to dtNe and dtNi
-    if( m_omega_source != 0 || m_omega_damping != 0)
+    //Add source terms
+    if( m_omega_source != 0)
     {
-        dg::blas1::subroutine( routines::ComputeSource(), m_sn[0], y[0][0],
-            m_profne, m_source, m_omega_source, m_damping, m_omega_damping);
+        dg::blas1::subroutine( routines::ComputeSource(), m_s[0], y[0][0],
+            m_profne, m_source, m_omega_source);
         //compute FLR correction
-        dg::blas2::gemv( m_lapperpN, m_sn[0], m_temp0);
-        dg::blas1::axpby( 1., m_sn[0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_sn[1]);
+        dg::blas2::gemv( m_lapperpN, m_s[0], m_temp0);
+        dg::blas1::axpby( 1., m_s[0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[1]);
 
-        dg::blas1::axpby( 1., m_sn[0], 1.0, yp[0][0]);
-        dg::blas1::axpby( 1., m_sn[1], 1.0, yp[0][1]);
+        dg::blas1::axpby( 1., m_s[0], 1.0, yp[0][0]);
+        dg::blas1::axpby( 1., m_s[1], 1.0, yp[0][1]);
     }
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( yp[0][0], dg::plus_equals(), manufactured::SNe{
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 6ac92d2b4..19efa2ce1 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -458,6 +458,9 @@ the $\varepsilon$ are free parameters to be specified by the user.
 We choose boundary conditions seperately on input for the variables
 $n_e$, $u_e$ and $\phi$. The boundary condition for $N_i$, $U_i$ and
 $\psi$ are equal to $n_e$, $u_e$ and $\phi$ respecitively.
+We choose $A_\parallel$ to have equal boundary conditions as $u_e$ and $U_i$.
+This will later enable us to treat the sum of $U$ and $A_\parallel$
+in the same way as $U$.
 Typically,
 \begin{align}
 n_e = n_0, \quad u_e = \phi = 0
@@ -527,19 +530,17 @@ wavelength $k_\psi$ aligned with the magnetic flux surfaces.
 \tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta(-\psi_p)H(Z-Z_X)
 \end{align}
 
-\subsection{Particle sinks/sources} \label{sec:sources}
+\subsection{Sinks and sources} \label{sec:sources}
 The idea for the source terms $S$ is to fix the profile $n_{prof}$ in the
-core and the wall shadow (the corners) of our domain, where our model does not apply.
+core of our domain, where our model does not apply.
 We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
   S_{n_e}(R,Z,\varphi, t) &= \omega_s
-    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho_{s} -\rho(R,Z)) H(Z-Z_X) \nonumber\\
-                            &+ \omega_d
-    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho(R,Z) - \rho_{d}) \\
+    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho_{s} -\rho(R,Z)) H(Z-Z_X)\\
     \rho(R,Z) &:= \frac{\psi_p(R_0,0)- \psi_p(R,Z) }{\psi_p(R_0,0)},
 \end{align}
-with $0 < \rho_{s}<1$ and $\rho_{d}>1$
-where $\omega_s$ is the source strength parameter and $\omega_d$ the damping parameter.
+with $0 < \rho_{s}<1$
+where $\omega_s$ is the source strength parameter.
 This will result in exponential adaption of the core and wall
 density profile of the form $n_e \propto n_{prof}+(n_{prof}-n_{e,0})e^{-\omega_st}$.
 For ions we use
@@ -551,7 +552,6 @@ Note that Eq.~\eqref{eq:ion_source} is explicitly chosen as to avoid vorticity g
 by the particle source (cf.~Section~\ref{sec:conservation}). $S_{n_e}$ needs to be smooth
 so that $\nabla_\perp^2 S_{n_e}$ is well defined.
 
-
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
 Integrating the density equation we directly get
@@ -590,7 +590,7 @@ boundary)
   \nonumber\\ &
 +\left[T_i\left( 1+\ln{(N_i)}\right) +e \psi_i + \frac{1}{2} m_i U_i^2 \right](\Lambda_{N_i}+S_{N_i})
 \nonumber \\ &
-+ m_e u_e n_e \Lambda_{u_e}+m_iU_i N_i \Lambda_{U_i} - \eta_\parallel J_{\parallel,s}^2\bigg\}.
++ m_e u_e n_e \Lambda_{u_e}+m_iU_i N_i \Lambda_{U_i}- \eta_\parallel J_{\parallel,s}^2\bigg\}.
 \end{align}
 The energy consists of the Helmholtz free energy density for electrons and ions, the \(\vec{E} \times \vec{B}\) energy density, the parallel energy densities for electrons and ions and the perturbed magnetic field energy density.
 In \(\Lambda\) we insert the dissipative terms of Section~\ref{sec:dissres}. \\
@@ -634,8 +634,10 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
         -\tau U\nabla\cdot\vec{ \mathcal K_\kappa}\nonumber\\&
         - \left(2\tau + {\mu}U^2\right) \mathcal K_\kappa (U)
         -2\tau U\mathcal K_\kappa(\ln N)
-        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e)
-        + \nu_\perp\Delta_\perp U + \nu_\parallel \Delta_\parallel U ,
+        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
+        + \nu_\perp\Delta_\perp W
+        - \nu_\perp\frac{\beta}{\mu}\Delta_\perp A_\parallel
+        + \nu_\parallel \Delta_\parallel U,
         \label{eq:EgyrofluidU} \\
         W&:= \left( U + \frac{\beta}{\mu}A_\parallel\right)
     \end{align}
@@ -654,7 +656,8 @@ and
     A_\parallel &= N_iW_i-n_e w_e
   \end{align}
 \end{subequations}
-Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite.
+Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite. Also note the peculiar notation of the perpendicular viscosity on $U$.
+We indicate that $\Delta_\perp W$ is treated implicitly while the correction $\Delta_\perp A_\parallel$ is treated explicitly.
 The energy theorem reads $\partial_t \mathcal E + \mathcal S = \Lambda$ (with $z_e=-1$ and $z_i=+1$)
 \begin{align}
   \mathcal{E}= \int  \dV & \left( z_e\tau_e n_e \ln{(n_e)} +z_i\tau_i N_i\ln{(N_i)}
@@ -670,10 +673,12 @@ The energy theorem reads $\partial_t \mathcal E + \mathcal S = \Lambda$ (with $z
 +z_i\left[\tau_i\left( 1+\ln{(N_i)}\right) + \psi_i + \frac{1}{2} \mu_i U_i^2 \right]
     \left(\nu_\perp\Delta_\perp N_i + \nu_\parallel\Delta_\parallel N_i +S_{N_i}\right)
 \nonumber \\ &
-    +z_e\mu_e u_e n_e \left(\nu_\perp\Delta_\perp w_e + \nu_\parallel \Delta_\parallel u_e\right)\nonumber \\ &
-    +z_i\mu_iU_i N_i \left(\nu_\perp\Delta_\perp W_i + \nu_\parallel \Delta_\parallel U_i\right)% \nonumber \\ &
+    +z_e\mu_e u_e n_e \left(\nu_\perp\Delta_\perp u_e + \nu_\parallel \Delta_\parallel u_e\right)\nonumber \\ &
+    +z_i\mu_iU_i N_i \left(\nu_\perp\Delta_\perp U_i + \nu_\parallel \Delta_\parallel U_i\right)% \nonumber \\ &
     - \eta \left(n_e(U_i-u_e)\right)^2\bigg\}.
 \end{align}
+Replace $\Delta_\perp$ with $-\Delta_\perp^2$ when hyperviscous diffusion is chosen
+for the diffusion terms in the above equations.
 \subsection{Manufactured Solution}
 In order to test the implementation we manufacture a solution to Eqs.~\eqref{eq:Egyrofluid} and \eqref{eq:elliptic} of the form
 \begin{align*}
@@ -755,11 +760,10 @@ nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$ \\
 resistivity & float &1e-4  & - & parallel resistivity parameter Eq.~\eqref{eq:resistivity}\\
 curvmode  & string & "low beta"  & "toroidal"& curvature mode ("low beta", "true" no approximation, "toroidal": toroidal field approx) \\
 symmetric & bool & false & false & If true, initialize all quantities symmetric in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used to construct the parallel derivatives and then overwtiten to $N_z\equiv 1$. \\
-bc & dict & & & Dictionary of boundary conditions \ldots\\
+bc & dict & & & Dictionary of boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
 \qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y for $n_e$ and $N_i$\\
 \qquad velocity  & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $u_e$ and $U_i$\\
 \qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $\phi$ and $\psi$\\
-\qquad induction & char[2] & [DIR,DIR] & [DIR,DIR] & boundary conditions in x and y for $A_\parallel$ \\
     boxscaleR  & float[2] & [1.1,1.1]     & [1.05,1.05] & $[\varepsilon^{R-}, \varepsilon^{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
     boxscaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon^{Z-}, \varepsilon^{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box} \\
 initne    & string & "turbulence"     & "blob"  & initial condition for the
@@ -778,10 +782,8 @@ sigma\_z    & float &0.25   & - & variance in units of $R_0$  \\
 k\_psi     & float &0    & - & zonal mode wave number  \\
 nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} \\
 source  & float & 0     & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
-damping  & float & 0     & 0 & profile damping rate $\omega_d$ in Eq.~\eqref{eq:electron_source} \\
 alpha     & float  & 0.02 & - & Width $\alpha$ of Heaviside profile in Eq.~\eqref{eq:heaviside_profile} \\
 rho\_source & float  & 0.2   & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source}  \\
-rho\_damping & float  & 1.1   & 1.1 & Damping region boundary $0<\rho_{d}<1$ in Eq.~\eqref{eq:electron_source}  \\
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 44382f129..dc6b73d02 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -140,14 +140,15 @@ int main( int argc, char* argv[])
         //then shift tanh
         p.rho_source-3.*p.alpha, p.alpha, -1.), grid);
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
-    if( p.omega_source != 0)
-        feltor.set_source( p.omega_source, profile, source_damping);
 
     HVec profile_damping = dg::pullback( dg::geo::TanhDamping(
         mag.psip(), -3.*p.alpha, p.alpha, -1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
+    if( p.omega_source != 0 )
+        feltor.set_source( profile, p.omega_source, source_damping);
+
     //Now perturbation
     HVec ntilde = dg::evaluate(dg::zero,grid);
     if( p.initne == "blob" || p.initne == "straight blob")
@@ -157,9 +158,21 @@ int main( int argc, char* argv[])
         if( p.symmetric)
             ntilde = dg::pullback( init0, grid);
         else if( p.initne == "blob")//rounds =3 ->2*3-1
-            ntilde = feltor.fieldalignedn( init0, gaussianZ, (unsigned)p.Nz/2, 3);
+        {
+            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+            //evaluate should always be used with mx,my > 1
+            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 3);
+        }
         else if( p.initne == "straight blob")//rounds =1 ->2*1-1
-            ntilde = feltor.fieldalignedn( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+        {
+            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+            //evaluate should always be used with mx,my > 1
+            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+        }
     }
     else if( p.initne == "turbulence")
     {
@@ -168,7 +181,13 @@ int main( int argc, char* argv[])
         if( p.symmetric)
             ntilde = dg::pullback( init0, grid);
         else
-            ntilde = feltor.fieldalignedn( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+        {
+            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+            //evaluate should always be used with mx,my > 1
+            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+        }
         dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
     }
     else if( p.initne == "zonal")
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index 8be4e7090..796bbd6b8 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -34,8 +34,7 @@
     {
         "density" : ["DIR", "DIR"],
         "velocity": ["DIR", "DIR"],
-        "potential":["DIR", "DIR"],
-        "induction":["DIR", "DIR"]
+        "potential":["DIR", "DIR"]
     },
     "boxscaleR" :  [1.1,1.1],
     "boxscaleZ" :  [1.2,1.1],
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 82c017af9..bf2fe3730 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -27,7 +27,7 @@ struct Parameters
     std::array<double,2> mu; // mu[0] = mu_e, m[1] = mu_i
     std::array<double,2> tau; // tau[0] = -1, tau[1] = tau_i
     double alpha, beta;
-    double rho_source, rho_damping;
+    double rho_source;
 
     double nu_perp, nu_parallel;
     double c;
@@ -38,12 +38,12 @@ struct Parameters
     double sigma_z;
     double k_psi;
 
-    double omega_source, omega_damping;
+    double omega_source;
     double nprofamp;
     double boxscaleRm, boxscaleRp;
     double boxscaleZm, boxscaleZp;
 
-    enum dg::bc bcxN, bcyN, bcxU, bcyU, bcxP, bcyP, bcxA, bcyA;
+    enum dg::bc bcxN, bcyN, bcxU, bcyU, bcxP, bcyP;
     std::string initne, initphi, curvmode, perp_diff;
     bool symmetric;
     Parameters( const Json::Value& js) {
@@ -88,10 +88,8 @@ struct Parameters
 
         nprofamp  = js["nprofileamp"].asDouble();
         omega_source  = js.get("source", 0.).asDouble();
-        omega_damping = js.get("damping",0.).asDouble();
         alpha        = js.get("alpha", 0.02).asDouble();
         rho_source   = js.get("rho_source", 0.2).asDouble();
-        rho_damping  = js.get("rho_damping", 1.1).asDouble();
 
         bcxN = dg::str2bc(js["bc"]["density"][0].asString());
         bcyN = dg::str2bc(js["bc"]["density"][1].asString());
@@ -99,8 +97,6 @@ struct Parameters
         bcyU = dg::str2bc(js["bc"]["velocity"][1].asString());
         bcxP = dg::str2bc(js["bc"]["potential"][0].asString());
         bcyP = dg::str2bc(js["bc"]["potential"][1].asString());
-        bcxA = dg::str2bc(js["bc"]["indcution"].get(0u, "DIR").asString());
-        bcyA = dg::str2bc(js["bc"]["indcution"].get(1u, "DIR").asString());
 
         boxscaleRm  = js["boxscaleR"].get(0u,1.05).asDouble();
         boxscaleRp  = js["boxscaleR"].get(1u,1.05).asDouble();
@@ -137,9 +133,7 @@ struct Parameters
             <<"    init Phi:     "<<initphi<<"\n";
         os << "Profile parameters are: \n"
             <<"     omega_source:                 "<<omega_source<<"\n"
-            <<"     omega_damping:                "<<omega_damping<<"\n"
             <<"     rho_source:                   "<<rho_source<<"\n"
-            <<"     rho_damping:                  "<<rho_damping<<"\n"
             <<"     alpha:                        "<<alpha<<"\n"
             <<"     density profile amplitude:    "<<nprofamp<<"\n"
             <<"     boxscale R+:                  "<<boxscaleRp<<"\n"
@@ -172,9 +166,7 @@ struct Parameters
             <<"     bc velocity x  = "<<dg::bc2str(bcxU)<<"\n"
             <<"     bc velocity y  = "<<dg::bc2str(bcyU)<<"\n"
             <<"     bc potential x = "<<dg::bc2str(bcxP)<<"\n"
-            <<"     bc potential y = "<<dg::bc2str(bcyP)<<"\n"
-            <<"     bc induction x = "<<dg::bc2str(bcxA)<<"\n"
-            <<"     bc induction y = "<<dg::bc2str(bcyA)<<"\n";
+            <<"     bc potential y = "<<dg::bc2str(bcyP)<<"\n";
         os << std::flush;
     }
 };
-- 
GitLab


From 60801e2c9860f001cece55e01c59b253559c54a1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 18 Jan 2019 17:46:55 +0100
Subject: [PATCH 005/540] Fix symmetric parameter in glfw feltor.cu

---
 src/feltor/feltor.cu | 44 +++++++++++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index e57119c96..9224fda37 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -43,7 +43,7 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
     //Make grid
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
-        p.n, p.Nx, p.Ny, p.Nz, p.bcxN, p.bcyN, dg::PER);
+        p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
 
     //create RHS
@@ -152,7 +152,7 @@ int main( int argc, char* argv[])
     std::map<std::string, const dg::DVec* > v4d;
     v4d["ne-1 / "] = &y0[0][0],               v4d["ni-1 / "] = &y0[0][1];
     v4d["Ue / "]   = &feltor.fields()[1][0],  v4d["Ui / "]   = &feltor.fields()[1][1];
-    v4d["Omega / "] = &feltor.potential()[0]; v4d["Apar / "] = &feltor.induction();
+    v4d["Phi / "] = &feltor.potential()[0]; v4d["Apar / "] = &feltor.induction();
     const feltor::Quantities& q = feltor.quantities();
     double dEdt = 0, accuracy = 0, dMdt = 0, accuracyM  = 0;
     std::map<std::string, const double*> v0d{
@@ -201,8 +201,11 @@ int main( int argc, char* argv[])
     is >> js;
     is.close();
     unsigned red = js.get("reduction", 1).asUInt();
-    GLFWwindow* w = draw::glfwInitAndCreateWindow( (p.Nz/red+1)*js["width"].asDouble(), js["rows"].asDouble()*js["height"].asDouble(), "");
-    draw::RenderHostData render(js["rows"].asDouble(), p.Nz/red + 1);
+    double rows = js["rows"].asDouble(), cols = p.Nz/red+1,
+           width = js["width"].asDouble(), height = js["height"].asDouble();
+    if ( p.symmetric ) cols = rows, rows = 1;
+    GLFWwindow* w = draw::glfwInitAndCreateWindow( cols*width, rows*height, "");
+    draw::RenderHostData render(rows, cols);
 
     std::cout << "Begin computation \n";
     std::cout << std::scientific << std::setprecision( 2);
@@ -212,10 +215,11 @@ int main( int argc, char* argv[])
     {
         for( auto pair : v4d)
         {
-            if(pair.first == "Omega / ")
+            if(pair.first == "Phi / ")
             {
-                dg::blas2::gemv( laplacianM, *pair.second, dvisual);
-                dg::assign( dvisual, hvisual);
+                //dg::blas2::gemv( laplacianM, *pair.second, dvisual);
+                //dg::assign( dvisual, hvisual);
+                dg::assign( *pair.second, hvisual);
             }
             else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
             {
@@ -229,18 +233,24 @@ int main( int argc, char* argv[])
                 visual.begin(), visual.end(), 0., thrust::maximum<double>() );
             colors.scalemin() = -colors.scalemax();
             title <<pair.first << colors.scalemax()<<"\t";
-            for( unsigned k=0; k<p.Nz/red;k++)
+            if ( p.symmetric )
+                render.renderQuad( hvisual, grid.n()*grid.Nx(),
+                                            grid.n()*grid.Ny(), colors);
+            else
             {
-                unsigned size=grid.n()*grid.n()*grid.Nx()*grid.Ny();
-                dg::HVec part( visual.begin() +  k*red   *size,
-                               visual.begin() + (k*red+1)*size);
-                render.renderQuad( part, grid.n()*grid.Nx(),
-                                         grid.n()*grid.Ny(), colors);
+                for( unsigned k=0; k<p.Nz/red;k++)
+                {
+                    unsigned size=grid.n()*grid.n()*grid.Nx()*grid.Ny();
+                    dg::HVec part( visual.begin() +  k*red   *size,
+                                   visual.begin() + (k*red+1)*size);
+                    render.renderQuad( part, grid.n()*grid.Nx(),
+                                             grid.n()*grid.Ny(), colors);
+                }
+                dg::blas1::scal(avisual,0.);
+                toroidal_average(visual,avisual);
+                render.renderQuad( avisual, grid.n()*grid.Nx(),
+                                            grid.n()*grid.Ny(), colors);
             }
-            dg::blas1::scal(avisual,0.);
-            toroidal_average(visual,avisual);
-            render.renderQuad( avisual, grid.n()*grid.Nx(),
-                                        grid.n()*grid.Ny(), colors);
         }
         title << std::fixed;
         title << " &&   time = "<<time;
-- 
GitLab


From bea0f6a93d458342410603251f8f8218f8d8c5bf Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 18 Jan 2019 19:07:01 +0100
Subject: [PATCH 006/540] Show time first in feltor and fix make clean

---
 src/feltor/Makefile  | 2 +-
 src/feltor/feltor.cu | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index f5d5175cf..7c18d4d16 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -25,4 +25,4 @@ feltor_mpi: feltor_hpc.cu feltor.cuh
 .PHONY: clean
 
 clean:
-	rm -f feltor_hpc feltor_mpi manufactured
+	rm -f feltor feltor_hpc feltor_mpi manufactured
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 9224fda37..567446403 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -213,6 +213,8 @@ int main( int argc, char* argv[])
     title << std::setprecision(2) << std::scientific;
     while ( !glfwWindowShouldClose( w ))
     {
+        title << std::fixed;
+        title << "t = "<<time<<"   ";
         for( auto pair : v4d)
         {
             if(pair.first == "Phi / ")
@@ -252,8 +254,6 @@ int main( int argc, char* argv[])
                                             grid.n()*grid.Ny(), colors);
             }
         }
-        title << std::fixed;
-        title << " &&   time = "<<time;
         glfwSetWindowTitle(w,title.str().c_str());
         title.str("");
         glfwPollEvents();
-- 
GitLab


From b472e44b760123fc875119c72df4aa3910bce640 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 21 Jan 2019 14:30:19 +0100
Subject: [PATCH 007/540] Add parallel friction terms

- and make Resistivity implicit
- and friction implicit
---
 src/feltor/feltor.cu          |  11 +++-
 src/feltor/feltor.cuh         | 111 ++++++++++++++++++++++++----------
 src/feltor/feltor.tex         |  37 ++++++++----
 src/feltor/feltor_hpc.cu      |  12 +++-
 src/feltor/input/default.json |   6 +-
 src/feltor/parameters.h       |  14 +++--
 6 files changed, 133 insertions(+), 58 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 567446403..452a32be9 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -66,6 +66,11 @@ int main( int argc, char* argv[])
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
         //then shift tanh
         p.rho_source-3.*p.alpha, p.alpha, -1.), grid);
+    dg::HVec damping_damping = dg::pullback(dg::geo::TanhDamping(
+        //first change coordinate from psi to (psi_0 - psip)/psi_0
+        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
+        //then shift tanh
+        p.rho_damping, p.alpha, 1.), grid);
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
     dg::HVec profile_damping = dg::pullback( dg::geo::TanhDamping(
@@ -73,8 +78,10 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
-    if( p.omega_source != 0 )
-        feltor.set_source( profile, p.omega_source, source_damping);
+    feltor.set_source( profile, p.omega_source, source_damping,
+        p.omega_damping, damping_damping);
+    im.set_damping( p.omega_damping, damping_damping);
+
 
     //Now perturbation
     dg::HVec ntilde = dg::evaluate(dg::zero,grid);
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index a4d16d052..b27d95058 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -21,9 +21,9 @@ struct AddResistivity{
     DG_DEVICE
     void operator()( double ne, double ni, double ue,
         double ui, double& dtUe, double& dtUi) const{
-        double current = ne*(ui-ue);
+        double current = (ne+1)*(ui-ue);
         dtUe += -m_C/m_mu[0] * current;
-        dtUi += -m_C/m_mu[1] * ne/ni * current;
+        dtUi += -m_C/m_mu[1] * (ne+1)/(ni+1) * current;
     }
     private:
     double m_C;
@@ -220,6 +220,24 @@ struct Implicit
 #else
         dg::blas1::copy( 0, yp);
 #endif
+        //------------------Add Resistivity--------------------------//
+        //Note that this works because N converges independently of W
+        //and the term is linear in W for fixed N
+        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+            y[0][0], y[0][1], y[1][0], y[1][1], yp[1][0], yp[1][1]);
+        //------------------Add Friction-----------------------------//
+        if( m_omega_damping != 0)
+        {
+            dg::blas1::pointwiseDot( -m_omega_damping, m_damping, y[1][0],
+                1., yp[1][0]);
+            dg::blas1::pointwiseDot( -m_omega_damping, m_damping, y[1][1],
+                1., yp[1][1]);
+        }
+    }
+
+    void set_damping( double omega_damping, container damping){
+        m_damping = damping;
+        m_omega_damping = omega_damping;
     }
 
     const container& weights() const{
@@ -233,8 +251,9 @@ struct Implicit
     }
 
   private:
+    double m_omega_damping=0;
     const feltor::Parameters m_p;
-    container m_temp;
+    container m_temp, m_damping;
     dg::Elliptic3d<Geometry, Matrix, container> m_lapM_perpN, m_lapM_perpU;
 };
 
@@ -247,7 +266,7 @@ struct Quantities
     //resisitive and diffusive terms
     double Dres = 0, Dpar[4] = {0,0,0,0}, Dperp[4] = {0,0,0,0};
     double aligned = 0; //alignment parameter
-    double source[2] = {0,0}; //source terms
+    double source[4] = {0,0,0,0}; //source terms
     void display( std::ostream& os = std::cout ) const
     {
         os << "Quantities: \n"
@@ -259,7 +278,7 @@ struct Quantities
            << "    Dres: "<<Dres<<"\n"
            << "    Dpar: ["<<Dpar[0]<<", "<<Dpar[1]<<", "<<Dpar[2]<<", "<<Dpar[3]<<"]\n"
            << "   Dperp: ["<<Dperp[0]<<", "<<Dperp[1]<<", "<<Dperp[2]<<", "<<Dperp[3]<<"]\n"
-           << " Sources: ["<<source[0]<<", "<<source[1]<<"]\n"
+           << " Sources: ["<<source[0]<<", "<<source[1]<<", "<<source[2]<<", "<<source[3]<<"]\n"
            << " aligned: "<<aligned<<"\n";
     }
 };
@@ -303,16 +322,19 @@ struct Explicit
     const std::array<std::array<container,2>,2>& fields() const{
         return m_fields;
     }
-    const std::array<container,2>& sources() const{
+    const std::array<std::array<container,2>,2>& sources() const{
         return m_s;
     }
 
     //source strength, profile - 1
-    void set_source( container profile, double omega_source, container source)
+    void set_source( container profile, double omega_source, container source,
+        double omega_damping, container damping)
     {
         m_profne = profile;
         m_omega_source = omega_source;
         m_source = source;
+        m_omega_damping = omega_damping;
+        m_damping = damping;
     }
   private:
     void compute_phi( double t, const std::array<container,2>& y);
@@ -347,14 +369,14 @@ struct Explicit
     std::array<container,3> m_curv, m_curvKappa, m_b;
     container m_divCurvKappa;
     container m_binv, m_divb;
-    container m_source, m_profne;
+    container m_source, m_profne, m_damping;
     container m_vol3d;
 
     container m_apar, m_dxA, m_dyA, m_dzA;
     std::array<container,2> m_phi, m_dxPhi, m_dyPhi, m_dzPhi;
     std::array<container,2> m_logn, m_dxN, m_dyN, m_dzN, m_dsN;
-    std::array<container,2> m_dxU, m_dyU, m_dzU, m_dsU, m_s;
-    std::array<std::array<container,2>,2> m_fields;
+    std::array<container,2> m_dxU, m_dyU, m_dzU, m_dsU;
+    std::array<std::array<container,2>,2> m_fields, m_s;
 
     std::vector<container> m_multi_chi;
 
@@ -373,7 +395,7 @@ struct Explicit
 
     const feltor::Parameters m_p;
     Quantities m_q;
-    double m_omega_source = 0.;
+    double m_omega_source = 0., m_omega_damping = 0.;
 
 };
 
@@ -507,7 +529,7 @@ Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g,
     m_UE2 = m_temp2 = m_temp1 = m_temp0;
     m_phi[0] = m_phi[1] = m_temp0;
     m_dxPhi = m_dyPhi = m_dzPhi = m_fields[0] = m_fields[1] = m_logn = m_phi;
-    m_s = m_phi;
+    m_s = m_fields;
     m_dxN = m_dyN = m_dzN = m_dsN = m_dxU = m_dyU = m_dzU = m_dsU = m_phi;
     m_apar = m_dxA = m_dyA = m_dzA = m_phi[0];
     //--------------------------Construct-------------------------//
@@ -570,7 +592,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_phi(
 #ifdef DG_MANUFACTURED
         dg::blas1::copy( y[1], m_temp1);
         dg::blas1::evaluate( m_temp1, dg::plus_equals(), manufactured::SGammaNi{
-            m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.c,
+            m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
             m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
         std::vector<unsigned> numberG = m_multigrid.direct_solve(
             m_multi_invgammaN, m_temp0, m_temp1, m_p.eps_gamma);
@@ -585,7 +607,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_phi(
     }
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( m_temp0, dg::plus_equals(), manufactured::SPhie{
-        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.c,
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
 #endif //DG_MANUFACTURED
     //----------Invert polarisation----------------------------//
@@ -603,7 +625,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_phi(
 #ifdef DG_MANUFACTURED
         dg::blas1::copy( m_phi[0], m_temp0);
         dg::blas1::evaluate( m_temp0, dg::plus_equals(), manufactured::SGammaPhie{
-            m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.c,
+            m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
             m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
         std::vector<unsigned> number = m_multigrid.direct_solve(
             m_multi_invgammaP, m_phi[1], m_temp0, m_p.eps_gamma);
@@ -632,7 +654,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_psi(
         m_UE2, m_temp0, m_temp1, m_binv);
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( m_phi[1], dg::plus_equals(), manufactured::SPhii{
-        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.c,
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
 #endif //DG_MANUFACTURED
     //m_UE2 now contains u_E^2; also update derivatives
@@ -661,7 +683,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_apar(
     m_old_apar.extrapolate( time, m_apar);
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( m_temp0, dg::plus_equals(), manufactured::SA{
-        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.c,
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
 #endif //DG_MANUFACTURED
     std::vector<unsigned> number = m_multigrid.direct_solve(
@@ -824,7 +846,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_dissipation(
     m_q.aligned = dg::blas2::dot( m_logn[0], m_vol3d, m_dsN[0]);
     /////////////////DISSIPATION TERMS//////////////////////////////
     m_q.diff = m_p.nu_parallel*dg::blas1::dot( m_vol3d, m_dsN[0]);
-    m_q.diff += dg::blas1::dot( m_vol3d, m_s[0]); //particle sources
+    m_q.diff += dg::blas1::dot( m_vol3d, m_s[0][0]); //particle sources
     // energy dissipation through diffusion
     double z[2] = {-1.0,1.0};
     for( unsigned i=0; i<2;i++)
@@ -834,7 +856,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_dissipation(
         dg::blas1::subroutine( routines::ComputeDiss(z[i], m_p.mu[i], m_p.tau[i]),
                 m_temp2, m_logn[i], m_phi[i], fields[1][i]);
         // Dissipation through sink/source terms
-        m_q.source[i] = dg::blas2::dot( m_temp2, m_vol3d, m_s[i]);
+        m_q.source[i] = dg::blas2::dot( m_temp2, m_vol3d, m_s[0][i]);
         // parallel dissipation for N: nu_parallel *(Delta_s N)
         m_q.Dpar[i] = m_p.nu_parallel*dg::blas2::dot(
                         m_temp2, m_vol3d, m_dsN[i]);
@@ -858,6 +880,8 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_dissipation(
         //Z*mu*N*U
         dg::blas1::pointwiseDot( z[i]*m_p.mu[i], fields[0][i], fields[1][i],
                 0, m_temp2);
+        // Dissipation through sink/source terms
+        m_q.source[i+2] = dg::blas2::dot( m_temp2, m_vol3d, m_s[1][i]);
         // parallel dissipation for U: nu_parallel *(Delta_s U)
         m_q.Dpar[i+2] = m_p.nu_parallel*dg::blas2::dot(
             m_temp2, m_vol3d, m_dsU[i]);
@@ -877,7 +901,7 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_dissipation(
     // resistive energy (quadratic current): -C (n_e (U_i-u_e))**2
     dg::blas1::pointwiseDot(1., fields[0][0], fields[1][1],
         -1., fields[0][0], fields[1][0], 0., m_temp0);
-    m_q.Dres = -m_p.c*dg::blas2::dot(m_temp0, m_vol3d, m_temp0);
+    m_q.Dres = -m_p.eta*dg::blas2::dot(m_temp0, m_vol3d, m_temp0);
     m_q.ediff = m_q.Dres + m_q.source[0] + m_q.source[1]
         + m_q.Dpar[0]+m_q.Dperp[0]+m_q.Dpar[1]+m_q.Dperp[1]
         + m_q.Dpar[2]+m_q.Dperp[2]+m_q.Dpar[3]+m_q.Dperp[3];
@@ -915,10 +939,23 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
 
     // Set perpendicular dynamics in yp
     compute_perp( t, y, m_fields, yp);
-    //------------------Add Resistivity--------------------------//
-    dg::blas1::subroutine( routines::AddResistivity( m_p.c, m_p.mu),
-        m_fields[0][0], m_fields[0][1], m_fields[1][0], m_fields[1][1],
-        yp[1][0], yp[1][1]);
+    //------------------Add Apar correction terms----------------//
+    if( m_p.beta != 0 )
+    {
+        dg::blas1::pointwiseDot( 1., m_fields[0][0],m_fields[0][0],m_apar,
+                                 0., m_temp0);
+        dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[0]*(
+            1./m_p.mu[0]-1./m_p.mu[1]), m_temp0, m_fields[0][0], 1., yp[1][0]);
+        dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[1]*(
+            1./m_p.mu[0]-1./m_p.mu[1]), m_temp0, m_fields[0][1], 1., yp[1][1]);
+        if( m_omega_damping != 0)
+        {
+            dg::blas1::pointwiseDot( m_omega_damping*m_p.beta/m_p.mu[0],
+                m_damping, m_apar, 1., yp[1][0]);
+            dg::blas1::pointwiseDot( m_omega_damping*m_p.beta/m_p.mu[1],
+                m_damping, m_apar, 1., yp[1][1]);
+        }
+    }
 
 #else
 
@@ -934,27 +971,35 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
     //Add source terms
     if( m_omega_source != 0)
     {
-        dg::blas1::subroutine( routines::ComputeSource(), m_s[0], y[0][0],
+        dg::blas1::subroutine( routines::ComputeSource(), m_s[0][0], y[0][0],
             m_profne, m_source, m_omega_source);
         //compute FLR correction
-        dg::blas2::gemv( m_lapperpN, m_s[0], m_temp0);
-        dg::blas1::axpby( 1., m_s[0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[1]);
+        dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
+        dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
 
-        dg::blas1::axpby( 1., m_s[0], 1.0, yp[0][0]);
-        dg::blas1::axpby( 1., m_s[1], 1.0, yp[0][1]);
+        dg::blas1::axpby( 1., m_s[0][0], 1.0, yp[0][0]);
+        dg::blas1::axpby( 1., m_s[0][1], 1.0, yp[0][1]);
+    }
+    if( m_omega_damping != 0)
+    {
+        //just compute m_s terms for energies, terms are implicit
+        dg::blas1::pointwiseDot( -m_omega_damping, m_fields[1][0],
+            m_damping, 0., m_s[1][0]);
+        dg::blas1::pointwiseDot( -m_omega_damping, m_fields[1][1],
+            m_damping, 0., m_s[1][1]);
     }
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( yp[0][0], dg::plus_equals(), manufactured::SNe{
-        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.c,
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
     dg::blas1::evaluate( yp[0][1], dg::plus_equals(), manufactured::SNi{
-        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.c,
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
     dg::blas1::evaluate( yp[1][0], dg::plus_equals(), manufactured::SUe{
-        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.c,
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
     dg::blas1::evaluate( yp[1][1], dg::plus_equals(), manufactured::SUi{
-        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.c,
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
 #endif //DG_MANUFACTURED
     timer.toc();
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 19efa2ce1..42074c22d 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -67,11 +67,11 @@ we can define various differential operations.
     $\Delta_\parallel f:= \vec{\nabla} \cdot ( \bhat\bhat\cdot\vec{\nabla} f )$\\
 \bottomrule
 \end{longtable}
-with $b^i$ the contra- and $b_i$ the co-variant components of $\bhat$, and 
+with $b^i$ the contra- and $b_i$ the co-variant components of $\bhat$, and
 $\eps^{ijk}$ the Levi-Civita symbols.
 Explicit expressions for the above expressions
 depend on the choice of the magnetic field and the underlying coordinate system.
-    Note that we have
+Note that we have
 \begin{align}
     \vec \nabla \cdot \vec{\mathcal K_\kappa}
     &= -\vec\nabla \cdot \vec{\mathcal K_{\nabla B}} = -\vec{ \mathcal K_\kappa}\cdot\nabla\ln B, \\
@@ -383,7 +383,7 @@ mN \frac{\partial}{\partial t} U &+ mN \left(
     &+ 2m\vec \nabla \cdot ( NU \vec v_\kappa)
     -mNU\vec \nabla\cdot \vec v_\kappa
     + mNU\mathcal K_\kappa(\phi) \nonumber\\
-    &= -T (\bhat + \tilde{\vec b})\cdot \nabla N -qN \left( (\bhat+\tilde{\vec b})\cdot \nabla \psi + \frac{\partial A_\parallel}{\partial t}\right) + mN R_{\eta_\parallel} + mN\Lambda_U
+    &= -T (\bhat + \tilde{\vec b})\cdot \nabla N -qN \left( (\bhat+\tilde{\vec b})\cdot \nabla \psi + \frac{\partial A_\parallel}{\partial t}\right) + mN R_{\eta_\parallel} + mN(\Lambda_U + S_U)
 \label{}
 \end{align}
 with
@@ -531,7 +531,7 @@ wavelength $k_\psi$ aligned with the magnetic flux surfaces.
 \end{align}
 
 \subsection{Sinks and sources} \label{sec:sources}
-The idea for the source terms $S$ is to fix the profile $n_{prof}$ in the
+The idea for the source terms $S_N$ is to fix the profile $n_{prof}$ in the
 core of our domain, where our model does not apply.
 We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
@@ -552,6 +552,15 @@ Note that Eq.~\eqref{eq:ion_source} is explicitly chosen as to avoid vorticity g
 by the particle source (cf.~Section~\ref{sec:conservation}). $S_{n_e}$ needs to be smooth
 so that $\nabla_\perp^2 S_{n_e}$ is well defined.
 
+The idea for the terms $S_U$ is mainly to provide more numerical stability
+in the corner regions of the domain, where the parallel derivative may lead
+to unfavourable numerical instabilities.
+For both electrons and ions we choose
+\begin{align} \label{eq:velocity_source}
+  S_{U}(R,Z,\varphi, t) := -\omega_d U \Theta( \rho(R,Z) - \rho_d)
+\end{align}
+with $\rho_d > 1$.
+
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
 Integrating the density equation we directly get
@@ -590,7 +599,7 @@ boundary)
   \nonumber\\ &
 +\left[T_i\left( 1+\ln{(N_i)}\right) +e \psi_i + \frac{1}{2} m_i U_i^2 \right](\Lambda_{N_i}+S_{N_i})
 \nonumber \\ &
-+ m_e u_e n_e \Lambda_{u_e}+m_iU_i N_i \Lambda_{U_i}- \eta_\parallel J_{\parallel,s}^2\bigg\}.
++ m_e u_e n_e (\Lambda_{u_e}+S_{u_e})+m_iU_i N_i (\Lambda_{U_i}+S_{U_i})- \eta_\parallel J_{\parallel,s}^2\bigg\}.
 \end{align}
 The energy consists of the Helmholtz free energy density for electrons and ions, the \(\vec{E} \times \vec{B}\) energy density, the parallel energy densities for electrons and ions and the perturbed magnetic field energy density.
 In \(\Lambda\) we insert the dissipative terms of Section~\ref{sec:dissres}. \\
@@ -637,7 +646,7 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
         - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
         + \nu_\perp\Delta_\perp W
         - \nu_\perp\frac{\beta}{\mu}\Delta_\perp A_\parallel
-        + \nu_\parallel \Delta_\parallel U,
+        + \nu_\parallel \Delta_\parallel U + S_U,
         \label{eq:EgyrofluidU} \\
         W&:= \left( U + \frac{\beta}{\mu}A_\parallel\right)
     \end{align}
@@ -673,8 +682,8 @@ The energy theorem reads $\partial_t \mathcal E + \mathcal S = \Lambda$ (with $z
 +z_i\left[\tau_i\left( 1+\ln{(N_i)}\right) + \psi_i + \frac{1}{2} \mu_i U_i^2 \right]
     \left(\nu_\perp\Delta_\perp N_i + \nu_\parallel\Delta_\parallel N_i +S_{N_i}\right)
 \nonumber \\ &
-    +z_e\mu_e u_e n_e \left(\nu_\perp\Delta_\perp u_e + \nu_\parallel \Delta_\parallel u_e\right)\nonumber \\ &
-    +z_i\mu_iU_i N_i \left(\nu_\perp\Delta_\perp U_i + \nu_\parallel \Delta_\parallel U_i\right)% \nonumber \\ &
+    +z_e\mu_e u_e n_e \left(\nu_\perp\Delta_\perp u_e + \nu_\parallel \Delta_\parallel u_e + S_{u_e}\right)\nonumber \\ &
+    +z_i\mu_i U_i N_i \left(\nu_\perp\Delta_\perp U_i + \nu_\parallel \Delta_\parallel U_i + S_{U_i}\right)% \nonumber \\ &
     - \eta \left(n_e(U_i-u_e)\right)^2\bigg\}.
 \end{align}
 Replace $\Delta_\perp$ with $-\Delta_\perp^2$ when hyperviscous diffusion is chosen
@@ -714,7 +723,7 @@ discontinuous Galerkin on structured grid
 Helmholtz and Elliptic matrix inversions & multigrid/ conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
 Parallel derivatives & refined  FCI & cf.~\cite{Held2016,Stegmeir2017} \\
 Curvature terms & direct dG & regular dG approximation of derivatives \\
-time & Adaptive ARKode stepper & $3rd$ order explicit, perpendicular diffusion $3rd$ order implicit \\
+time & Adaptive ARKode stepper & $3rd$ order explicit, perpendicular diffusion and resistance/friction $3rd$ order implicit \\
 \bottomrule
 \end{longtable}
 \section{Input/Output}
@@ -781,9 +790,11 @@ posY       & float &0.0    & - & blob Z-position in units of $a$ \\
 sigma\_z    & float &0.25   & - & variance in units of $R_0$  \\
 k\_psi     & float &0    & - & zonal mode wave number  \\
 nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} \\
-source  & float & 0     & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
-alpha     & float  & 0.02 & - & Width $\alpha$ of Heaviside profile in Eq.~\eqref{eq:heaviside_profile} \\
-rho\_source & float  & 0.2   & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source}  \\
+alpha       & float & 0.02 & - & Width $\alpha$ of Heaviside profile in Eq.~\eqref{eq:heaviside_profile} \\
+source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
+rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source}  \\
+damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
+rho\_damping& float & 0.2  & 1.2 & Friction region boundary $\rho_{d}>1$ in Eq.~\eqref{eq:velocity_source}  \\
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
@@ -827,7 +838,7 @@ z\_XYZ           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 Psip             & Dataset & 3 (z,y,x) & Flux function $\psi_p(R,Z)$ \\
 Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ \\
 Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta(\rho_{s} - \rho(R,Z)) H(Z-Z_X)$\\
-Damping          & Dataset & 3 (z,y,x) & Damping  profile $\Theta(\rho(R,Z) - \rho_{d} )$\\
+Damping          & Dataset & 3 (z,y,x) & Damping profile $\Theta(\rho(R,Z) - \rho_{d} )$\\
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index dc6b73d02..15e29fdf1 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -139,6 +139,11 @@ int main( int argc, char* argv[])
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
         //then shift tanh
         p.rho_source-3.*p.alpha, p.alpha, -1.), grid);
+    dg::HVec damping_damping = dg::pullback(dg::geo::TanhDamping(
+        //first change coordinate from psi to (psi_0 - psip)/psi_0
+        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
+        //then shift tanh
+        p.rho_damping, p.alpha, 1.), grid);
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
     HVec profile_damping = dg::pullback( dg::geo::TanhDamping(
@@ -146,8 +151,9 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
-    if( p.omega_source != 0 )
-        feltor.set_source( profile, p.omega_source, source_damping);
+    feltor.set_source( profile, p.omega_source, source_damping,
+        p.omega_damping, damping_damping);
+    im.set_damping( p.omega_damping, damping_damping);
 
     //Now perturbation
     HVec ntilde = dg::evaluate(dg::zero,grid);
@@ -258,7 +264,7 @@ int main( int argc, char* argv[])
         std::map<std::string, const HVec*> v3d{
             {"BR", &vecR}, {"BZ", &vecZ}, {"BP", &vecP},
             {"Psip", &psip}, {"Nprof", &profile },
-            {"Source", &source_damping }
+            {"Source", &source_damping }, {"Damping", &damping_damping}
         };
         IHMatrix project = dg::create::projection( grid_out, grid);
         HVec transferH( dg::evaluate(dg::zero, grid_out));
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index 796bbd6b8..b6a243c78 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -42,7 +42,9 @@
     "initphi"    : "zero",
     "curvmode"   : "true",
     "symmetric"  : false,
-    "source"     : 0,
     "alpha"      : 0.02,
-    "rho_source" : 0.2
+    "source"     : 0,
+    "rho_source" : 0.2,
+    "damping"     : 0,
+    "rho_daming" : 1.2
 }
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index bf2fe3730..739397d23 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -27,10 +27,10 @@ struct Parameters
     std::array<double,2> mu; // mu[0] = mu_e, m[1] = mu_i
     std::array<double,2> tau; // tau[0] = -1, tau[1] = tau_i
     double alpha, beta;
-    double rho_source;
+    double rho_source, rho_damping;
 
     double nu_perp, nu_parallel;
-    double c;
+    double eta;
 
     double amp;
     double sigma;
@@ -38,7 +38,7 @@ struct Parameters
     double sigma_z;
     double k_psi;
 
-    double omega_source;
+    double omega_source, omega_damping;
     double nprofamp;
     double boxscaleRm, boxscaleRp;
     double boxscaleZm, boxscaleZp;
@@ -77,7 +77,7 @@ struct Parameters
         nu_perp     = js["nu_perp"].asDouble();
         perp_diff   = js.get("perp_diff", "viscous").asString();
         nu_parallel = js["nu_parallel"].asDouble();
-        c           = js["resistivity"].asDouble();
+        eta         = js["resistivity"].asDouble();
 
         amp         = js["amplitude"].asDouble();
         sigma       = js["sigma"].asDouble();
@@ -88,8 +88,10 @@ struct Parameters
 
         nprofamp  = js["nprofileamp"].asDouble();
         omega_source  = js.get("source", 0.).asDouble();
+        omega_damping = js.get("damping", 0.).asDouble();
         alpha        = js.get("alpha", 0.02).asDouble();
         rho_source   = js.get("rho_source", 0.2).asDouble();
+        rho_damping  = js.get("rho_damping", 1.2).asDouble();
 
         bcxN = dg::str2bc(js["bc"]["density"][0].asString());
         bcyN = dg::str2bc(js["bc"]["density"][1].asString());
@@ -117,7 +119,7 @@ struct Parameters
             <<"     Ion-temperature   = "<<tau[1]<<"\n"
             <<"     perp. Viscosity   = "<<nu_perp<<"\n"
             <<"     perp. Viscosity   = "<<perp_diff<<"\n"
-            <<"     par. Resistivity  = "<<c<<"\n"
+            <<"     par. Resistivity  = "<<eta<<"\n"
             <<"     beta              = "<<beta<<"\n"
             <<"     par. Viscosity    = "<<nu_parallel<<"\n"
             <<"     curvature mode    = "<<curvmode<<"\n"
@@ -134,6 +136,8 @@ struct Parameters
         os << "Profile parameters are: \n"
             <<"     omega_source:                 "<<omega_source<<"\n"
             <<"     rho_source:                   "<<rho_source<<"\n"
+            <<"     omega_damping:                "<<omega_damping<<"\n"
+            <<"     rho_damping:                  "<<rho_damping<<"\n"
             <<"     alpha:                        "<<alpha<<"\n"
             <<"     density profile amplitude:    "<<nprofamp<<"\n"
             <<"     boxscale R+:                  "<<boxscaleRp<<"\n"
-- 
GitLab


From 317d11381435b35553741e2524c096da06660ea6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 21 Jan 2019 15:05:25 +0100
Subject: [PATCH 008/540] Correct sign of Apar correction

---
 src/feltor/feltor.cuh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index b27d95058..f0b1b13d4 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -15,18 +15,18 @@ namespace routines{
 //Resistivity (consistent density dependency,
 //parallel momentum conserving, quadratic current energy conservation dependency)
 struct AddResistivity{
-    AddResistivity( double C, std::array<double,2> mu): m_C(C){
+    AddResistivity( double eta, std::array<double,2> mu): m_eta(eta){
         m_mu[0] = mu[0], m_mu[1] = mu[1];
     }
     DG_DEVICE
     void operator()( double ne, double ni, double ue,
         double ui, double& dtUe, double& dtUi) const{
         double current = (ne+1)*(ui-ue);
-        dtUe += -m_C/m_mu[0] * current;
-        dtUi += -m_C/m_mu[1] * (ne+1)/(ni+1) * current;
+        dtUe += -m_eta/m_mu[0] * current;
+        dtUi += -m_eta/m_mu[1] * (ne+1)/(ni+1) * current;
     }
     private:
-    double m_C;
+    double m_eta;
     double m_mu[2];
 };
 struct ComputePerpDrifts{
@@ -945,9 +945,9 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
         dg::blas1::pointwiseDot( 1., m_fields[0][0],m_fields[0][0],m_apar,
                                  0., m_temp0);
         dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[0]*(
-            1./m_p.mu[0]-1./m_p.mu[1]), m_temp0, m_fields[0][0], 1., yp[1][0]);
+            1./m_p.mu[1]-1./m_p.mu[0]), m_temp0, m_fields[0][0], 1., yp[1][0]);
         dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[1]*(
-            1./m_p.mu[0]-1./m_p.mu[1]), m_temp0, m_fields[0][1], 1., yp[1][1]);
+            1./m_p.mu[1]-1./m_p.mu[0]), m_temp0, m_fields[0][1], 1., yp[1][1]);
         if( m_omega_damping != 0)
         {
             dg::blas1::pointwiseDot( m_omega_damping*m_p.beta/m_p.mu[0],
-- 
GitLab


From ba6e5546ad126a8e80f6a2b7d1ba3d2a4f71e3a8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 22 Jan 2019 12:54:59 +0100
Subject: [PATCH 009/540] Add Xpoint geometry parameters in feltor3d

and remove bug in feltor_mpi compilation
---
 src/feltor/feltor_hpc.cu                     |  2 +-
 src/feltor/geometry/geometry_params.js       |  2 +-
 src/feltor/geometry/geometry_paramsXpoint.js | 34 ++++++++++++++++++++
 3 files changed, 36 insertions(+), 2 deletions(-)
 create mode 100644 src/feltor/geometry/geometry_paramsXpoint.js

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 15e29fdf1..a44658331 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -139,7 +139,7 @@ int main( int argc, char* argv[])
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
         //then shift tanh
         p.rho_source-3.*p.alpha, p.alpha, -1.), grid);
-    dg::HVec damping_damping = dg::pullback(dg::geo::TanhDamping(
+    HVec damping_damping = dg::pullback(dg::geo::TanhDamping(
         //first change coordinate from psi to (psi_0 - psip)/psi_0
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
         //then shift tanh
diff --git a/src/feltor/geometry/geometry_params.js b/src/feltor/geometry/geometry_params.js
index c41c22dae..7d160c23f 100644
--- a/src/feltor/geometry/geometry_params.js
+++ b/src/feltor/geometry/geometry_params.js
@@ -6,7 +6,7 @@
 	"A" : 1,
 	"R_0" : 91.884200000000007,
 	"alpha" : 0.02,
-	"c" : 
+	"c" :
 	[
 		0.13104522088367745,
 		-0.1051324097610889,
diff --git a/src/feltor/geometry/geometry_paramsXpoint.js b/src/feltor/geometry/geometry_paramsXpoint.js
new file mode 100644
index 000000000..90ee9b27a
--- /dev/null
+++ b/src/feltor/geometry/geometry_paramsXpoint.js
@@ -0,0 +1,34 @@
+//
+//          * Input-File for TJ-K axisymmetric solovev equilibrium *
+//    Iconst without X-point, elong AUG,  Te_10eV,B0_0.07T,deuterium,Cref_1e-5
+//          -----------------------------------------------------------
+{
+	"A" : 0,
+	"R_0" : 91.884233908188605,
+	"alpha" : 0.02,
+	"c" :
+	[
+		0.58136855188608583,
+		9.2360756664978574,
+		-9.1888338426687337,
+		-4.1872227595243885,
+		6.544638217090343,
+		-5.7648048417281998,
+		-0.23576721602133818,
+		1.3692553155542757,
+		6.3214049719456709,
+		-3.9167432232703687,
+		-0.97303855303064757,
+		0.10615175705576452
+	],
+	"elongation" : 1.7,
+	"equilibrium" : "solovev",
+	"inverseaspectratio" : 0.16666666666666669,
+	"psip_max" : 0,
+	"psip_max_cut" : 10000000000,
+	"psip_max_lim" : 10000000000,
+	"psip_min" : -1.1,
+	"qampl" : 1,
+	"rk4eps" : 0.01,
+	"triangularity" : 0.33000000000000002
+}
-- 
GitLab


From 1d179b9d64326f4f34d5afaa7e8226691f340a3a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 22 Jan 2019 14:09:44 +0100
Subject: [PATCH 010/540] Inner loop for feltor_hpc

- count total number of failed steps
- construct an inner loop between outputs for energy
---
 src/feltor/feltor.tex         |  5 +--
 src/feltor/feltor_hpc.cu      | 61 ++++++++++++++++++++---------------
 src/feltor/input/default.json |  5 +--
 src/feltor/parameters.h       | 15 +++++----
 4 files changed, 50 insertions(+), 36 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 42074c22d..5c169d124 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -748,8 +748,9 @@ dt     & integer &1e-2& - &initial time stepsize in units of $c_s/\rho_s$ \\
 compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing points in x and y: output contains n*Nx/c[0] points in x,
     (has to divde Nx evenly), and n*Ny/c[1] points in y,
     (has to divde Ny evenly)\\
-itstp       & integer & 2  & - & Number of time steps between outputs \\
-maxout      & integer & 10 & - & Total Number of outputs excluding first \\
+inner\_loop & integer & 2  & 1 & Number of time steps between each energy output \\
+itstp       & integer & 2  & - & Number of energy outputs for each fields outputs \\
+maxout      & integer & 10 & - & Total Number of fields outputs excluding first (The total number of time steps is maxout$\cdot$itstp$\cdot$inner\_loop) \\
 eps\_time   & float & 1e-9  & - & Accuracy of solver for implicit part in time-stepper \\
 rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper \\
 eps\_pol    & float & 1e-5  & - &  Accuracy of residual of the inversion of polarisation and induction Eq. \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index a44658331..c186b9e40 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -314,7 +314,7 @@ int main( int argc, char* argv[])
     }
     err = nc_enddef(ncid);
     ///////////////////////////////////first output/////////////////////////
-    double time = 0, dt_new = p.dt, dt = 0;
+    double time = 0, dt = p.dt;
     MPI_OUT std::cout << "First output ... \n";
     //first, update quantities in feltor
     {
@@ -377,7 +377,7 @@ int main( int argc, char* argv[])
         "ARK-4-2-3", y0, grid.size(), p.eps_time);
     dg::Timer t;
     t.tic();
-    unsigned step = 0;
+    unsigned step = 0, failed_counter = 0;
     MPI_OUT q.display(std::cout);
     for( unsigned i=1; i<=p.maxout; i++)
     {
@@ -386,29 +386,36 @@ int main( int argc, char* argv[])
         ti.tic();
         for( unsigned j=0; j<p.itstp; j++)
         {
-            try{
-                do
-                {
-                    dt = dt_new;
-                    adaptive.step( feltor, im, time, y0, time, y0, dt_new,
-                        dg::pid_control, dg::l2norm, p.rtol, 1e-10);
-                    if( adaptive.failed())
-                        MPI_OUT std::cout << "FAILED STEP! REPEAT!\n";
-                }while ( adaptive.failed());
+            double previous_time = time;
+            for( unsigned k=0; k<p.inner_loop; k++)
+            {
+                try{
+                    do
+                    {
+                        adaptive.step( feltor, im, time, y0, time, y0, dt,
+                            dg::pid_control, dg::l2norm, p.rtol, 1e-10);
+                        if( adaptive.failed())
+                        {
+                            MPI_OUT std::cout << "FAILED STEP! REPEAT!\n";
+                            failed_counter++;
+                        }
+                    }while ( adaptive.failed());
+                }
+                catch( dg::Fail& fail) {
+                    MPI_OUT std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
+                    MPI_OUT std::cerr << "Does Simulation respect CFL condition?\n";
+                    #ifdef FELTOR_MPI
+                    err = nc_close(ncid);
+                    #endif //FELTOR_MPI
+                    return -1;
+                }
+                step++;
             }
-            catch( dg::Fail& fail) {
-                MPI_OUT std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-                MPI_OUT std::cerr << "Does Simulation respect CFL condition?\n";
-                #ifdef FELTOR_MPI
-                err = nc_close(ncid);
-                #endif //FELTOR_MPI
-                return -1;
-            }
-            step++;
+            double deltat = time - previous_time;
 
             feltor.update_quantities();
-            MPI_OUT std::cout << "Timestep "<<dt<<"\n";
-            dEdt = (*v0d["energy"] - E0)/dt, dMdt = (*v0d["mass"] - M0)/dt;
+            MPI_OUT std::cout << "Timestep "<<deltat<<"\n";
+            dEdt = (*v0d["energy"] - E0)/deltat, dMdt = (*v0d["mass"] - M0)/deltat;
             E0 = *v0d["energy"], M0 = *v0d["mass"];
             accuracy  = 2.*fabs( (dEdt - *v0d["ediff"])/( dEdt + *v0d["ediff"]));
             accuracyM = 2.*fabs( (dMdt - *v0d["diff"])/( dMdt + *v0d["diff"]));
@@ -436,10 +443,12 @@ int main( int argc, char* argv[])
 
         }
         ti.toc();
-        MPI_OUT std::cout << "\n\t Step "<<step <<" of "<<p.itstp*p.maxout
-                  << " at time "<<time;
+        MPI_OUT std::cout << "\n\t Step "<<step <<" of "
+                    << p.inner_loop*p.itstp*p.maxout << " at time "<<time;
         MPI_OUT std::cout << "\n\t Average time for one step: "
-                  << ti.diff()/(double)p.itstp<<"s";
+                    << ti.diff()/(double)p.itstp/(double)p.inner_loop<<"s";
+        MPI_OUT std::cout << "\n\t Total number of failed steps: "
+                    << failed_counter;
         ti.tic();
         //////////////////////////write fields////////////////////////
         start[0] = i;
@@ -471,7 +480,7 @@ int main( int argc, char* argv[])
     double second = t.diff() - hour*3600 - minute*60;
     MPI_OUT std::cout << std::fixed << std::setprecision(2) <<std::setfill('0');
     MPI_OUT std::cout <<"Computation Time \t"<<hour<<":"<<std::setw(2)<<minute<<":"<<second<<"\n";
-    MPI_OUT std::cout <<"which is         \t"<<t.diff()/p.itstp/p.maxout<<"s/step\n";
+    MPI_OUT std::cout <<"which is         \t"<<t.diff()/p.itstp/p.maxout/p.inner_loop<<"s/step\n";
 #ifdef FELTOR_MPI
     err = nc_close(ncid);
     MPI_Finalize();
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index b6a243c78..2c0651467 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -7,8 +7,9 @@
     "compression" : [2,2],
     "refineDS" : [5,5],
     "rk4eps" : 1e-5,
-    "itstp"  : 5,
-    "maxout" : 100,
+    "inner_loop" : 1,
+    "itstp"      : 5,
+    "maxout"     : 100,
     "eps_pol"    : 1e-5,
     "jumpfactor" : 1,
     "eps_gamma"  : 1e-6,
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 739397d23..c82492bd3 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -12,6 +12,7 @@ struct Parameters
     unsigned n_out, Nx_out, Ny_out, Nz_out;
     double dt;
     unsigned cx, cy;
+    unsigned inner_loop;
     unsigned itstp;
     unsigned maxout;
 
@@ -55,6 +56,7 @@ struct Parameters
         cx      = js["compression"].get(0u,1).asUInt();
         cy      = js["compression"].get(1u,1).asUInt();
         n_out = n, Nx_out = Nx/cx, Ny_out = Ny/cy, Nz_out = Nz;
+        inner_loop = js.get("inner_loop",1).asUInt();
         itstp   = js["itstp"].asUInt();
         maxout  = js["maxout"].asUInt();
         eps_time    = js["eps_time"].asDouble();
@@ -158,12 +160,13 @@ struct Parameters
             <<"     Accuracy Fieldline    "<<rk4eps<<"\n"
             <<"     Refined DS            "<<mx<<" "<<my<<"\n";
         os << "Output parameters are: \n"
-            <<"     n_out  =              "<<n_out<<"\n"
-            <<"     Nx_out =              "<<Nx_out<<"\n"
-            <<"     Ny_out =              "<<Ny_out<<"\n"
-            <<"     Nz_out =              "<<Nz_out<<"\n"
-            <<"     Steps between output: "<<itstp<<"\n"
-            <<"     Number of outputs:    "<<maxout<<"\n";
+            <<"     n_out  =                 "<<n_out<<"\n"
+            <<"     Nx_out =                 "<<Nx_out<<"\n"
+            <<"     Ny_out =                 "<<Ny_out<<"\n"
+            <<"     Nz_out =                 "<<Nz_out<<"\n"
+            <<"     Steps between energies:  "<<inner_loop<<"\n"
+            <<"     Energies between output: "<<itstp<<"\n"
+            <<"     Number of outputs:       "<<maxout<<"\n";
         os << "Boundary conditions are: \n"
             <<"     bc density x   = "<<dg::bc2str(bcxN)<<"\n"
             <<"     bc density y   = "<<dg::bc2str(bcyN)<<"\n"
-- 
GitLab


From 234556f5a80a9e6c04f0cce686ecfc930dd406e8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 25 Jan 2019 13:58:13 +0100
Subject: [PATCH 011/540] Various little updates

- BENCHMARK in feltor3d Makefile
- failed_counter and inner_loop in feltor.cu
- corrections in default.json and feltor.tex
---
 src/feltor/Makefile           |  4 +--
 src/feltor/feltor.cu          | 52 +++++++++++++++++++++--------------
 src/feltor/feltor.tex         | 21 ++++++++------
 src/feltor/input/default.json | 18 ++++++------
 src/feltor/parameters.h       |  2 +-
 5 files changed, 55 insertions(+), 42 deletions(-)

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index 7c18d4d16..a2ed7df2f 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -17,10 +17,10 @@ feltor: feltor.cu feltor.cuh
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g
 
 feltor_hpc: feltor_hpc.cu feltor.cuh
-	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB)
+	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
 
 feltor_mpi: feltor_hpc.cu feltor.cuh
-	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DFELTOR_MPI
+	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DFELTOR_MPI -DDG_BENCHMARK
 
 .PHONY: clean
 
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 452a32be9..3491477e1 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -218,6 +218,7 @@ int main( int argc, char* argv[])
     std::cout << std::scientific << std::setprecision( 2);
     dg::Average<dg::HVec> toroidal_average( grid, dg::coo3d::z);
     title << std::setprecision(2) << std::scientific;
+    unsigned failed_counter = 0;
     while ( !glfwWindowShouldClose( w ))
     {
         title << std::fixed;
@@ -239,9 +240,9 @@ int main( int argc, char* argv[])
                 dg::assign( *pair.second, hvisual);
             dg::blas2::gemv( equi, hvisual, visual);
             colors.scalemax() = (double)thrust::reduce(
-                visual.begin(), visual.end(), 0., thrust::maximum<double>() );
+                visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
             colors.scalemin() = -colors.scalemax();
-            title <<pair.first << colors.scalemax()<<"\t";
+            title <<pair.first << colors.scalemax()<<"   ";
             if ( p.symmetric )
                 render.renderQuad( hvisual, grid.n()*grid.Nx(),
                                             grid.n()*grid.Ny(), colors);
@@ -270,25 +271,33 @@ int main( int argc, char* argv[])
         t.tic();
         for( unsigned i=0; i<p.itstp; i++)
         {
-            try{
-                do
-                {
-                    dt = dt_new;
-                    adaptive.step( feltor, im, time, y0, time, y0, dt_new,
-                        dg::pid_control, dg::l2norm, p.rtol, 1e-10);
-                    if( adaptive.failed())
-                        std::cout << "FAILED STEP! REPEAT!\n";
-                }while ( adaptive.failed());
-            }
-            catch( dg::Fail& fail) {
-                std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-                std::cerr << "Does Simulation respect CFL condition?\n";
-                glfwSetWindowShouldClose( w, GL_TRUE);
-                break;
+            double current_time = time;
+            for( unsigned k=0; k<p.inner_loop; k++)
+            {
+                try{
+                    do
+                    {
+                        dt = dt_new;
+                        adaptive.step( feltor, im, time, y0, time, y0, dt_new,
+                            dg::pid_control, dg::l2norm, p.rtol, 1e-10);
+                        if( adaptive.failed())
+                        {
+                            failed_counter++;
+                            std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
+                        }
+                    }while ( adaptive.failed());
+                }
+                catch( dg::Fail& fail) {
+                    std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
+                    std::cerr << "Does Simulation respect CFL condition?\n";
+                    glfwSetWindowShouldClose( w, GL_TRUE);
+                    break;
+                }
+                step++;
             }
-            step++;
+            dt = time - current_time;
             feltor.update_quantities();
-            std::cout << "Timestep "<<dt<<"\n";
+            std::cout << "Timestep "<<dt_new<<"\n";
             dEdt = (*v0d["energy"] - E0)/dt, dMdt = (*v0d["mass"] - M0)/dt;
             E0 = *v0d["energy"], M0 = *v0d["mass"];
             accuracy  = 2.*fabs( (dEdt - *v0d["ediff"])/( dEdt + *v0d["ediff"]));
@@ -306,8 +315,9 @@ int main( int argc, char* argv[])
 
         }
         t.toc();
-        std::cout << "\n\t Step "<<step;
-        std::cout << "\n\t Average time for one step: "<<t.diff()/(double)p.itstp<<"s\n\n";
+        std::cout << "\n\t Step "<<step << " at time  "<<time;
+        std::cout << "\n\t Average time for one step: "<<t.diff()/(double)p.itstp/(double)p.inner_loop;
+        std::cout << "\n\t Total # of failed steps:   "<<failed_counter<<"\n\n";
     }
     glfwTerminate();
     ////////////////////////////////////////////////////////////////////
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 5c169d124..92b9f32f1 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -273,7 +273,7 @@ simplify to
 In this approximation we apply the toroidal field line approximation
 as in Section
 \ref{sec:torfieldlineapprox}
-but approximate the curvature operator $\mathcal K_\kappa \approx \bhat\times\kappa$.
+but approximate the curvature operator $\mathcal K_\kappa \approx \bhat\times\kappa$
   with
   $\vec \kappa := \bhat \cdot \vec \nabla\bhat = -\bhat \times(\vec \nabla\times \bhat)$.
 For an isotropic pressure plasma \(\vec{P} = \vec{I} P_\perp + \vec{b} \vec{b} P_\Delta \approx \vec{I} P_\perp\) and with the definition of the plasma beta parameter
@@ -455,9 +455,9 @@ where we define
 where $a$ is the minor radius, $e$ is the elongation of the flux surfaces and
 the $\varepsilon$ are free parameters to be specified by the user.
 
-We choose boundary conditions seperately on input for the variables
+We choose boundary conditions separately on input for the variables
 $n_e$, $u_e$ and $\phi$. The boundary condition for $N_i$, $U_i$ and
-$\psi$ are equal to $n_e$, $u_e$ and $\phi$ respecitively.
+$\psi$ are equal to $n_e$, $u_e$ and $\phi$ respectively.
 We choose $A_\parallel$ to have equal boundary conditions as $u_e$ and $U_i$.
 This will later enable us to treat the sum of $U$ and $A_\parallel$
 in the same way as $U$.
@@ -468,7 +468,7 @@ n_e = n_0, \quad u_e = \phi = 0
 \end{align}
 where $\hat n$ is the normal vector to the boundary.
 
-We initialize the parallel velocitiy to zero
+We initialize the parallel velocity to zero
 \begin{align}
   u_e(R,Z,\varphi,0) = U_i(R,Z,\varphi,0) = 0
   \label{}
@@ -709,7 +709,7 @@ We then symbolically compute (with the help of Mathematica) source terms that we
 the corresponding equation in code (\texttt{manufactured.h}) and simulate from $t=0...10^{-3}$.
 By comparing the numerical solution to the manufactured one we can observe the convergence of our numerical methods. Note that in order to better distinguish
 the convergence of the dG discretized terms from our parallel derivative
-we can selectively choose to only activate perpendicular (including $A_\parallel$ temrs) or parallel terms (those that involve derivatives along $\bhat$).
+we can selectively choose to only activate perpendicular (including $A_\parallel$ terms) or parallel terms (those that involve derivatives along $\bhat$).
 
 Unfortunately, we were unable to find a closed solution for the energy integrals with the above fields.
 
@@ -764,12 +764,12 @@ mu         & float & -0.000272121& - & $\mu_e =-m_e/m_i$.
     One of $\left\{ -0.000544617, -0.000272121, -0.000181372 \right\}$\\
 tau        & float &1      & - & $\tau = T_i/T_e$  \\
 beta       & float & 5e-6  & 0 & Plasma beta $5\cdot 10^{-6}$ (TJK), $4\cdot 10^{-3}$ (Compass), If $0$, then the model is electrostatic \\
-nu\_perp   & float &1e-3   & - & pependicular viscosity $\nu_\perp$ \\
+nu\_perp   & float &1e-3   & - & perpendicular viscosity $\nu_\perp$ \\
 perp\_diff & string & "viscous" & "viscous" & "viscous": $\Lambda_\perp\propto \nu_\perp\Delta_\perp$ , "hyperviscous": $\Lambda_\perp \propto -\nu_\perp\Delta_\perp^2$\\
 nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$ \\
 resistivity & float &1e-4  & - & parallel resistivity parameter Eq.~\eqref{eq:resistivity}\\
 curvmode  & string & "low beta"  & "toroidal"& curvature mode ("low beta", "true" no approximation, "toroidal": toroidal field approx) \\
-symmetric & bool & false & false & If true, initialize all quantities symmetric in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used to construct the parallel derivatives and then overwtiten to $N_z\equiv 1$. \\
+symmetric & bool & false & false & If true, initialize all quantities symmetric in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used to construct the parallel derivatives and then overwritten to $N_z\equiv 1$. \\
 bc & dict & & & Dictionary of boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
 \qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y for $n_e$ and $N_i$\\
 \qquad velocity  & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $u_e$ and $U_i$\\
@@ -796,6 +796,9 @@ source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq
 rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source}  \\
 damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
 rho\_damping& float & 0.2  & 1.2 & Friction region boundary $\rho_{d}>1$ in Eq.~\eqref{eq:velocity_source}  \\
+%restart    & bool & false & false & If true, all input and geometry parameters except maxout,
+%    itstp and inner\_loop are ignored. Instead, the parameters and all fields are initialized from output.nc
+%    The actual output is then appended to output.nc This effectively continues an existing simulation.
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
@@ -863,7 +866,7 @@ Uperp     & Dataset & 1 (energy\_time) & total perpendicular energy integral Eq.
 Apar     & Dataset & 1 (energy\_time) & total magnetic energy integral Eq.~\eqref{eq:energy_conservation} \\
 dEdt      & Dataset & 1 (energy\_time) & change of energy per time  \\
 accuracy  & Dataset & 1 (energy\_time) & accuracy of energy theorem in time  \\
-aligned   & Dataset & 1 (energy\_time) & Alignment paramter $\int\dV \ln n_e\Delta_\parallel n_e$\\
+aligned   & Dataset & 1 (energy\_time) & Alignment parameter $\int\dV \ln n_e\Delta_\parallel n_e$\\
 \bottomrule
 \end{longtable}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -905,7 +908,7 @@ total\_flux      & Dataset & 1 (time) & $\int \dV f_e$\\
 \bottomrule
 \end{longtable}
 X $\in$ \{ electrons, ions, Ue, Ui, potential,
-induction, vorticity, fluxe, Lpeprinv, Lparallelinv\} corresponding to \{
+induction, vorticity, fluxe, Lperpinv, Lparallelinv\} corresponding to \{
     $n_e$, $N_i$, $u_e$, $U_i$, $\phi$, $A_\parallel$, $-\Delta_\perp \phi$,
     $f_e$, $L_\perp^{-1}$, $L_\parallel^{-1}$\}
 
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index 2c0651467..660aff232 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -27,7 +27,7 @@
     "sigma"     : 2.0,
     "posX"      : 0.3,
     "posY"      : 0,
-    "sigma_z"   : 0.25,
+    "sigma_z"   : 0.5,
     "k_psi"     : 0,
     "nprofileamp" : 4,
     "bgprofamp"   : 1,
@@ -39,13 +39,13 @@
     },
     "boxscaleR" :  [1.1,1.1],
     "boxscaleZ" :  [1.2,1.1],
-    "initne"     : "turbulence",
-    "initphi"    : "zero",
-    "curvmode"   : "true",
-    "symmetric"  : false,
-    "alpha"      : 0.02,
-    "source"     : 0,
-    "rho_source" : 0.2,
+    "initne"    : "turbulence",
+    "initphi"   : "zero",
+    "curvmode"  : "true",
+    "symmetric" : false,
+    "alpha"       : 0.05,
+    "source"      : 0,
+    "rho_source"  : 0.2,
     "damping"     : 0,
-    "rho_daming" : 1.2
+    "rho_damping" : 1.2
 }
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index c82492bd3..cfab428dd 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -110,7 +110,7 @@ struct Parameters
         initne      = js.get( "initne", "blob").asString();
         initphi     = js.get( "initphi", "zero").asString();
         curvmode    = js.get( "curvmode", "toroidal").asString();
-        symmetric  = js.get( "symmetric", false).asBool();
+        symmetric   = js.get( "symmetric", false).asBool();
     }
     void display( std::ostream& os = std::cout ) const
     {
-- 
GitLab


From b48a077a43a425bd808088e0228fc8c67b647ae4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 25 Jan 2019 18:30:28 +0100
Subject: [PATCH 012/540] Add DIRKStep time stepper and FixedPointIteration

Further
- add stepper method to Adaptive
- ignore_fsal and enable_fsal in ERKStep and RungeKutta
- an ImplicitRungeKutta analogous to RungeKutta
- implicit test in runge_kutta_t.cu
---
 inc/dg/adaptive.h       |   7 +
 inc/dg/implicit.h       |  65 ++++++++-
 inc/dg/runge_kutta.h    | 293 +++++++++++++++++++++++++++++++++++++++-
 inc/dg/runge_kutta_t.cu |  30 ++++
 4 files changed, 389 insertions(+), 6 deletions(-)

diff --git a/inc/dg/adaptive.h b/inc/dg/adaptive.h
index 01e5c9137..7e7ea1f64 100644
--- a/inc/dg/adaptive.h
+++ b/inc/dg/adaptive.h
@@ -188,6 +188,13 @@ struct Adaptive
     template<class Explicit, class ErrorNorm = value_type(const container_type&)>
     value_type guess_stepsize( Explicit& ex, value_type t0, const container_type& u0, enum direction dir, ErrorNorm& norm, value_type rtol, value_type atol);
 
+    ///@brief Allow write access to internal stepper
+    ///
+    ///Maybe useful to set options in the stepper
+    stepper_type& stepper() { return m_stepper;}
+    ///@brief Read access to internal stepper
+    const stepper_type& stepper() const { return m_stepper;}
+
     /*!@brief Explicit or Implicit adaptive step
      *
      * @param rhs The right hand side of the equation to integrate
diff --git a/inc/dg/implicit.h b/inc/dg/implicit.h
index 6ccb07133..096b51247 100644
--- a/inc/dg/implicit.h
+++ b/inc/dg/implicit.h
@@ -63,7 +63,7 @@ struct TensorTraits< detail::Implicit<M, V> >
  * works only for linear positive definite operators as it uses a conjugate
  * gradient solver to invert the equation
  * @copydoc hide_ContainerType
- * @sa Karniadakis ARKStep
+ * @sa Karniadakis ARKStep DIRKStep
  */
 template<class ContainerType>
 struct DefaultSolver
@@ -113,4 +113,67 @@ struct DefaultSolver
     ContainerType m_rhs;
     value_type m_eps;
 };
+
+/*!@brief Fixed point iterator for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
+ *
+ * @copydoc hide_ContainerType
+ * @sa Karniadakis ARKStep DIRKStep
+ */
+template<class ContainerType>
+struct FixedPointSolver
+{
+    using container_type = ContainerType;
+    using value_type = get_value_type<ContainerType>;//!< value type of vectors
+    ///No memory allocation
+    FixedPointSolver(){}
+    /*!
+    * @param copyable vector of the size that is later used in \c solve (
+     it does not matter what values \c copyable contains, but its size is important;
+     the \c solve method can only be called with vectors of the same size)
+    * @param max_iter maimum iteration number
+    * @param eps accuracy parameter. Convergence is in the l2 norm
+    */
+    FixedPointSolver( const ContainerType& copyable, unsigned max_iter, value_type eps):
+        m_current( copyable), m_eps(eps), m_max_iter(max_iter)
+        {}
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_current;}
+
+    template< class Implicit>
+    void solve( value_type alpha, Implicit im, value_type t, ContainerType& y, const ContainerType& rhs)
+    {
+#ifdef DG_BENCHMARK
+#ifdef MPI_VERSION
+        int rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+#endif//MPI
+        Timer ti;
+        ti.tic();
+#endif //DG_BENCHMARK
+        unsigned number = 0;
+        value_type error = 0;
+        do
+        {
+            dg::blas1::copy( y, m_current);
+            im( t, m_current, y);
+            dg::blas1::axpby( 1., rhs, -alpha, y);
+            dg::blas1::axpby( 1., y, -1., m_current); //the difference
+            number++;
+            error = sqrt( dg::blas1::dot( m_current, m_current));
+        }while ( error > m_eps && number < m_max_iter);
+        //std::cout << " Error it "<<number<<" is "<<error<<"\n";
+#ifdef DG_BENCHMARK
+        ti.toc();
+#ifdef MPI_VERSION
+        if(rank==0)
+#endif//MPI
+        std::cout << "# of iterations time solver: "<<number<<"/"<<m_pcg.get_max()<<" took "<<ti.diff()<<"s\n";
+#endif //DG_BENCHMARK
+    }
+    private:
+    ContainerType m_current;
+    value_type m_eps;
+    unsigned m_max_iter;
+};
 }//namespace dg
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 0aad62b86..586d0ad9e 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -66,6 +66,12 @@ struct ERKStep
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_k[0];}
+
+    ///All susequqent calls to \c step method will ignore the first same as last property
+    void ignore_fsal(){ m_ignore_fsal = true;}
+    ///All susequqent calls to \c step method will enable the check for the first same as last property
+    void enable_fsal(){ m_ignore_fsal = false;}
+
     ///@copydoc RungeKutta::step()
     ///@param delta Contains error estimate on output (must have equal size as \c u0)
     template<class RHS>
@@ -86,6 +92,7 @@ struct ERKStep
     ButcherTableau<value_type> m_rk;
     std::vector<ContainerType> m_k;
     value_type m_t1 = 1e300;//remember the last timestep at which ERK is called
+    bool m_ignore_fsal = false;
 };
 
 template< class ContainerType>
@@ -93,10 +100,9 @@ template< class RHS>
 void ERKStep<ContainerType>::step( RHS& f, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt, ContainerType& delta)
 {
     unsigned s = m_rk.num_stages();
-    //this behaviour must be documented
-    //0 stage: probe fsal
+    //0 stage: probe
     value_type tu = t0;
-    if( t0 != m_t1)
+    if( t0 != m_t1 || m_ignore_fsal)
         f(t0, u0, m_k[0]); //freshly compute k_0
     //else take from last call
     //1 stage
@@ -277,9 +283,9 @@ struct ARKStep
              SolverParams&& ...ps
              ):
          m_solver( std::forward<SolverParams>(ps)...),
+         m_rhs( m_solver.copyable()),
          m_rkE(ex_tableau),
          m_rkI(im_tableau),
-         m_rhs( m_solver.copyable()),
          m_kE(m_rkE.num_stages(), m_rhs),
          m_kI(m_rkI.num_stages(), m_rhs)
     {
@@ -347,7 +353,7 @@ struct ARKStep
     }
     private:
     SolverType m_solver;
-    ContainerType m_rhs, m_u1;
+    ContainerType m_rhs;
     ButcherTableau<value_type> m_rkE, m_rkI;
     std::vector<ContainerType> m_kE, m_kI;
     value_type m_t1 = 1e300;
@@ -502,11 +508,23 @@ struct RungeKutta
     * @param u1 (write only) contains result on output (may alias u0)
     * @param dt timestep
     * @note on return \c rhs(t1, u1) will be the last call to \c rhs (this is useful if \c RHS holds state, which is then updated to the current timestep)
+    * @note About the first same as last property (fsal): Some Butcher tableaus
+    * (e.g. Dormand-Prince) have the property that the last value k_s of a
+    * timestep is the same as the first value k_0 of the next timestep. This
+    * means that we can save one call to the right hand side. This property is
+    * automatically activated if \c tableau.isFsal() returns \c true and \c t0
+    * equals \c t1 of the last call to \c step. You can deactivate it by
+    * calling the \c ignore_fsal() method, which is useful for splitting methods
+    * but increases the number of rhs calls by 1.
     */
     template<class RHS>
     void step( RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
         m_erk.step( rhs, t0, u0, t1, u1, dt, m_delta);
     }
+    ///All susequqent calls to \c step method will ignore the first same as last property
+    void ignore_fsal(){ m_erk.ignore_fsal();}
+    ///All susequqent calls to \c step method will enable the check for the first same as last property
+    void enable_fsal(){ m_erk.enable_fsal();}
     ///@copydoc ERKStep::order
     unsigned order() const {
         return m_erk.order();
@@ -519,6 +537,271 @@ struct RungeKutta
     ERKStep<ContainerType> m_erk;
     ContainerType m_delta;
 };
+/*!
+ * @brief Struct for diagonally implicit Runge Kutta time-step with error estimate
+* \f[
+ \begin{align}
+    k_i = f\left( t^n + c_i \Delta t, u^n + \Delta t \sum_{j=1}^{s} a_{ij} k_j\right) \\
+    u^{n+1} = u^{n} + \Delta t\sum_{j=1}^s b_j k_j \\
+    \tilde u^{n+1} = u^{n} + \Delta t\sum_{j=1}^s \tilde b_j k_j
+ \end{align}
+\f]
+ *
+ * So far we did not implement the use of a mass matrix \c M.
+ * You can provide your own coefficients or use one of the embedded methods
+ * in the following table:
+ * @copydoc hide_implicit_butcher_tableaus
+ *
+ * @copydoc hide_SolverType
+ * @copydoc hide_ContainerType
+ * @ingroup time
+ */
+template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
+struct DIRKStep
+{
+    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
+    using container_type = ContainerType; //!< the type of the vector class in use
+    ///@copydoc RungeKutta::RungeKutta()
+    DIRKStep(){ }
+
+    ///@copydoc construct()
+    template<class ...SolverParams>
+    DIRKStep( ConvertsToButcherTableau<value_type> im_tableau,
+             SolverParams&& ...ps
+             ):
+         m_solver( std::forward<SolverParams>(ps)...),
+         m_rhs( m_solver.copyable()),
+         m_rkI(im_tableau),
+         m_kI(m_rkI.num_stages(), m_rhs)
+    {
+    }
+
+    /*!@brief Construct with a diagonally implicit Butcher Tableau
+     *
+     * The tableau may be one of the implict methods listed in
+     * \c ConvertsToButcherTableau, or you provide your own tableau.
+     *
+     * @param im_tableau diagonally implicit tableau, name or identifier that \c ConvertsToButcherTableau
+     * @param ps Parameters that
+     * are forwarded to the constructor of \c SolverType
+     * @tparam SolverParams Type of parameters (deduced by the compiler)
+     */
+    template<class ...SolverParams>
+    void construct(
+             ConvertsToButcherTableau<value_type> im_tableau,
+             SolverParams&& ...ps
+             )
+    {
+        m_rkI = im_tableau;
+        m_solver = SolverType( std::forward<SolverParams>(ps)...);
+        m_rhs = m_solver.copyable();
+        m_kI.assign(m_rkI.num_stages(), m_rhs);
+    }
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_kI[0];}
+
+    /**
+    * @brief Advance one step
+    *
+    * @copydoc hide_rhs
+    * @param t0 start time
+    * @param u0 value at \c t0
+    * @param t1 (write only) end time ( equals \c t0+dt on output
+    *   unless \c freeze_time() was called, then it equals \c t0 on output, may alias \c t0)
+    * @param u1 (write only) contains result on output (may alias u0)
+    * @param dt timestep
+    * @param delta Contains error estimate on output (must have equal size as \c u0)
+    */
+    template< class RHS>
+    void step( RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt, ContainerType& delta);
+    ///@copydoc ERKStep::order()
+    unsigned order() const {
+        return m_rkI.order();
+    }
+    ///@copydoc ERKStep::embedded_order()
+    unsigned embedded_order() const {
+        return m_rkI.order();
+    }
+    ///@copydoc ERKStep::num_stages()
+    unsigned num_stages() const{
+        return m_rkI.num_stages();
+    }
+
+    ///All subsequent calls to \c step method will not advance time
+    ///This is useful in an operator splitting
+    void freeze_time() { m_freeze_time=true;}
+    ///All subsequent calls to \c step method will advance time again
+    void unfreeze_time() { m_freeze_time=false;}
+    private:
+    SolverType m_solver;
+    ContainerType m_rhs;
+    ButcherTableau<value_type> m_rkI;
+    std::vector<ContainerType> m_kI;
+    bool m_freeze_time = false;
+};
+
+template<class ContainerType, class SolverType>
+template< class RHS>
+void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt, ContainerType& delta)
+{
+    unsigned s = m_rkI.num_stages();
+    value_type tu = t0;
+    //0 stage
+    //rhs = u0
+    tu = DG_FMA( m_rkI.c(0),dt, t0);
+    if( m_freeze_time) tu = t0;
+    blas1::copy( u0, delta); //better init with rhs
+    m_solver.solve( -dt*m_rkI.a(0,0), rhs, tu, delta, u0);
+
+    //1 stage
+    if( s>1){
+        rhs(tu, delta, m_kI[0]);
+        blas1::evaluate( m_rhs, dg::equals(), PairSum(), 1., u0,
+                dt*m_rkI.a(1,0), m_kI[0]);
+        tu = DG_FMA( m_rkI.c(1),dt, t0);
+        if( m_freeze_time) tu = t0;
+        //store solution in delta, init with last solution
+        blas1::copy( m_rhs, delta); //better init with rhs
+        m_solver.solve( -dt*m_rkI.a(1,1), rhs, tu, delta, m_rhs);
+        rhs(tu, delta, m_kI[1]);
+    }
+    //2 stage
+    if( s>2){
+        blas1::evaluate( m_rhs, dg::equals(), PairSum(), 1., u0,
+                 dt*m_rkI.a(2,0), m_kI[0],
+                 dt*m_rkI.a(2,1), m_kI[1]);
+        tu = DG_FMA( m_rkI.c(2),dt, t0);
+        if( m_freeze_time) tu = t0;
+        //just take last solution as init
+        blas1::copy( m_rhs, delta); //better init with rhs
+        m_solver.solve( -dt*m_rkI.a(2,2), rhs, tu, delta, m_rhs);
+        rhs(tu, delta, m_kI[2]);
+    }
+    //3 stage and higher
+    if( s>3){
+        blas1::evaluate( m_rhs, dg::equals(), PairSum(), 1., u0,
+                 dt*m_rkI.a(3,0), m_kI[0],
+                 dt*m_rkI.a(3,1), m_kI[1],
+                 dt*m_rkI.a(3,2), m_kI[2]);
+        tu = DG_FMA( m_rkI.c(3),dt, t0);
+        blas1::copy( m_rhs, delta); //better init with rhs
+        m_solver.solve( -dt*m_rkI.a(3,3), rhs, tu, delta, m_rhs);
+        rhs(tu, delta, m_kI[3]);
+        for( unsigned i=4; i<s; i++)
+        {
+            dg::blas1::copy( u0, m_rhs);
+            for( unsigned j=0; j<i; j++)
+                dg::blas1::axpby( dt*m_rkI.a(i,j), m_kI[j], 1., m_rhs);
+            tu = DG_FMA( m_rkI.c(i),dt, t0);
+            if( m_freeze_time) tu = t0;
+            blas1::copy( m_rhs, delta); //better init with rhs
+            m_solver.solve( -dt*m_rkI.a(i,i), rhs, tu, delta, m_rhs);
+            rhs(tu, delta, m_kI[i]);
+        }
+    }
+    t1 = t0 + dt;
+    //Now compute result and error estimate
+    switch( s)
+    {
+        case 1: dg::blas1::copy( delta, u1); break; //implicit Euler
+        case 2:
+            blas1::subroutine( dg::EmbeddedPairSum(),
+                            u1, delta,
+                            1., 0., u0,
+                            dt*m_rkI.b(0), dt*m_rkI.d(0), m_kI[0],
+                            dt*m_rkI.b(1), dt*m_rkI.d(1), m_kI[1]); break;
+        case 3: blas1::subroutine( dg::EmbeddedPairSum(),
+                            u1, delta,
+                            1., 0., u0,
+                            dt*m_rkI.b(0), dt*m_rkI.d(0), m_kI[0],
+                            dt*m_rkI.b(1), dt*m_rkI.d(1), m_kI[1],
+                            dt*m_rkI.b(2), dt*m_rkI.d(2), m_kI[2]); break;
+        default: blas1::subroutine( dg::EmbeddedPairSum(),
+                            u1, delta,
+                            1., 0., u0,
+                            dt*m_rkI.b(0), dt*m_rkI.d(0), m_kI[0],
+                            dt*m_rkI.b(1), dt*m_rkI.d(1), m_kI[1],
+                            dt*m_rkI.b(2), dt*m_rkI.d(2), m_kI[2],
+                            dt*m_rkI.b(3), dt*m_rkI.d(3), m_kI[3]);
+            //sum the rest
+            for( unsigned i=4; i<s; i++)
+            {
+                dg::blas1::axpby( dt*m_rkI.b(i), m_kI[i], 1., u1);
+                dg::blas1::axpby( dt*m_rkI.d(i), m_kI[i], 1., delta);
+            }
+    }
+}
+/**
+* @brief Struct for Runge-Kutta fixed-step implicit time-integration
+* \f[
+ \begin{align}
+    k_i = f\left( t^n + c_i \Delta t, u^n + \Delta t \sum_{j=1}^{s} a_{ij} k_j\right) \\
+    u^{n+1} = u^{n} + \Delta t\sum_{j=1}^s b_j k_j
+ \end{align}
+\f]
+
+The method is defined by its (implicit) ButcherTableau, given by
+the coefficients \c a, \c b and \c c,  and \c s is the number
+of stages.
+
+You can provide your own coefficients or use one of our predefined methods:
+@copydoc hide_implicit_butcher_tableaus
+* @ingroup time
+*
+* @note Uses only \c dg::blas1 routines to integrate one step.
+* @copydoc hide_ContainerType
+*/
+template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
+struct ImplicitRungeKutta
+{
+    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
+    using container_type = ContainerType; //!< the type of the vector class in use
+    ///@brief No memory allocation, Call \c construct before using the object
+    ImplicitRungeKutta(){}
+
+    ///@copydoc DIRKStep::construct()
+    template<class ...SolverParams>
+    ImplicitRungeKutta( ConvertsToButcherTableau<value_type> im_tableau,
+             SolverParams&& ...ps
+             ): m_dirk( im_tableau, std::forward<SolverParams>(ps)...), m_delta(m_dirk.copyable())
+             {}
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_delta;}
+    ///All subsequent calls to \c step method will not advance time
+    ///This is useful in an operator splitting
+    void freeze_time() { m_dirk.freeze_time();}
+    ///All subsequent calls to \c step method will advance time again
+    void unfreeze_time() { m_dirk.unfreeze_time();}
+    /**
+    * @brief Advance one step
+    *
+    * @copydoc hide_rhs
+    * @param rhs right hand side subroutine
+    * @param t0 start time
+    * @param u0 value at \c t0
+    * @param t1 (write only) end time ( equals \c t0+dt on output
+    *   unless \c freeze_time() was called, then it equals \c t0 on output, may alias \c t0)
+    * @param u1 (write only) contains result on output (may alias u0)
+    * @param dt timestep
+    */
+    template<class RHS>
+    void step( RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
+        m_dirk.step( rhs, t0, u0, t1, u1, dt, m_delta);
+    }
+    ///@copydoc ERKStep::order
+    unsigned order() const {
+        return m_dirk.order();
+    }
+    ///@copydoc ERKStep::num_stages()
+    unsigned num_stages() const{
+        return m_dirk.num_stages();
+    }
+  private:
+    DIRKStep<ContainerType, SolverType> m_dirk;
+    ContainerType m_delta;
+};
 
 ///@addtogroup time
 ///@{
diff --git a/inc/dg/runge_kutta_t.cu b/inc/dg/runge_kutta_t.cu
index da331510c..993456ccb 100644
--- a/inc/dg/runge_kutta_t.cu
+++ b/inc/dg/runge_kutta_t.cu
@@ -58,6 +58,7 @@ int main()
     dg::blas1::axpby( 1., solution(t_end, damping, omega_0, omega_drive), -1., u);
     std::cout << "Norm of error is "<<sqrt(dg::blas1::dot( u, u))<<"\n";
     //![doxygen]
+    std::cout << "Explicit Methods with "<<N<<" steps:\n";
     std::vector<std::string> names{
         "Euler",
         "Midpoint-2-2",
@@ -85,5 +86,34 @@ int main()
         dg::blas1::axpby( 1., sol , -1., u1);
         std::cout << "Norm of error in "<<std::setw(24) <<name<<"\t"<<sqrt(dg::blas1::dot( u1, u1))<<"\n";
     }
+    ///-------------------------------Implicit Methods----------------------//
+    const unsigned N_im = 10; //we can take fewer steps
+    const double dt_im = (t_end - t_start)/(double)N_im;
+    std::cout << "Implicit Methods with "<<N_im<<" steps:\n";
+    std::vector<std::string> implicit_names{
+        "Euler (implicit)",
+        "SDIRK-2-1-2",
+        "Billington-3-3-2",
+        "TRBDF2-3-3-2",
+        "Kvaerno-4-2-3",
+        "ARK-4-2-3 (implicit)",
+        "Cash-5-2-4",
+        "Cash-5-3-4",
+        "SDIRK-5-3-4",
+        "ARK-6-3-4 (implicit)",
+        "Kvaerno-7-4-5",
+        "ARK-8-4-5 (implicit)",
+    };
+    for( auto name : implicit_names)
+    {
+        u = solution(t_start, damping, omega_0, omega_drive);
+        std::array<double, 2> u1(u), sol = solution(t_end, damping, omega_0, omega_drive);
+        dg::ImplicitRungeKutta<std::array<double,2>, dg::FixedPointSolver<std::array<double,2>> > irk( name, u, 100, 1e-14);
+        double t=t_start;
+        for( unsigned i=0; i<N_im; i++)
+            irk.step( functor, t, u1, t, u1, dt_im); //step inplace
+        dg::blas1::axpby( 1., sol , -1., u1);
+        std::cout << "Norm of error in "<<std::setw(24) <<name<<"\t"<<sqrt(dg::blas1::dot( u1, u1))<<"\n";
+    }
     return 0;
 }
-- 
GitLab


From 61e89ffc621359b344361817dd6c47244535a619 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 27 Jan 2019 22:09:20 +0100
Subject: [PATCH 013/540] Add small docu for time-integrators

---
 inc/dg/implicit.h    | 10 +++++++++-
 inc/dg/runge_kutta.h |  1 +
 inc/dg/tableau.h     | 11 +++++++++--
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/inc/dg/implicit.h b/inc/dg/implicit.h
index 096b51247..bfbfac217 100644
--- a/inc/dg/implicit.h
+++ b/inc/dg/implicit.h
@@ -49,7 +49,7 @@ struct TensorTraits< detail::Implicit<M, V> >
  * @tparam SolverType
     The task of this class is to solve the equation \f$ (y+\alpha\hat I(t,y)) = \rho\f$
     for the given implicit part I, parameter alpha, time t and
-    right hand side rho. For example \c dg::DefaultSolver
+    right hand side rho. For example \c dg::DefaultSolver or \c dg::FixedPointSolver
     If you write your own class:
  * it must have a solve method of type:
     \c void \c solve( value_type alpha, Implicit im, value_type t, ContainerType& y, const ContainerType& rhs);
@@ -60,10 +60,12 @@ struct TensorTraits< detail::Implicit<M, V> >
 
 /*!@brief Default Solver class for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
  *
+ * for given t, alpha and rho.
  * works only for linear positive definite operators as it uses a conjugate
  * gradient solver to invert the equation
  * @copydoc hide_ContainerType
  * @sa Karniadakis ARKStep DIRKStep
+ * @ingroup invert
  */
 template<class ContainerType>
 struct DefaultSolver
@@ -116,8 +118,14 @@ struct DefaultSolver
 
 /*!@brief Fixed point iterator for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
  *
+ * for given t, alpha and rho.
+ * The fixed point iteration is given by
+ * \f[
+ *  y_{k+1}  = \rho - \alpha\hat I(t,y_k)
+ *  \f]
  * @copydoc hide_ContainerType
  * @sa Karniadakis ARKStep DIRKStep
+ * @ingroup invert
  */
 template<class ContainerType>
 struct FixedPointSolver
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 586d0ad9e..c08f6d476 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -605,6 +605,7 @@ struct DIRKStep
     * @brief Advance one step
     *
     * @copydoc hide_rhs
+    * @param rhs right hand side subroutine
     * @param t0 start time
     * @param u0 value at \c t0
     * @param t1 (write only) end time ( equals \c t0+dt on output
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index e19dd28c7..ef7a6021b 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -15,9 +15,16 @@ namespace dg{
  * https://en.wikipedia.org/wiki/Runge%E2%80%93Kutta_methods
  * for an introduction.
  * The coefficients of the tableau
- * should be easily constructible and accessible afterwards.
- * Furthermore, it provides utilities like the number of stages, whether the
+ * are easily constructible and accessible.
+ * Furthermore, we provide utilities like the number of stages, whether the
  * tableau is embedded or not and the order of the method.
+ *
+ * Currently available are
+ *
+ * Explicit methods
+ * @copydoc hide_explicit_butcher_tableaus
+ * Implicit methods
+ * @copydoc hide_implicit_butcher_tableaus
  * @tparam real_type type of the coefficients
  * @sa RungeKutta, ERKStep, ARKStep
  * @ingroup time
-- 
GitLab


From 73866306decd0148dafcc1f52fd75df22a65274f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 27 Jan 2019 23:50:39 +0100
Subject: [PATCH 014/540] Implement a modification to the solovev Psi

using an approximate Heaviside function
test in geometry_diag compiles
---
 inc/dg/functors.h               | 131 ++++++++++++++--
 inc/dg/runge_kutta.h            |   8 +-
 inc/geometries/geometry_diag.cu |   2 +-
 inc/geometries/solovev.h        | 267 +++++++++++++++++---------------
 4 files changed, 264 insertions(+), 144 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 926db4677..268f134e4 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -757,24 +757,33 @@ struct PsiPupil
     double psimax_;
 };
 /**
- * @brief Zero up to psimax, then one
+ * @brief Zero up to xb, then one
      \f[ \begin{cases}
-        1  \text{ if } \psi > \psi_{\max} \\
-        0  \text{ else}
+        0  \text{ if } x < x_b \\
+        1  \text{ else}
      \end{cases}\f]
  */
 struct Heaviside
 {
-    Heaviside( double psimax):
-        psimax_(psimax){ }
 
-    double operator()(double psi)const
+    /**
+     * @brief Construct with xb and sign
+     *
+     * @param xb boundary value
+     * @param sign either +1 or -1, If -1, \c x<x_b is replaced with \c x>x_b
+     * thus approximating H(xb-x) if H(x-xb) is the regular Heaviside function
+     */
+    Heaviside( double xb, int sign = +1):
+        m_xb(xb){ }
+
+    double operator()(double x)const
     {
-        if( psi > psimax_) return 1.;
-        return 0.;
+        if( (x < m_xb && m_s == 1) || (x > m_xb && m_s == -1)) return 0.;
+        return 1.;
     }
     private:
-    double psimax_;
+    double m_xb;
+    int m_s;
 };
 
 /**
@@ -800,7 +809,7 @@ struct GaussianDamping
     double m_psimax, m_alpha;
 };
 /**
- * @brief Step function using tanh
+ * @brief An approximation to Heaviside using tanh
  * \f[ f(x) = B + 0.5 A(1+ \text{sign} \tanh((x-x_b)/\alpha ) ) \f]
  */
 struct TanhProfX {
@@ -833,6 +842,108 @@ struct TanhProfX {
     double profa_;
 };
 
+/**
+ * @brief An approximation to Heaviside using cosh
+     \f[ \begin{cases}
+        \cosh^{-2}((x-x_b)/\alpha)  \text{ if } x < x_b \\
+        1  \text{ else }
+     \end{cases}\f]
+ */
+struct Sech2 {
+    /**
+     * @brief Construct with xb, width and sign
+     *
+     * @param xb boundary value
+     * @param width damping width \c alpha
+     * @param sign either +1 or -1, If -1, \c x<x_b is replaced with \c x>x_b
+     * thus approximating H(xb-x) if H(x-xb) is the regular Heaviside function
+     */
+    Sech2(double xb, double width, int sign = +1) :
+        m_xb(xb), m_w(width), m_s(sign){}
+    DG_DEVICE
+    double operator() (double x)const
+    {
+        if( ( x > m_xb && m_s == 1 ) || (x < m_xb && m_s == -1 )) return 1;
+        return 1./cosh( (x-m_xb)/m_w)/cosh( (x-m_xb)/m_w);
+    }
+    DG_DEVICE
+    double operator()( double x, double y)const{ return this->operator()(x);}
+    DG_DEVICE
+    double operator()( double x, double y, double z)const{ return this->operator()(x);}
+    private:
+    double m_xb, m_w;
+    int m_s;
+};
+
+/**
+ * @brief The integral of Sech2,  approximates the integrated Heaviside using tanh
+     \f[ \begin{cases}
+        x_b + \alpha \tanh((x-x_b)/\alpha)  \text{ if } x < x_b \\
+        x  \text{ else }
+     \end{cases}\f]
+
+     The maximum is at \c x_b+alpha
+ */
+struct ISech2 {
+    /**
+     * @brief Construct with xb, width and sign
+     *
+     * @param xb boundary value
+     * @param width damping width \c alpha
+     * @param sign either +1 or -1, If -1, \c x<x_b is replaced with \c x>x_b
+     */
+    ISech2(double xb, double width, int sign = +1) :
+        m_xb(xb), m_w(width), m_s(sign){}
+    DG_DEVICE
+    double operator() (double x)const
+    {
+        if( ( x > m_xb && m_s == 1 ) || (x < m_xb && m_s == -1 )) return x;
+        return m_xb + m_w*tanh( (x-m_xb)/m_w);
+    }
+    DG_DEVICE
+    double operator()( double x, double y)const{ return this->operator()(x);}
+    DG_DEVICE
+    double operator()( double x, double y, double z)const{ return this->operator()(x);}
+    private:
+    double m_xb, m_w;
+    int m_s;
+};
+
+/**
+ * @brief The derivative of Sech2
+     \f[ \begin{cases}
+        -\frac{2}{\alpha} \cosh^{-2}((x-x_b)/\alpha)\tanh((x-x_b)/\alpha)  \text{ if } x < x_b \\
+        0  \text{ else }
+     \end{cases}\f]
+ */
+struct DSech2 {
+    /**
+     * @brief Construct with xb, width and sign
+     *
+     * @param xb boundary value
+     * @param width damping width \c alpha
+     * @param sign either +1 or -1, If -1, \c x<x_b is replaced with \c x>x_b
+     */
+    DSech2(double xb, double width, int sign = +1) :
+        m_xb(xb), m_w(width), m_s(sign){}
+    DG_DEVICE
+    double operator() (double x)const
+    {
+        if( ( x > m_xb && m_s == 1 ) || (x < m_xb && m_s == -1 )) return 0.;
+        return -2./m_w*tanh( (x-m_xb)/m_w)/cosh( (x-m_xb)/m_w)/cosh( (x-m_xb)/m_w);
+    }
+    DG_DEVICE
+    double operator()( double x, double y)const{ return this->operator()(x);}
+    DG_DEVICE
+    double operator()( double x, double y, double z)const{ return this->operator()(x);}
+    private:
+    double m_xb, m_w;
+    int m_s;
+};
+
+
+
+
 /**
  * @brief Exponential \f[ f(x) = A \exp(\lambda x)\f]
  *
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index c08f6d476..e7ba1c233 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -67,9 +67,9 @@ struct ERKStep
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_k[0];}
 
-    ///All susequqent calls to \c step method will ignore the first same as last property
+    ///All subsequent calls to \c step method will ignore the first same as last property
     void ignore_fsal(){ m_ignore_fsal = true;}
-    ///All susequqent calls to \c step method will enable the check for the first same as last property
+    ///All subsequent calls to \c step method will enable the check for the first same as last property
     void enable_fsal(){ m_ignore_fsal = false;}
 
     ///@copydoc RungeKutta::step()
@@ -521,9 +521,9 @@ struct RungeKutta
     void step( RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
         m_erk.step( rhs, t0, u0, t1, u1, dt, m_delta);
     }
-    ///All susequqent calls to \c step method will ignore the first same as last property
+    ///All subsequent calls to \c step method will ignore the first same as last property
     void ignore_fsal(){ m_erk.ignore_fsal();}
-    ///All susequqent calls to \c step method will enable the check for the first same as last property
+    ///All subsequent calls to \c step method will enable the check for the first same as last property
     void enable_fsal(){ m_erk.enable_fsal();}
     ///@copydoc ERKStep::order
     unsigned order() const {
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 6e9554c3d..9c1c1e810 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -137,7 +137,7 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
 
     //Test coefficients
-    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+    dg::geo::TokamakMagneticField c = dg::geo::createModifiedSolovevField(gp, 1, 0.1);
     const double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     const double Z_X = -1.1*gp.elongation*gp.a;
     const double R_H = gp.R_0-gp.triangularity*gp.a;
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 3b6ae4079..77fa04367 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -374,177 +374,162 @@ namespace mod
 
 struct Psip: public aCylindricalFunctor<Psip>
 {
-    Psip( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
-        psip_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
-    {
-        psipZZ_X_ = psipZZ_(R_X, Z_X);
-        psipRZ_X_ = psipRZ_(R_X, Z_X);
-        psipRR_X_ = psipRR_(R_X, Z_X);
-
-    }
+    Psip( Parameters gp, double psi0, double alpha) :
+        m_isech2( psi0, alpha, -1), m_psip(gp)
+    { }
     double do_compute(double R, double Z) const
     {
-        double psip_RZ = psip_(R,Z);
-        double Rbar = R - R_X, Zbar = Z - Z_X;
-        double psip_2 =  0.5*(- psipZZ_X_*Rbar*Rbar + 2.*psipRZ_X_*Rbar*Zbar - psipRR_X_*Zbar*Zbar) - psip_RZ ;
-        return  psip_RZ + 0.5*psip_2*cauchy_(R,Z);
+        double psip = m_psip(R,Z);
+        return m_isech2( psip);
     }
     private:
-    double R_X, Z_X;
-    double psipZZ_X_, psipRZ_X_, psipRR_X_;
-    solovev::Psip psip_;
-    solovev::PsipRR psipRR_;
-    solovev::PsipRZ psipRZ_;
-    solovev::PsipZZ psipZZ_;
-    dg::Cauchy cauchy_;
+    dg::ISech2 m_isech2;
+    solovev::Psip m_psip;
 };
 struct PsipR: public aCylindricalFunctor<PsipR>
 {
-    PsipR( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
-        psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50,1.)
-    {
-        psipZZ_X_ = psipZZ_(R_X, Z_X);
-        psipRZ_X_ = psipRZ_(R_X, Z_X);
-        psipRR_X_ = psipRR_(R_X, Z_X);
-    }
+    PsipR( Parameters gp, double psi0, double alpha) :
+        m_sech2( psi0, alpha, -1), m_psip(gp), m_psipR(gp)
+    { }
     double do_compute(double R, double Z) const
     {
-        double psipR_RZ = psipR_(R,Z);
-        if( !cauchy_.inside(R,Z)) return psipR_RZ;
-        double Rbar = R - R_X, Zbar = Z - Z_X;
-        double psip_2 =  0.5*(- psipZZ_X_*Rbar*Rbar + 2.*psipRZ_X_*Rbar*Zbar - psipRR_X_*Zbar*Zbar) - psip_(R,Z) ;
-        double psip_2R =  - psipZZ_X_*Rbar + psipRZ_X_*Zbar - psipR_RZ;
-        return psipR_RZ + 0.5*(psip_2R*cauchy_(R,Z) + psip_2*cauchy_.dx(R,Z)  );
+        double psip = m_psip(R,Z);
+        double psipR = m_psipR(R,Z);
+        return psipR*m_sech2( psip);
     }
     private:
-    double R_X, Z_X;
-    double psipZZ_X_, psipRZ_X_, psipRR_X_;
-    solovev::Psip psip_;
-    solovev::PsipR psipR_;
-    solovev::PsipRR psipRR_;
-    solovev::PsipRZ psipRZ_;
-    solovev::PsipZZ psipZZ_;
-    dg::Cauchy cauchy_;
+    dg::Sech2 m_sech2;
+    solovev::Psip m_psip;
+    solovev::PsipR m_psipR;
 };
 struct PsipZ: public aCylindricalFunctor<PsipZ>
 {
-    PsipZ( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
-        psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
-    {
-        psipZZ_X_ = psipZZ_(R_X, Z_X);
-        psipRZ_X_ = psipRZ_(R_X, Z_X);
-        psipRR_X_ = psipRR_(R_X, Z_X);
-    }
+    PsipZ( Parameters gp, double psi0, double alpha) :
+        m_sech2( psi0, alpha, -1), m_psip(gp), m_psipZ(gp)
+    { }
     double do_compute(double R, double Z) const
     {
-        double psipZ_RZ = psipZ_(R,Z);
-        if( !cauchy_.inside(R,Z)) return psipZ_RZ;
-        double Rbar = R - R_X, Zbar = Z - Z_X;
-        double psip_2 =  0.5*(- psipZZ_X_*Rbar*Rbar + 2.*psipRZ_X_*Rbar*Zbar - psipRR_X_*Zbar*Zbar) - psip_(R,Z) ;
-        double psip_2Z =  - psipRR_X_*Zbar + psipRZ_X_*Rbar - psipZ_RZ;
-        return psipZ_RZ + 0.5*(psip_2Z*cauchy_(R,Z) + psip_2*cauchy_.dy(R,Z));
+        double psip = m_psip(R,Z);
+        double psipZ = m_psipZ(R,Z);
+        return psipZ*m_sech2( psip);
     }
     private:
-    double R_X, Z_X;
-    double psipZZ_X_, psipRZ_X_, psipRR_X_;
-    solovev::Psip psip_;
-    solovev::PsipZ psipZ_;
-    solovev::PsipRR psipRR_;
-    solovev::PsipRZ psipRZ_;
-    solovev::PsipZZ psipZZ_;
-    dg::Cauchy cauchy_;
+    dg::Sech2 m_sech2;
+    solovev::Psip m_psip;
+    solovev::PsipZ m_psipZ;
 };
 
 struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 {
-    PsipZZ( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
-        psip_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
-    {
-        psipZZ_X_ = psipZZ_(R_X, Z_X);
-        psipRZ_X_ = psipRZ_(R_X, Z_X);
-        psipRR_X_ = psipRR_(R_X, Z_X);
-    }
+    PsipZZ( Parameters gp, double psi0, double alpha) :
+        m_sech2( psi0, alpha, -1), m_dsech2( psi0, alpha, -1), m_psip(gp), m_psipZ(gp), m_psipZZ(gp)
+    { }
     double do_compute(double R, double Z) const
     {
-        double psipZZ_RZ = psipZZ_(R,Z);
-        if( !cauchy_.inside(R, Z)) return psipZZ_RZ;
-        double Rbar = R - R_X, Zbar = Z - Z_X;
-        double psip_2 =  0.5*(- psipZZ_X_*Rbar*Rbar + 2.*psipRZ_X_*Rbar*Zbar - psipRR_X_*Zbar*Zbar) - psip_(R,Z) ;
-        double psip_2Z =  - psipRR_X_*Zbar + psipRZ_X_*Rbar - psipZ_(R,Z);
-        double psip_2ZZ =  - psipRR_X_ - psipZZ_RZ;
-        return psipZZ_RZ + 0.5*(psip_2ZZ*cauchy_(R,Z) + 2.*cauchy_.dy(R,Z)*psip_2Z +  psip_2*cauchy_.dyy(R,Z));
+        double psip = m_psip(R,Z);
+        double psipZ = m_psipZ(R,Z);
+        double psipZZ = m_psipZZ(R,Z);
+        return psipZZ*m_sech2( psip) + psipZ*psipZ*m_dsech2(psip);
     }
     private:
-    double R_X, Z_X;
-    double psipZZ_X_, psipRZ_X_, psipRR_X_;
-    solovev::Psip psip_;
-    solovev::PsipZ psipZ_;
-    solovev::PsipRR psipRR_;
-    solovev::PsipRZ psipRZ_;
-    solovev::PsipZZ psipZZ_;
-    dg::Cauchy cauchy_;
+    dg::Sech2 m_sech2;
+    dg::DSech2 m_dsech2;
+    solovev::Psip m_psip;
+    solovev::PsipZ m_psipZ;
+    solovev::PsipZZ m_psipZZ;
 };
 struct PsipRR: public aCylindricalFunctor<PsipRR>
 {
-    PsipRR( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
-        psip_(gp), psipR_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
+    PsipRR( Parameters gp, double psi0, double alpha) :
+        m_sech2( psi0, alpha, -1), m_dsech2( psi0, alpha, -1), m_psip(gp), m_psipR(gp), m_psipRR(gp)
+    { }
+    double do_compute(double R, double Z) const
     {
-        psipZZ_X_ = psipZZ_(R_X, Z_X);
-        psipRZ_X_ = psipRZ_(R_X, Z_X);
-        psipRR_X_ = psipRR_(R_X, Z_X);
+        double psip = m_psip(R,Z);
+        double psipR = m_psipR(R,Z);
+        double psipRR = m_psipRR(R,Z);
+        return psipRR*m_sech2( psip) + psipR*psipR*m_dsech2(psip);
     }
+    private:
+    dg::Sech2 m_sech2;
+    dg::DSech2 m_dsech2;
+    solovev::Psip m_psip;
+    solovev::PsipR m_psipR;
+    solovev::PsipRR m_psipRR;
+};
+struct PsipRZ: public aCylindricalFunctor<PsipRZ>
+{
+    PsipRZ( Parameters gp, double psi0, double alpha) :
+        m_sech2( psi0, alpha, -1), m_dsech2( psi0, alpha, -1), m_psip(gp), m_psipR(gp), m_psipZ(gp), m_psipRZ(gp)
+    { }
     double do_compute(double R, double Z) const
     {
-        double psipRR_RZ = psipRR_(R,Z);
-        if( !cauchy_.inside(R,Z)) return psipRR_RZ;
-        double Rbar = R - R_X, Zbar = Z - Z_X;
-        double psip_2 =  0.5*(- psipZZ_X_*Rbar*Rbar + 2.*psipRZ_X_*Rbar*Zbar - psipRR_X_*Zbar*Zbar) - psip_(R,Z) ;
-        double psip_2R =  - psipZZ_X_*Rbar + psipRZ_X_*Zbar - psipR_(R,Z);
-        double psip_2RR =  - psipZZ_X_ - psipRR_RZ;
-        return psipRR_RZ + 0.5*(psip_2RR*cauchy_(R,Z) + 2.*cauchy_.dx(R,Z)*psip_2R +  psip_2*cauchy_.dxx(R,Z));
+        double psip = m_psip(R,Z);
+        double psipR = m_psipR(R,Z);
+        double psipZ = m_psipZ(R,Z);
+        double psipRZ = m_psipRZ(R,Z);
+        return psipRZ*m_sech2( psip) + psipR*psipZ*m_dsech2(psip);
     }
     private:
-    double R_X, Z_X;
-    double psipZZ_X_, psipRZ_X_, psipRR_X_;
-    solovev::Psip psip_;
-    solovev::PsipR psipR_;
-    solovev::PsipRR psipRR_;
-    solovev::PsipRZ psipRZ_;
-    solovev::PsipZZ psipZZ_;
-    dg::Cauchy cauchy_;
+    dg::Sech2 m_sech2;
+    dg::DSech2 m_dsech2;
+    solovev::Psip m_psip;
+    solovev::PsipR m_psipR;
+    solovev::PsipZ m_psipZ;
+    solovev::PsipRZ m_psipRZ;
 };
-struct PsipRZ: public aCylindricalFunctor<PsipRZ>
+
+struct Ipol: public aCylindricalFunctor<Ipol>
 {
-    PsipRZ( Parameters gp): R_X( gp.R_0-1.1*gp.triangularity*gp.a), Z_X(-1.1*gp.elongation*gp.a),
-        psip_(gp), psipR_(gp), psipZ_(gp), psipRR_(gp), psipRZ_(gp), psipZZ_(gp), cauchy_( R_X, Z_X, 50, 50, 1)
+    Ipol(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp, psi0, alpha) { }
+    double do_compute(double R, double Z) const
     {
-        psipZZ_X_ = psipZZ_(R_X, Z_X);
-        psipRZ_X_ = psipRZ_(R_X, Z_X);
-        psipRR_X_ = psipRR_(R_X, Z_X);
+        //sign before A changed to -
+        return qampl_*sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.);
     }
+  private:
+    double R_0_, A_,qampl_;
+    mod::Psip psip_;
+};
+struct IpolR: public aCylindricalFunctor<IpolR>
+{
+    IpolR(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp, psi0, alpha), psipR_(gp, psi0, alpha) { }
     double do_compute(double R, double Z) const
     {
-        double psipRZ_RZ = psipRZ_(R,Z);
-        if( !cauchy_.inside(R,Z)) return psipRZ_RZ;
-        double Rbar = R - R_X, Zbar = Z - Z_X;
-        double psip_2 =  0.5*(- psipZZ_(R_X, Z_X)*Rbar*Rbar + 2.*psipRZ_(R_X, Z_X)*Rbar*Zbar - psipRR_(R_X, Z_X)*Zbar*Zbar) - psip_(R,Z);
-        double psip_2R =  - psipZZ_X_*Rbar + psipRZ_X_*Zbar - psipR_(R,Z);
-        double psip_2Z =  - psipRR_X_*Zbar + psipRZ_X_*Rbar - psipZ_(R,Z);
-        double psip_2RZ =  - psipRZ_X_ - psipRZ_RZ;
-        return psipRZ_RZ + 0.5*(psip_2RZ*cauchy_(R,Z) + cauchy_.dx(R,Z)*psip_2Z + cauchy_.dy(R,Z)*psip_2R  +  psip_2*cauchy_.dxy(R,Z));
+        return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipR_(R,Z)/R_0_);
     }
-    private:
-    double R_X, Z_X;
-    double psipZZ_X_, psipRZ_X_, psipRR_X_;
-    solovev::Psip psip_;
-    solovev::PsipR psipR_;
-    solovev::PsipZ psipZ_;
-    solovev::PsipRR psipRR_;
-    solovev::PsipRZ psipRZ_;
-    solovev::PsipZZ psipZZ_;
-    dg::Cauchy cauchy_;
+  private:
+    double R_0_, A_,qampl_;
+    mod::Psip psip_;
+    mod::PsipR psipR_;
+};
+struct IpolZ: public aCylindricalFunctor<IpolZ>
+{
+    IpolZ(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp, psi0, alpha), psipZ_(gp, psi0, alpha) { }
+    double do_compute(double R, double Z) const
+    {
+        return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipZ_(R,Z)/R_0_);
+    }
+  private:
+    double R_0_, A_,qampl_;
+    mod::Psip psip_;
+    mod::PsipZ psipZ_;
 };
 
+static inline dg::geo::CylindricalFunctorsLvl2 createPsip( Parameters gp,
+    double psi0, double alpha)
+{
+    return CylindricalFunctorsLvl2( Psip(gp, psi0, alpha), PsipR(gp, psi0,
+    alpha), PsipZ(gp, psi0, alpha), PsipRR(gp, psi0, alpha), PsipRZ(gp,
+    psi0, alpha), PsipZZ(gp, psi0, alpha));
+}
+static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp,
+    double psi0, double alpha)
+{
+    return CylindricalFunctorsLvl1( Ipol(gp, psi0, alpha), IpolR(gp, psi0,
+    alpha), IpolZ(gp, psi0, alpha));
+}
+
 } //namespace mod
 ///@endcond
 
@@ -561,9 +546,33 @@ struct PsipRZ: public aCylindricalFunctor<PsipRZ>
  * @return A magnetic field object
  * @ingroup geom
  */
-static inline dg::geo::TokamakMagneticField createSolovevField( dg::geo::solovev::Parameters gp)
+static inline dg::geo::TokamakMagneticField createSolovevField(
+    dg::geo::solovev::Parameters gp)
+{
+    return TokamakMagneticField( gp.R_0, solovev::createPsip(gp),
+        solovev::createIpol(gp));
+}
+/**
+ * @brief Create a modified Solovev Magnetic field
+ *
+ * Based on \c dg::geo::solovev::mod::Psip(gp) and
+ * \c dg::geo::solovev::mod::Ipol(gp)
+ * We modify psi above a certain value to a constant using the
+ * \c dg::ISech2 function (an approximation to the integrated Heaviside
+ * function with width alpha), i.e. we replace psi with ISech2(psi).
+ * This subsequently modifies all derivatives of psi and the poloidal
+ * current of course.
+ * @param gp Solovev parameters
+ * @param psi0 above this value psi is modified to a constant
+ * @param alpha determines how quickly the modification acts (smaller is quicker)
+ * @return A magnetic field object
+ * @ingroup geom
+ */
+static inline dg::geo::TokamakMagneticField createModifiedSolovevField(
+    dg::geo::solovev::Parameters gp, double psi0, double alpha)
 {
-    return TokamakMagneticField( gp.R_0, solovev::createPsip(gp), solovev::createIpol(gp));
+    return TokamakMagneticField( gp.R_0, solovev::mod::createPsip(gp,
+        psi0, alpha), solovev::mod::createIpol(gp, psi0, alpha));
 }
 
 } //namespace geo
-- 
GitLab


From 1c49f0213fe0e1fb4b9ec55d400b44626ac3f647 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 29 Jan 2019 00:00:21 +0100
Subject: [PATCH 015/540] Add implicit midpoint and trapezoidal tableaus

- also mark fsal in explicit tableau description
- also catch explicit first step in DIRK implementation
---
 inc/dg/cg.h             |  2 +-
 inc/dg/runge_kutta.h    | 13 +++++---
 inc/dg/runge_kutta_t.cu |  2 ++
 inc/dg/tableau.h        | 69 +++++++++++++++++++++++++++--------------
 4 files changed, 57 insertions(+), 29 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index c718eaa63..26abb9009 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -338,7 +338,7 @@ struct Extrapolation
 
     /**
     * @brief Extrapolate value to given time
-    * @param t time to which to extrapolate (must be different from the times used in the update function, else division by zero occurs)
+    * @param t time to which to extrapolate
     * @param new_x (write only) contains extrapolated value on output ( may alias the tail)
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
     */
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index e7ba1c233..eaeb5d078 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -653,11 +653,12 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
     tu = DG_FMA( m_rkI.c(0),dt, t0);
     if( m_freeze_time) tu = t0;
     blas1::copy( u0, delta); //better init with rhs
-    m_solver.solve( -dt*m_rkI.a(0,0), rhs, tu, delta, u0);
+    if( !(m_rkI.a(0,0)==0) )
+        m_solver.solve( -dt*m_rkI.a(0,0), rhs, tu, delta, u0);
+    rhs(tu, delta, m_kI[0]);
 
     //1 stage
     if( s>1){
-        rhs(tu, delta, m_kI[0]);
         blas1::evaluate( m_rhs, dg::equals(), PairSum(), 1., u0,
                 dt*m_rkI.a(1,0), m_kI[0]);
         tu = DG_FMA( m_rkI.c(1),dt, t0);
@@ -705,9 +706,11 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
     //Now compute result and error estimate
     switch( s)
     {
-        case 1: dg::blas1::copy( delta, u1); break; //implicit Euler
-        case 2:
-            blas1::subroutine( dg::EmbeddedPairSum(),
+        case 1: blas1::subroutine( dg::EmbeddedPairSum(),
+                            u1, delta,
+                            1., 0., u0,
+                            dt*m_rkI.b(0), dt*m_rkI.d(0), m_kI[0]); break;
+        case 2: blas1::subroutine( dg::EmbeddedPairSum(),
                             u1, delta,
                             1., 0., u0,
                             dt*m_rkI.b(0), dt*m_rkI.d(0), m_kI[0],
diff --git a/inc/dg/runge_kutta_t.cu b/inc/dg/runge_kutta_t.cu
index 993456ccb..ace3ab338 100644
--- a/inc/dg/runge_kutta_t.cu
+++ b/inc/dg/runge_kutta_t.cu
@@ -92,6 +92,8 @@ int main()
     std::cout << "Implicit Methods with "<<N_im<<" steps:\n";
     std::vector<std::string> implicit_names{
         "Euler (implicit)",
+        "Midpoint (implicit)",
+        "Trapezoidal-2-2",
         "SDIRK-2-1-2",
         "Billington-3-3-2",
         "TRBDF2-3-3-2",
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index ef7a6021b..060cf0d81 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -178,24 +178,12 @@ ButcherTableau<real_type> explicit_euler_1_1()
     return ButcherTableau<real_type>( 1,1, a,b,c);
 }
 template<class real_type>
-ButcherTableau<real_type> implicit_euler_1_1()
-{
-    real_type a[1] = {1};
-    real_type b[1] = {1};
-    real_type c[1] = {1};
-    return ButcherTableau<real_type>( 1,1, a,b,c);
-}
-template<class real_type>
 ButcherTableau<real_type> midpoint_2_2()
 {
-    real_type a[4] = {   0, 0, 
+    real_type a[4] = {   0, 0,
                        0.5, 0 };
     real_type b[2] = {0., 1.};
     real_type c[2] = {0, 0.5};
-    //real_type a[4] = {   0, 0, 
-    //                    1, 0 };
-    //real_type b[2] = {0.5,0.5};
-    //real_type c[2] = {0, 1};
     return ButcherTableau<real_type>( 2,2, a,b,c);
 }
 template<class real_type>
@@ -222,7 +210,7 @@ ButcherTableau<real_type> classic_4_4()
     return ButcherTableau<real_type>( 4,4, a,b,c);
 }
 //From Yoh and Zhong (AIAA 42, 2004)
-//!Attention! assumes another form of implementation 
+//!Attention! assumes another form of implementation
 //than ARK tableaus
 template<class real_type>
 ButcherTableau<real_type> sirk3a_ex_3_3()
@@ -637,14 +625,39 @@ ButcherTableau<real_type> feagin_17_8_10()
     return ButcherTableau<real_type>( 17, 8, 10, a, b,bt, c);
 }
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%%Implicit Butcher tables%%%%%%%%%%%%%%%%%%
+template<class real_type>
+ButcherTableau<real_type> implicit_euler_1_1()
+{
+    real_type a[1] = {1};
+    real_type b[1] = {1};
+    real_type c[1] = {1};
+    return ButcherTableau<real_type>( 1,1, a,b,c);
+}
+template<class real_type>
+ButcherTableau<real_type> implicit_midpoint_1_2()
+{
+    real_type a[1] = { 0.5};
+    real_type b[1] = { 1.};
+    real_type c[1] = { 0.5};
+    return ButcherTableau<real_type>( 1,2, a,b,c);
+}
+template<class real_type>
+ButcherTableau<real_type> trapezoidal_2_2()
+{
+    real_type a[4] = { 0, 0, 0.5, 0.5};
+    real_type b[2] = { 0.5, 0.5};
+    real_type c[2] = { 0, 1.};
+    return ButcherTableau<real_type>( 2,2, a,b,c);
+}
 
 template<class real_type>
 ButcherTableau<real_type> sdirk_2_1_2()
 {
+    real_type x = (2.-sqrt(2.))/2.;
     real_type data[] = {
-        1 , 1 , 0 ,
-  0 , -1 , 1 ,
-  2 , 1./2. , 1./2. ,
+  x , x , 0 ,
+  1.-x , 1.-2.*x , x,
+  2 , 0.5 , 0.5,
   1 , 1 , 0
     };
     return ButcherTableau<real_type>( 2, data);
@@ -842,7 +855,9 @@ enum tableau_identifier{
     FEAGIN_17_8_10,//!< <a href="http://sce.uhcl.edu/rungekutta/">Feagin-17-8-10</a>
     //implicit ARKode tableaus
     IMPLICIT_EULER_1_1,//!< <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Euler (implicit)</a>
-    SDIRK_2_1_2,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">SDIRK-2-1-2</a>
+    IMPLICIT_MIDPOINT_1_2, //!<  <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">implicit Midpoint</a>
+    TRAPEZOIDAL_2_2,//!<  <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Crank-Nicolson method</a>
+    SDIRK_2_1_2, //!<  <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">generic 2nd order A and L-stable</a>
     BILLINGTON_3_3_2,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Billington-3-3-2</a>
     TRBDF2_3_3_2,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">TRBDF2-3-3-2</a>
     KVAERNO_4_2_3,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Kvaerno-4-2-3</a>
@@ -865,8 +880,6 @@ ButcherTableau<real_type> tableau( enum tableau_identifier id)
     switch(id){
         case EXPLICIT_EULER_1_1:
             return dg::tableau::explicit_euler_1_1<real_type>();
-        case IMPLICIT_EULER_1_1:
-            return dg::tableau::implicit_euler_1_1<real_type>();
         case MIDPOINT_2_2:
             return dg::tableau::midpoint_2_2<real_type>();
         case KUTTA_3_3:
@@ -899,6 +912,12 @@ ButcherTableau<real_type> tableau( enum tableau_identifier id)
             return dg::tableau::fehlberg_13_7_8<real_type>();
         case FEAGIN_17_8_10:
             return dg::tableau::feagin_17_8_10<real_type>();
+        case IMPLICIT_EULER_1_1:
+            return dg::tableau::implicit_euler_1_1<real_type>();
+        case IMPLICIT_MIDPOINT_1_2:
+            return dg::tableau::implicit_midpoint_1_2<real_type>();
+        case TRAPEZOIDAL_2_2:
+            return dg::tableau::trapezoidal_2_2<real_type>();
         case SDIRK_2_1_2:
             return dg::tableau::sdirk_2_1_2<real_type>();
         case BILLINGTON_3_3_2:
@@ -952,6 +971,8 @@ ButcherTableau<real_type> tableau( std::string name)
         {"Feagin-17-8-10", FEAGIN_17_8_10},
         //Implicit methods
         {"Euler (implicit)", IMPLICIT_EULER_1_1},
+        {"Midpoint (implicit)", IMPLICIT_MIDPOINT_1_2},
+        {"Trapezoidal-2-2", TRAPEZOIDAL_2_2},
         {"SDIRK-2-1-2", SDIRK_2_1_2},
         {"Billington-3-3-2", BILLINGTON_3_3_2},
         {"TRBDF2-3-3-2", TRBDF2_3_3_2},
@@ -982,14 +1003,14 @@ ButcherTableau<real_type> tableau( std::string name)
  *   Kutta-3-3              | dg::KUTTA_3_3              | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Kutta-3-3</a>
  *   Runge-Kutta-4-4        | dg::CLASSIC_4_4            | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods"> "The" Runge-Kutta method</a>
  *   Heun-Euler-2-1-2       | dg::HEUN_EULER_2_1_2       | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Heun-Euler-2-1-2</a>
- *   Bogacki-Shampine-4-2-3 | dg::BOGACKI_SHAMPINE_4_2_3 | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Bogacki-Shampine</a>
+ *   Bogacki-Shampine-4-2-3 | dg::BOGACKI_SHAMPINE_4_2_3 | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Bogacki-Shampine</a> (fsal)
  *   ARK-4-2-3 (explicit)   | dg::ARK324L2SA_ERK_4_2_3   | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-4-2-3 (explicit)</a>
  *   Zonneveld-5-3-4        | dg::ZONNEVELD_5_3_4        | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Zonnveld-5-3-4</a>
  *   ARK-6-3-4 (explicit)   | dg::ARK436L2SA_ERK_6_3_4   | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-6-3-4 (explicit)</a>
  *   Sayfy_Aburub-6-3-4     | dg::SAYFY_ABURUB_6_3_4     | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Sayfy_Aburub_6_3_4</a>
  *   Cash_Karp-6-4-5        | dg::CASH_KARP_6_4_5        | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Cash-Karp</a>
  *   Fehlberg-6-4-5         | dg::FEHLBERG_6_4_5         | <a href="https://en.wikipedia.org/wiki/Runge%E2%80%93Kutta%E2%80%93Fehlberg_method">Runge-Kutta-Fehlberg</a>
- *   Dormand-Prince-7-4-5   | dg::DORMAND_PRINCE_7_4_5   | <a href="https://en.wikipedia.org/wiki/Dormand%E2%80%93Prince_method">Dormand-Prince method</a>
+ *   Dormand-Prince-7-4-5   | dg::DORMAND_PRINCE_7_4_5   | <a href="https://en.wikipedia.org/wiki/Dormand%E2%80%93Prince_method">Dormand-Prince method</a> (fsal)
  *   ARK-8-4-5 (explicit)   | dg::ARK548L2SA_ERK_8_4_5   | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-4-2-3 (explicit)</a>
  *   Verner-8-5-6           | dg::VERNER_8_5_6           | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Verner-8-5-6</a>
  *   Fehlberg-13-7-8        | dg::FEHLBERG_13_7_8        | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Fehlberg-13-7-8</a>
@@ -1002,7 +1023,9 @@ ButcherTableau<real_type> tableau( std::string name)
  *    Name  | Identifier | Description
  *   -------|------------| -----------
  *   Euler (implicit)     | dg::IMPLICIT_EULER_1_1     |  <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">backward Euler</a>
- *   SDIRK-2-1-2          | dg::SDIRK_2_1_2            |  <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">SDIRK-2-1-2</a>
+ *   Midpoint (implicit)     | dg::IMPLICIT_MIDPOINT_1_2     |  <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">implicit Midpoint</a>
+ *   Trapezoidal-2-2     | dg::TRAPEZOIDAL_2_2     |  <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Crank-Nicolson method</a>
+ *   SDIRK-2-1-2          | dg::SDIRK_2_1_2            | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">generic 2nd order A and L-stable</a>
  *   Billington-3-3-2     | dg::BILLINGTON_3_3_2       |  <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Billington-3-3-2</a>
  *   TRBDF2-3-3-2         | dg::TRBDF2_3_3_2           |  <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">TRBDF2-3-3-2</a>
  *   Kvaerno-4-2-3        | dg::KVAERNO_4_2_3          |  <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Kvaerno-4-2-3</a>
-- 
GitLab


From d815916f56b476ee64e84aec3bd590cc0ce067d9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 28 Jan 2019 16:14:27 +0100
Subject: [PATCH 016/540] More tests for the new timesteppers

- adaptive_t contains dirk stepper
- multistep_t contains operator splitting (IMEX_a)
- confirm small timestep in ARKode if eps is too small
---
 inc/dg/adaptive_t.cu            | 30 +++++++++++++++++++
 inc/dg/multistep_t.cu           | 51 +++++++++++++++++++++++++++++++--
 inc/dg/runge_kutta.h            | 25 ++++++++--------
 inc/geometries/geometry_diag.cu | 10 +++----
 4 files changed, 96 insertions(+), 20 deletions(-)

diff --git a/inc/dg/adaptive_t.cu b/inc/dg/adaptive_t.cu
index 2a21ff956..c4bbe0e49 100644
--- a/inc/dg/adaptive_t.cu
+++ b/inc/dg/adaptive_t.cu
@@ -54,6 +54,7 @@ int main()
     dg::blas1::axpby( 1., solution(t_end, damping, omega_0, omega_drive), -1., u_end);
     std::cout << "With "<<counter<<"\t Dormand Prince steps norm of error is "<<sqrt(dg::blas1::dot( u_end, u_end))<<"\n";
     //![doxygen.]
+    std::cout << "Explicit Methods \n";
     std::vector<std::string> names{
         "Heun-Euler-2-1-2",
         "Bogacki-Shampine-4-2-3",
@@ -79,5 +80,34 @@ int main()
         dg::blas1::axpby( 1.,sol  , -1., u_end);
         std::cout << "With "<<std::setw(6)<<counter<<" steps norm of error in "<<std::setw(24)<<name<<"\t"<<dg::l2norm( u_end)<<"\n";
     }
+    ///-------------------------------Implicit Methods----------------------//
+    std::cout << "Implicit Methods \n";
+    std::vector<std::string> implicit_names{
+        "SDIRK-2-1-2",
+        "Billington-3-3-2",
+        "TRBDF2-3-3-2",
+        "Kvaerno-4-2-3",
+        "ARK-4-2-3 (implicit)",
+        "Cash-5-2-4",
+        "Cash-5-3-4",
+        "SDIRK-5-3-4",
+        "ARK-6-3-4 (implicit)",
+        "Kvaerno-7-4-5",
+        "ARK-8-4-5 (implicit)",
+    };
+    for( auto name : implicit_names)
+    {
+        dt = 0;
+        u_start = solution(t_start, damping, omega_0, omega_drive);
+        dg::Adaptive< dg::DIRKStep<
+            std::array<double,2>, dg::FixedPointSolver<std::array<double,2> > > >
+                pd( name, u_start, 100, 1e-14);
+        counter = integrateAdaptive( pd, functor, t_start, u_start, t_end,
+            u_end, dt, dg::pid_control, dg::l2norm, 1e-6, 1e-10);
+
+        std::array<double, 2> sol = solution(t_end, damping, omega_0, omega_drive);
+        dg::blas1::axpby( 1.,sol  , -1., u_end);
+        std::cout << "With "<<std::setw(6)<<counter<<" steps norm of error in "<<std::setw(24)<<name<<"\t"<<dg::l2norm( u_end)<<"\n";
+    }
     return 0;
 }
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 878f0a0d3..12ff85876 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -1,4 +1,5 @@
 #include <iostream>
+#include <iomanip>
 #include <functional>
 
 #undef DG_DEBUG
@@ -102,8 +103,11 @@ int main()
 {
     unsigned n = 3, Nx = 50 , Ny = 50;
     std::cout << "Program tests Multistep and Semi-Implicit methods on a manufactured PDE\n";
+    //std::cout << "Type n (3), Nx (50), Ny (50)\n";
+    //std::cin >> n >> Nx >> Ny;
+    std::cout << "Computing on "<<n<<" x "<<Nx<<" x "<<Ny<<"\n";
     const double T = 0.1;
-    const double NT= 40, eps = 1e-8;
+    const double NT= 40, eps = 1e-6;
     const double dt = (T/NT);
     const double nu = 0.01;
     //construct the grid and the explicit and implicit parts
@@ -120,6 +124,7 @@ int main()
     double time = 0.;
     dg::DVec error( sol);
     exblas::udouble res;
+    std::cout << "### Test explicit multistep methods with "<<NT<<"steps\n";
     for( unsigned s=1; s<6; s++)
     {
         time = 0., y0 = init;
@@ -134,6 +139,7 @@ int main()
     }
     Explicit<dg::DVec> ex( grid, nu);
     Implicit<dg::DMatrix, dg::DVec> im( grid, nu);
+    std::cout << "### Test semi-implicit Karniadakis methods with "<<NT<<"steps\n";
     //![karniadakis]
     //construct time stepper
     dg::Karniadakis< dg::DVec > karniadakis( y0, y0.size(), eps);
@@ -149,14 +155,15 @@ int main()
     std::cout << "Relative error Karniadakis is "<< res.d<<"\t"<<res.i<<std::endl;
 
 
-    //TEST ARK methods
+    std::cout << "### Test semi-implicit ARK methods\n";
     std::vector<std::string> names{"ARK-4-2-3", "ARK-6-3-4", "ARK-8-4-5"};
+    double rtol = 1e-7, atol = 1e-10;
     for( auto name : names)
     {
         //![adaptive]
         time = 0., y0 = init;
         dg::Adaptive<dg::ARKStep<dg::DVec>> adapt( name, y0, y0.size(), eps);
-        double time = 0, rtol = 1e-5, atol = 1e-10;
+        double time = 0;
         double dt = adapt.guess_stepsize( ex, time, y0, dg::forward, dg::l2norm, rtol, atol);
         int counter=0;
         while( time < T )
@@ -172,5 +179,43 @@ int main()
         std::cout << counter <<" steps! ";
         std::cout << "Relative error "<<name<<" is "<< res.d<<"\t"<<res.i<<std::endl;
     }
+    std::cout << "### Test first order operator splitting\n";
+
+    std::vector<std::string> ex_names{
+        "Heun-Euler-2-1-2",
+        "Bogacki-Shampine-4-2-3",
+        "ARK-4-2-3 (explicit)",
+        "Zonneveld-5-3-4",
+        "ARK-6-3-4 (explicit)",
+        "Sayfy-Aburub-6-3-4",
+        "Cash-Karp-6-4-5",
+        "Fehlberg-6-4-5",
+        "Dormand-Prince-7-4-5",
+        "ARK-8-4-5 (explicit)"
+    };
+    for( auto name : ex_names)
+    {
+        time = 0., y0 = init;
+        dg::Adaptive<dg::ERKStep<dg::DVec>> adapt( name, y0);
+        dg::ImplicitRungeKutta<dg::DVec> dirk( "Euler (implicit)", y0, y0.size(), eps );
+        double time = 0;
+        double dt = adapt.guess_stepsize( ex, time, y0, dg::forward, dg::l2norm, rtol, atol);
+        int counter=0;
+        adapt.stepper().ignore_fsal();
+        dirk.freeze_time();
+        while( time < T )
+        {
+            if( time + dt > T)
+                dt = T-time;
+            dirk.step( im, time, y0, time, y0, dt);
+            adapt.step( ex, time, y0, time, y0, dt, dg::pid_control,
+                dg::l2norm, rtol, atol);
+            counter ++;
+        }
+        dg::blas1::axpby( -1., sol, 1., y0);
+        res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
+        std::cout << counter <<" steps! ";
+        std::cout << "Relative error "<<std::setw(24) <<name<<"\t"<<res.d<<"\n";
+    }
     return 0;
 }
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index eaeb5d078..e7209b846 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -67,7 +67,7 @@ struct ERKStep
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_k[0];}
 
-    ///All subsequent calls to \c step method will ignore the first same as last property
+    ///All subsequent calls to \c step method will ignore the first same as last property (useful if you want to implement an operator splitting)
     void ignore_fsal(){ m_ignore_fsal = true;}
     ///All subsequent calls to \c step method will enable the check for the first same as last property
     void enable_fsal(){ m_ignore_fsal = false;}
@@ -243,6 +243,11 @@ void ERKStep<ContainerType>::step( RHS& f, value_type t0, const ContainerType& u
  * Currently, the possible Butcher Tableaus for a fully implicit-explicit scheme
  * are the "ARK-4-2-3", "ARK-6-3-4" and "ARK-8-4-5" combinations.
  * So far we did not implement the use of a mass matrix \c M.
+ * @attention When you use the ARKStep in combination with the Adaptive time
+ * step algorithm pay attention to solve the implicit part with sufficient
+ * accuracy. Else, the error propagates into the time controller, which will
+ * then choose the timestep as if the implicit part was explicit i.e. far too
+ * small (don't really know fully why though).
  *
  * @copydoc hide_SolverType
  * @copydoc hide_ContainerType
@@ -521,7 +526,7 @@ struct RungeKutta
     void step( RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
         m_erk.step( rhs, t0, u0, t1, u1, dt, m_delta);
     }
-    ///All subsequent calls to \c step method will ignore the first same as last property
+    ///All subsequent calls to \c step method will ignore the first same as last property (useful if you want to implement an operator splitting)
     void ignore_fsal(){ m_erk.ignore_fsal();}
     ///All subsequent calls to \c step method will enable the check for the first same as last property
     void enable_fsal(){ m_erk.enable_fsal();}
@@ -630,7 +635,7 @@ struct DIRKStep
     }
 
     ///All subsequent calls to \c step method will not advance time
-    ///This is useful in an operator splitting
+    ///This is useful if you want to implement operator splitting
     void freeze_time() { m_freeze_time=true;}
     ///All subsequent calls to \c step method will advance time again
     void unfreeze_time() { m_freeze_time=false;}
@@ -650,8 +655,7 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
     value_type tu = t0;
     //0 stage
     //rhs = u0
-    tu = DG_FMA( m_rkI.c(0),dt, t0);
-    if( m_freeze_time) tu = t0;
+    tu = m_freeze_time ? t0 : DG_FMA( m_rkI.c(0),dt, t0);
     blas1::copy( u0, delta); //better init with rhs
     if( !(m_rkI.a(0,0)==0) )
         m_solver.solve( -dt*m_rkI.a(0,0), rhs, tu, delta, u0);
@@ -661,8 +665,7 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
     if( s>1){
         blas1::evaluate( m_rhs, dg::equals(), PairSum(), 1., u0,
                 dt*m_rkI.a(1,0), m_kI[0]);
-        tu = DG_FMA( m_rkI.c(1),dt, t0);
-        if( m_freeze_time) tu = t0;
+        tu = m_freeze_time ? t0 : DG_FMA( m_rkI.c(1),dt, t0);
         //store solution in delta, init with last solution
         blas1::copy( m_rhs, delta); //better init with rhs
         m_solver.solve( -dt*m_rkI.a(1,1), rhs, tu, delta, m_rhs);
@@ -673,8 +676,7 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
         blas1::evaluate( m_rhs, dg::equals(), PairSum(), 1., u0,
                  dt*m_rkI.a(2,0), m_kI[0],
                  dt*m_rkI.a(2,1), m_kI[1]);
-        tu = DG_FMA( m_rkI.c(2),dt, t0);
-        if( m_freeze_time) tu = t0;
+        tu = m_freeze_time ? t0 : DG_FMA( m_rkI.c(2),dt, t0);
         //just take last solution as init
         blas1::copy( m_rhs, delta); //better init with rhs
         m_solver.solve( -dt*m_rkI.a(2,2), rhs, tu, delta, m_rhs);
@@ -695,14 +697,13 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
             dg::blas1::copy( u0, m_rhs);
             for( unsigned j=0; j<i; j++)
                 dg::blas1::axpby( dt*m_rkI.a(i,j), m_kI[j], 1., m_rhs);
-            tu = DG_FMA( m_rkI.c(i),dt, t0);
-            if( m_freeze_time) tu = t0;
+            tu = m_freeze_time ? t0 : DG_FMA( m_rkI.c(i),dt, t0);
             blas1::copy( m_rhs, delta); //better init with rhs
             m_solver.solve( -dt*m_rkI.a(i,i), rhs, tu, delta, m_rhs);
             rhs(tu, delta, m_kI[i]);
         }
     }
-    t1 = t0 + dt;
+    t1 = m_freeze_time ? t0 : t0 + dt;
     //Now compute result and error estimate
     switch( s)
     {
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 9c1c1e810..48b24b141 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -30,10 +30,10 @@ struct Parameters
         Nx = js.get("Nx",100).asUInt();
         Ny = js.get("Ny",100).asUInt();
         Nz = js.get("Nz", 1).asUInt();
-        boxscaleRm = js.get("boxscaleRm", 1.).asDouble();
-        boxscaleRp = js.get("boxscaleRp", 1.).asDouble();
-        boxscaleZm = js.get("boxscaleZm", 1.3).asDouble();
-        boxscaleZp = js.get("boxscaleZp", 1.).asDouble();
+        boxscaleRm = js.get("boxscaleRm", 1.1).asDouble();
+        boxscaleRp = js.get("boxscaleRp", 1.1).asDouble();
+        boxscaleZm = js.get("boxscaleZm", 1.2).asDouble();
+        boxscaleZp = js.get("boxscaleZp", 1.1).asDouble();
         amp = js.get("amplitude", 1.).asDouble();
         k_psi = js.get("k_psi", 1.).asDouble();
         bgprofamp = js.get("bgprofamp", 1.).asDouble();
@@ -137,7 +137,7 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
 
     //Test coefficients
-    dg::geo::TokamakMagneticField c = dg::geo::createModifiedSolovevField(gp, 1, 0.1);
+    dg::geo::TokamakMagneticField c = dg::geo::createModifiedSolovevField(gp, 0.16, 0.1);
     const double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     const double Z_X = -1.1*gp.elongation*gp.a;
     const double R_H = gp.R_0-gp.triangularity*gp.a;
-- 
GitLab


From 9a30436f9111d4e8d760a8b7ab4957ccec21b21b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 29 Jan 2019 16:57:33 +0100
Subject: [PATCH 017/540] Strang splitting in feltor and feltor_hpc

- remove freeze_time again
- use ModifiedSolovev and feltor and feltor_hpc
- use Strang in feltor and feltor_hpc
- implement signal_handler for feltor_mpi
---
 inc/dg/implicit.h        |  3 +--
 inc/dg/multistep_t.cu    | 13 +++++++------
 inc/dg/runge_kutta.h     | 25 +++++++------------------
 src/feltor/Makefile      |  2 +-
 src/feltor/feltor.cu     | 22 ++++++++++++++--------
 src/feltor/feltor_hpc.cu | 40 ++++++++++++++++++++++++++++++++++------
 6 files changed, 64 insertions(+), 41 deletions(-)

diff --git a/inc/dg/implicit.h b/inc/dg/implicit.h
index bfbfac217..899064432 100644
--- a/inc/dg/implicit.h
+++ b/inc/dg/implicit.h
@@ -170,13 +170,12 @@ struct FixedPointSolver
             number++;
             error = sqrt( dg::blas1::dot( m_current, m_current));
         }while ( error > m_eps && number < m_max_iter);
-        //std::cout << " Error it "<<number<<" is "<<error<<"\n";
 #ifdef DG_BENCHMARK
         ti.toc();
 #ifdef MPI_VERSION
         if(rank==0)
 #endif//MPI
-        std::cout << "# of iterations time solver: "<<number<<"/"<<m_pcg.get_max()<<" took "<<ti.diff()<<"s\n";
+        std::cout << "# of iterations time solver: "<<number<<"/"<<m_max_iter<<" took "<<ti.diff()<<"s\n";
 #endif //DG_BENCHMARK
     }
     private:
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 12ff85876..956c1754e 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -179,7 +179,7 @@ int main()
         std::cout << counter <<" steps! ";
         std::cout << "Relative error "<<name<<" is "<< res.d<<"\t"<<res.i<<std::endl;
     }
-    std::cout << "### Test first order operator splitting\n";
+    std::cout << "### Test Strang operator splitting\n";
 
     std::vector<std::string> ex_names{
         "Heun-Euler-2-1-2",
@@ -197,24 +197,25 @@ int main()
     {
         time = 0., y0 = init;
         dg::Adaptive<dg::ERKStep<dg::DVec>> adapt( name, y0);
-        dg::ImplicitRungeKutta<dg::DVec> dirk( "Euler (implicit)", y0, y0.size(), eps );
+        dg::ImplicitRungeKutta<dg::DVec> dirk( "Trapezoidal-2-2", y0, y0.size(), eps );
         double time = 0;
         double dt = adapt.guess_stepsize( ex, time, y0, dg::forward, dg::l2norm, rtol, atol);
         int counter=0;
         adapt.stepper().ignore_fsal();
-        dirk.freeze_time();
         while( time < T )
         {
             if( time + dt > T)
                 dt = T-time;
-            dirk.step( im, time, y0, time, y0, dt);
-            adapt.step( ex, time, y0, time, y0, dt, dg::pid_control,
+            double dt_old = dt;
+            dirk.step( im, time, y0, time, y0, dt_old/2.);
+            adapt.step( ex, time-dt_old/2., y0, time, y0, dt, dg::pid_control,
                 dg::l2norm, rtol, atol);
+            dirk.step( im, time-dt_old/2., y0, time, y0, dt_old/2.);
             counter ++;
         }
         dg::blas1::axpby( -1., sol, 1., y0);
         res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
-        std::cout << counter <<" steps! ";
+        std::cout << std::setw(4)<<counter <<" steps! ";
         std::cout << "Relative error "<<std::setw(24) <<name<<"\t"<<res.d<<"\n";
     }
     return 0;
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index e7209b846..a21354d0c 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -614,7 +614,7 @@ struct DIRKStep
     * @param t0 start time
     * @param u0 value at \c t0
     * @param t1 (write only) end time ( equals \c t0+dt on output
-    *   unless \c freeze_time() was called, then it equals \c t0 on output, may alias \c t0)
+    *   may alias \c t0)
     * @param u1 (write only) contains result on output (may alias u0)
     * @param dt timestep
     * @param delta Contains error estimate on output (must have equal size as \c u0)
@@ -634,17 +634,11 @@ struct DIRKStep
         return m_rkI.num_stages();
     }
 
-    ///All subsequent calls to \c step method will not advance time
-    ///This is useful if you want to implement operator splitting
-    void freeze_time() { m_freeze_time=true;}
-    ///All subsequent calls to \c step method will advance time again
-    void unfreeze_time() { m_freeze_time=false;}
     private:
     SolverType m_solver;
     ContainerType m_rhs;
     ButcherTableau<value_type> m_rkI;
     std::vector<ContainerType> m_kI;
-    bool m_freeze_time = false;
 };
 
 template<class ContainerType, class SolverType>
@@ -655,7 +649,7 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
     value_type tu = t0;
     //0 stage
     //rhs = u0
-    tu = m_freeze_time ? t0 : DG_FMA( m_rkI.c(0),dt, t0);
+    tu = DG_FMA( m_rkI.c(0),dt, t0);
     blas1::copy( u0, delta); //better init with rhs
     if( !(m_rkI.a(0,0)==0) )
         m_solver.solve( -dt*m_rkI.a(0,0), rhs, tu, delta, u0);
@@ -665,7 +659,7 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
     if( s>1){
         blas1::evaluate( m_rhs, dg::equals(), PairSum(), 1., u0,
                 dt*m_rkI.a(1,0), m_kI[0]);
-        tu = m_freeze_time ? t0 : DG_FMA( m_rkI.c(1),dt, t0);
+        tu = DG_FMA( m_rkI.c(1),dt, t0);
         //store solution in delta, init with last solution
         blas1::copy( m_rhs, delta); //better init with rhs
         m_solver.solve( -dt*m_rkI.a(1,1), rhs, tu, delta, m_rhs);
@@ -676,7 +670,7 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
         blas1::evaluate( m_rhs, dg::equals(), PairSum(), 1., u0,
                  dt*m_rkI.a(2,0), m_kI[0],
                  dt*m_rkI.a(2,1), m_kI[1]);
-        tu = m_freeze_time ? t0 : DG_FMA( m_rkI.c(2),dt, t0);
+        tu = DG_FMA( m_rkI.c(2),dt, t0);
         //just take last solution as init
         blas1::copy( m_rhs, delta); //better init with rhs
         m_solver.solve( -dt*m_rkI.a(2,2), rhs, tu, delta, m_rhs);
@@ -697,13 +691,13 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
             dg::blas1::copy( u0, m_rhs);
             for( unsigned j=0; j<i; j++)
                 dg::blas1::axpby( dt*m_rkI.a(i,j), m_kI[j], 1., m_rhs);
-            tu = m_freeze_time ? t0 : DG_FMA( m_rkI.c(i),dt, t0);
+            tu = DG_FMA( m_rkI.c(i),dt, t0);
             blas1::copy( m_rhs, delta); //better init with rhs
             m_solver.solve( -dt*m_rkI.a(i,i), rhs, tu, delta, m_rhs);
             rhs(tu, delta, m_kI[i]);
         }
     }
-    t1 = m_freeze_time ? t0 : t0 + dt;
+    t1 = t0 + dt;
     //Now compute result and error estimate
     switch( s)
     {
@@ -774,11 +768,6 @@ struct ImplicitRungeKutta
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_delta;}
-    ///All subsequent calls to \c step method will not advance time
-    ///This is useful in an operator splitting
-    void freeze_time() { m_dirk.freeze_time();}
-    ///All subsequent calls to \c step method will advance time again
-    void unfreeze_time() { m_dirk.unfreeze_time();}
     /**
     * @brief Advance one step
     *
@@ -787,7 +776,7 @@ struct ImplicitRungeKutta
     * @param t0 start time
     * @param u0 value at \c t0
     * @param t1 (write only) end time ( equals \c t0+dt on output
-    *   unless \c freeze_time() was called, then it equals \c t0 on output, may alias \c t0)
+    *   may alias \c t0)
     * @param u1 (write only) contains result on output (may alias u0)
     * @param dt timestep
     */
diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index a2ed7df2f..bbcacab85 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -14,7 +14,7 @@ manufactured: manufactured.cu manufactured.h feltor.cuh
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
 
 feltor: feltor.cu feltor.cuh
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
 feltor_hpc: feltor_hpc.cu feltor.cuh
 	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 3491477e1..dbed7850f 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -44,7 +44,7 @@ int main( int argc, char* argv[])
     //Make grid
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    dg::geo::TokamakMagneticField mag = dg::geo::createModifiedSolovevField(gp, 0.16, 0.1);
 
     //create RHS
     std::cout << "Constructing Explicit...\n";
@@ -151,15 +151,18 @@ int main( int argc, char* argv[])
     dg::Timer t;
     double time = 0, dt_new = p.dt, dt =0;
     unsigned step = 0;
-    dg::Adaptive< dg::ARKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
-        "ARK-4-2-3", y0, grid.size(), p.eps_time);
+    dg::Adaptive< dg::ERKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
+        "Bogacki-Shampine-4-2-3", y0);
+    adaptive.stepper().ignore_fsal();//necessary for splitting
+    dg::ImplicitRungeKutta<std::array<std::array<dg::DVec,2>,2>> dirk(
+        "Trapezoidal-2-2", y0, grid.size(), p.eps_time);
 
     //since we map pointers we don't need to update those later
 
     std::map<std::string, const dg::DVec* > v4d;
     v4d["ne-1 / "] = &y0[0][0],               v4d["ni-1 / "] = &y0[0][1];
     v4d["Ue / "]   = &feltor.fields()[1][0],  v4d["Ui / "]   = &feltor.fields()[1][1];
-    v4d["Phi / "] = &feltor.potential()[0]; v4d["Apar / "] = &feltor.induction();
+    v4d["Ome / "] = &feltor.potential()[0]; v4d["Apar / "] = &feltor.induction();
     const feltor::Quantities& q = feltor.quantities();
     double dEdt = 0, accuracy = 0, dMdt = 0, accuracyM  = 0;
     std::map<std::string, const double*> v0d{
@@ -225,10 +228,10 @@ int main( int argc, char* argv[])
         title << "t = "<<time<<"   ";
         for( auto pair : v4d)
         {
-            if(pair.first == "Phi / ")
+            if(pair.first == "Ome / ")
             {
-                //dg::blas2::gemv( laplacianM, *pair.second, dvisual);
-                //dg::assign( dvisual, hvisual);
+                dg::blas2::gemv( laplacianM, *pair.second, dvisual);
+                dg::assign( dvisual, hvisual);
                 dg::assign( *pair.second, hvisual);
             }
             else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
@@ -278,7 +281,9 @@ int main( int argc, char* argv[])
                     do
                     {
                         dt = dt_new;
-                        adaptive.step( feltor, im, time, y0, time, y0, dt_new,
+                        //Strang splitting
+                        dirk.step( im, time, y0, time, y0, dt/2.);
+                        adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
                             dg::pid_control, dg::l2norm, p.rtol, 1e-10);
                         if( adaptive.failed())
                         {
@@ -286,6 +291,7 @@ int main( int argc, char* argv[])
                             std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
                         }
                     }while ( adaptive.failed());
+                    dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
                 }
                 catch( dg::Fail& fail) {
                     std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index c186b9e40..9b2b8fbcb 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -4,6 +4,7 @@
 #include <map>
 #include <sstream>
 #include <cmath>
+#include <csignal>
 
 #ifdef FELTOR_MPI
 #include <mpi.h>
@@ -31,6 +32,24 @@ using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #endif //FELTOR_MPI
 
+int ncid = -1; //netcdf id (signal handler can close the file)
+
+#ifdef FELTOR_MPI
+void sigterm_handler(int signal)
+{
+    file :: NC_Error_Handle err;
+    std::cout << "sigterm_handler, got signal " << signal << std::endl;
+    std::cout << "ncid = " << ncid << std::endl;
+    if(ncid != -1)
+    {
+        err = nc_close(ncid);
+        std::cerr << "SIGTERM caught. Closing NetCDF file with id " << ncid << std::endl;
+    }
+    MPI_Finalize();
+    exit(signal);
+}
+#endif //FELTOR_MPI
+
 int main( int argc, char* argv[])
 {
 #ifdef FELTOR_MPI
@@ -75,6 +94,9 @@ int main( int argc, char* argv[])
     MPI_Bcast( np, 3, MPI_INT, 0, MPI_COMM_WORLD);
     MPI_Comm comm;
     MPI_Cart_create( MPI_COMM_WORLD, 3, np, periods, true, &comm);
+    ////////////////////////////// Install signal handler ///////////////////
+    std::signal(SIGINT, sigterm_handler);
+    std::signal(SIGTERM, sigterm_handler);
 #endif //FELTOR_MPI
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js, gs;
@@ -117,7 +139,7 @@ int main( int argc, char* argv[])
         , comm
         #endif //FELTOR_MPI
         );
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    dg::geo::TokamakMagneticField mag = dg::geo::createModifiedSolovevField(gp, 0.16 ,0.1);
 
     //create RHS
     MPI_OUT std::cout << "Constructing Explicit...\n";
@@ -236,7 +258,6 @@ int main( int argc, char* argv[])
     };
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
-    int ncid;
 #ifdef FELTOR_MPI
     MPI_Info info = MPI_INFO_NULL;
     err = nc_create_par( argv[3], NC_NETCDF4|NC_MPIIO|NC_CLOBBER, comm, info, &ncid);
@@ -314,7 +335,7 @@ int main( int argc, char* argv[])
     }
     err = nc_enddef(ncid);
     ///////////////////////////////////first output/////////////////////////
-    double time = 0, dt = p.dt;
+    double time = 0, dt_new = p.dt, dt =0;
     MPI_OUT std::cout << "First output ... \n";
     //first, update quantities in feltor
     {
@@ -373,8 +394,11 @@ int main( int argc, char* argv[])
 #endif //FELTOR_MPI
     MPI_OUT std::cout << "First write successful!\n";
     ///////////////////////////////////////Timeloop/////////////////////////////////
-    dg::Adaptive< dg::ARKStep<std::array<std::array<DVec,2>,2>> > adaptive(
-        "ARK-4-2-3", y0, grid.size(), p.eps_time);
+    dg::Adaptive< dg::ERKStep<std::array<std::array<DVec,2>,2>> > adaptive(
+        "Bogacki-Shampine-4-2-3", y0);
+    adaptive.stepper().ignore_fsal();//necessary for splitting
+    dg::ImplicitRungeKutta<std::array<std::array<DVec,2>,2>> dirk(
+        "Trapezoidal-2-2", y0, grid.size(), p.eps_time);
     dg::Timer t;
     t.tic();
     unsigned step = 0, failed_counter = 0;
@@ -392,7 +416,10 @@ int main( int argc, char* argv[])
                 try{
                     do
                     {
-                        adaptive.step( feltor, im, time, y0, time, y0, dt,
+                        //Strang splitting
+                        dt = dt_new;
+                        dirk.step( im, time, y0, time, y0, dt/2.);
+                        adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
                             dg::pid_control, dg::l2norm, p.rtol, 1e-10);
                         if( adaptive.failed())
                         {
@@ -400,6 +427,7 @@ int main( int argc, char* argv[])
                             failed_counter++;
                         }
                     }while ( adaptive.failed());
+                    dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
                 }
                 catch( dg::Fail& fail) {
                     MPI_OUT std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-- 
GitLab


From c33df5b16b7c8e336baf3f572b3dd6e8fc5bec53 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 30 Jan 2019 01:02:51 +0100
Subject: [PATCH 018/540] Use input parameter for ModifiedSolovev

and document the new modification
---
 inc/dg/functors.h             |  7 ++--
 src/feltor/feltor.cu          |  3 +-
 src/feltor/feltor.tex         | 65 +++++++++++++++++++++++++++--------
 src/feltor/feltor_hpc.cu      |  3 +-
 src/feltor/input/default.json |  1 -
 5 files changed, 59 insertions(+), 20 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 268f134e4..6741c9d57 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -132,9 +132,12 @@ struct Gaussian
 };
 
 /**
- * @brief A blob that drops to zero
+ * @brief A bump that drops to zero and is infinitely continuously differentiable
  * \f[
-   f(x,y) = Ae^{1 + \left(\frac{(x-x_0)^2}{\sigma_x^2} + \frac{(y-y_0)^2}{\sigma_y^2} - 1\right)^{-1}}
+   f(x,y) = \begin{cases}
+   Ae^{1 + \left(\frac{(x-x_0)^2}{\sigma_x^2} + \frac{(y-y_0)^2}{\sigma_y^2} - 1\right)^{-1}} \text{ if } \frac{(x-x_0)^2}{\sigma_x^2} + \frac{(y-y_0)^2}{\sigma_y^2} < 1\\
+   0 \text{ else}
+   \end{cases}
    \f]
  */
 struct Cauchy
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index dbed7850f..893affd81 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -44,7 +44,8 @@ int main( int argc, char* argv[])
     //Make grid
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
-    dg::geo::TokamakMagneticField mag = dg::geo::createModifiedSolovevField(gp, 0.16, 0.1);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha);
 
     //create RHS
     std::cout << "Constructing Explicit...\n";
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 92b9f32f1..3a6aac9cb 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -91,13 +91,14 @@ Note that in any arbitrary coordinate system we have
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsection{Coordinate system}\label{sec:cylmetric}
-We employ cylindrical coordinates \( (R,Z,\varphi) \), with \(\varphi\) anti directed to the geometric toroidal angle to
+We employ cylindrical coordinates \( (R,Z,\varphi) \), with \(\varphi\) anti directed to the geometric toroidal angle (clockwise if viewed from above) to
 obtain a right handed system. The parametric representation in Cartesian \((x,y,z)\) coordinates is therefore simply:
 \begin{align}
  x &= R \hspace{1 mm} \sin{(\varphi)}, &
  y &= R \hspace{1 mm} \cos{(\varphi)}, &
  z &= Z .
 \end{align}
+Note here that the angle $\varphi = 0$ corresponds to the Cartesian $y$-axis.
 Covariant
 basis vectors and metric tensor:
 \begin{align}
@@ -124,6 +125,7 @@ In cylindrical coordinates the general axisymmetric magnetic field normalized wi
  \psi_p}{\partial Z} \ehat_R -  \frac{\partial \psi_p}{\partial R} \ehat_Z\right] ,
 \end{align}
 which can obviously not be manipulated to be in Clebsch form. Hence we are dealing with a non-flux aligned coordinate system.
+Note that with a typically convex function $\psi$, $I(\psi)>0$ and the previously defined coordinate system the field line winding is a {\bf left handed screw} in the positive $\vec B$-direction.
 For the sake of clarity we define the poloidal magnetic field \( \vec{B}_p = \frac{R_0}{R}\left( \frac{\partial \psi}{\partial Z}\ehat_R - \frac{\partial \psi}{\partial R}\ehat_Z\right)
 \) and the toroidal magnetic field \(\vec{B}_t =\frac{R_0I}{R} \ehat_{\varphi}\).
 %The unit vectors are denoted by \(\ehat_{R}\), \(\ehat_{Z}\), \(\ehat_{\varphi}\).
@@ -357,6 +359,36 @@ q:=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=
 = \frac{1}{2\pi}\int \frac{I(\psi_p)}{R}\delta(\psi_p-\psi_{p0}) \d R\d Z
 \end{align}
 
+\subsection{ Modified $\psi_p$}
+Our computational domain is a box and in particular not aligned with the
+magnetic flux surfaces. This means that particularly in the corners of
+the domain the field lines enter and leave the box very quickly.
+It turns out that this behaviour is numerically disadvantageous in the
+computation of parallel derivatives. In order to remedy this situation
+we propse to modify the flux surfaces $\psi_p$ to a constant value
+if $\psi_p$ exceeds a certain critical value. In this way the poloidal
+field component vanishes in the corners of the domain.
+
+We define an approximation to the step function with width $\alpha$
+\begin{align}
+\Theta(\psi) := \begin{cases}
+0 &\text{ for } \psi < \psi_0 \\
+\cosh^{-2}\left( \frac{\psi-\psi_0}{\alpha}  \right) &\text{ else}
+\end{cases}
+\label{}
+\end{align}
+The integral of this function gives
+\begin{align}
+\theta(\psi) := \begin{cases}
+\psi &\text{ for } \psi < \psi_0 \\
+\alpha \tanh\left( \frac{\psi-\psi_0}{\alpha}  \right) &\text{ else}
+\end{cases}
+\label{eq:modified_psi}
+\end{align}
+
+We now use $\theta(\psi)$ instead of $\psi$ for the computation of the
+magnetic field, which introduces a shear layer at $\psi_0$ where the
+fieldlines are straightened to match $\ehat_\varphi$.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
@@ -480,7 +512,8 @@ and initialize the electron density with
 \end{align}
 consisting of a toroidally symmetric background profile $n_{\text{prof}}(R,Z)$ and a perturbation
 $\tilde n(R,Z,\varphi)$.
-Let us define an approximate Heaviside function
+Note that we should take care to intitialize a smooth profile with ideally well-defined $\Delta^2_\perp n_e$.
+Let us define another approximation to the Heaviside function
 \begin{align}
   \Theta(x) := \frac{1}{2}\left( 1 + \tanh\left( \frac{x-3\alpha}{ \alpha} \right) \right) \quad \Theta(x) \approx H(x)
   \label{eq:heaviside_profile}
@@ -492,6 +525,7 @@ We can then define a flux-aligned density profile as
   n_{\text{prof}}(R,Z)=
       n_0 + \triangle n_{peak}\frac{\psi_p(R,Z)}{\psi_p(R_0, 0)} \Theta( -\psi_p(R,Z)) H(Z-Z_X)
 \end{align}
+%NOTE: this profile is slightly below n_0 outside the LCFS! (not sure whether this does anything for nabla_parallel)
 The second Heaviside is multiplied only if the equilibrium $\psi_p$ has an
 X-point and avoids a profile in the private flux region.
 
@@ -537,7 +571,8 @@ We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
   S_{n_e}(R,Z,\varphi, t) &= \omega_s
     (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho_{s} -\rho(R,Z)) H(Z-Z_X)\\
-    \rho(R,Z) &:= \frac{\psi_p(R_0,0)- \psi_p(R,Z) }{\psi_p(R_0,0)},
+    \rho(R,Z) &:= \frac{\psi_p(R_0,0)- \psi_p(R,Z) }{\psi_p(R_0,0)},\\
+    \psi_p(R,Z)&:= (1-\rho(R,Z))\psi_p(R_0,0)
 \end{align}
 with $0 < \rho_{s}<1$
 where $\omega_s$ is the source strength parameter.
@@ -552,14 +587,14 @@ Note that Eq.~\eqref{eq:ion_source} is explicitly chosen as to avoid vorticity g
 by the particle source (cf.~Section~\ref{sec:conservation}). $S_{n_e}$ needs to be smooth
 so that $\nabla_\perp^2 S_{n_e}$ is well defined.
 
-The idea for the terms $S_U$ is mainly to provide more numerical stability
-in the corner regions of the domain, where the parallel derivative may lead
-to unfavourable numerical instabilities.
-For both electrons and ions we choose
-\begin{align} \label{eq:velocity_source}
-  S_{U}(R,Z,\varphi, t) := -\omega_d U \Theta( \rho(R,Z) - \rho_d)
-\end{align}
-with $\rho_d > 1$.
+%The idea for the terms $S_U$ is mainly to provide more numerical stability
+%in the corner regions of the domain, where the parallel derivative may lead
+%to unfavourable numerical instabilities.
+%For both electrons and ions we choose
+%\begin{align} \label{eq:velocity_source}
+%  S_{U}(R,Z,\varphi, t) := -\omega_d U \Theta( \rho(R,Z) - \rho_d)
+%\end{align}
+%with $\rho_d > 1$.
 
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
@@ -791,11 +826,11 @@ posY       & float &0.0    & - & blob Z-position in units of $a$ \\
 sigma\_z    & float &0.25   & - & variance in units of $R_0$  \\
 k\_psi     & float &0    & - & zonal mode wave number  \\
 nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} \\
-alpha       & float & 0.02 & - & Width $\alpha$ of Heaviside profile in Eq.~\eqref{eq:heaviside_profile} \\
+alpha       & float & 0.02 & - & Width $\alpha$ of Heaviside profile in Eq.~\eqref{eq:heaviside_profile} and \eqref{eq:modified_psi} \\
 source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
 rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source}  \\
-damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
-rho\_damping& float & 0.2  & 1.2 & Friction region boundary $\rho_{d}>1$ in Eq.~\eqref{eq:velocity_source}  \\
+%damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
+rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0$ in Eq.~\eqref{eq:modified_psi}  \\
 %restart    & bool & false & false & If true, all input and geometry parameters except maxout,
 %    itstp and inner\_loop are ignored. Instead, the parameters and all fields are initialized from output.nc
 %    The actual output is then appended to output.nc This effectively continues an existing simulation.
@@ -842,7 +877,7 @@ z\_XYZ           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 Psip             & Dataset & 3 (z,y,x) & Flux function $\psi_p(R,Z)$ \\
 Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ \\
 Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta(\rho_{s} - \rho(R,Z)) H(Z-Z_X)$\\
-Damping          & Dataset & 3 (z,y,x) & Damping profile $\Theta(\rho(R,Z) - \rho_{d} )$\\
+%Damping          & Dataset & 3 (z,y,x) & Damping profile $\Theta(\rho(R,Z) - \rho_{d} )$\\
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 9b2b8fbcb..f8ef49160 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -139,7 +139,8 @@ int main( int argc, char* argv[])
         , comm
         #endif //FELTOR_MPI
         );
-    dg::geo::TokamakMagneticField mag = dg::geo::createModifiedSolovevField(gp, 0.16 ,0.1);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha);
 
     //create RHS
     MPI_OUT std::cout << "Constructing Explicit...\n";
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index 660aff232..00c820046 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -46,6 +46,5 @@
     "alpha"       : 0.05,
     "source"      : 0,
     "rho_source"  : 0.2,
-    "damping"     : 0,
     "rho_damping" : 1.2
 }
-- 
GitLab


From fab6c15a57acd3a057745ad04a88903393a046ea Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 30 Jan 2019 18:43:20 +0100
Subject: [PATCH 019/540] Design a polynomial Heaviside and use in feltor

- untested in feltor.cu and feltor_hpc yet
---
 inc/dg/functors.h               | 129 ++++++++++++++++++--------------
 inc/geometries/geometry_diag.cu |   2 +-
 inc/geometries/solovev.h        |  50 ++++++-------
 src/feltor/feltor.cu            |  16 ++--
 src/feltor/feltor.tex           |  59 ++++++++-------
 src/feltor/feltor_hpc.cu        |  16 ++--
 6 files changed, 149 insertions(+), 123 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 6741c9d57..fb5492965 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -846,107 +846,124 @@ struct TanhProfX {
 };
 
 /**
- * @brief An approximation to Heaviside using cosh
+ * @brief An approximation to Heaviside using polynomials
      \f[ \begin{cases}
-        \cosh^{-2}((x-x_b)/\alpha)  \text{ if } x < x_b \\
-        1  \text{ else }
+     0 \text{ if } x < x_0-a \\
+        ((16 a^3 - 29 a^2 (x - x_0) + 20 a (x - x_0)^2 - 5 (x - x_0)^3) (a + x - 
+   x_0)^4)/(32 a^7) \text{ if } |x-x_0| < a \\
+        1  \text{ if } x > x_0 + a
      \end{cases}\f]
+
+     This function is 3 times continuously differentiable, takes the value 0.5 at x0 and
+     has a transition width a on both sides of x0.
  */
-struct Sech2 {
+struct PolynomialHeaviside {
     /**
-     * @brief Construct with xb, width and sign
+     * @brief Construct with x0, width and sign
      *
-     * @param xb boundary value
-     * @param width damping width \c alpha
-     * @param sign either +1 or -1, If -1, \c x<x_b is replaced with \c x>x_b
-     * thus approximating H(xb-x) if H(x-xb) is the regular Heaviside function
+     * @param x0 boundary value
+     * @param a transition width
+     * @param sign either +1 (original) or -1 (the function is mirrored at the \c x=x0 axis: f(2x0-x))
      */
-    Sech2(double xb, double width, int sign = +1) :
-        m_xb(xb), m_w(width), m_s(sign){}
+    PolynomialHeaviside(double x0, double a, int sign = +1) :
+        x0(x0), a(a), m_s(sign){}
     DG_DEVICE
     double operator() (double x)const
     {
-        if( ( x > m_xb && m_s == 1 ) || (x < m_xb && m_s == -1 )) return 1;
-        return 1./cosh( (x-m_xb)/m_w)/cosh( (x-m_xb)/m_w);
+        if( m_s == -1) x = 2*x0-x; //mirror 
+        if ( x < x0-a) return 0;
+        if ( x > x0+a) return 1;
+        return ((16.*a*a*a - 29.*a*a*(x - x0)
+               + 20.*a*(x - x0)*(x - x0) 
+               - 5.*(x - x0)*(x-x0)*(x-x0))
+               *(a + x - x0)*(a + x - x0)
+               *(a + x - x0)*(a + x - x0))/(32.*a*a*a * a*a*a*a);
     }
-    DG_DEVICE
-    double operator()( double x, double y)const{ return this->operator()(x);}
-    DG_DEVICE
-    double operator()( double x, double y, double z)const{ return this->operator()(x);}
     private:
-    double m_xb, m_w;
+    double x0, a;
     int m_s;
 };
 
 /**
- * @brief The integral of Sech2,  approximates the integrated Heaviside using tanh
+ * @brief The integral of PolynomialHeaviside approximates xH(x)
      \f[ \begin{cases}
-        x_b + \alpha \tanh((x-x_b)/\alpha)  \text{ if } x < x_b \\
-        x  \text{ else }
+     x_0 \text{ if } x < x_0-a \\
+     x_0 + ((35 a^3 - 47 a^2 (x - x0) + 25 a (x - x0)^2 - 5 (x - x0)^3) (a + x - x0)^5)/(256 a^7)
+        \text{ if } |x-x_0| < a \\
+        x  \text{ if } x > x_0 + a
      \end{cases}\f]
 
-     The maximum is at \c x_b+alpha
+     This function is 4 times continuously differentiable,
+     has a transition width \c a on both sides of \c x0, where it transitions from the
+     constant \c x0 to the linear function \c x.
  */
-struct ISech2 {
+struct IPolynomialHeaviside {
     /**
-     * @brief Construct with xb, width and sign
+     * @brief Construct with x0, width and sign
      *
-     * @param xb boundary value
-     * @param width damping width \c alpha
-     * @param sign either +1 or -1, If -1, \c x<x_b is replaced with \c x>x_b
+     * @param x0 boundary value
+     * @param a transition width
+     * @param sign either +1 (original) or -1 (the function is point mirrored at \c x=x0: 2*x0-f(2x0-x))
      */
-    ISech2(double xb, double width, int sign = +1) :
-        m_xb(xb), m_w(width), m_s(sign){}
+    IPolynomialHeaviside(double x0, double a, int sign = +1) :
+        x0(x0), a(a), m_s(sign){}
     DG_DEVICE
     double operator() (double x)const
     {
-        if( ( x > m_xb && m_s == 1 ) || (x < m_xb && m_s == -1 )) return x;
-        return m_xb + m_w*tanh( (x-m_xb)/m_w);
+        if( m_s == -1) x = 2*x0-x; //mirror 
+        double result;
+        if ( x < x0-a) result =  x0;
+        else if ( x > x0+a) result =  x;
+        else
+            result =  x0 + ((35.* a*a*a - 47.* a*a*(x - x0) + 25.*a*(x - x0)*(x-x0)
+                - 5.*(x - x0)*(x-x0)*(x-x0))
+                *(a+x-x0)*(a+x-x0)*(a+x-x0)*(a+x-x0)*(a+x-x0))
+            /(256.*a*a*a * a*a*a*a);
+        if ( m_s == +1) return result;
+        return 2*x0 - result;
+
     }
-    DG_DEVICE
-    double operator()( double x, double y)const{ return this->operator()(x);}
-    DG_DEVICE
-    double operator()( double x, double y, double z)const{ return this->operator()(x);}
     private:
-    double m_xb, m_w;
+    double x0, a;
     int m_s;
 };
 
 /**
- * @brief The derivative of Sech2
+ * @brief The derivative of PolynomialHeaviside approximates delta(x)
      \f[ \begin{cases}
-        -\frac{2}{\alpha} \cosh^{-2}((x-x_b)/\alpha)\tanh((x-x_b)/\alpha)  \text{ if } x < x_b \\
-        0  \text{ else }
+     0 \text{ if } x < x_0-a || x > x_0+a \\
+     (35 (a + x - x0)^3 (a - x + x0)^3)/(32 a^7)
+        \text{ if } |x-x_0| < a 
      \end{cases}\f]
+
+     This function is 2 times continuously differentiable, is symmetric around \c x0
+     and has a width \c a on both sides of \c x0.
+     The integral over this function yields 1.
  */
-struct DSech2 {
+struct DPolynomialHeaviside {
     /**
-     * @brief Construct with xb, width and sign
+     * @brief Construct with x0, width and sign
      *
-     * @param xb boundary value
-     * @param width damping width \c alpha
-     * @param sign either +1 or -1, If -1, \c x<x_b is replaced with \c x>x_b
+     * @param x0 boundary value
+     * @param a transition width
+     * @param sign either +1 (original) or -1 (the function is mirrored at \c x=x0)
+     * (since this function is symmetric this parameter is ignored, it's there to be
+     * consistent with PolynomialHeaviside)
      */
-    DSech2(double xb, double width, int sign = +1) :
-        m_xb(xb), m_w(width), m_s(sign){}
+    DPolynomialHeaviside(double x0, double a, int sign = +1) :
+        x0(x0), a(a){}
     DG_DEVICE
     double operator() (double x)const
     {
-        if( ( x > m_xb && m_s == 1 ) || (x < m_xb && m_s == -1 )) return 0.;
-        return -2./m_w*tanh( (x-m_xb)/m_w)/cosh( (x-m_xb)/m_w)/cosh( (x-m_xb)/m_w);
+        if ( (x < x0-a) || (x > x0+a)) return 0;
+        return (35.*(a+x-x0)*(a+x-x0)*(a+x-x0)*(a-x+x0)*(a-x+x0)*(a-x+x0))
+            /(32.*a*a*a * a*a*a*a);
     }
-    DG_DEVICE
-    double operator()( double x, double y)const{ return this->operator()(x);}
-    DG_DEVICE
-    double operator()( double x, double y, double z)const{ return this->operator()(x);}
     private:
-    double m_xb, m_w;
-    int m_s;
+    double x0, a;
 };
 
 
-
-
 /**
  * @brief Exponential \f[ f(x) = A \exp(\lambda x)\f]
  *
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 48b24b141..4b54f6d15 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -137,7 +137,7 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
 
     //Test coefficients
-    dg::geo::TokamakMagneticField c = dg::geo::createModifiedSolovevField(gp, 0.16, 0.1);
+    dg::geo::TokamakMagneticField c = dg::geo::createModifiedSolovevField(gp, 0.16, 0.05);
     const double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     const double Z_X = -1.1*gp.elongation*gp.a;
     const double R_H = gp.R_0-gp.triangularity*gp.a;
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 77fa04367..05ec43c66 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -375,46 +375,46 @@ namespace mod
 struct Psip: public aCylindricalFunctor<Psip>
 {
     Psip( Parameters gp, double psi0, double alpha) :
-        m_isech2( psi0, alpha, -1), m_psip(gp)
+        m_ipoly( psi0, alpha, -1), m_psip(gp)
     { }
     double do_compute(double R, double Z) const
     {
         double psip = m_psip(R,Z);
-        return m_isech2( psip);
+        return m_ipoly( psip);
     }
     private:
-    dg::ISech2 m_isech2;
+    dg::IPolynomialHeaviside m_ipoly;
     solovev::Psip m_psip;
 };
 struct PsipR: public aCylindricalFunctor<PsipR>
 {
     PsipR( Parameters gp, double psi0, double alpha) :
-        m_sech2( psi0, alpha, -1), m_psip(gp), m_psipR(gp)
+        m_poly( psi0, alpha, -1), m_psip(gp), m_psipR(gp)
     { }
     double do_compute(double R, double Z) const
     {
         double psip = m_psip(R,Z);
         double psipR = m_psipR(R,Z);
-        return psipR*m_sech2( psip);
+        return psipR*m_poly( psip);
     }
     private:
-    dg::Sech2 m_sech2;
+    dg::PolynomialHeaviside m_poly;
     solovev::Psip m_psip;
     solovev::PsipR m_psipR;
 };
 struct PsipZ: public aCylindricalFunctor<PsipZ>
 {
     PsipZ( Parameters gp, double psi0, double alpha) :
-        m_sech2( psi0, alpha, -1), m_psip(gp), m_psipZ(gp)
+        m_poly( psi0, alpha, -1), m_psip(gp), m_psipZ(gp)
     { }
     double do_compute(double R, double Z) const
     {
         double psip = m_psip(R,Z);
         double psipZ = m_psipZ(R,Z);
-        return psipZ*m_sech2( psip);
+        return psipZ*m_poly( psip);
     }
     private:
-    dg::Sech2 m_sech2;
+    dg::PolynomialHeaviside m_poly;
     solovev::Psip m_psip;
     solovev::PsipZ m_psipZ;
 };
@@ -422,18 +422,18 @@ struct PsipZ: public aCylindricalFunctor<PsipZ>
 struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 {
     PsipZZ( Parameters gp, double psi0, double alpha) :
-        m_sech2( psi0, alpha, -1), m_dsech2( psi0, alpha, -1), m_psip(gp), m_psipZ(gp), m_psipZZ(gp)
+        m_poly( psi0, alpha, -1), m_dpoly( psi0, alpha, -1), m_psip(gp), m_psipZ(gp), m_psipZZ(gp)
     { }
     double do_compute(double R, double Z) const
     {
         double psip = m_psip(R,Z);
         double psipZ = m_psipZ(R,Z);
         double psipZZ = m_psipZZ(R,Z);
-        return psipZZ*m_sech2( psip) + psipZ*psipZ*m_dsech2(psip);
+        return psipZZ*m_poly( psip) + psipZ*psipZ*m_dpoly(psip);
     }
     private:
-    dg::Sech2 m_sech2;
-    dg::DSech2 m_dsech2;
+    dg::PolynomialHeaviside m_poly;
+    dg::DPolynomialHeaviside m_dpoly;
     solovev::Psip m_psip;
     solovev::PsipZ m_psipZ;
     solovev::PsipZZ m_psipZZ;
@@ -441,18 +441,18 @@ struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 struct PsipRR: public aCylindricalFunctor<PsipRR>
 {
     PsipRR( Parameters gp, double psi0, double alpha) :
-        m_sech2( psi0, alpha, -1), m_dsech2( psi0, alpha, -1), m_psip(gp), m_psipR(gp), m_psipRR(gp)
+        m_poly( psi0, alpha, -1), m_dpoly( psi0, alpha, -1), m_psip(gp), m_psipR(gp), m_psipRR(gp)
     { }
     double do_compute(double R, double Z) const
     {
         double psip = m_psip(R,Z);
         double psipR = m_psipR(R,Z);
         double psipRR = m_psipRR(R,Z);
-        return psipRR*m_sech2( psip) + psipR*psipR*m_dsech2(psip);
+        return psipRR*m_poly( psip) + psipR*psipR*m_dpoly(psip);
     }
     private:
-    dg::Sech2 m_sech2;
-    dg::DSech2 m_dsech2;
+    dg::PolynomialHeaviside m_poly;
+    dg::DPolynomialHeaviside m_dpoly;
     solovev::Psip m_psip;
     solovev::PsipR m_psipR;
     solovev::PsipRR m_psipRR;
@@ -460,7 +460,7 @@ struct PsipRR: public aCylindricalFunctor<PsipRR>
 struct PsipRZ: public aCylindricalFunctor<PsipRZ>
 {
     PsipRZ( Parameters gp, double psi0, double alpha) :
-        m_sech2( psi0, alpha, -1), m_dsech2( psi0, alpha, -1), m_psip(gp), m_psipR(gp), m_psipZ(gp), m_psipRZ(gp)
+        m_poly( psi0, alpha, -1), m_dpoly( psi0, alpha, -1), m_psip(gp), m_psipR(gp), m_psipZ(gp), m_psipRZ(gp)
     { }
     double do_compute(double R, double Z) const
     {
@@ -468,11 +468,11 @@ struct PsipRZ: public aCylindricalFunctor<PsipRZ>
         double psipR = m_psipR(R,Z);
         double psipZ = m_psipZ(R,Z);
         double psipRZ = m_psipRZ(R,Z);
-        return psipRZ*m_sech2( psip) + psipR*psipZ*m_dsech2(psip);
+        return psipRZ*m_poly( psip) + psipR*psipZ*m_dpoly(psip);
     }
     private:
-    dg::Sech2 m_sech2;
-    dg::DSech2 m_dsech2;
+    dg::PolynomialHeaviside m_poly;
+    dg::DPolynomialHeaviside m_dpoly;
     solovev::Psip m_psip;
     solovev::PsipR m_psipR;
     solovev::PsipZ m_psipZ;
@@ -558,12 +558,12 @@ static inline dg::geo::TokamakMagneticField createSolovevField(
  * Based on \c dg::geo::solovev::mod::Psip(gp) and
  * \c dg::geo::solovev::mod::Ipol(gp)
  * We modify psi above a certain value to a constant using the
- * \c dg::ISech2 function (an approximation to the integrated Heaviside
- * function with width alpha), i.e. we replace psi with ISech2(psi).
+ * \c dg::IPolynomialHeaviside function (an approximation to the integrated Heaviside
+ * function with width alpha), i.e. we replace psi with IPolynomialHeaviside(psi).
  * This subsequently modifies all derivatives of psi and the poloidal
- * current of course.
+ * current.
  * @param gp Solovev parameters
- * @param psi0 above this value psi is modified to a constant
+ * @param psi0 above this value psi is modified to a constant psi0
  * @param alpha determines how quickly the modification acts (smaller is quicker)
  * @return A magnetic field object
  * @ingroup geom
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 893affd81..f25fd6fc3 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -62,20 +62,20 @@ int main( int argc, char* argv[])
     if( gp.hasXpoint() )
         xpoint_damping = dg::pullback(
             dg::geo::ZCutter(-1.1*gp.elongation*gp.a), grid);
-    dg::HVec source_damping = dg::pullback(dg::geo::TanhDamping(
+    dg::HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
         //first change coordinate from psi to (psi_0 - psip)/psi_0
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift tanh
-        p.rho_source-3.*p.alpha, p.alpha, -1.), grid);
-    dg::HVec damping_damping = dg::pullback(dg::geo::TanhDamping(
+        //then shift
+        p.rho_source, p.alpha, -1), grid);
+    dg::HVec damping_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
         //first change coordinate from psi to (psi_0 - psip)/psi_0
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift tanh
-        p.rho_damping, p.alpha, 1.), grid);
+        //then shift
+        p.rho_damping, p.alpha, +1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
-    dg::HVec profile_damping = dg::pullback( dg::geo::TanhDamping(
-        mag.psip(), -3.*p.alpha, p.alpha, -1), grid);
+    dg::HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
+        mag.psip(), -p.alpha, p.alpha, -1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 3a6aac9cb..503409087 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -99,12 +99,12 @@ obtain a right handed system. The parametric representation in Cartesian \((x,y,
  z &= Z .
 \end{align}
 Note here that the angle $\varphi = 0$ corresponds to the Cartesian $y$-axis.
-Covariant
-basis vectors and metric tensor:
+The unit
+basis vectors and metric tensor are:
 \begin{align}
  \ehat_R      &= (\sin{(\varphi)} ,   \cos{(\varphi)},0)^T, &
  \ehat_Z      &= ( 0 ,0 ,1 )^T, &
- \ehat_{\varphi} &= R ( \cos{(\varphi)} , -\sin{(\varphi)} , 0 )^T,
+ \ehat_{\varphi} &= ( \cos{(\varphi)} , -\sin{(\varphi)} , 0 )^T,
 \\
  g &= \begin{pmatrix}
   1 & 0 & 0 \\
@@ -147,19 +147,19 @@ $\vec\nabla p = \vec j\times \vec B$ and $\vec \nabla\times\vec B = \vec j$ )
 \end{align}
 from where we recover the Grad-Shafranov equation
 \begin{align}\label{eq:GSEdimless}
- -\Delta^*_\perp  \psi_p &= \frac{R^2}{R_0^2} \frac{d p}{d  \psi_p } + I \frac{d I}{d  \psi_p }
+    -\Delta^*_\perp  \psi_p &= \frac{R^2}{R_0^2} \frac{d p}{d  \psi_p } + I \frac{d I}{d  \psi_p } \equiv \frac{R}{R_0} j_{\hat\varphi}
 \end{align}
 with $\Delta^*_\perp \psi_p = R\partial_R (R^{-1}\psi_R) + \psi_{ZZ}$.
 Note that $R$ and $Z$ are normalized
 with $\rho_s$, $\psi_p$ with $\psi_{p0}$, $p$ with
 $p_0 := \psi_{p0}^2/\mu_0\rho_s^4$, $I$ with $I_0:=\psi_{p0}/\rho_s$,
 and $j_\varphi$ with $\j_{\varphi 0} = \psi_{p0}/\rho_s^3\mu_0$.
-The Solov'ev assumptions consist of \(A/R_0 = -I \frac{d I}{d  \psi_p }\) and \((1-A)/R_0^3 = -\frac{d p}{d  \psi_p }\), where \(A\) is a constant~\cite{Cerfon2010,Cerfon2014}.
+The Solov'ev assumptions consist of \(A/R_0 = -I \frac{d I}{d  \psi_p }\) and \((1-A)/R_0 = -\frac{d p}{d  \psi_p }\), where \(A\) is a constant~\cite{Cerfon2010,Cerfon2014}.
 By integration over \(\psi_p\) we find
 \begin{align}\label{eq:solovevassumption}
- p(\psi_p) &= (A-1)\psi_p/R_0^3,  &
+ p(\psi_p) &= (A-1)\psi_p/R_0,  &
  I(\psi_p) &= \sqrt{-2 A \psi_p/R_0 + 1}, &
- j_\varphi &= \left[(A-1)R - A R_0^2 / R\right]/R_0^3.
+    j_{\hat\varphi} &= \left[(A-1)R^2/R_0^2 - A \right]/R.
 \end{align}
 Now, we introduce \(\bar{R} \equiv \frac{R}{R_0}\) and \(\bar{Z} \equiv\frac{Z}{R_0}\)
 and solve Equations~\eqref{eq:GSEdimless} and~\eqref{eq:solovevassumption} to obtain
@@ -365,30 +365,38 @@ magnetic flux surfaces. This means that particularly in the corners of
 the domain the field lines enter and leave the box very quickly.
 It turns out that this behaviour is numerically disadvantageous in the
 computation of parallel derivatives. In order to remedy this situation
-we propse to modify the flux surfaces $\psi_p$ to a constant value
+we propose to modify the flux surfaces $\psi_p$ to a constant value
 if $\psi_p$ exceeds a certain critical value. In this way the poloidal
 field component vanishes in the corners of the domain.
 
 We define an approximation to the step function with width $\alpha$
 \begin{align}
 \Theta(\psi) := \begin{cases}
-0 &\text{ for } \psi < \psi_0 \\
-\cosh^{-2}\left( \frac{\psi-\psi_0}{\alpha}  \right) &\text{ else}
+    1 & \text{ for } \psi < - \alpha  \\
+    \frac{1}{32 \alpha^7}  \left(16 \alpha^3+29 \alpha^2 \psi+20 \alpha \psi^2+5 \psi^3\right) (\alpha-\psi)^4
+    &\text{ for } -\alpha<\psi<+\alpha \\
+    0 & \text{ for } \psi > \alpha 
 \end{cases}
-\label{}
+    \approx H(-\psi)
+\label{eq:approx_heaviside}
 \end{align}
-The integral of this function gives
+if $H(\psi)$ is the Heaviside step function.
+An integral of this function is
 \begin{align}
 \theta(\psi) := \begin{cases}
-\psi &\text{ for } \psi < \psi_0 \\
-\alpha \tanh\left( \frac{\psi-\psi_0}{\alpha}  \right) &\text{ else}
+\psi &\text{ for } \psi < -\alpha \\
+    - \frac{1}{256 \alpha^7} \left(35 \alpha^3+47 \alpha^2 \psi+25 \alpha \psi^2+5 \psi^3\right) (\alpha-\psi)^5
+     &\text{ for } -\alpha<\psi<+\alpha \\
+    0 &\text{ for } \psi > \alpha
 \end{cases}
+    \approx \psi H(-\psi)
 \label{eq:modified_psi}
 \end{align}
 
-We now use $\theta(\psi)$ instead of $\psi$ for the computation of the
-magnetic field, which introduces a shear layer at $\psi_0$ where the
+We now use $\theta(\psi-\psi_0)+\psi_0$ instead of $\psi$ for the computation of the
+magnetic field, which introduces a shear layer around $\psi_0$ where the
 fieldlines are straightened to match $\ehat_\varphi$.
+Note that $\Theta(0) = 0.5$ and $\theta(0) = 35\alpha/256$.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
@@ -513,17 +521,18 @@ and initialize the electron density with
 consisting of a toroidally symmetric background profile $n_{\text{prof}}(R,Z)$ and a perturbation
 $\tilde n(R,Z,\varphi)$.
 Note that we should take care to intitialize a smooth profile with ideally well-defined $\Delta^2_\perp n_e$.
-Let us define another approximation to the Heaviside function
-\begin{align}
-  \Theta(x) := \frac{1}{2}\left( 1 + \tanh\left( \frac{x-3\alpha}{ \alpha} \right) \right) \quad \Theta(x) \approx H(x)
-  \label{eq:heaviside_profile}
-\end{align}
-where $H(x)$ is the actual Heaviside function and
-$\alpha$ is a (small) width parameter.
-We can then define a flux-aligned density profile as
+%Let us define another approximation to the Heaviside function
+%\begin{align}
+%  \Theta(x) := \frac{1}{2}\left( 1 + \tanh\left( \frac{x-3\alpha}{ \alpha} \right) \right) \quad \Theta(x) \approx H(x)
+%  \label{eq:heaviside_profile}
+%\end{align}
+%where $H(x)$ is the actual Heaviside function and
+%$\alpha$ is a (small) width parameter.
+
+Let us define a flux-aligned density profile as
 \begin{align} \label{eq:density_profile}
   n_{\text{prof}}(R,Z)=
-      n_0 + \triangle n_{peak}\frac{\psi_p(R,Z)}{\psi_p(R_0, 0)} \Theta( -\psi_p(R,Z)) H(Z-Z_X)
+      n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) \Theta(\psi_p(R, Z)+\alpha)}{\psi_p(R_0,0)} H(Z-Z_X)
 \end{align}
 %NOTE: this profile is slightly below n_0 outside the LCFS! (not sure whether this does anything for nabla_parallel)
 The second Heaviside is multiplied only if the equilibrium $\psi_p$ has an
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index f8ef49160..26526dff0 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -157,20 +157,20 @@ int main( int argc, char* argv[])
     if( gp.hasXpoint() )
         xpoint_damping = dg::pullback(
             dg::geo::ZCutter(-1.1*gp.elongation*gp.a), grid);
-    HVec source_damping = dg::pullback(dg::geo::TanhDamping(
+    HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
         //first change coordinate from psi to (psi_0 - psip)/psi_0
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift tanh
-        p.rho_source-3.*p.alpha, p.alpha, -1.), grid);
-    HVec damping_damping = dg::pullback(dg::geo::TanhDamping(
+        //then shift
+        p.rho_source, p.alpha, -1), grid);
+    HVec damping_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
         //first change coordinate from psi to (psi_0 - psip)/psi_0
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift tanh
-        p.rho_damping, p.alpha, 1.), grid);
+        //then shift
+        p.rho_damping, p.alpha, +1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
-    HVec profile_damping = dg::pullback( dg::geo::TanhDamping(
-        mag.psip(), -3.*p.alpha, p.alpha, -1), grid);
+    dg::HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
+        mag.psip(), -p.alpha, p.alpha, -1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
-- 
GitLab


From 524e115975656214351a9a5ba233fbc9ff4c8331 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 31 Jan 2019 17:46:49 +0100
Subject: [PATCH 020/540] Small utility functions in time interface

like stepper and copyable
---
 inc/dg/implicit.h       | 6 +++++-
 inc/dg/multigrid.h      | 7 +++++--
 inc/dg/multistep.h      | 5 +++++
 inc/dg/runge_kutta.h    | 9 +++++++++
 src/feltor/parameters.h | 1 +
 5 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/inc/dg/implicit.h b/inc/dg/implicit.h
index 899064432..a9b793441 100644
--- a/inc/dg/implicit.h
+++ b/inc/dg/implicit.h
@@ -10,7 +10,11 @@ template< class LinearOp, class ContainerType>
 struct Implicit
 {
     using value_type = get_value_type<ContainerType>;
+    Implicit(){}
     Implicit( value_type alpha, value_type t, LinearOp& f): f_(f), alpha_(alpha), t_(t){}
+    void construct( value_type alpha, value_type t, LinearOp& f){
+        f_ = f; alpha_=alpha; t_=t;
+    }
     void symv( const ContainerType& x, ContainerType& y)
     {
         if( alpha_ != 0)
@@ -88,7 +92,7 @@ struct DefaultSolver
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_rhs;}
 
-    template< class Implicit>
+    template< class Implicit> //going to be a reference type
     void solve( value_type alpha, Implicit im, value_type t, ContainerType& y, const ContainerType& rhs)
     {
         detail::Implicit<Implicit, ContainerType> implicit( alpha, t, im);
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 896223bc1..e388ba7d4 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -276,7 +276,11 @@ struct MultigridCG2d
     ///(if the solution method returns this number, failure is indicated)
     unsigned max_iter() const{return m_cg[0].get_max();}
 
-private:
+    ///@brief Return an object of same size as the object used for construction on the finest grid
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const Container& copyable() const {return m_x[0];}
+
+  private:
 
 	void set_scheme(const int scheme_type)
 	{
@@ -368,5 +372,4 @@ private:
     unsigned m_startStage;
     std::vector<stepinfo> m_schemeLayout;
 };
-
 }//namespace dg
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 1a29a1f4b..394441352 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -218,6 +218,11 @@ struct Karniadakis
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return u_[0];}
 
+    ///Write access to the internal solver for the implicit part
+    SolverType& solver() { return m_solver;}
+    ///Read access to the internal solver for the implicit part
+    const SolverType& solver() const { return m_solver;}
+
     /**
      * @brief Initialize by integrating two timesteps backward in time
      *
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index a21354d0c..4386d1da1 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -606,6 +606,11 @@ struct DIRKStep
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_kI[0];}
 
+    ///Write access to the internal solver for the implicit part
+    SolverType& solver() { return m_solver;}
+    ///Read access to the internal solver for the implicit part
+    const SolverType& solver() const { return m_solver;}
+
     /**
     * @brief Advance one step
     *
@@ -768,6 +773,10 @@ struct ImplicitRungeKutta
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_delta;}
+    ///Write access to the internal solver for the implicit part
+    SolverType& solver() { return m_dirk.solver();}
+    ///Read access to the internal solver for the implicit part
+    const SolverType& solver() const { return m_dirk.solver();}
     /**
     * @brief Advance one step
     *
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index cfab428dd..f168478cf 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -47,6 +47,7 @@ struct Parameters
     enum dg::bc bcxN, bcyN, bcxU, bcyU, bcxP, bcyP;
     std::string initne, initphi, curvmode, perp_diff;
     bool symmetric;
+    Parameters() = default;
     Parameters( const Json::Value& js) {
         n       = js["n"].asUInt();
         Nx      = js["Nx"].asUInt();
-- 
GitLab


From 6ebbd507d6897da820979202e3d9334e0ec373fa Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 31 Jan 2019 17:47:54 +0100
Subject: [PATCH 021/540] Start splitting implicit part in feltor

into one for density and one for velocity
however this does not work properly yet
Also found error in previous feltor since apar correction was still
active
---
 src/feltor/feltor.cu  |  18 +-
 src/feltor/feltor.cuh | 370 ++++++++++++++++++++++++++++--------------
 2 files changed, 261 insertions(+), 127 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index f25fd6fc3..3b5ff3fa7 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -51,7 +51,8 @@ int main( int argc, char* argv[])
     std::cout << "Constructing Explicit...\n";
     feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, mag);
     std::cout << "Constructing Implicit...\n";
-    feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> im( grid, p, mag);
+    feltor::ImplicitDensity<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> im_dens( grid, p, mag);
+    feltor::ImplicitVelocity<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> im_velo( grid, p, mag);
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
@@ -81,7 +82,6 @@ int main( int argc, char* argv[])
 
     feltor.set_source( profile, p.omega_source, source_damping,
         p.omega_damping, damping_damping);
-    im.set_damping( p.omega_damping, damping_damping);
 
 
     //Now perturbation
@@ -155,8 +155,10 @@ int main( int argc, char* argv[])
     dg::Adaptive< dg::ERKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
         "Bogacki-Shampine-4-2-3", y0);
     adaptive.stepper().ignore_fsal();//necessary for splitting
-    dg::ImplicitRungeKutta<std::array<std::array<dg::DVec,2>,2>> dirk(
-        "Trapezoidal-2-2", y0, grid.size(), p.eps_time);
+    dg::ImplicitRungeKutta<std::array<dg::DVec,2>> dirk_dens(
+        "Trapezoidal-2-2", y0[0], grid.size(), p.eps_time);
+    dg::ImplicitRungeKutta<std::array<dg::DVec,2>> dirk_velo(
+        "Trapezoidal-2-2", y0[0], grid.size(), p.eps_time);
 
     //since we map pointers we don't need to update those later
 
@@ -283,7 +285,9 @@ int main( int argc, char* argv[])
                     {
                         dt = dt_new;
                         //Strang splitting
-                        dirk.step( im, time, y0, time, y0, dt/2.);
+                        dirk_dens.step( im_dens, time, y0[0], time, y0[0], dt/2.);
+                        im_velo.set_density( y0[0]);
+                        dirk_velo.step( im_velo, time-dt/2., y0[1], time, y0[1], dt/2.);
                         adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
                             dg::pid_control, dg::l2norm, p.rtol, 1e-10);
                         if( adaptive.failed())
@@ -292,7 +296,9 @@ int main( int argc, char* argv[])
                             std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
                         }
                     }while ( adaptive.failed());
-                    dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
+                    dirk_dens.step( im_dens, time-dt/2., y0[0], time, y0[0], dt/2.);
+                    im_velo.set_density( y0[0]);
+                    dirk_velo.step( im_velo, time-dt/2., y0[1], time, y0[1], dt/2.);
                 }
                 catch( dg::Fail& fail) {
                     std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index f0b1b13d4..f09999e40 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -21,9 +21,9 @@ struct AddResistivity{
     DG_DEVICE
     void operator()( double ne, double ni, double ue,
         double ui, double& dtUe, double& dtUi) const{
-        double current = (ne+1)*(ui-ue);
+        double current = (ne)*(ui-ue);
         dtUe += -m_eta/m_mu[0] * current;
-        dtUi += -m_eta/m_mu[1] * (ne+1)/(ni+1) * current;
+        dtUi += -m_eta/m_mu[1] * (ne)/(ni) * current;
     }
     private:
     double m_eta;
@@ -172,91 +172,6 @@ struct ComputeSource{
 }//namespace routines
 
 
-template<class Geometry, class IMatrix, class Matrix, class container>
-struct Implicit
-{
-
-    Implicit( const Geometry& g, feltor::Parameters p,
-            dg::geo::TokamakMagneticField mag):
-        m_p(p),
-        m_lapM_perpN( g, p.bcxN,p.bcyN,dg::PER, dg::normed, dg::centered),
-        m_lapM_perpU( g, p.bcxU,p.bcyU,dg::PER, dg::normed, dg::centered)
-    {
-        dg::assign( dg::evaluate( dg::zero, g), m_temp);
-        auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
-        if( p.curvmode == "true")
-            bhat = dg::geo::createBHat(mag);
-        dg::SparseTensor<container> hh
-            = dg::geo::createProjectionTensor( bhat, g);
-        //set perpendicular projection tensor h
-        m_lapM_perpN.set_chi( hh);
-        m_lapM_perpU.set_chi( hh);
-    }
-
-    void operator()( double t, const std::array<std::array<container,2>,2>& y, std::array<std::array<container,2>,2>& yp)
-    {
-#if FELTORPERP == 1
-        /* y[0][0] := N_e - 1
-           y[0][1] := N_i - 1
-           y[1][0] := w_e
-           y[1][1] := W_i
-        */
-        for( unsigned i=0; i<2; i++)
-        {
-            //dissipation acts on w!
-            if( m_p.perp_diff == "hyperviscous")
-            {
-                dg::blas2::symv( m_lapM_perpN, y[0][i],      m_temp);
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, m_temp, 0., yp[0][i]);
-                dg::blas2::symv( m_lapM_perpU, y[1][i],      m_temp);
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, m_temp, 0., yp[1][i]);
-            }
-            else // m_p.perp_diff == "viscous"
-            {
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, y[0][i],  0., yp[0][i]);
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, y[1][i],  0., yp[1][i]);
-            }
-        }
-#else
-        dg::blas1::copy( 0, yp);
-#endif
-        //------------------Add Resistivity--------------------------//
-        //Note that this works because N converges independently of W
-        //and the term is linear in W for fixed N
-        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
-            y[0][0], y[0][1], y[1][0], y[1][1], yp[1][0], yp[1][1]);
-        //------------------Add Friction-----------------------------//
-        if( m_omega_damping != 0)
-        {
-            dg::blas1::pointwiseDot( -m_omega_damping, m_damping, y[1][0],
-                1., yp[1][0]);
-            dg::blas1::pointwiseDot( -m_omega_damping, m_damping, y[1][1],
-                1., yp[1][1]);
-        }
-    }
-
-    void set_damping( double omega_damping, container damping){
-        m_damping = damping;
-        m_omega_damping = omega_damping;
-    }
-
-    const container& weights() const{
-        return m_lapM_perpU.weights();
-    }
-    const container& inv_weights() const {
-        return m_lapM_perpU.inv_weights();
-    }
-    const container& precond() const {
-        return m_lapM_perpU.precond();
-    }
-
-  private:
-    double m_omega_damping=0;
-    const feltor::Parameters m_p;
-    container m_temp, m_damping;
-    dg::Elliptic3d<Geometry, Matrix, container> m_lapM_perpN, m_lapM_perpU;
-};
-
 struct Quantities
 {
     double mass = 0, diff = 0; //mass and mass diffusion
@@ -336,10 +251,10 @@ struct Explicit
         m_omega_damping = omega_damping;
         m_damping = damping;
     }
+    void compute_apar( double t, std::array<std::array<container,2>,2>& fields);
   private:
     void compute_phi( double t, const std::array<container,2>& y);
     void compute_psi( double t, const std::array<container,2>& y);
-    void compute_apar( double t, std::array<std::array<container,2>,2>& fields);
     void compute_energies(
         const std::array<std::array<container,2>,2>& fields);
     void compute_dissipation(
@@ -749,24 +664,26 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_perp(
             );
         }
     }
+    //dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+    //    y[0][0], y[0][1], fields[1][0], fields[1][1], yp[1][0], yp[1][1]);
     // Add correction to perpendicular viscosity in W
-    if( m_p.beta != 0)
-    {
-        if( m_p.perp_diff == "viscous")
-        {
-            dg::blas2::gemv(  m_lapperpU, m_apar, m_temp0); //!minus
-        }
-        else
-        {
-            dg::blas2::gemv( m_lapperpU, m_apar, m_temp1);
-            dg::blas2::gemv( m_lapperpU, m_temp1, m_temp0); //!plus
-        }
-        for( unsigned i=0; i<2; i++)
-        {
-            //-nu_perp*beta/mu Delta_perp A_par
-            dg::blas1::axpby( m_p.nu_perp*m_p.beta/m_p.mu[i], m_temp0, 1., yp[1][i]);
-        }
-    }
+    //if( m_p.beta != 0)
+    //{
+    //    if( m_p.perp_diff == "viscous")
+    //    {
+    //        dg::blas2::gemv(  m_lapperpU, m_apar, m_temp0); //!minus
+    //    }
+    //    else
+    //    {
+    //        dg::blas2::gemv( m_lapperpU, m_apar, m_temp1);
+    //        dg::blas2::gemv( m_lapperpU, m_temp1, m_temp0); //!plus
+    //    }
+    //    for( unsigned i=0; i<2; i++)
+    //    {
+    //        //-nu_perp*beta/mu Delta_perp A_par
+    //        dg::blas1::axpby( m_p.nu_perp*m_p.beta/m_p.mu[i], m_temp0, 1., yp[1][i]);
+    //    }
+    //}
 }
 
 template<class Geometry, class IMatrix, class Matrix, class container>
@@ -940,22 +857,22 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
     // Set perpendicular dynamics in yp
     compute_perp( t, y, m_fields, yp);
     //------------------Add Apar correction terms----------------//
-    if( m_p.beta != 0 )
-    {
-        dg::blas1::pointwiseDot( 1., m_fields[0][0],m_fields[0][0],m_apar,
-                                 0., m_temp0);
-        dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[0]*(
-            1./m_p.mu[1]-1./m_p.mu[0]), m_temp0, m_fields[0][0], 1., yp[1][0]);
-        dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[1]*(
-            1./m_p.mu[1]-1./m_p.mu[0]), m_temp0, m_fields[0][1], 1., yp[1][1]);
-        if( m_omega_damping != 0)
-        {
-            dg::blas1::pointwiseDot( m_omega_damping*m_p.beta/m_p.mu[0],
-                m_damping, m_apar, 1., yp[1][0]);
-            dg::blas1::pointwiseDot( m_omega_damping*m_p.beta/m_p.mu[1],
-                m_damping, m_apar, 1., yp[1][1]);
-        }
-    }
+    //if( m_p.beta != 0 )
+    //{
+    //    dg::blas1::pointwiseDot( 1., m_fields[0][0],m_fields[0][0],m_apar,
+    //                             0., m_temp0);
+    //    dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[0]*(
+    //        1./m_p.mu[1]-1./m_p.mu[0]), m_temp0, m_fields[0][0], 1., yp[1][0]);
+    //    dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[1]*(
+    //        1./m_p.mu[1]-1./m_p.mu[0]), m_temp0, m_fields[0][1], 1., yp[1][1]);
+    //    if( m_omega_damping != 0)
+    //    {
+    //        dg::blas1::pointwiseDot( m_omega_damping*m_p.beta/m_p.mu[0],
+    //            m_damping, m_apar, 1., yp[1][0]);
+    //        dg::blas1::pointwiseDot( m_omega_damping*m_p.beta/m_p.mu[1],
+    //            m_damping, m_apar, 1., yp[1][1]);
+    //    }
+    //}
 
 #else
 
@@ -1011,4 +928,215 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
     //std::cout << "#One rhs took "<<timer.diff()<<"s\n";
 }
 
+template<class Geometry, class IMatrix, class Matrix, class container>
+struct ImplicitDensity
+{
+    ImplicitDensity(){}
+
+    ImplicitDensity( const Geometry& g, feltor::Parameters p,
+            dg::geo::TokamakMagneticField mag)
+    {
+        construct( g, p, mag);
+    }
+
+    void construct( const Geometry& g, feltor::Parameters p,
+        dg::geo::TokamakMagneticField mag)
+    {
+        m_p = p;
+        m_lapM_perpN.construct( g, p.bcxN, p.bcyN,dg::PER, dg::normed, dg::centered);
+        dg::assign( dg::evaluate( dg::zero, g), m_temp);
+        auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
+        if( p.curvmode == "true")
+            bhat = dg::geo::createBHat(mag);
+        dg::SparseTensor<container> hh
+            = dg::geo::createProjectionTensor( bhat, g);
+        //set perpendicular projection tensor h
+        m_lapM_perpN.set_chi( hh);
+    }
+
+    void operator()( double t, const std::array<container,2>& y, std::array<container,2>& yp)
+    {
+#if FELTORPERP == 1
+        /* y[0][0] := N_e - 1
+           y[0][1] := N_i - 1
+        */
+        for( unsigned i=0; i<2; i++)
+        {
+            //dissipation acts on w!
+            if( m_p.perp_diff == "hyperviscous")
+            {
+                dg::blas2::symv( m_lapM_perpN, y[i],      m_temp);
+                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, m_temp, 0., yp[i]);
+            }
+            else // m_p.perp_diff == "viscous"
+            {
+                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, y[i],  0., yp[i]);
+            }
+        }
+#else
+        dg::blas1::copy( 0, yp);
+#endif
+    }
+
+    const container& weights() const{
+        return m_lapM_perpN.weights();
+    }
+    const container& inv_weights() const {
+        return m_lapM_perpN.inv_weights();
+    }
+    const container& precond() const {
+        return m_lapM_perpN.precond();
+    }
+
+  private:
+    feltor::Parameters m_p;
+    container m_temp;
+    dg::Elliptic3d<Geometry, Matrix, container> m_lapM_perpN;
+};
+
+template<class Geometry, class IMatrix, class Matrix, class container>
+struct ImplicitVelocity
+{
+
+    ImplicitVelocity(){}
+    ImplicitVelocity( const Geometry& g, feltor::Parameters p,
+            dg::geo::TokamakMagneticField mag){
+        construct( g, p, mag);
+    }
+    void construct( const Geometry& g, feltor::Parameters p,
+            dg::geo::TokamakMagneticField mag)
+    {
+        m_p=p;
+        m_lapM_perpU.construct( g, p.bcxU,p.bcyU,dg::PER, dg::normed, dg::centered);
+        dg::assign( dg::evaluate( dg::zero, g), m_temp);
+        m_apar = m_temp;
+        m_fields[0][0] = m_fields[0][1] = m_temp;
+        m_fields[1][0] = m_fields[1][1] = m_temp;
+        auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
+        if( p.curvmode == "true")
+            bhat = dg::geo::createBHat(mag);
+        dg::SparseTensor<container> hh
+            = dg::geo::createProjectionTensor( bhat, g);
+        //set perpendicular projection tensor h
+        m_lapM_perpU.set_chi( hh);
+        m_induction.construct(  g,
+            p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
+        m_induction.elliptic().set_chi( hh);
+        m_invert.construct( m_temp, g.size(), p.eps_pol,1 );
+    }
+    void set_density( const std::array<container, 2>& dens){
+        dg::blas1::copy( dens, m_fields[0]);
+        dg::blas1::plus( m_fields[0][0], (+1));
+        dg::blas1::plus( m_fields[0][1], (+1));
+        dg::blas1::axpby(  m_p.beta/m_p.mu[1], m_fields[0][1],
+                          -m_p.beta/m_p.mu[0], m_fields[0][0], m_temp);
+        m_induction.set_chi( m_temp);
+    }
+
+    void operator()( double t, const std::array<container,2>& y, std::array<container,2>& yp)
+    {
+#if FELTORPERP == 1
+        /*
+           y[0] := w_e
+           y[1] := W_i
+        */
+        dg::blas1::copy( y, m_fields[1]);
+        if( m_p.beta != 0){
+            dg::blas1::pointwiseDot(  1., m_fields[0][1], m_fields[1][1],
+                                     -1., m_fields[0][0], m_fields[1][0],
+                                      0., m_temp);
+            m_invert( m_induction, m_apar, m_temp, weights(),
+                inv_weights(), precond());
+            dg::blas1::axpby( 1., m_fields[1][0], -m_p.beta/m_p.mu[0],
+                m_apar, m_fields[1][0]);
+            dg::blas1::axpby( 1., m_fields[1][1], -m_p.beta/m_p.mu[1],
+                m_apar, m_fields[1][1]);
+        }
+
+        for( unsigned i=0; i<2; i++)
+        {
+            if( m_p.perp_diff == "hyperviscous")
+            {
+                dg::blas2::symv( m_lapM_perpU, m_fields[1][i],      m_temp);
+                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, m_temp, 0., yp[i]);
+            }
+            else // m_p.perp_diff == "viscous"
+            {
+                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, m_fields[1][i],  0., yp[i]);
+            }
+        }
+#else
+        dg::blas1::copy( 0, yp);
+#endif
+        //------------------Add Resistivity--------------------------//
+        //Note that this works because N converges independently of W
+        //and the term is linear in W for fixed N
+        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+            m_fields[0][0], m_fields[0][1], m_fields[1][0], m_fields[1][1], yp[0], yp[1]);
+    }
+
+    const container& weights() const{
+        return m_lapM_perpU.weights();
+    }
+    const container& inv_weights() const {
+        return m_lapM_perpU.inv_weights();
+    }
+    const container& precond() const {
+        return m_lapM_perpU.precond();
+    }
+
+  private:
+    feltor::Parameters m_p;
+    container m_temp, m_apar;
+    dg::Invert<container> m_invert;
+    dg::Helmholtz3d<Geometry, Matrix, container> m_induction;
+    std::array<std::array<container,2>,2> m_fields;
+    dg::Elliptic3d<Geometry, Matrix, container> m_lapM_perpU;
+};
+
+
+/*!@brief Multigrid Solver class for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
+*
+*/
+template< class Implicit, class Geometry, class Matrix, class Container>
+struct MultigridCG2dSolver
+{
+    using geometry_type = Geometry;
+    using matrix_type = Matrix;
+    using container_type = Container;
+    using value_type = dg::get_value_type<Container>;//!< value type of vectors
+    ///No memory allocation
+    MultigridCG2dSolver(){}
+
+    template<class ...Params>
+    MultigridCG2dSolver( const Geometry& grid, const unsigned stages, Params&& ... ps):
+        m_multigrid(grid, stages, std::forward<Params>(ps)...),
+        m_multi_implicit(stages)
+    {
+    }
+
+    template<class ...Params>
+    void construct_implicit( Params&& ...ps)
+    {
+        for( unsigned u=0; u<m_multigrid.stages(); u++)
+            m_multi_implicit[u].construct( m_multigrid.grid(u), std::forward<Params>(ps)...);
+    }
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const Container& copyable()const{ return m_multigrid.copyable();}
+
+    void solve( value_type alpha, Implicit& im, value_type t, Container& y, const Container& rhs)
+    {
+        std::vector<
+            dg::detail::Implicit<Implicit, Container>> implicit( m_multigrid.stages());
+        for( unsigned i=0; i<m_multigrid.stages(); i++)
+            implicit[i].construct( alpha, t, m_multi_implicit[i]);
+        m_multigrid.direct_solve( implicit, y, rhs, m_eps);
+    }
+    private:
+    dg::MultigridCG2d< Geometry, Matrix, Container> m_multigrid;
+    std::vector<Implicit> m_multi_implicit;
+    value_type m_eps;
+};
+
 } //namespace feltor
-- 
GitLab


From de699ef0f28c5ad8c52e17d47edb909d9ab74b19 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 31 Jan 2019 22:31:20 +0100
Subject: [PATCH 022/540] Implement a multigrid time solver for feltor

- begin a new file implicit.h moving implicit part from feltor.cuh
- and change feltor and feltor_hpc to use it
---
 src/feltor/feltor.cu     |  37 ++--
 src/feltor/feltor.cuh    | 457 ++++++++-------------------------------
 src/feltor/feltor_hpc.cu |  17 +-
 src/feltor/implicit.h    | 344 +++++++++++++++++++++++++++++
 4 files changed, 455 insertions(+), 400 deletions(-)
 create mode 100644 src/feltor/implicit.h

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 3b5ff3fa7..f30631389 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -7,6 +7,7 @@
 #include "draw/host_window.h"
 
 #include "feltor.cuh"
+#include "implicit.h"
 
 int main( int argc, char* argv[])
 {
@@ -51,8 +52,7 @@ int main( int argc, char* argv[])
     std::cout << "Constructing Explicit...\n";
     feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, mag);
     std::cout << "Constructing Implicit...\n";
-    feltor::ImplicitDensity<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> im_dens( grid, p, mag);
-    feltor::ImplicitVelocity<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> im_velo( grid, p, mag);
+    feltor::Implicit<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> im( grid, p, mag);
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
@@ -68,11 +68,6 @@ int main( int argc, char* argv[])
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
         //then shift
         p.rho_source, p.alpha, -1), grid);
-    dg::HVec damping_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-        //first change coordinate from psi to (psi_0 - psip)/psi_0
-        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift
-        p.rho_damping, p.alpha, +1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
     dg::HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
@@ -80,8 +75,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
-    feltor.set_source( profile, p.omega_source, source_damping,
-        p.omega_damping, damping_damping);
+    feltor.set_source( profile, p.omega_source, source_damping);
 
 
     //Now perturbation
@@ -155,11 +149,10 @@ int main( int argc, char* argv[])
     dg::Adaptive< dg::ERKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
         "Bogacki-Shampine-4-2-3", y0);
     adaptive.stepper().ignore_fsal();//necessary for splitting
-    dg::ImplicitRungeKutta<std::array<dg::DVec,2>> dirk_dens(
-        "Trapezoidal-2-2", y0[0], grid.size(), p.eps_time);
-    dg::ImplicitRungeKutta<std::array<dg::DVec,2>> dirk_velo(
-        "Trapezoidal-2-2", y0[0], grid.size(), p.eps_time);
-
+    dg::ImplicitRungeKutta<std::array<std::array<dg::DVec,2>,2>,
+        feltor::FeltorSpecialSolver<
+        dg::CylindricalGrid3d, dg::DMatrix, dg::DVec>> dirk(
+            "Trapezoidal-2-2", grid, p, mag);
     //since we map pointers we don't need to update those later
 
     std::map<std::string, const dg::DVec* > v4d;
@@ -277,17 +270,15 @@ int main( int argc, char* argv[])
         t.tic();
         for( unsigned i=0; i<p.itstp; i++)
         {
-            double current_time = time;
+            double previous_time = time;
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
                     do
                     {
-                        dt = dt_new;
                         //Strang splitting
-                        dirk_dens.step( im_dens, time, y0[0], time, y0[0], dt/2.);
-                        im_velo.set_density( y0[0]);
-                        dirk_velo.step( im_velo, time-dt/2., y0[1], time, y0[1], dt/2.);
+                        dt = dt_new;
+                        dirk.step( im, time, y0, time, y0, dt/2.);
                         adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
                             dg::pid_control, dg::l2norm, p.rtol, 1e-10);
                         if( adaptive.failed())
@@ -296,9 +287,7 @@ int main( int argc, char* argv[])
                             std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
                         }
                     }while ( adaptive.failed());
-                    dirk_dens.step( im_dens, time-dt/2., y0[0], time, y0[0], dt/2.);
-                    im_velo.set_density( y0[0]);
-                    dirk_velo.step( im_velo, time-dt/2., y0[1], time, y0[1], dt/2.);
+                    dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
                 }
                 catch( dg::Fail& fail) {
                     std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
@@ -308,10 +297,10 @@ int main( int argc, char* argv[])
                 }
                 step++;
             }
-            dt = time - current_time;
+            double deltat = time - previous_time;
             feltor.update_quantities();
             std::cout << "Timestep "<<dt_new<<"\n";
-            dEdt = (*v0d["energy"] - E0)/dt, dMdt = (*v0d["mass"] - M0)/dt;
+            dEdt = (*v0d["energy"] - E0)/deltat, dMdt = (*v0d["mass"] - M0)/deltat;
             E0 = *v0d["energy"], M0 = *v0d["mass"];
             accuracy  = 2.*fabs( (dEdt - *v0d["ediff"])/( dEdt + *v0d["ediff"]));
             accuracyM = 2.*fabs( (dMdt - *v0d["diff"])/( dMdt + *v0d["diff"]));
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index f09999e40..e2e0649c1 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -12,23 +12,6 @@ namespace feltor
 {
 
 namespace routines{
-//Resistivity (consistent density dependency,
-//parallel momentum conserving, quadratic current energy conservation dependency)
-struct AddResistivity{
-    AddResistivity( double eta, std::array<double,2> mu): m_eta(eta){
-        m_mu[0] = mu[0], m_mu[1] = mu[1];
-    }
-    DG_DEVICE
-    void operator()( double ne, double ni, double ue,
-        double ui, double& dtUe, double& dtUi) const{
-        double current = (ne)*(ui-ue);
-        dtUe += -m_eta/m_mu[0] * current;
-        dtUi += -m_eta/m_mu[1] * (ne)/(ni) * current;
-    }
-    private:
-    double m_eta;
-    double m_mu[2];
-};
 struct ComputePerpDrifts{
     ComputePerpDrifts( double mu, double tau, double beta):
         m_mu(mu), m_tau(tau), m_beta(beta){}
@@ -198,27 +181,27 @@ struct Quantities
     }
 };
 
-template< class Geometry, class IMatrix, class Matrix, class container >
+template< class Geometry, class IMatrix, class Matrix, class Container >
 struct Explicit
 {
     Explicit( const Geometry& g, feltor::Parameters p,
         dg::geo::TokamakMagneticField mag);
 
     //potential[0]: electron potential, potential[1]: ion potential
-    const std::array<container,2>& potential( ) const {
+    const std::array<Container,2>& potential( ) const {
         return m_phi;
     }
-    const container& induction() const {
+    const Container& induction() const {
         return m_apar;
     }
     //Given N_i-1 initialize n_e-1 such that phi=0
-    void initializene( const container& ni, container& ne);
+    void initializene( const Container& ni, Container& ne);
     //Given n_e-1 initialize N_i-1 such that phi=0
-    void initializeni( const container& ne, container& ni);
+    void initializeni( const Container& ne, Container& ni);
 
     void operator()( double t,
-        const std::array<std::array<container,2>,2>& y,
-        std::array<std::array<container,2>,2>& yp);
+        const std::array<std::array<Container,2>,2>& y,
+        std::array<std::array<Container,2>,2>& yp);
 
     // update quantities to the state of the last call to operator()
     // This is to possibly save some computation time in the timestepper
@@ -234,39 +217,36 @@ struct Explicit
     const Quantities& quantities( ) const{
         return m_q;
     }
-    const std::array<std::array<container,2>,2>& fields() const{
+    const std::array<std::array<Container,2>,2>& fields() const{
         return m_fields;
     }
-    const std::array<std::array<container,2>,2>& sources() const{
+    const std::array<std::array<Container,2>,2>& sources() const{
         return m_s;
     }
 
     //source strength, profile - 1
-    void set_source( container profile, double omega_source, container source,
-        double omega_damping, container damping)
+    void set_source( Container profile, double omega_source, Container source)
     {
         m_profne = profile;
         m_omega_source = omega_source;
         m_source = source;
-        m_omega_damping = omega_damping;
-        m_damping = damping;
     }
-    void compute_apar( double t, std::array<std::array<container,2>,2>& fields);
+    void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
   private:
-    void compute_phi( double t, const std::array<container,2>& y);
-    void compute_psi( double t, const std::array<container,2>& y);
+    void compute_phi( double t, const std::array<Container,2>& y);
+    void compute_psi( double t, const std::array<Container,2>& y);
     void compute_energies(
-        const std::array<std::array<container,2>,2>& fields);
+        const std::array<std::array<Container,2>,2>& fields);
     void compute_dissipation(
-        const std::array<std::array<container,2>,2>& fields);
+        const std::array<std::array<Container,2>,2>& fields);
     void compute_perp( double t,
-        const std::array<std::array<container,2>,2>& y,
-        const std::array<std::array<container,2>,2>& fields,
-        std::array<std::array<container,2>,2>& yp);
+        const std::array<std::array<Container,2>,2>& y,
+        const std::array<std::array<Container,2>,2>& fields,
+        std::array<std::array<Container,2>,2>& yp);
     void compute_parallel( double t,
-        const std::array<std::array<container,2>,2>& y,
-        const std::array<std::array<container,2>,2>& fields,
-        std::array<std::array<container,2>,2>& yp);
+        const std::array<std::array<Container,2>,2>& y,
+        const std::array<std::array<Container,2>,2>& fields,
+        std::array<std::array<Container,2>,2>& yp);
     void construct_mag( const Geometry&, feltor::Parameters,
         dg::geo::TokamakMagneticField);
     void construct_bhat( const Geometry&, feltor::Parameters,
@@ -274,48 +254,48 @@ struct Explicit
     void construct_invert( const Geometry&, feltor::Parameters,
         dg::geo::TokamakMagneticField);
 
-    container m_UE2;
-    container m_temp0, m_temp1, m_temp2;//helper variables
+    Container m_UE2;
+    Container m_temp0, m_temp1, m_temp2;//helper variables
 #ifdef DG_MANUFACTURED
-    container m_R, m_Z, m_P; //coordinates
+    Container m_R, m_Z, m_P; //coordinates
 #endif //DG_MANUFACTURED
 
     //these should be considered const
-    std::array<container,3> m_curv, m_curvKappa, m_b;
-    container m_divCurvKappa;
-    container m_binv, m_divb;
-    container m_source, m_profne, m_damping;
-    container m_vol3d;
+    std::array<Container,3> m_curv, m_curvKappa, m_b;
+    Container m_divCurvKappa;
+    Container m_binv, m_divb;
+    Container m_source, m_profne;
+    Container m_vol3d;
 
-    container m_apar, m_dxA, m_dyA, m_dzA;
-    std::array<container,2> m_phi, m_dxPhi, m_dyPhi, m_dzPhi;
-    std::array<container,2> m_logn, m_dxN, m_dyN, m_dzN, m_dsN;
-    std::array<container,2> m_dxU, m_dyU, m_dzU, m_dsU;
-    std::array<std::array<container,2>,2> m_fields, m_s;
+    Container m_apar, m_dxA, m_dyA, m_dzA;
+    std::array<Container,2> m_phi, m_dxPhi, m_dyPhi, m_dzPhi;
+    std::array<Container,2> m_logn, m_dxN, m_dyN, m_dzN, m_dsN;
+    std::array<Container,2> m_dxU, m_dyU, m_dzU, m_dsU;
+    std::array<std::array<Container,2>,2> m_fields, m_s;
 
-    std::vector<container> m_multi_chi;
+    std::vector<Container> m_multi_chi;
 
     //matrices and solvers
     Matrix m_dx_N, m_dx_U, m_dx_P, m_dy_N, m_dy_U, m_dy_P, m_dz;
-    dg::geo::DS<Geometry, IMatrix, Matrix, container> m_ds_P, m_ds_N, m_ds_U;
-    dg::Elliptic3d< Geometry, Matrix, container> m_lapperpN, m_lapperpU;
-    std::vector<dg::Elliptic3d< Geometry, Matrix, container> > m_multi_pol;
-    std::vector<dg::Helmholtz3d<Geometry, Matrix, container> > m_multi_invgammaP,
+    dg::geo::DS<Geometry, IMatrix, Matrix, Container> m_ds_P, m_ds_N, m_ds_U;
+    dg::Elliptic3d< Geometry, Matrix, Container> m_lapperpN, m_lapperpU;
+    std::vector<dg::Elliptic3d< Geometry, Matrix, Container> > m_multi_pol;
+    std::vector<dg::Helmholtz3d<Geometry, Matrix, Container> > m_multi_invgammaP,
         m_multi_invgammaN, m_multi_induction;
 
-    dg::MultigridCG2d<Geometry, Matrix, container> m_multigrid;
-    dg::Extrapolation<container> m_old_phi, m_old_psi, m_old_gammaN, m_old_apar;
+    dg::MultigridCG2d<Geometry, Matrix, Container> m_multigrid;
+    dg::Extrapolation<Container> m_old_phi, m_old_psi, m_old_gammaN, m_old_apar;
 
-    dg::SparseTensor<container> m_hh;
+    dg::SparseTensor<Container> m_hh;
 
     const feltor::Parameters m_p;
     Quantities m_q;
-    double m_omega_source = 0., m_omega_damping = 0.;
+    double m_omega_source = 0.;
 
 };
 
-template<class Grid, class IMatrix, class Matrix, class container>
-void Explicit<Grid, IMatrix, Matrix, container>::construct_mag(
+template<class Grid, class IMatrix, class Matrix, class Container>
+void Explicit<Grid, IMatrix, Matrix, Container>::construct_mag(
     const Grid& g, feltor::Parameters p, dg::geo::TokamakMagneticField mag)
 {
     //due to the various approximations bhat and mag not always correspond
@@ -348,8 +328,8 @@ void Explicit<Grid, IMatrix, Matrix, container>::construct_mag(
     dg::assign(  dg::pullback(dg::geo::Divb(mag), g), m_divb);
 
 }
-template<class Grid, class IMatrix, class Matrix, class container>
-void Explicit<Grid, IMatrix, Matrix, container>::construct_bhat(
+template<class Grid, class IMatrix, class Matrix, class Container>
+void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
     const Grid& g, feltor::Parameters p, dg::geo::TokamakMagneticField mag)
 {
     //in DS we take the true bhat
@@ -374,10 +354,10 @@ void Explicit<Grid, IMatrix, Matrix, container>::construct_bhat(
     if( p.curvmode == "true")
         bhat = dg::geo::createBHat(mag);
     dg::pushForward(bhat.x(), bhat.y(), bhat.z(), m_b[0], m_b[1], m_b[2], g);
-    dg::SparseTensor<container> metric = g.metric();
+    dg::SparseTensor<Container> metric = g.metric();
     dg::tensor::inv_multiply3d( metric, m_b[0], m_b[1], m_b[2],
                                         m_b[0], m_b[1], m_b[2]);
-    container vol = dg::tensor::volume( metric);
+    Container vol = dg::tensor::volume( metric);
     dg::blas1::pointwiseDivide( m_binv, vol, vol); //1/vol/B
     for( int i=0; i<3; i++)
         dg::blas1::pointwiseDot( vol, m_b[i], m_b[i]); //b_i/vol/B
@@ -387,8 +367,8 @@ void Explicit<Grid, IMatrix, Matrix, container>::construct_bhat(
     m_lapperpN.set_chi( m_hh);
     m_lapperpU.set_chi( m_hh);
 }
-template<class Grid, class IMatrix, class Matrix, class container>
-void Explicit<Grid, IMatrix, Matrix, container>::construct_invert(
+template<class Grid, class IMatrix, class Matrix, class Container>
+void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
     const Grid& g, feltor::Parameters p, dg::geo::TokamakMagneticField mag)
 {
     /////////////////////////init elliptic and helmholtz operators/////////
@@ -402,7 +382,7 @@ void Explicit<Grid, IMatrix, Matrix, container>::construct_invert(
     m_multi_induction.resize(p.stages);
     for( unsigned u=0; u<p.stages; u++)
     {
-        dg::SparseTensor<container> hh = dg::geo::createProjectionTensor(
+        dg::SparseTensor<Container> hh = dg::geo::createProjectionTensor(
             bhat, m_multigrid.grid(u));
         m_multi_pol[u].construct( m_multigrid.grid(u),
             p.bcxP, p.bcyP, dg::PER, dg::not_normed,
@@ -419,8 +399,8 @@ void Explicit<Grid, IMatrix, Matrix, container>::construct_invert(
         m_multi_induction[u].elliptic().set_chi( hh);
     }
 }
-template<class Grid, class IMatrix, class Matrix, class container>
-Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g,
+template<class Grid, class IMatrix, class Matrix, class Container>
+Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     feltor::Parameters p, dg::geo::TokamakMagneticField mag):
 #ifdef DG_MANUFACTURED
     m_R( dg::pullback( dg::cooX3d, g)),
@@ -455,9 +435,9 @@ Explicit<Grid, IMatrix, Matrix, container>::Explicit( const Grid& g,
     dg::assign( dg::create::volume(g), m_vol3d);
 }
 
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::initializene(
-    const container& src, container& target)
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::initializene(
+    const Container& src, Container& target)
 {
     // ne = Ni
     dg::blas1::copy( src, target);
@@ -466,12 +446,12 @@ void Explicit<Geometry, IMatrix, Matrix, container>::initializene(
         std::vector<unsigned> number = m_multigrid.direct_solve(
             m_multi_invgammaN, target, src, m_p.eps_gamma);
         if(  number[0] == m_multigrid.max_iter())
-        throw dg::Fail( m_p.eps_gamma);
+            throw dg::Fail( m_p.eps_gamma);
     }
 }
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::initializeni(
-    const container& src, container& target)
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::initializeni(
+    const Container& src, Container& target)
 {
     // Ni = ne
     dg::blas1::copy( src, target);
@@ -482,9 +462,9 @@ void Explicit<Geometry, IMatrix, Matrix, container>::initializeni(
     }
 }
 
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::compute_phi(
-    double time, const std::array<container,2>& y)
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::compute_phi(
+    double time, const std::array<Container,2>& y)
 {
     //y[0]:= n_e - 1
     //y[1]:= N_i - 1
@@ -554,9 +534,9 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_phi(
     }
 }
 
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::compute_psi(
-    double time, const std::array<container,2>& y)
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
+    double time, const std::array<Container,2>& y)
 {
     //-------Compute Psi and derivatives
     dg::blas2::symv( m_dx_P, m_phi[0], m_dxPhi[0]);
@@ -577,9 +557,9 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_psi(
     dg::blas2::symv( m_dy_P, m_phi[1], m_dyPhi[1]);
     if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[1], m_dzPhi[1]);
 }
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::compute_apar(
-    double time, std::array<std::array<container,2>,2>& fields)
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
+    double time, std::array<std::array<Container,2>,2>& fields)
 {
     //on input
     //fields[0][0] = n_e, fields[1][0]:= w_e
@@ -616,12 +596,12 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_apar(
     dg::blas1::axpby( 1., fields[1][1], -m_p.beta/m_p.mu[1], m_apar, fields[1][1]);
 }
 
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::compute_perp(
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
     double t,
-    const std::array<std::array<container,2>,2>& y,
-    const std::array<std::array<container,2>,2>& fields,
-    std::array<std::array<container,2>,2>& yp)
+    const std::array<std::array<Container,2>,2>& y,
+    const std::array<std::array<Container,2>,2>& fields,
+    std::array<std::array<Container,2>,2>& yp)
 {
     //y[0] = N-1, y[1] = W; fields[0] = N, fields[1] = U
     for( unsigned i=0; i<2; i++)
@@ -664,34 +644,14 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_perp(
             );
         }
     }
-    //dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
-    //    y[0][0], y[0][1], fields[1][0], fields[1][1], yp[1][0], yp[1][1]);
-    // Add correction to perpendicular viscosity in W
-    //if( m_p.beta != 0)
-    //{
-    //    if( m_p.perp_diff == "viscous")
-    //    {
-    //        dg::blas2::gemv(  m_lapperpU, m_apar, m_temp0); //!minus
-    //    }
-    //    else
-    //    {
-    //        dg::blas2::gemv( m_lapperpU, m_apar, m_temp1);
-    //        dg::blas2::gemv( m_lapperpU, m_temp1, m_temp0); //!plus
-    //    }
-    //    for( unsigned i=0; i<2; i++)
-    //    {
-    //        //-nu_perp*beta/mu Delta_perp A_par
-    //        dg::blas1::axpby( m_p.nu_perp*m_p.beta/m_p.mu[i], m_temp0, 1., yp[1][i]);
-    //    }
-    //}
 }
 
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::compute_parallel(
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
     double t,
-    const std::array<std::array<container,2>,2>& y,
-    const std::array<std::array<container,2>,2>& fields,
-    std::array<std::array<container,2>,2>& yp)
+    const std::array<std::array<Container,2>,2>& y,
+    const std::array<std::array<Container,2>,2>& fields,
+    std::array<std::array<Container,2>,2>& yp)
 {
     //y[0] = N-1, y[1] = W; fields[0] = N, fields[1] = U
     for( unsigned i=0; i<2; i++)
@@ -725,9 +685,9 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_parallel(
     }
 }
 
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::compute_energies(
-    const std::array<std::array<container,2>,2>& fields)
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::compute_energies(
+    const std::array<std::array<Container,2>,2>& fields)
 {
     double z[2]    = {-1.0,1.0};
     m_q.mass = dg::blas1::dot( m_vol3d, fields[0][0]);
@@ -755,9 +715,9 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_energies(
                  + m_q.Tpar[0] + m_q.Tpar[1];
 }
 
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::compute_dissipation(
-    const std::array<std::array<container,2>,2>& fields)
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::compute_dissipation(
+    const std::array<std::array<Container,2>,2>& fields)
 {
     //alignement: lnN * Delta_s N
     m_q.aligned = dg::blas2::dot( m_logn[0], m_vol3d, m_dsN[0]);
@@ -829,11 +789,11 @@ void Explicit<Geometry, IMatrix, Matrix, container>::compute_dissipation(
    y[1][0] := w_e
    y[1][1] := W_i
 */
-template<class Geometry, class IMatrix, class Matrix, class container>
-void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     double t,
-    const std::array<std::array<container,2>,2>& y,
-    std::array<std::array<container,2>,2>& yp)
+    const std::array<std::array<Container,2>,2>& y,
+    std::array<std::array<Container,2>,2>& yp)
 {
     dg::Timer timer;
     timer.tic();
@@ -856,23 +816,6 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
 
     // Set perpendicular dynamics in yp
     compute_perp( t, y, m_fields, yp);
-    //------------------Add Apar correction terms----------------//
-    //if( m_p.beta != 0 )
-    //{
-    //    dg::blas1::pointwiseDot( 1., m_fields[0][0],m_fields[0][0],m_apar,
-    //                             0., m_temp0);
-    //    dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[0]*(
-    //        1./m_p.mu[1]-1./m_p.mu[0]), m_temp0, m_fields[0][0], 1., yp[1][0]);
-    //    dg::blas1::pointwiseDivide( m_p.beta*m_p.eta/m_p.mu[1]*(
-    //        1./m_p.mu[1]-1./m_p.mu[0]), m_temp0, m_fields[0][1], 1., yp[1][1]);
-    //    if( m_omega_damping != 0)
-    //    {
-    //        dg::blas1::pointwiseDot( m_omega_damping*m_p.beta/m_p.mu[0],
-    //            m_damping, m_apar, 1., yp[1][0]);
-    //        dg::blas1::pointwiseDot( m_omega_damping*m_p.beta/m_p.mu[1],
-    //            m_damping, m_apar, 1., yp[1][1]);
-    //    }
-    //}
 
 #else
 
@@ -897,14 +840,6 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
         dg::blas1::axpby( 1., m_s[0][0], 1.0, yp[0][0]);
         dg::blas1::axpby( 1., m_s[0][1], 1.0, yp[0][1]);
     }
-    if( m_omega_damping != 0)
-    {
-        //just compute m_s terms for energies, terms are implicit
-        dg::blas1::pointwiseDot( -m_omega_damping, m_fields[1][0],
-            m_damping, 0., m_s[1][0]);
-        dg::blas1::pointwiseDot( -m_omega_damping, m_fields[1][1],
-            m_damping, 0., m_s[1][1]);
-    }
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( yp[0][0], dg::plus_equals(), manufactured::SNe{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
@@ -927,216 +862,4 @@ void Explicit<Geometry, IMatrix, Matrix, container>::operator()(
     //#endif
     //std::cout << "#One rhs took "<<timer.diff()<<"s\n";
 }
-
-template<class Geometry, class IMatrix, class Matrix, class container>
-struct ImplicitDensity
-{
-    ImplicitDensity(){}
-
-    ImplicitDensity( const Geometry& g, feltor::Parameters p,
-            dg::geo::TokamakMagneticField mag)
-    {
-        construct( g, p, mag);
-    }
-
-    void construct( const Geometry& g, feltor::Parameters p,
-        dg::geo::TokamakMagneticField mag)
-    {
-        m_p = p;
-        m_lapM_perpN.construct( g, p.bcxN, p.bcyN,dg::PER, dg::normed, dg::centered);
-        dg::assign( dg::evaluate( dg::zero, g), m_temp);
-        auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
-        if( p.curvmode == "true")
-            bhat = dg::geo::createBHat(mag);
-        dg::SparseTensor<container> hh
-            = dg::geo::createProjectionTensor( bhat, g);
-        //set perpendicular projection tensor h
-        m_lapM_perpN.set_chi( hh);
-    }
-
-    void operator()( double t, const std::array<container,2>& y, std::array<container,2>& yp)
-    {
-#if FELTORPERP == 1
-        /* y[0][0] := N_e - 1
-           y[0][1] := N_i - 1
-        */
-        for( unsigned i=0; i<2; i++)
-        {
-            //dissipation acts on w!
-            if( m_p.perp_diff == "hyperviscous")
-            {
-                dg::blas2::symv( m_lapM_perpN, y[i],      m_temp);
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, m_temp, 0., yp[i]);
-            }
-            else // m_p.perp_diff == "viscous"
-            {
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, y[i],  0., yp[i]);
-            }
-        }
-#else
-        dg::blas1::copy( 0, yp);
-#endif
-    }
-
-    const container& weights() const{
-        return m_lapM_perpN.weights();
-    }
-    const container& inv_weights() const {
-        return m_lapM_perpN.inv_weights();
-    }
-    const container& precond() const {
-        return m_lapM_perpN.precond();
-    }
-
-  private:
-    feltor::Parameters m_p;
-    container m_temp;
-    dg::Elliptic3d<Geometry, Matrix, container> m_lapM_perpN;
-};
-
-template<class Geometry, class IMatrix, class Matrix, class container>
-struct ImplicitVelocity
-{
-
-    ImplicitVelocity(){}
-    ImplicitVelocity( const Geometry& g, feltor::Parameters p,
-            dg::geo::TokamakMagneticField mag){
-        construct( g, p, mag);
-    }
-    void construct( const Geometry& g, feltor::Parameters p,
-            dg::geo::TokamakMagneticField mag)
-    {
-        m_p=p;
-        m_lapM_perpU.construct( g, p.bcxU,p.bcyU,dg::PER, dg::normed, dg::centered);
-        dg::assign( dg::evaluate( dg::zero, g), m_temp);
-        m_apar = m_temp;
-        m_fields[0][0] = m_fields[0][1] = m_temp;
-        m_fields[1][0] = m_fields[1][1] = m_temp;
-        auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
-        if( p.curvmode == "true")
-            bhat = dg::geo::createBHat(mag);
-        dg::SparseTensor<container> hh
-            = dg::geo::createProjectionTensor( bhat, g);
-        //set perpendicular projection tensor h
-        m_lapM_perpU.set_chi( hh);
-        m_induction.construct(  g,
-            p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
-        m_induction.elliptic().set_chi( hh);
-        m_invert.construct( m_temp, g.size(), p.eps_pol,1 );
-    }
-    void set_density( const std::array<container, 2>& dens){
-        dg::blas1::copy( dens, m_fields[0]);
-        dg::blas1::plus( m_fields[0][0], (+1));
-        dg::blas1::plus( m_fields[0][1], (+1));
-        dg::blas1::axpby(  m_p.beta/m_p.mu[1], m_fields[0][1],
-                          -m_p.beta/m_p.mu[0], m_fields[0][0], m_temp);
-        m_induction.set_chi( m_temp);
-    }
-
-    void operator()( double t, const std::array<container,2>& y, std::array<container,2>& yp)
-    {
-#if FELTORPERP == 1
-        /*
-           y[0] := w_e
-           y[1] := W_i
-        */
-        dg::blas1::copy( y, m_fields[1]);
-        if( m_p.beta != 0){
-            dg::blas1::pointwiseDot(  1., m_fields[0][1], m_fields[1][1],
-                                     -1., m_fields[0][0], m_fields[1][0],
-                                      0., m_temp);
-            m_invert( m_induction, m_apar, m_temp, weights(),
-                inv_weights(), precond());
-            dg::blas1::axpby( 1., m_fields[1][0], -m_p.beta/m_p.mu[0],
-                m_apar, m_fields[1][0]);
-            dg::blas1::axpby( 1., m_fields[1][1], -m_p.beta/m_p.mu[1],
-                m_apar, m_fields[1][1]);
-        }
-
-        for( unsigned i=0; i<2; i++)
-        {
-            if( m_p.perp_diff == "hyperviscous")
-            {
-                dg::blas2::symv( m_lapM_perpU, m_fields[1][i],      m_temp);
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, m_temp, 0., yp[i]);
-            }
-            else // m_p.perp_diff == "viscous"
-            {
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, m_fields[1][i],  0., yp[i]);
-            }
-        }
-#else
-        dg::blas1::copy( 0, yp);
-#endif
-        //------------------Add Resistivity--------------------------//
-        //Note that this works because N converges independently of W
-        //and the term is linear in W for fixed N
-        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
-            m_fields[0][0], m_fields[0][1], m_fields[1][0], m_fields[1][1], yp[0], yp[1]);
-    }
-
-    const container& weights() const{
-        return m_lapM_perpU.weights();
-    }
-    const container& inv_weights() const {
-        return m_lapM_perpU.inv_weights();
-    }
-    const container& precond() const {
-        return m_lapM_perpU.precond();
-    }
-
-  private:
-    feltor::Parameters m_p;
-    container m_temp, m_apar;
-    dg::Invert<container> m_invert;
-    dg::Helmholtz3d<Geometry, Matrix, container> m_induction;
-    std::array<std::array<container,2>,2> m_fields;
-    dg::Elliptic3d<Geometry, Matrix, container> m_lapM_perpU;
-};
-
-
-/*!@brief Multigrid Solver class for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
-*
-*/
-template< class Implicit, class Geometry, class Matrix, class Container>
-struct MultigridCG2dSolver
-{
-    using geometry_type = Geometry;
-    using matrix_type = Matrix;
-    using container_type = Container;
-    using value_type = dg::get_value_type<Container>;//!< value type of vectors
-    ///No memory allocation
-    MultigridCG2dSolver(){}
-
-    template<class ...Params>
-    MultigridCG2dSolver( const Geometry& grid, const unsigned stages, Params&& ... ps):
-        m_multigrid(grid, stages, std::forward<Params>(ps)...),
-        m_multi_implicit(stages)
-    {
-    }
-
-    template<class ...Params>
-    void construct_implicit( Params&& ...ps)
-    {
-        for( unsigned u=0; u<m_multigrid.stages(); u++)
-            m_multi_implicit[u].construct( m_multigrid.grid(u), std::forward<Params>(ps)...);
-    }
-    ///@brief Return an object of same size as the object used for construction
-    ///@return A copyable object; what it contains is undefined, its size is important
-    const Container& copyable()const{ return m_multigrid.copyable();}
-
-    void solve( value_type alpha, Implicit& im, value_type t, Container& y, const Container& rhs)
-    {
-        std::vector<
-            dg::detail::Implicit<Implicit, Container>> implicit( m_multigrid.stages());
-        for( unsigned i=0; i<m_multigrid.stages(); i++)
-            implicit[i].construct( alpha, t, m_multi_implicit[i]);
-        m_multigrid.direct_solve( implicit, y, rhs, m_eps);
-    }
-    private:
-    dg::MultigridCG2d< Geometry, Matrix, Container> m_multigrid;
-    std::vector<Implicit> m_multi_implicit;
-    value_type m_eps;
-};
-
 } //namespace feltor
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 26526dff0..66fc10599 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -13,6 +13,7 @@
 
 #include "dg/file/nc_utilities.h"
 #include "feltor.cuh"
+#include "implicit.h"
 
 #ifdef FELTOR_MPI
 using HVec = dg::MHVec;
@@ -146,7 +147,7 @@ int main( int argc, char* argv[])
     MPI_OUT std::cout << "Constructing Explicit...\n";
     feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
     MPI_OUT std::cout << "Constructing Implicit...\n";
-    feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
+    feltor::Implicit< Geometry, DMatrix, DVec> im( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
@@ -174,9 +175,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
-    feltor.set_source( profile, p.omega_source, source_damping,
-        p.omega_damping, damping_damping);
-    im.set_damping( p.omega_damping, damping_damping);
+    feltor.set_source( profile, p.omega_source, source_damping);
 
     //Now perturbation
     HVec ntilde = dg::evaluate(dg::zero,grid);
@@ -398,8 +397,9 @@ int main( int argc, char* argv[])
     dg::Adaptive< dg::ERKStep<std::array<std::array<DVec,2>,2>> > adaptive(
         "Bogacki-Shampine-4-2-3", y0);
     adaptive.stepper().ignore_fsal();//necessary for splitting
-    dg::ImplicitRungeKutta<std::array<std::array<DVec,2>,2>> dirk(
-        "Trapezoidal-2-2", y0, grid.size(), p.eps_time);
+    dg::ImplicitRungeKutta<std::array<std::array<DVec,2>,2>,
+        feltor::FeltorSpecialSolver< Geometry, DMatrix, DVec>> dirk(
+            "Trapezoidal-2-2", grid, p, mag);
     dg::Timer t;
     t.tic();
     unsigned step = 0, failed_counter = 0;
@@ -424,8 +424,8 @@ int main( int argc, char* argv[])
                             dg::pid_control, dg::l2norm, p.rtol, 1e-10);
                         if( adaptive.failed())
                         {
-                            MPI_OUT std::cout << "FAILED STEP! REPEAT!\n";
                             failed_counter++;
+                            MPI_OUT std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
                         }
                     }while ( adaptive.failed());
                     dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
@@ -441,9 +441,8 @@ int main( int argc, char* argv[])
                 step++;
             }
             double deltat = time - previous_time;
-
             feltor.update_quantities();
-            MPI_OUT std::cout << "Timestep "<<deltat<<"\n";
+            MPI_OUT std::cout << "Timestep "<<dt_new<<"\n";
             dEdt = (*v0d["energy"] - E0)/deltat, dMdt = (*v0d["mass"] - M0)/deltat;
             E0 = *v0d["energy"], M0 = *v0d["mass"];
             accuracy  = 2.*fabs( (dEdt - *v0d["ediff"])/( dEdt + *v0d["ediff"]));
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
new file mode 100644
index 000000000..d62fdedc8
--- /dev/null
+++ b/src/feltor/implicit.h
@@ -0,0 +1,344 @@
+#pragma once
+
+#include "dg/algorithm.h"
+#include "parameters.h"
+#include "dg/geometries/geometries.h"
+
+//This contains the implicit part of the feltor equations
+//We even write a custom solver object for it
+
+namespace feltor
+{
+namespace routines
+{
+//Resistivity (consistent density dependency,
+//parallel momentum conserving, quadratic current energy conservation dependency)
+struct AddResistivity{
+    AddResistivity( double eta, std::array<double,2> mu): m_eta(eta){
+        m_mu[0] = mu[0], m_mu[1] = mu[1];
+    }
+    DG_DEVICE
+    void operator()( double ne, double ni, double ue,
+        double ui, double& dtUe, double& dtUi) const{
+        double current = (ne)*(ui-ue);
+        dtUe += -m_eta/m_mu[0] * current;
+        dtUi += -m_eta/m_mu[1] * (ne)/(ni) * current;
+    }
+    private:
+    double m_eta;
+    double m_mu[2];
+};
+}//namespace routines
+
+template<class Geometry, class Matrix, class Container>
+struct ImplicitDensity
+{
+    ImplicitDensity() = default;
+    ImplicitDensity( const Geometry& g, feltor::Parameters p,
+            dg::geo::TokamakMagneticField mag)
+    {
+        construct( g, p, mag);
+    }
+
+    void construct( const Geometry& g, feltor::Parameters p,
+        dg::geo::TokamakMagneticField mag)
+    {
+        m_p = p;
+        m_lapM_perpN.construct( g, p.bcxN, p.bcyN,dg::PER, dg::normed, dg::centered);
+        dg::assign( dg::evaluate( dg::zero, g), m_temp);
+        auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
+        if( p.curvmode == "true")
+            bhat = dg::geo::createBHat(mag);
+        dg::SparseTensor<Container> hh
+            = dg::geo::createProjectionTensor( bhat, g);
+        //set perpendicular projection tensor h
+        m_lapM_perpN.set_chi( hh);
+    }
+
+    void operator()( double t, const std::array<Container,2>& y,
+        std::array<Container,2>& yp)
+    {
+#if FELTORPERP == 1
+        /* y[0] := n_e - 1
+           y[1] := N_i - 1
+        */
+        for( unsigned i=0; i<2; i++)
+        {
+            //dissipation acts on w!
+            if( m_p.perp_diff == "hyperviscous")
+            {
+                dg::blas2::symv( m_lapM_perpN, y[i],      m_temp);
+                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, m_temp, 0., yp[i]);
+            }
+            else // m_p.perp_diff == "viscous"
+                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, y[i],  0., yp[i]);
+        }
+#else
+        dg::blas1::copy( 0, yp);
+#endif
+    }
+
+    const Container& weights() const{
+        return m_lapM_perpN.weights();
+    }
+    const Container& inv_weights() const {
+        return m_lapM_perpN.inv_weights();
+    }
+    const Container& precond() const {
+        return m_lapM_perpN.precond();
+    }
+
+  private:
+    feltor::Parameters m_p;
+    Container m_temp;
+    dg::Elliptic3d<Geometry, Matrix, Container> m_lapM_perpN;
+};
+
+template<class Geometry, class Matrix, class Container>
+struct ImplicitVelocity
+{
+    ImplicitVelocity() = default;
+    ImplicitVelocity( const Geometry& g, feltor::Parameters p,
+            dg::geo::TokamakMagneticField mag){
+        construct( g, p, mag);
+    }
+    void construct( const Geometry& g, feltor::Parameters p,
+            dg::geo::TokamakMagneticField mag)
+    {
+        m_p=p;
+        m_lapM_perpU.construct( g, p.bcxU,p.bcyU,dg::PER,
+            dg::normed, dg::centered);
+        dg::assign( dg::evaluate( dg::zero, g), m_temp);
+        m_apar = m_temp;
+        m_fields[0][0] = m_fields[0][1] = m_temp;
+        m_fields[1][0] = m_fields[1][1] = m_temp;
+        auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
+        if( p.curvmode == "true")
+            bhat = dg::geo::createBHat(mag);
+        dg::SparseTensor<Container> hh
+            = dg::geo::createProjectionTensor( bhat, g);
+        //set perpendicular projection tensor h
+        m_lapM_perpU.set_chi( hh);
+        m_induction.construct(  g,
+            p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
+        m_induction.elliptic().set_chi( hh);
+        m_invert.construct( m_temp, g.size(), p.eps_pol,1 );
+    }
+    void set_density( const std::array<Container, 2>& dens){
+        dg::blas1::transform( dens, m_fields[0], dg::PLUS<double>(+1));
+        dg::blas1::axpby(  m_p.beta/m_p.mu[1], m_fields[0][1],
+                          -m_p.beta/m_p.mu[0], m_fields[0][0], m_temp);
+        m_induction.set_chi( m_temp);
+    }
+
+    void operator()( double t, const std::array<Container,2>& w,
+        std::array<Container,2>& wp)
+    {
+        //Note that this operator works because N converges
+        //independently of W and these terms are linear in W for fixed N
+#if FELTORPERP == 1
+        /* w[0] := w_e
+           w[1] := W_i
+        */
+        dg::blas1::copy( w, m_fields[1]);
+        if( m_p.beta != 0){
+            //let us solve for apar
+            dg::blas1::pointwiseDot(  1., m_fields[0][1], m_fields[1][1],
+                                     -1., m_fields[0][0], m_fields[1][0],
+                                      0., m_temp);
+            m_invert( m_induction, m_apar, m_temp, weights(),
+                inv_weights(), precond());
+            //compute u_e and U_i
+            dg::blas1::axpby( 1., m_fields[1][0], -m_p.beta/m_p.mu[0],
+                m_apar, m_fields[1][0]);
+            dg::blas1::axpby( 1., m_fields[1][1], -m_p.beta/m_p.mu[1],
+                m_apar, m_fields[1][1]);
+        }
+        /* fields[0] := u_e
+           fields[1] := U_i
+        */
+
+        for( unsigned i=0; i<2; i++)
+        {
+            if( m_p.perp_diff == "hyperviscous")
+            {
+                dg::blas2::symv( m_lapM_perpU, m_fields[1][i],      m_temp);
+                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, m_temp, 0., wp[i]);
+            }
+            else // m_p.perp_diff == "viscous"
+                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU,
+                    m_fields[1][i],  0., wp[i]);
+        }
+#else
+        dg::blas1::copy( 0, wp);
+#endif
+        //------------------Add Resistivity--------------------------//
+        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+            m_fields[0][0], m_fields[0][1],
+            m_fields[1][0], m_fields[1][1], wp[0], wp[1]);
+    }
+
+    const Container& weights() const{
+        return m_lapM_perpU.weights();
+    }
+    const Container& inv_weights() const {
+        return m_lapM_perpU.inv_weights();
+    }
+    const Container& precond() const {
+        return m_lapM_perpU.precond();
+    }
+
+  private:
+    feltor::Parameters m_p;
+    Container m_temp, m_apar;
+    dg::Invert<Container> m_invert;
+    dg::Helmholtz3d<Geometry, Matrix, Container> m_induction;
+    std::array<std::array<Container,2>,2> m_fields;
+    dg::Elliptic3d<Geometry, Matrix, Container> m_lapM_perpU;
+};
+
+template<class Geometry, class Matrix, class Container>
+struct Implicit
+{
+    Implicit() = default;
+    Implicit( const Geometry& g, feltor::Parameters p,
+            dg::geo::TokamakMagneticField mag):
+            m_dens( g,p,mag), m_velo( g,p,mag){}
+    void operator()( double t, const std::array<std::array<Container,2>,2>& y,
+        std::array<std::array<Container,2>,2>& yp)
+    {
+        m_dens( t,y[0], yp[0]);
+        m_velo.set_density( y[0]);
+        m_velo( t,y[1], yp[1]);
+    }
+    private:
+    ImplicitDensity <Geometry, Matrix, Container> m_dens;
+    ImplicitVelocity<Geometry, Matrix, Container> m_velo;
+};
+
+//compute unnormalized y + alpha f(y,t)
+template< class LinearOp, class Container>
+struct Helmholtz
+{
+    using value_type = dg::get_value_type<Container>;
+    Helmholtz(){}
+    Helmholtz( value_type alpha, value_type t, LinearOp* f): f_(f), alpha_(alpha), t_(t){}
+    void construct( value_type alpha, value_type t, LinearOp* f){
+        f_ = f; alpha_=alpha; t_=t;
+    }
+    void symv( const std::array<Container,2>& x, std::array<Container,2>& y)
+    {
+        if( alpha_ != 0)
+            (*f_)(t_,x,y);
+        dg::blas1::axpby( 1., x, alpha_, y, y);
+        dg::blas2::symv( f_->weights(), y, y);
+    }
+    const Container& weights() const{
+        return f_->weights();
+    }
+    const Container& inv_weights() const {
+        return f_->inv_weights();
+    }
+    const Container& precond() const {
+        return f_->precond();
+    }
+  private:
+    LinearOp* f_;
+    value_type alpha_;
+    value_type t_;
+};
+
+/*!@brief Multigrid Solver class for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
+*
+*/
+template< class Geometry, class Matrix, class Container>
+struct FeltorSpecialSolver
+{
+    using geometry_type = Geometry;
+    using matrix_type = Matrix;
+    using container_type = Container;
+    using value_type = dg::get_value_type<Container>;
+    FeltorSpecialSolver(){}
+
+    FeltorSpecialSolver( const Geometry& grid, Parameters p,
+        dg::geo::TokamakMagneticField mag ):
+        m_multigrid(grid, p.stages),
+        m_multi_imdens(p.stages),
+        m_multi_imvelo(p.stages)
+    {
+        for( unsigned u=0; u<m_multigrid.stages(); u++)
+        {
+            m_multi_imdens[u].construct( m_multigrid.grid(u),
+                p, mag);
+            m_multi_imvelo[u].construct( m_multigrid.grid(u),
+                p, mag);
+        }
+        std::array<Container,2> temp = dg::construct<std::array<Container,2>>( dg::evaluate( dg::zero, grid));
+        m_multi_n = m_multigrid.project( temp);
+        m_eps = p.eps_time;
+    }
+
+    //this must return the class used in the timestepper
+    std::array<std::array<Container,2>,2> copyable()const{
+        return std::array<std::array<Container,2>,2>{ m_multi_n[0], m_multi_n[0]};
+    }
+
+    //Solve y + a I(t,y) = rho
+    void solve( value_type alpha,
+        Implicit<Geometry, Matrix,Container>& im,
+        value_type t,
+        std::array<std::array<Container,2>,2>& y,
+        const std::array<std::array<Container,2>,2>& rhs)
+    {
+        //1. First construct Helmholtz type functor hierarchy
+        std::vector<
+          Helmholtz<ImplicitDensity<Geometry,Matrix,Container>,
+          Container>> multi_helm_dens( m_multigrid.stages());
+        std::vector<
+          Helmholtz<ImplicitVelocity<Geometry,Matrix,Container>,
+          Container>> multi_helm_velo(
+        m_multigrid.stages());
+        for( unsigned i=0; i<m_multigrid.stages(); i++)
+        {
+            multi_helm_dens[i].construct( alpha, t, &m_multi_imdens[i]);
+            multi_helm_velo[i].construct( alpha, t, &m_multi_imvelo[i]);
+        }
+        //2. Solve the density equation
+
+        std::vector<unsigned> number =
+            m_multigrid.direct_solve( multi_helm_dens, y[0], rhs[0], m_eps);
+        if(  number[0] == m_multigrid.max_iter())
+            throw dg::Fail( m_eps);
+#ifdef DG_BENCHMARK
+        //std::cout << "# iterations implicit density:  "<<number[0]<<"\n";
+#endif
+        //3. project density to all grids and pass to velocity Equation
+        m_multigrid.project( y[0], m_multi_n);
+        for( unsigned i=0; i<m_multigrid.stages(); i++)
+            m_multi_imvelo[i].set_density( m_multi_n[i]);
+        //4. Solve Velocity equation
+        number = m_multigrid.direct_solve( multi_helm_velo, y[1], rhs[1], m_eps);
+        if(  number[0] == m_multigrid.max_iter())
+            throw dg::Fail( m_eps);
+#ifdef DG_BENCHMARK
+        //std::cout << "# iterations implicit velocity: "<<number[0]<<"\n";
+#endif
+
+    }
+    private:
+    dg::MultigridCG2d< Geometry, Matrix, std::array<Container,2>> m_multigrid;
+    std::vector<ImplicitDensity<Geometry,Matrix,Container>> m_multi_imdens;
+    std::vector<ImplicitVelocity<Geometry,Matrix,Container>> m_multi_imvelo;
+    std::vector<std::array<Container,2>> m_multi_n;
+    value_type m_eps;
+};
+
+}//namespace feltor
+namespace dg{
+template< class M, class V>
+struct TensorTraits< feltor::Helmholtz<M, V> >
+{
+    using value_type = get_value_type<V>;
+    using tensor_category = SelfMadeMatrixTag;
+};
+} //namespace dg
-- 
GitLab


From 7c4dfa9e68e3328d85caaea1e2f3e3fcf3198dd0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 1 Feb 2019 15:19:45 +0100
Subject: [PATCH 023/540] Revert to dg::Invert in time solver

seems to be the most effecitve. Multigrid does not really seem
to help at all, at least in the first three timesteps. Also initial
simulations show that in the end time inversion should be fast.
---
 src/feltor/feltor.cu     |   4 +-
 src/feltor/feltor.cuh    |   3 +
 src/feltor/feltor.tex    |   6 +-
 src/feltor/feltor_hpc.cu |   4 +-
 src/feltor/implicit.h    | 119 +++++++--------------------------------
 5 files changed, 32 insertions(+), 104 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index f30631389..722893a21 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -52,7 +52,7 @@ int main( int argc, char* argv[])
     std::cout << "Constructing Explicit...\n";
     feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, mag);
     std::cout << "Constructing Implicit...\n";
-    feltor::Implicit<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec> im( grid, p, mag);
+    feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> im( grid, p, mag);
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
@@ -151,7 +151,7 @@ int main( int argc, char* argv[])
     adaptive.stepper().ignore_fsal();//necessary for splitting
     dg::ImplicitRungeKutta<std::array<std::array<dg::DVec,2>,2>,
         feltor::FeltorSpecialSolver<
-        dg::CylindricalGrid3d, dg::DMatrix, dg::DVec>> dirk(
+        dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>> dirk(
             "Trapezoidal-2-2", grid, p, mag);
     //since we map pointers we don't need to update those later
 
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index e2e0649c1..2790d2cef 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -8,6 +8,9 @@
 #define FELTORPERP 1
 #endif
 
+
+//Latest measurement: m = 60.000 per step
+
 namespace feltor
 {
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 503409087..92b960dd7 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -765,9 +765,11 @@ discontinuous Galerkin on structured grid
 \rowcolor{gray!50}\textbf{Term} &  \textbf{Method} & \textbf{Description}  \\ \midrule
     coordinate system & Cylindrical & equidistant discretization of $[R_{\min},R_{\max}] \times [Z_{\min},Z_{\max}] \times [0,2\pi]$ (Eq.~\eqref{eq:box}, equal number of Gaussian nodes in $R$ and $Z$, equidistant planes in $\varphi$ with one Gaussian node \\
 Helmholtz and Elliptic matrix inversions & multigrid/ conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
-Parallel derivatives & refined  FCI & cf.~\cite{Held2016,Stegmeir2017} \\
+Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017} \\
 Curvature terms & direct dG & regular dG approximation of derivatives \\
-time & Adaptive ARKode stepper & $3rd$ order explicit, perpendicular diffusion and resistance/friction $3rd$ order implicit \\
+time & Strang Splitting & $2$nd order\\
+\qquad explicit & Bogacki-Shampine embedded & $3$rd order explicit\\
+\qquad implicit & Trapezoidal & perp. Diffusion and Resistive term. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
 \bottomrule
 \end{longtable}
 \section{Input/Output}
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 66fc10599..e8a3ebc84 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -147,7 +147,7 @@ int main( int argc, char* argv[])
     MPI_OUT std::cout << "Constructing Explicit...\n";
     feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
     MPI_OUT std::cout << "Constructing Implicit...\n";
-    feltor::Implicit< Geometry, DMatrix, DVec> im( grid, p, mag);
+    feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
@@ -398,7 +398,7 @@ int main( int argc, char* argv[])
         "Bogacki-Shampine-4-2-3", y0);
     adaptive.stepper().ignore_fsal();//necessary for splitting
     dg::ImplicitRungeKutta<std::array<std::array<DVec,2>,2>,
-        feltor::FeltorSpecialSolver< Geometry, DMatrix, DVec>> dirk(
+        feltor::FeltorSpecialSolver< Geometry, IDMatrix, DMatrix, DVec>> dirk(
             "Trapezoidal-2-2", grid, p, mag);
     dg::Timer t;
     t.tic();
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index d62fdedc8..9c78636e6 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -30,7 +30,7 @@ struct AddResistivity{
 };
 }//namespace routines
 
-template<class Geometry, class Matrix, class Container>
+template<class Geometry, class IMatrix, class Matrix, class Container>
 struct ImplicitDensity
 {
     ImplicitDensity() = default;
@@ -94,7 +94,7 @@ struct ImplicitDensity
     dg::Elliptic3d<Geometry, Matrix, Container> m_lapM_perpN;
 };
 
-template<class Geometry, class Matrix, class Container>
+template<class Geometry, class IMatrix, class Matrix, class Container>
 struct ImplicitVelocity
 {
     ImplicitVelocity() = default;
@@ -157,7 +157,6 @@ struct ImplicitVelocity
         /* fields[0] := u_e
            fields[1] := U_i
         */
-
         for( unsigned i=0; i<2; i++)
         {
             if( m_p.perp_diff == "hyperviscous")
@@ -197,13 +196,14 @@ struct ImplicitVelocity
     dg::Elliptic3d<Geometry, Matrix, Container> m_lapM_perpU;
 };
 
-template<class Geometry, class Matrix, class Container>
+template<class Geometry, class IMatrix, class Matrix, class Container>
 struct Implicit
 {
     Implicit() = default;
     Implicit( const Geometry& g, feltor::Parameters p,
             dg::geo::TokamakMagneticField mag):
             m_dens( g,p,mag), m_velo( g,p,mag){}
+
     void operator()( double t, const std::array<std::array<Container,2>,2>& y,
         std::array<std::array<Container,2>,2>& yp)
     {
@@ -212,46 +212,14 @@ struct Implicit
         m_velo( t,y[1], yp[1]);
     }
     private:
-    ImplicitDensity <Geometry, Matrix, Container> m_dens;
-    ImplicitVelocity<Geometry, Matrix, Container> m_velo;
-};
-
-//compute unnormalized y + alpha f(y,t)
-template< class LinearOp, class Container>
-struct Helmholtz
-{
-    using value_type = dg::get_value_type<Container>;
-    Helmholtz(){}
-    Helmholtz( value_type alpha, value_type t, LinearOp* f): f_(f), alpha_(alpha), t_(t){}
-    void construct( value_type alpha, value_type t, LinearOp* f){
-        f_ = f; alpha_=alpha; t_=t;
-    }
-    void symv( const std::array<Container,2>& x, std::array<Container,2>& y)
-    {
-        if( alpha_ != 0)
-            (*f_)(t_,x,y);
-        dg::blas1::axpby( 1., x, alpha_, y, y);
-        dg::blas2::symv( f_->weights(), y, y);
-    }
-    const Container& weights() const{
-        return f_->weights();
-    }
-    const Container& inv_weights() const {
-        return f_->inv_weights();
-    }
-    const Container& precond() const {
-        return f_->precond();
-    }
-  private:
-    LinearOp* f_;
-    value_type alpha_;
-    value_type t_;
+    ImplicitDensity <Geometry, IMatrix, Matrix, Container> m_dens;
+    ImplicitVelocity<Geometry, IMatrix, Matrix, Container> m_velo;
 };
 
 /*!@brief Multigrid Solver class for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
 *
 */
-template< class Geometry, class Matrix, class Container>
+template< class Geometry, class IMatrix, class Matrix, class Container>
 struct FeltorSpecialSolver
 {
     using geometry_type = Geometry;
@@ -261,84 +229,39 @@ struct FeltorSpecialSolver
     FeltorSpecialSolver(){}
 
     FeltorSpecialSolver( const Geometry& grid, Parameters p,
-        dg::geo::TokamakMagneticField mag ):
-        m_multigrid(grid, p.stages),
-        m_multi_imdens(p.stages),
-        m_multi_imvelo(p.stages)
+        dg::geo::TokamakMagneticField mag)
     {
-        for( unsigned u=0; u<m_multigrid.stages(); u++)
-        {
-            m_multi_imdens[u].construct( m_multigrid.grid(u),
-                p, mag);
-            m_multi_imvelo[u].construct( m_multigrid.grid(u),
-                p, mag);
-        }
         std::array<Container,2> temp = dg::construct<std::array<Container,2>>( dg::evaluate( dg::zero, grid));
-        m_multi_n = m_multigrid.project( temp);
         m_eps = p.eps_time;
+        m_solver = dg::DefaultSolver<std::array<Container,2>>(
+            temp, grid.size(), p.eps_time);
+        m_imdens.construct( grid, p, mag);
+        m_imvelo.construct( grid, p, mag);
     }
 
     //this must return the class used in the timestepper
     std::array<std::array<Container,2>,2> copyable()const{
-        return std::array<std::array<Container,2>,2>{ m_multi_n[0], m_multi_n[0]};
+
+        return std::array<std::array<Container,2>,2>{ m_solver.copyable(), m_solver.copyable()};
     }
 
     //Solve y + a I(t,y) = rho
     void solve( value_type alpha,
-        Implicit<Geometry, Matrix,Container>& im,
+        Implicit<Geometry, IMatrix, Matrix,Container>& im,
         value_type t,
         std::array<std::array<Container,2>,2>& y,
         const std::array<std::array<Container,2>,2>& rhs)
     {
-        //1. First construct Helmholtz type functor hierarchy
-        std::vector<
-          Helmholtz<ImplicitDensity<Geometry,Matrix,Container>,
-          Container>> multi_helm_dens( m_multigrid.stages());
-        std::vector<
-          Helmholtz<ImplicitVelocity<Geometry,Matrix,Container>,
-          Container>> multi_helm_velo(
-        m_multigrid.stages());
-        for( unsigned i=0; i<m_multigrid.stages(); i++)
-        {
-            multi_helm_dens[i].construct( alpha, t, &m_multi_imdens[i]);
-            multi_helm_velo[i].construct( alpha, t, &m_multi_imvelo[i]);
-        }
-        //2. Solve the density equation
-
-        std::vector<unsigned> number =
-            m_multigrid.direct_solve( multi_helm_dens, y[0], rhs[0], m_eps);
-        if(  number[0] == m_multigrid.max_iter())
-            throw dg::Fail( m_eps);
-#ifdef DG_BENCHMARK
-        //std::cout << "# iterations implicit density:  "<<number[0]<<"\n";
-#endif
-        //3. project density to all grids and pass to velocity Equation
-        m_multigrid.project( y[0], m_multi_n);
-        for( unsigned i=0; i<m_multigrid.stages(); i++)
-            m_multi_imvelo[i].set_density( m_multi_n[i]);
-        //4. Solve Velocity equation
-        number = m_multigrid.direct_solve( multi_helm_velo, y[1], rhs[1], m_eps);
-        if(  number[0] == m_multigrid.max_iter())
-            throw dg::Fail( m_eps);
-#ifdef DG_BENCHMARK
-        //std::cout << "# iterations implicit velocity: "<<number[0]<<"\n";
-#endif
 
+        m_solver.solve( alpha, m_imdens, t, y[0], rhs[0]);
+        m_imvelo.set_density( y[0]);
+        m_solver.solve( alpha, m_imvelo, t, y[1], rhs[1]);
     }
     private:
-    dg::MultigridCG2d< Geometry, Matrix, std::array<Container,2>> m_multigrid;
-    std::vector<ImplicitDensity<Geometry,Matrix,Container>> m_multi_imdens;
-    std::vector<ImplicitVelocity<Geometry,Matrix,Container>> m_multi_imvelo;
-    std::vector<std::array<Container,2>> m_multi_n;
+    dg::DefaultSolver<std::array<Container,2>> m_solver;
+    ImplicitDensity<Geometry,IMatrix, Matrix,Container> m_imdens;
+    ImplicitVelocity<Geometry,IMatrix, Matrix,Container> m_imvelo;
     value_type m_eps;
 };
 
 }//namespace feltor
-namespace dg{
-template< class M, class V>
-struct TensorTraits< feltor::Helmholtz<M, V> >
-{
-    using value_type = get_value_type<V>;
-    using tensor_category = SelfMadeMatrixTag;
-};
-} //namespace dg
-- 
GitLab


From dcda5e161fdd09df27092a885f393b57e36d5283 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Fri, 1 Feb 2019 15:34:33 +0100
Subject: [PATCH 024/540] Correct dg::HVec bug in feltor_hpc

---
 src/feltor/feltor_hpc.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index e8a3ebc84..539be5d6a 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -170,7 +170,7 @@ int main( int argc, char* argv[])
         p.rho_damping, p.alpha, +1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
-    dg::HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
+    HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
         mag.psip(), -p.alpha, p.alpha, -1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
-- 
GitLab


From 5dad6fe2cb2843870494b0b17b28ad5a70abd443 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 4 Feb 2019 16:13:49 +0100
Subject: [PATCH 025/540] Implement multigrid and full-multigrid in comments

so far in pseudocode
---
 inc/dg/multigrid.h | 182 ++++++++++++++-------------------------------
 1 file changed, 54 insertions(+), 128 deletions(-)

diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index e388ba7d4..784f5c14d 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -99,65 +99,6 @@ struct MultigridCG2d
             m_cg[u].construct(m_x[u], 1);
     }
 
-    /*
-	template<class SymmetricOp>
-	std::vector<unsigned> solve( std::vector<SymmetricOp>& op, Container& x, const Container& b, const double eps)
-	{
-        //project initial guess down to coarse grid
-        project(x, m_x);
-        dg::blas2::symv(op[0].weights(), b, m_b[0]);
-        // project b down to coarse grid
-        for( unsigned u=0; u<m_stages-1; u++)
-            dg::blas2::gemv( m_interT[u], m_b[u], m_b[u+1]);
-
-
-        unsigned int numStageSteps = m_schemeLayout.size();
-		std::vector<unsigned> number(numStageSteps);
-
-        unsigned u = m_startStage;
-
-        for (unsigned i = 0; i < numStageSteps; i++)
-        {
-            unsigned w = u + m_schemeLayout[i].m_step;
-            //
-            // iterate the solver on the system A x = b, with x = 0 as inital guess
-            m_cg[u].set_max(m_schemeLayout[i].m_niter);
-            number[i] = m_cg[u](op[u], m_x[u], m_b[u], op[u].precond(), op[u].inv_weights(), eps);
-
-            //
-            // debug print
-            //std::cout << "pass: " << i << ", stage: " << u << ", max iter: " << m_schemeLayout[i].m_niter << ", iter: " << number[i] << std::endl;
-
-            if (m_schemeLayout[i].m_step > 0)
-            {
-                //
-                // compute residual r = Wb - A x
-                dg::blas2::symv(op[u], m_x[u], m_r[u]);
-                dg::blas1::axpby(-1.0, m_r[u], 1.0, m_b[u], m_r[u]);
-                //
-                // transfer residual to the rhs of the coarser grid
-                dg::blas2::symv(m_interT[u], m_r[u], m_b[w]);
-                //dg::blas2::symv(m_project[u], m_x[u], m_x[w]);
-                //std::cout << "zeroed " << w << ", ";
-                dg::blas1::scal(m_x[w], 0.0);
-            }
-            else if (m_schemeLayout[i].m_step < 0)
-            {
-                //
-                // correct the solution vector of the finer grid
-                // x[w] = x[w] + P^{-1} x[u]
-                dg::blas2::symv(1., m_inter[w], m_x[u], 1., m_x[w]);
-                //dg::blas2::symv(m_inter[w], m_x[u], m_x[w]);
-            }
-
-            u = w;
-		}
-
-		m_x[0].swap(x);
-		return number;
-	}
-    */
-
     /**
      * @brief Nested iterations
      *
@@ -282,73 +223,69 @@ struct MultigridCG2d
 
   private:
 
-	void set_scheme(const int scheme_type)
-	{
-        assert(scheme_type <= 1 && scheme_type >= 0);
-
-        // initialize one conjugate-gradient for each grid size
-        for (unsigned u = 0; u < m_stages; u++)
-            m_cg[u].construct(m_x[u], 1);
-
-        switch (scheme_type)
+    /*
+	template<class SymmetricOp, class ContainerType0, class ContainerType1>
+    void full_multigrid( std::vector<SymmetricOp>& op,
+        ContainerType0& x, ContainerType1& b,
+        unsigned nu1, unsigned nu2, unsigned gamma, unsigned mu, double eps)
+    {
+        dg::blas2::symv(op[0].weights(), b, m_b[0]);
+        // project x down to coarse grid
+        dg::blas1::copy( x, m_x[0]);
+        dg::blas1::copy( b, m_b[0]);
+        for( unsigned u=0; u<m_stages-1; u++)
         {
-            // from coarse to fine
-        case(0):
-
-            //m_mode = nestediteration;
-            m_startStage = m_stages - 1;
-            for (int u = m_stages - 1; u >= 0; u--)
-                m_schemeLayout.push_back(stepinfo(-1, m_x[u].size()));
-            break;
-
-        case(1):
-
-            //m_mode = correctionscheme;
-            m_startStage = 0;
-            for (unsigned u = 0; u < m_stages-1; u++)
-                m_schemeLayout.push_back(stepinfo(1, 5));
-
-            m_schemeLayout.push_back(stepinfo(-1, 1000));
-
-            for (int u = m_stages - 2; u >= 0; u--)
-                m_schemeLayout.push_back(stepinfo(-1, 1000));
-
-            break;
-
-        default:
-            break;
+            dg::blas2::gemv( m_interT[u], m_x[u], m_x[u+1]);
+            dg::blas2::gemv( m_interT[u], m_b[u], m_b[u+1]);
         }
+        solve( op[m_stages-1], m_x[m_stages-1], m_b[m_stages-1], eps);
 
-        // there is no step at the last stage so the step must be zero
-        m_schemeLayout.back().m_niter = m_x[0].size();
-        m_schemeLayout.back().m_step = 0;
-
-        //PrintScheme();
-
-        // checks:
-        // (0) the last entry should be the stage before the original grid
-        // (1) there can not be more than n-1 interpolations in succession
-
-        unsigned u = m_startStage;
-        assert( u <= m_stages - 1);
-        for (unsigned i = 0; i < m_schemeLayout.size(); i++)
+		for( int p=m_stages-2; p>=0; p--)
         {
-            u += m_schemeLayout[i].m_step;
-            assert( u <= m_stages - 1);
+            dg::blas2::gemv( m_inter[p], m_x[p+1],  m_x[p]);
+            for( unsigned u=0; u<mu; u++)
+                multigrid_cycle( op, m_x, m_b, nu1, nu2, gamma, p, eps);
         }
-        assert(u == 0);
-	}
+    }
 
-    void PrintScheme(void)
+    template<class SymmetricOp>
+    void multigrid_cycle( std::vector<SymmetricOp>& op,
+        std::vector<Container>& x, std::vector<Container>& b,
+        unsigned nu1, unsigned nu2, unsigned gamma, unsigned p, double eps)
     {
-        std::cout << "Scheme: " << std::endl;
-        unsigned u = m_startStage;
-        for (unsigned i = 0; i < m_schemeLayout.size(); i++)
+        // p < m_stages-1
+        // x[p]   initial condition on input, solution on output
+        // x[p+1] write only (solution of lower stage)
+        // b[p]   read only,
+        // b[p+1] write only (residual after Pre-smooting at lower stage)
+        // m_r[p] write only
+        // gamma: typically 1 (V-cycle) or 2 (W-cycle)
+        // nu1, nu2: typically in {0,1,2,3}
+
+        // 1. Pre-Smooth
+        smooth( op[p], x[p], b[p]); //nu1 times
+        // 2. Residual
+        dg::blas2:symv( op[p], x[p], m_r[p]);
+        dg::blas1::axpby( 1., b[p], -1., m_r[p]);
+        // 3. Coarsen
+        dg::blas2::symv( m_interT[p], m_r[p], b[p+1]);
+        // 4. Solve or recursive call to get x[p+1] with initial guess 0
+        dg::blas1::scal( x[p+1], 0.);
+        if( p+1 == m_stages-1)
+            solve ( op[p+1], x[p+1], b[p+1], eps);
+        else
         {
-            std::cout << "num " << i << ", stage: " << u << ", iterations on current stage: " << m_schemeLayout[i].m_niter << ", step direction " << m_schemeLayout[i].m_step << std::endl;
-            u += m_schemeLayout[i].m_step;
+            //update x[p+1] gamma times
+            for( unsigned u=0; u<gamma; u++)
+                multigrid_cycle( op, x, b, nu1, nu2, gamma, p+1, eps);
         }
+
+        // 5. Correct
+        dg::blas2::symv( 1., m_inter[p], x[p+1], 1., x[p]);
+        // 6. Post-Smooth
+        smooth( op[p], x[p], b[p]); //nu2 times
     }
+    */
 
 private:
     unsigned m_stages;
@@ -359,17 +296,6 @@ private:
     std::vector< CG<Container> > m_cg;
     std::vector< Container> m_x, m_r, m_b;
 
-    struct stepinfo
-    {
-        stepinfo(int step, const unsigned niter) : m_step(step), m_niter(niter)
-        {
-        };
-
-        int m_step; // {+1,-1}
-        unsigned int m_niter;
-    };
-
-    unsigned m_startStage;
-    std::vector<stepinfo> m_schemeLayout;
 };
+
 }//namespace dg
-- 
GitLab


From e40a3a350eba6e056ffa596f5dbe9cfacafe072a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 2 Feb 2019 19:45:18 +0100
Subject: [PATCH 026/540] Implement Chebyshev iteration

and test in cg2d_t.cu
---
 inc/dg/cg.h      | 71 ++++++++++++++++++++++++++++++++++++++++++++++++
 inc/dg/cg2d_t.cu | 26 +++++++++++++++++-
 2 files changed, 96 insertions(+), 1 deletion(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 26abb9009..1d587b94a 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -250,6 +250,77 @@ unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const Con
     return max_iter;
 }
 ///@endcond
+//
+//implement the classical 3-term recursion with explicit residual
+template< class ContainerType>
+class ChebyshevIteration
+{
+  public:
+    using container_type = ContainerType;
+    using value_type = get_value_type<ContainerType>; //!< value type of the ContainerType class
+    ///@brief Allocate nothing, Call \c construct method before usage
+    ChebyshevIteration(){}
+    ///@copydoc construct()
+    ChebyshevIteration( const ContainerType& copyable):
+        m_r(copyable), m_x1(m_r){}
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_r;}
+
+    /**
+     * @brief Allocate memory for the pcg method
+     *
+     * @param copyable A ContainerType must be copy-constructible from this
+     * @param max_iterations Maximum number of iterations to be used
+     */
+    void construct( const ContainerType& copyable) {
+        m_x1 = m_r = copyable;
+    }
+    /**
+     * @brief Solve the system A*x = b using Chebyshev iteration
+     *
+     * The iteration stops when the maximum number of iterations is reached
+     * @param A A symmetric, positive definit matrix
+     * @param x Contains an initial value on input and the solution on output.
+     * @param b The right hand side vector. x and b may be the same vector.
+     * @param lmin the minimum Eigenvalue
+     * @param lmax the minimum Eigenvalue
+     * @param num_iter the number of iterations
+     *
+     * @copydoc hide_matrix
+     * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
+     */
+    template< class MatrixType, class ContainerType0, class ContainerType1>
+    void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b, double lmin, double lmax, unsigned num_iter)
+    {
+        assert ( lmin < lmax);
+        double a = (lmin+lmax)/2., c = (lmin-lmax)/2.;
+        double betan = -c*c/2./a;
+        double gamman = -a;
+        dg::blas1::copy( x, m_x1);
+        dg::blas2::symv( A, x, m_r);
+        dg::blas1::axpby( 1., b, -1., m_r);
+        //-(r0+a*x0)/gamma0, m_x1=0
+        dg::blas1::axpby( -1./gamman, m_r, -a/gamman, x);
+        dg::blas1::copy( 0., m_x1);
+        for ( unsigned u=1; u<num_iter; u++)
+        {
+            betan = (c/2.)*(c/2.)/gamman;
+            gamman = -(a+betan);
+
+            dg::blas2::symv( A, x, m_r);
+            dg::blas1::axpby( 1., b, -1., m_r);
+            dg::blas1::evaluate( m_x1, dg::equals(), PairSum(),
+                            -1./gamman, m_r,
+                            -a/gamman, x,
+                            -betan/gamman,m_x1);
+            x.swap(m_x1);
+        }
+    }
+    //version of CG where Preconditioner is not trivial
+  private:
+    ContainerType m_r, m_x1;
+};
 
 
 /**
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index 2e982184e..ff281cd0a 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -48,7 +48,6 @@ int main()
     const double eps = 1e-6;
     std::cout << "Number of pcg iterations "<< pcg( A, x, b, v2d, eps)<<std::endl;
 //! [doxygen]
-    //std::cout << "Number of cg iterations "<< pcg( A, x, b, dg::Identity<double>(), eps)<<std::endl;
     std::cout << "For a precision of "<< eps<<std::endl;
     //compute error
     dg::HVec error( solution);
@@ -68,6 +67,31 @@ int main()
     res.d = sqrt(dg::blas2::dot( w2d, resi));
     std::cout << "L2 Norm of Residuum is        " << res.d<<"\t"<<res.i << std::endl;
     //Fehler der Integration des Sinus ist vernachlässigbar (vgl. evaluation_t)
+    dg::blas1::copy( 0., x);
+    dg::ChebyshevIteration<dg::HVec> cheby( copyable_vector);
+    double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
+    std::cout << "Type number of Chebyshev iterations\n";
+    unsigned num_iter;
+    std::cin >> num_iter;
+    cheby.solve( A, x, b, lmin, lmax, num_iter);
+    std::cout << "After "<<num_iter<<" Chebyshev iterations we have:\n";
+
+    dg::blas1::copy( solution, error);
+    dg::blas1::axpby( 1.,x,-1.,error);
+
+    dg::blas1::copy( b, resi);
+    dg::blas2::symv(  A, x, Ax);
+    dg::blas1::axpby( 1.,Ax,-1.,resi);
+
+    res.d = sqrt(dg::blas2::dot( w2d, x));
+    std::cout << "L2 Norm of x0 is              " << res.d<<"\n";
+    res.d = sqrt(dg::blas2::dot(w2d , solution));
+    std::cout << "L2 Norm of Solution is        " << res.d<<"\n";
+    res.d = sqrt(dg::blas2::dot(w2d , error));
+    std::cout << "L2 Norm of Error is           " << res.d<<"\n";
+    res.d = sqrt(dg::blas2::dot( w2d, resi));
+    std::cout << "L2 Norm of Residuum is        " << res.d<<"\n";
+
 
     return 0;
 }
-- 
GitLab


From 20f16d3d0f0146a7b9067bf1573f064d1147f35d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 4 Feb 2019 22:56:07 +0100
Subject: [PATCH 027/540] Documentation for Chebyshev

and better cg2d_t lmax
uncomment implemented multigrid solver
---
 inc/dg/cg.h        | 20 +++++++++++++++++++-
 inc/dg/cg2d_t.cu   |  2 ++
 inc/dg/multigrid.h | 41 ++++++++++++++++++++++++-----------------
 3 files changed, 45 insertions(+), 18 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 1d587b94a..f5890f9bd 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -252,6 +252,23 @@ unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const Con
 ///@endcond
 //
 //implement the classical 3-term recursion with explicit residual
+/**
+* @brief Three-term recursion of the Chebyshev iteration for solving
+* \f[ Ax=b\f]
+*
+* Given the minimum and maximum Eigenvalue of the matrix A we define
+* \f[ a = (l_\min+l_\max)/2 \quad c = (l_\max - l_\min)/2 \\
+*     \beta_{-1} = 0\ \beta_0 = -c^2/2/a\ \gamma_0 = -a \\
+*     \beta_{n-1} = (c/2)^2/\gamma_{n-1} \quad \gamma_n = -(a+\beta_{n-1}) \\
+*     x_0 := x \ x_{-1} = r_{-1} = 0 \\
+*     x_{n+1} := -(r_n + ax_n + \beta_{n-1} x_{n-1})/\gamma_{n} \\
+*     r_{n+1} :=  b - Ax_{n+1}
+* \f]
+*
+* @ingroup invert
+*
+* @copydoc hide_ContainerType
+*/
 template< class ContainerType>
 class ChebyshevIteration
 {
@@ -291,7 +308,8 @@ class ChebyshevIteration
      * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
      */
     template< class MatrixType, class ContainerType0, class ContainerType1>
-    void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b, double lmin, double lmax, unsigned num_iter)
+    void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b,
+        double lmin, double lmax, unsigned num_iter)
     {
         assert ( lmin < lmax);
         double a = (lmin+lmax)/2., c = (lmin-lmax)/2.;
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index ff281cd0a..aa85daf23 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -70,6 +70,8 @@ int main()
     dg::blas1::copy( 0., x);
     dg::ChebyshevIteration<dg::HVec> cheby( copyable_vector);
     double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
+    double hxhy = lx*ly/(n*n*Nx*Ny);
+    lmin *= hxhy, lmax *= hxhy; //we multiplied the matrix by w2d
     std::cout << "Type number of Chebyshev iterations\n";
     unsigned num_iter;
     std::cin >> num_iter;
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 784f5c14d..9c75597c8 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -159,7 +159,8 @@ struct MultigridCG2d
         //update initial guess
         dg::blas1::axpby( 1., m_x[0], 1., x);
         m_cg[0].set_max(m_grids[0]->size());
-        number[0] = m_cg[0]( op[0], x, m_b[0], op[0].precond(), op[0].inv_weights(), eps);
+        number[0] = m_cg[0]( op[0], x, m_b[0], op[0].precond(),
+            op[0].inv_weights(), eps);
 #ifdef DG_BENCHMARK
         t.toc();
 #ifdef MPI_VERSION
@@ -223,7 +224,6 @@ struct MultigridCG2d
 
   private:
 
-    /*
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
     void full_multigrid( std::vector<SymmetricOp>& op,
         ContainerType0& x, ContainerType1& b,
@@ -238,7 +238,9 @@ struct MultigridCG2d
             dg::blas2::gemv( m_interT[u], m_x[u], m_x[u+1]);
             dg::blas2::gemv( m_interT[u], m_b[u], m_b[u+1]);
         }
-        solve( op[m_stages-1], m_x[m_stages-1], m_b[m_stages-1], eps);
+        unsigned s = m_stages-1;
+        unsigned number = m_cg[s]( op[s], m_x[s], m_b[s], op[s].precond(),
+            op[s].inv_weights(), eps);
 
 		for( int p=m_stages-2; p>=0; p--)
         {
@@ -254,25 +256,30 @@ struct MultigridCG2d
         unsigned nu1, unsigned nu2, unsigned gamma, unsigned p, double eps)
     {
         // p < m_stages-1
-        // x[p]   initial condition on input, solution on output
-        // x[p+1] write only (solution of lower stage)
-        // b[p]   read only,
-        // b[p+1] write only (residual after Pre-smooting at lower stage)
-        // m_r[p] write only
-        // gamma: typically 1 (V-cycle) or 2 (W-cycle)
+        // x[p]    initial condition on input, solution on output
+        // x[p+1]  write only (solution of lower stage)
+        // b[p]    read only,
+        // b[p+1]  write only (residual after Pre-smooting at lower stage)
+        // m_r[p]  write only
+        //
+        // gamma:  typically 1 (V-cycle) or 2 (W-cycle)
         // nu1, nu2: typically in {0,1,2,3}
+        double lmin = 0, lmax = 1; //determine lmin and lmax
 
-        // 1. Pre-Smooth
-        smooth( op[p], x[p], b[p]); //nu1 times
+        // 1. Pre-Smooth nu1 times
+        m_cheby[p].solve( op[p], x[p], b[p], 0.1*lmax, 1.1*lmax, nu1);
         // 2. Residual
-        dg::blas2:symv( op[p], x[p], m_r[p]);
+        dg::blas2::symv( op[p], x[p], m_r[p]);
         dg::blas1::axpby( 1., b[p], -1., m_r[p]);
         // 3. Coarsen
         dg::blas2::symv( m_interT[p], m_r[p], b[p+1]);
         // 4. Solve or recursive call to get x[p+1] with initial guess 0
         dg::blas1::scal( x[p+1], 0.);
         if( p+1 == m_stages-1)
-            solve ( op[p+1], x[p+1], b[p+1], eps);
+        {
+            m_cg[p+1]( op[p+1], x[p+1], b[p+1], op[p+1].precond(),
+                op[p+1].inv_weights(), eps);
+        }
         else
         {
             //update x[p+1] gamma times
@@ -282,18 +289,18 @@ struct MultigridCG2d
 
         // 5. Correct
         dg::blas2::symv( 1., m_inter[p], x[p+1], 1., x[p]);
-        // 6. Post-Smooth
-        smooth( op[p], x[p], b[p]); //nu2 times
+        // 6. Post-Smooth nu2 times
+        m_cheby[p].solve( op[p], x[p], b[p], 0.1*lmax, 1.1*lmax, nu2);
     }
-    */
 
-private:
+  private:
     unsigned m_stages;
     std::vector< dg::ClonePtr< Geometry> > m_grids;
     std::vector< MultiMatrix<Matrix, Container> >  m_inter;
     std::vector< MultiMatrix<Matrix, Container> >  m_interT;
     std::vector< MultiMatrix<Matrix, Container> >  m_project;
     std::vector< CG<Container> > m_cg;
+    std::vector< ChebyshevIteration<Container>> m_cheby;
     std::vector< Container> m_x, m_r, m_b;
 
 };
-- 
GitLab


From 3fbb3a8f19f7d1527b21cddefb7f2c0c18da7639 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 6 Feb 2019 18:34:24 +0100
Subject: [PATCH 028/540] Fix bug in fieldaligned initialization

and accomodate rho_damping in geometry_diag
- also found that in order for nu_parallel to have an effect on dynamics
during the timescale of the perp dynamics it should be much larger than
1
---
 inc/geometries/fieldaligned.h   | 12 +++++++-----
 inc/geometries/geometry_diag.cu |  6 +++++-
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 426af82c9..aa635b40e 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -466,9 +466,9 @@ Fieldaligned<Geometry, IMatrix, container>::Fieldaligned(
     dg::split( m_hm_inv, m_temp, grid); //3d vector
     for( unsigned i=0; i<m_Nz; i++)
         dg::blas1::copy( m_hm, m_temp[i]);
-    dg::blas1::axpby( 1., m_hp_inv, -1., m_hm_inv, m_h0_inv);//hm is negative
-    dg::blas1::pointwiseDivide( -1., m_hm_inv, m_hm_inv);
-    dg::blas1::pointwiseDivide(  1., m_hp_inv, m_hp_inv);
+    dg::blas1::axpby( 1., m_hp_inv, -1., m_hm_inv, m_h0_inv); //hp-hm
+    dg::blas1::pointwiseDivide( -1., m_hm_inv, m_hm_inv); //0-hm
+    dg::blas1::pointwiseDivide(  1., m_hp_inv, m_hp_inv); //hp-0
     dg::blas1::pointwiseDivide(  1., m_h0_inv, m_h0_inv);
 }
 
@@ -497,9 +497,11 @@ container Fieldaligned<G, I,container>::evaluate( const BinaryOp& binary, const
             unsigned rep = r*m_Nz + i0;
             for(unsigned k=0; k<rep; k++)
             {
-                dg::blas2::symv( m_plus, tempP, temp);
+                //!!! The value of f at the plus plane is I^- of the current plane
+                dg::blas2::symv( m_minus, tempP, temp);
                 temp.swap( tempP);
-                dg::blas2::symv( m_minus, tempM, temp);
+                //!!! The value of f at the minus plane is I^+ of the current plane
+                dg::blas2::symv( m_plus, tempM, temp);
                 temp.swap( tempM);
             }
             dg::blas1::scal( tempP, unary(  (double)rep*m_g->hz() ) );
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 4b54f6d15..aa0a689ac 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -25,6 +25,7 @@ struct Parameters
     double boxscaleZm, boxscaleZp;
     double amp, k_psi, bgprofamp, nprofileamp;
     double sigma, posX, posY;
+    double rho_damping, alpha;
     Parameters( const Json::Value& js){
         n = js.get("n",3).asUInt();
         Nx = js.get("Nx",100).asUInt();
@@ -41,6 +42,8 @@ struct Parameters
         sigma = js.get("sigma", 10).asDouble();
         posX = js.get("posX", 0.5).asDouble();
         posY = js.get("posY", 0.5).asDouble();
+        rho_damping = js.get("rho_damping", 1.2).asDouble();
+        alpha = js.get("alpha", 0.05).asDouble();
     }
     void display( std::ostream& os = std::cout ) const
     {
@@ -137,7 +140,8 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
 
     //Test coefficients
-    dg::geo::TokamakMagneticField c = dg::geo::createModifiedSolovevField(gp, 0.16, 0.05);
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+    c = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*c.psip()(c.R0(), 0.), p.alpha);
     const double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     const double Z_X = -1.1*gp.elongation*gp.a;
     const double R_H = gp.R_0-gp.triangularity*gp.a;
-- 
GitLab


From 0cc2f2c2cc584f2b1838730220da85b43333bc9f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 8 Feb 2019 00:13:40 +0100
Subject: [PATCH 029/540] Add dg::blas1::reduce function

test omp in blas_t and blas_mpit seems to work
---
 inc/dg/backend/blas1_cuda.cuh          | 20 +++++-
 inc/dg/backend/blas1_dispatch_mpi.h    | 15 +++++
 inc/dg/backend/blas1_dispatch_scalar.h |  8 +++
 inc/dg/backend/blas1_dispatch_shared.h |  8 +++
 inc/dg/backend/blas1_dispatch_vector.h | 11 ++++
 inc/dg/backend/blas1_omp.h             |  8 +++
 inc/dg/backend/blas1_serial.h          |  8 +++
 inc/dg/blas1.h                         | 88 +++++++++++++++++---------
 inc/dg/blas_mpit.cu                    |  3 +
 inc/dg/blas_t.cu                       |  6 +-
 10 files changed, 142 insertions(+), 33 deletions(-)

diff --git a/inc/dg/backend/blas1_cuda.cuh b/inc/dg/backend/blas1_cuda.cuh
index 6d964f812..cb8a0e3b4 100644
--- a/inc/dg/backend/blas1_cuda.cuh
+++ b/inc/dg/backend/blas1_cuda.cuh
@@ -1,5 +1,7 @@
 #ifndef _DG_BLAS_CUDA_
 #define _DG_BLAS_CUDA_
+#include <thrust/reduce.h>
+#include <thrust/system/cuda/execution_policy.h>
 #include "exblas/exdot_cuda.cuh"
 namespace dg
 {
@@ -45,9 +47,9 @@ template<class Subroutine, class PointerOrValue, class ...PointerOrValues>
     const int grid_size = gridDim.x*blockDim.x;
     //every thread takes num_is/grid_size is
     for( int i = thread_id; i<size; i += grid_size)
+        f(get_device_element(x,i), get_device_element(xs,i)...);
         //f(x[i], xs[i]...);
         //f(thrust::raw_reference_cast(*(x+i)), thrust::raw_reference_cast(*(xs+i))...);
-        f(get_device_element(x,i), get_device_element(xs,i)...);
 }
 
 template< class Subroutine, class PointerOrValue, class ...PointerOrValues>
@@ -58,6 +60,22 @@ inline void doSubroutine_dispatch( CudaTag, int size, Subroutine f, PointerOrVal
     subroutine_kernel<Subroutine, PointerOrValue, PointerOrValues...><<<NUM_BLOCKS, BLOCK_SIZE>>>(size, f, x, xs...);
 }
 
+template<class T, class PointerOrValue, class BinaryOp>
+ __global__ T reduction_kernel( int size, T init, PointerOrValue x, BinaryOp op)
+{
+    const int thread_id = blockDim.x * blockIdx.x + threadIdx.x;
+    const int grid_size = gridDim.x*blockDim.x;
+    //every thread takes num_is/grid_size is
+    for( int i = thread_id; i<size; i += grid_size)
+
+}
+
+template<class T, class Pointer, class BinaryOp>
+inline T doReduce_dispatch( CudaTag, int size, Pointer x, T init, BinaryOp op)
+{
+    return thrust::reduce(thrust::cuda::par, x, x+size, init, op);
+}
+
 
 }//namespace detail
 }//namespace blas1
diff --git a/inc/dg/backend/blas1_dispatch_mpi.h b/inc/dg/backend/blas1_dispatch_mpi.h
index c3b2fd33a..5945d8df4 100644
--- a/inc/dg/backend/blas1_dispatch_mpi.h
+++ b/inc/dg/backend/blas1_dispatch_mpi.h
@@ -11,6 +11,8 @@
 
 ///@cond
 namespace dg {
+template<class value_type>
+static inline MPI_Datatype getMPIDataType();
 
 template<class to_ContainerType, class from_ContainerType, class ...Params>
 inline to_ContainerType construct( const from_ContainerType& src, Params&& ...ps);
@@ -88,6 +90,19 @@ inline void doSubroutine( MPIVectorTag, Subroutine f, container&& x, Containers&
         do_get_data(std::forward<Containers>(xs), get_tensor_category<Containers>())...);
 }
 
+template<class T, class ContainerType, class BinaryOp>
+inline T doReduce( MPIVectorTag, const ContainerType& x, T init, BinaryOp op)
+{
+    init = dg::blas1::reduce( x.data(), init, op);
+    //now do the MPI reduction
+    int size;
+    MPI_Comm_size( x.communicator(), &size);
+    thrust::host_vector<T> reduction( size);
+    MPI_Allgather( &init, 1, getMPIDataType<T>(), thrust::raw_pointer_cast(reduction.data()), 1, getMPIDataType<T>(), x.communicator());
+    //reduce received data (serial execution)
+    return dg::blas1::reduce( reduction, init, op) ;
+}
+
 } //namespace detail
 } //namespace blas1
 } //namespace dg
diff --git a/inc/dg/backend/blas1_dispatch_scalar.h b/inc/dg/backend/blas1_dispatch_scalar.h
index 7b0f95fd2..6e5e11f8d 100644
--- a/inc/dg/backend/blas1_dispatch_scalar.h
+++ b/inc/dg/backend/blas1_dispatch_scalar.h
@@ -35,6 +35,14 @@ inline void doSubroutine( AnyScalarTag, Subroutine f, ContainerType&& x, Contain
     f(x,xs...);
 }
 
+template<class T, class ContainerType, class BinaryOp>
+inline T doReduce( AnyScalarTag, ContainerType x, T init, BinaryOp op)
+{
+    init = op( init, x);
+    return init;
+}
+
+
 } //namespace detail
 } //namespace blas1
 } //namespace dg
diff --git a/inc/dg/backend/blas1_dispatch_shared.h b/inc/dg/backend/blas1_dispatch_shared.h
index 52112f4af..0df707c3d 100644
--- a/inc/dg/backend/blas1_dispatch_shared.h
+++ b/inc/dg/backend/blas1_dispatch_shared.h
@@ -42,6 +42,8 @@ namespace blas1
 {
 template< class Subroutine, class ContainerType, class ...ContainerTypes>
 inline void subroutine( Subroutine f, ContainerType&& x, ContainerTypes&&... xs);
+template<class ContainerType, class BinaryOp>
+inline get_value_type<ContainerType> reduce( const ContainerType& x, get_value_type<ContainerType> init, BinaryOp op);
 namespace detail
 {
 template< class ContainerType1, class ContainerType2>
@@ -97,6 +99,12 @@ inline void doSubroutine( SharedVectorTag, Subroutine f, ContainerType&& x, Cont
             );
 }
 
+template<class T, class ContainerType, class BinaryOp>
+inline T doReduce( SharedVectorTag, const ContainerType& x, T init, BinaryOp op)
+{
+    return doReduce_dispatch( get_execution_policy<ContainerType>(), x.size(), thrust::raw_pointer_cast( x.data()), init, op);
+}
+
 } //namespace detail
 } //namespace blas1
 } //namespace dg
diff --git a/inc/dg/backend/blas1_dispatch_vector.h b/inc/dg/backend/blas1_dispatch_vector.h
index bb7006aaf..bfc548884 100644
--- a/inc/dg/backend/blas1_dispatch_vector.h
+++ b/inc/dg/backend/blas1_dispatch_vector.h
@@ -137,6 +137,17 @@ inline void doSubroutine( RecursiveVectorTag, Subroutine f, container&& x, Conta
     doSubroutine_dispatch( RecursiveVectorTag(), get_execution_policy<vector_type>(), size, f, std::forward<container>( x), std::forward<Containers>( xs)...);
 }
 
+template<class T, class ContainerType, class BinaryOp>
+inline T doReduce( RecursiveVectorTag, const ContainerType& x, T init, BinaryOp op)
+{
+    //reduce sequentially recursively
+    for ( unsigned u=0; u<x.size(); u++)
+    {
+        init = op( init, dg::blas1::reduce( x[u], init, op));
+    }
+    return init;
+}
+
 } //namespace detail
 } //namespace blas1
 } //namespace dg
diff --git a/inc/dg/backend/blas1_omp.h b/inc/dg/backend/blas1_omp.h
index 2ceb197fc..262379933 100644
--- a/inc/dg/backend/blas1_omp.h
+++ b/inc/dg/backend/blas1_omp.h
@@ -1,6 +1,8 @@
 #ifndef _DG_BLAS_OMP_
 #define _DG_BLAS_OMP_
 #include <omp.h>
+#include <thrust/reduce.h>
+#include <thrust/system/omp/execution_policy.h>
 #include "config.h"
 #include "blas1_serial.h"
 #include "exblas/exdot_omp.h"
@@ -60,6 +62,12 @@ inline void doSubroutine_dispatch( OmpTag, int size, Subroutine f, PointerOrValu
         doSubroutine_dispatch( SerialTag(), size, f, x, xs...);
 }
 
+template<class T, class Pointer, class BinaryOp>
+inline T doReduce_dispatch( OmpTag, int size, Pointer x, T init, BinaryOp op)
+{
+    return thrust::reduce(thrust::omp::par, x, x+size, init, op);
+}
+
 }//namespace detail
 }//namespace blas1
 }//namespace dg
diff --git a/inc/dg/backend/blas1_serial.h b/inc/dg/backend/blas1_serial.h
index 9d3a340cf..59163749b 100644
--- a/inc/dg/backend/blas1_serial.h
+++ b/inc/dg/backend/blas1_serial.h
@@ -42,6 +42,14 @@ inline void doSubroutine_dispatch( SerialTag, int size, Subroutine f, PointerOrV
     }
 }
 
+template<class T, class Pointer, class BinaryOp>
+inline T doReduce_dispatch( SerialTag, int size, Pointer x, T init, BinaryOp op)
+{
+    for(int i=0; i<size; i++)
+        init = op( init, x[i]);
+    return init;
+}
+
 
 }//namespace detail
 }//namespace blas1
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 5ec704822..a542de172 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -45,6 +45,64 @@ namespace blas1
  * The compiler chooses the implementation and parallelization of this function based on given template parameters. For a full set of rules please refer to \ref dispatch.
  */
 
+/*! @brief \f$ x^T y\f$ Binary reproducible Euclidean dot product between two vectors
+ *
+ * This routine computes \f[ x^T y = \sum_{i=0}^{N-1} x_i y_i \f]
+ * @copydoc hide_iterations
+ *
+ * @note Our implementation guarantees binary reproducible results.
+ * The sum is computed with infinite precision and the result is rounded
+ * to the nearest double precision number.
+ * This is possible with the help of an adapted version of the \c ::exblas library.
+
+For example
+@code
+dg::DVec two( 100,2), three(100,3);
+double result = dg::blas1::dot( two, three); // result = 600 (100*(2*3))
+@endcode
+ * @param x Left Container
+ * @param y Right Container may alias x
+ * @return Scalar product as defined above
+ * @note This routine is always executed synchronously due to the
+        implicit memcpy of the result. With mpi the result is broadcasted to all processes
+ * @copydoc hide_ContainerType
+ */
+template< class ContainerType1, class ContainerType2>
+inline get_value_type<ContainerType1> dot( const ContainerType1& x, const ContainerType2& y)
+{
+    std::vector<int64_t> acc = dg::blas1::detail::doDot_superacc( x,y);
+    return exblas::cpu::Round(acc.data());
+}
+
+/*! @brief \f$ x_0 \otimes x_1 \otimes \dots \otimes x_{N-1} \f$ Custom reduction
+ *
+ * This routine computes \f[ s = s_0 + x_0 \otimes x_1 \otimes \dots \otimes x_i \otimes \dots \otimes x_{N-1} \f]
+ * where \f$ \otimes \f$ is an arbitrary **commutative** and **associative** binary operator, \f$ s_0\f$ is the initial value and
+ * @copydoc hide_iterations
+ *
+ * @note numerical addition/multiplication is **not** exactly associative
+ * which means that the associated reduction looses precision due to inexact arithmetic. For binary reproducible exactly rounded results use the dg::blas1::dot function.
+ * However, this function is more general and faster to execute than dg::blas1::dot.
+
+For example
+@code
+dg::DVec x( 100,2);
+double result = dg::blas1::reduce( x, 0., thrust::plus<double>()); // result = 200 ( 0 + 2 + 2 + ... + 2 ))
+@endcode
+ * @param x Left Container
+ * @param init initial value of the reduction
+ * @return Custom reduction as defined above
+ * @note This routine is always executed synchronously due to the
+        implicit memcpy of the result. With mpi the result is broadcasted to all processes
+ * @tparam BinaryOp Functor with signature: \c value_type \c operator()( value_type, value_type), must be associative and commutative
+ * @copydoc hide_ContainerType
+ */
+template< class ContainerType, class BinaryOp>
+inline get_value_type<ContainerType> reduce( const ContainerType& x, get_value_type<ContainerType> init, BinaryOp op )
+{
+    return dg::blas1::detail::doReduce( dg::get_tensor_category<ContainerType>(), x, init, op);
+}
+
 /**
  * @brief \f$ y=x \f$
  *
@@ -515,36 +573,6 @@ inline void subroutine( Subroutine f, ContainerType&& x, ContainerTypes&&... xs)
     dg::blas1::detail::doSubroutine(tensor_category(), f, std::forward<ContainerType>(x), std::forward<ContainerTypes>(xs)...);
 }
 
-/*! @brief \f$ x^T y\f$ Binary reproducible Euclidean dot product between two vectors
- *
- * This routine computes \f[ x^T y = \sum_{i=0}^{N-1} x_i y_i \f]
- * @copydoc hide_iterations
- *
- * @note Our implementation guarantees binary reproducible results.
- * The sum is computed with infinite precision and the result is rounded
- * to the nearest double precision number.
- * This is possible with the help of an adapted version of the \c ::exblas library.
-
-For example
-@code
-dg::DVec two( 100,2), three(100,3);
-double temp = dg::blas1::dot( two, three); // temp = 30 (5*(2*3))
-@endcode
- * @param x Left ContainerType
- * @param y Right ContainerType may alias x
- * @return Scalar product as defined above
- * @note This routine is always executed synchronously due to the
-        implicit memcpy of the result. With mpi the result is broadcasted to all processes
- * @copydoc hide_ContainerType
- */
-template< class ContainerType1, class ContainerType2>
-inline get_value_type<ContainerType1> dot( const ContainerType1& x, const ContainerType2& y)
-{
-    std::vector<int64_t> acc = dg::blas1::detail::doDot_superacc( x,y);
-    return exblas::cpu::Round(acc.data());
-//    return dg::blas1::detail::doDot( x, y, get_tensor_category<ContainerType1>(), get_tensor_category<ContainerType2>() );
-}
-
 ///@brief Deprecated: Use \c dg::construct<ContainerType>() instead
 ///@attention This function is deprecated! Please replace with \c dg::construct (data transfer between different devices)
 template<class ContainerType, class from_ContainerType>
diff --git a/inc/dg/blas_mpit.cu b/inc/dg/blas_mpit.cu
index b97fdec11..536d173cc 100644
--- a/inc/dg/blas_mpit.cu
+++ b/inc/dg/blas_mpit.cu
@@ -36,10 +36,13 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "Test trivial parallel functions:\n"<<std::boolalpha;
     dg::blas1::axpby( 2., x1, 3., x2);
     if(rank==0)std::cout << "Scalar addition                   "<< (x2 == 8.)<<std::endl;
+    if(rank==0)std::cout << "Reduction                         " << (dg::blas1::reduce( arr1, 0, thrust::maximum<float>()) == 4) <<std::endl;
     dg::blas1::axpby( 2., arr1, 3., vec2);
     if(rank==0)std::cout << "Recursive Vec Scalar addition     "<< (vec2[0] == 34.)<<std::endl;
     dg::blas1::axpby( 2., vec1, 3., arr2);
     if(rank==0)std::cout << "Recursive Arr Scalar addition     "<< (arr2[0] == 26.)<<std::endl;
+    double max = dg::blas1::reduce( dvec1, 0, thrust::maximum<double>());
+    if(rank==0)std::cout << "Recursive DVec reduction          " << (max == 30) <<std::endl;
     dg::blas1::copy( 2., arrdvec1);
     if(rank==0)std::cout << "Recursive DVec Copy Scalar to     "<< (arrdvec1[0].data()[0] == 2 && arrdvec1[1].data()[0]==2)<<std::endl;
     dg::blas1::axpby( 2., vec1 , 3, arrdvec1);
diff --git a/inc/dg/blas_t.cu b/inc/dg/blas_t.cu
index 37dcbcb6a..00730455b 100644
--- a/inc/dg/blas_t.cu
+++ b/inc/dg/blas_t.cu
@@ -29,10 +29,12 @@ int main()
     std::cout << "Test trivial parallel functions:\n"<<std::boolalpha;
     dg::blas1::axpby( 2., x1, 3., x2);
     std::cout << "Scalar addition                   "<< (x2 == 8.)<<std::endl;
+    std::cout << "Reduction                         " << (dg::blas1::reduce( arr1, 0, thrust::maximum<float>()) == 4) <<std::endl;
     dg::blas1::axpby( 2., arr1, 3., vec2);
     std::cout << "Recursive Vec Scalar addition     "<< (vec2[0] == 34.)<<std::endl;
     dg::blas1::axpby( 2., vec1, 3., arr2);
     std::cout << "Recursive Arr Scalar addition     "<< (arr2[0] == 26.)<<std::endl;
+    std::cout << "Recursive DVec reduction          " << (dg::blas1::reduce( dvec1, 0, thrust::maximum<double>()) == 30) <<std::endl;
     dg::blas1::copy( 2., arrdvec1);
     std::cout << "Recursive DVec Copy Scalar to     "<< (arrdvec1[0][0] == 2 && arrdvec1[1][0]==2)<<std::endl;
     dg::blas1::axpby( 2., vec1 , 3, arrdvec1);
@@ -42,9 +44,9 @@ int main()
     std::array<dg::DVec, 3> array_v{ dvec1, dvec1, dvec1}, array_w(array_v);
     std::array<double, 3> array_p{ 1,2,3};
     dg::blas1::subroutine( Expression(), dvec1, array_w[2], 3);
-    std::cout << "Example in documentation          "<< (dvec1[0] ==310)<<std::endl;
+    std::cout << "Example in documentation          "<< (dvec1[0] ==374)<<std::endl;
     dg::blas1::subroutine( Expression(), array_v, array_w, array_p);
-    std::cout << "Example in documentation          "<< (array_v[0][0] == 110 && array_v[1][1] == 820)<<std::endl;
+    std::cout << "Example in documentation          "<< (array_v[0][0] == 132 && array_v[1][1] == 903)<<std::endl;
     std::cout << "Test DOT functions:\n"<<std::boolalpha;
     double result = dg::blas1::dot( 1., array_p);
     std::cout << "blas1 dot recursive Scalar          "<< (result == 6) <<"\n";
-- 
GitLab


From f8ba2d3c5253a17139260f6b528f7944cf05a978 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 8 Feb 2019 09:58:55 +0100
Subject: [PATCH 030/540] Fix little cuda bug

---
 inc/dg/backend/blas1_cuda.cuh | 10 ----------
 src/feltor/feltor.tex         |  2 +-
 2 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/inc/dg/backend/blas1_cuda.cuh b/inc/dg/backend/blas1_cuda.cuh
index cb8a0e3b4..307f7a27c 100644
--- a/inc/dg/backend/blas1_cuda.cuh
+++ b/inc/dg/backend/blas1_cuda.cuh
@@ -60,16 +60,6 @@ inline void doSubroutine_dispatch( CudaTag, int size, Subroutine f, PointerOrVal
     subroutine_kernel<Subroutine, PointerOrValue, PointerOrValues...><<<NUM_BLOCKS, BLOCK_SIZE>>>(size, f, x, xs...);
 }
 
-template<class T, class PointerOrValue, class BinaryOp>
- __global__ T reduction_kernel( int size, T init, PointerOrValue x, BinaryOp op)
-{
-    const int thread_id = blockDim.x * blockIdx.x + threadIdx.x;
-    const int grid_size = gridDim.x*blockDim.x;
-    //every thread takes num_is/grid_size is
-    for( int i = thread_id; i<size; i += grid_size)
-
-}
-
 template<class T, class Pointer, class BinaryOp>
 inline T doReduce_dispatch( CudaTag, int size, Pointer x, T init, BinaryOp op)
 {
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 92b960dd7..fc625a562 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -91,7 +91,7 @@ Note that in any arbitrary coordinate system we have
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsection{Coordinate system}\label{sec:cylmetric}
-We employ cylindrical coordinates \( (R,Z,\varphi) \), with \(\varphi\) anti directed to the geometric toroidal angle (clockwise if viewed from above) to
+We employ cylindrical coordinates \( (R,Z,\varphi) \), with \(\varphi\) anti directed to the geometric toroidal angle ({\bf clockwise} if viewed from above) to
 obtain a right handed system. The parametric representation in Cartesian \((x,y,z)\) coordinates is therefore simply:
 \begin{align}
  x &= R \hspace{1 mm} \sin{(\varphi)}, &
-- 
GitLab


From 59d3c75f24e152e1b5b2d3c436c685e8a0a30856 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 12 Feb 2019 22:57:41 +0100
Subject: [PATCH 031/540] Add Multigrid algorithm

Chebyshev iterations, Eigenvalue estimator, multigrid cycle, full
multigrid, tests in cg2d_t and multigrid_b
---
 inc/dg/cg.h           |  90 ------------------------
 inc/dg/cg2d_t.cu      |   7 +-
 inc/dg/chebyshev.h    | 101 +++++++++++++++++++++++++++
 inc/dg/eve.h          |  84 ++++++++++++++++++++++
 inc/dg/multigrid.h    | 157 ++++++++++++++++++++++++++++++++++--------
 inc/dg/multigrid_b.cu | 126 +++++++++++++++++++++++++++++++++
 6 files changed, 443 insertions(+), 122 deletions(-)
 create mode 100644 inc/dg/chebyshev.h
 create mode 100644 inc/dg/eve.h
 create mode 100644 inc/dg/multigrid_b.cu

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index f5890f9bd..94b1599a6 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -250,96 +250,6 @@ unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const Con
     return max_iter;
 }
 ///@endcond
-//
-//implement the classical 3-term recursion with explicit residual
-/**
-* @brief Three-term recursion of the Chebyshev iteration for solving
-* \f[ Ax=b\f]
-*
-* Given the minimum and maximum Eigenvalue of the matrix A we define
-* \f[ a = (l_\min+l_\max)/2 \quad c = (l_\max - l_\min)/2 \\
-*     \beta_{-1} = 0\ \beta_0 = -c^2/2/a\ \gamma_0 = -a \\
-*     \beta_{n-1} = (c/2)^2/\gamma_{n-1} \quad \gamma_n = -(a+\beta_{n-1}) \\
-*     x_0 := x \ x_{-1} = r_{-1} = 0 \\
-*     x_{n+1} := -(r_n + ax_n + \beta_{n-1} x_{n-1})/\gamma_{n} \\
-*     r_{n+1} :=  b - Ax_{n+1}
-* \f]
-*
-* @ingroup invert
-*
-* @copydoc hide_ContainerType
-*/
-template< class ContainerType>
-class ChebyshevIteration
-{
-  public:
-    using container_type = ContainerType;
-    using value_type = get_value_type<ContainerType>; //!< value type of the ContainerType class
-    ///@brief Allocate nothing, Call \c construct method before usage
-    ChebyshevIteration(){}
-    ///@copydoc construct()
-    ChebyshevIteration( const ContainerType& copyable):
-        m_r(copyable), m_x1(m_r){}
-    ///@brief Return an object of same size as the object used for construction
-    ///@return A copyable object; what it contains is undefined, its size is important
-    const ContainerType& copyable()const{ return m_r;}
-
-    /**
-     * @brief Allocate memory for the pcg method
-     *
-     * @param copyable A ContainerType must be copy-constructible from this
-     * @param max_iterations Maximum number of iterations to be used
-     */
-    void construct( const ContainerType& copyable) {
-        m_x1 = m_r = copyable;
-    }
-    /**
-     * @brief Solve the system A*x = b using Chebyshev iteration
-     *
-     * The iteration stops when the maximum number of iterations is reached
-     * @param A A symmetric, positive definit matrix
-     * @param x Contains an initial value on input and the solution on output.
-     * @param b The right hand side vector. x and b may be the same vector.
-     * @param lmin the minimum Eigenvalue
-     * @param lmax the minimum Eigenvalue
-     * @param num_iter the number of iterations
-     *
-     * @copydoc hide_matrix
-     * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
-     */
-    template< class MatrixType, class ContainerType0, class ContainerType1>
-    void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b,
-        double lmin, double lmax, unsigned num_iter)
-    {
-        assert ( lmin < lmax);
-        double a = (lmin+lmax)/2., c = (lmin-lmax)/2.;
-        double betan = -c*c/2./a;
-        double gamman = -a;
-        dg::blas1::copy( x, m_x1);
-        dg::blas2::symv( A, x, m_r);
-        dg::blas1::axpby( 1., b, -1., m_r);
-        //-(r0+a*x0)/gamma0, m_x1=0
-        dg::blas1::axpby( -1./gamman, m_r, -a/gamman, x);
-        dg::blas1::copy( 0., m_x1);
-        for ( unsigned u=1; u<num_iter; u++)
-        {
-            betan = (c/2.)*(c/2.)/gamman;
-            gamman = -(a+betan);
-
-            dg::blas2::symv( A, x, m_r);
-            dg::blas1::axpby( 1., b, -1., m_r);
-            dg::blas1::evaluate( m_x1, dg::equals(), PairSum(),
-                            -1./gamman, m_r,
-                            -a/gamman, x,
-                            -betan/gamman,m_x1);
-            x.swap(m_x1);
-        }
-    }
-    //version of CG where Preconditioner is not trivial
-  private:
-    ContainerType m_r, m_x1;
-};
-
 
 /**
 * @brief Extrapolate based on up to three past solutions
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index aa85daf23..905534e0e 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -3,6 +3,7 @@
 
 #include "cg.h"
 #include "elliptic.h"
+#include "chebyshev.h"
 
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
@@ -68,13 +69,11 @@ int main()
     std::cout << "L2 Norm of Residuum is        " << res.d<<"\t"<<res.i << std::endl;
     //Fehler der Integration des Sinus ist vernachlässigbar (vgl. evaluation_t)
     dg::blas1::copy( 0., x);
-    dg::ChebyshevIteration<dg::HVec> cheby( copyable_vector);
+    dg::Chebyshev<dg::HVec> cheby( copyable_vector);
     double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
     double hxhy = lx*ly/(n*n*Nx*Ny);
     lmin *= hxhy, lmax *= hxhy; //we multiplied the matrix by w2d
-    std::cout << "Type number of Chebyshev iterations\n";
-    unsigned num_iter;
-    std::cin >> num_iter;
+    unsigned num_iter = 200;
     cheby.solve( A, x, b, lmin, lmax, num_iter);
     std::cout << "After "<<num_iter<<" Chebyshev iterations we have:\n";
 
diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
new file mode 100644
index 000000000..31f9e9c10
--- /dev/null
+++ b/inc/dg/chebyshev.h
@@ -0,0 +1,101 @@
+#ifndef _DG_CHEB_
+#define _DG_CHEB_
+
+#include <cmath>
+
+#include "blas.h"
+
+
+namespace dg
+{
+//
+//implement the classical 3-term recursion with explicit residual
+/**
+* @brief Three-term recursion of the Chebyshev iteration for solving
+* \f[ Ax=b\f]
+*
+* Given the minimum and maximum Eigenvalue of the matrix A we define
+* \f[ \theta = (\lambda_\min+\lambda_\max)/2 \quad \delta = (\lambda_\max - \lambda_\min)/2 \\
+*     \rho_0 := \frac{\delta}{\theta},\ x_0 := x, \ x_{1} = x_0+\frac{1}{\theta} (b-Ax_0) \\
+*     \rho_{k}:=\left(\frac{2\theta}{\delta}-\rho_{k-1}\right)^{-1} \\
+*     x_{k+1} := x_k + \rho_k\left( \rho_{k-1}(x_k - x_{k-1})
+*     + \frac{2}{\delta} ( b - Ax_k) \right)
+* \f]
+* For more information see the book "Iteratvie Methods for Sparse
+* Linear Systems" 2nd edition by Yousef Saad
+*
+* @ingroup invert
+*
+* @copydoc hide_ContainerType
+*/
+template< class ContainerType>
+class Chebyshev
+{
+  public:
+    using container_type = ContainerType;
+    using value_type = get_value_type<ContainerType>; //!< value type of the ContainerType class
+    ///@brief Allocate nothing, Call \c construct method before usage
+    Chebyshev(){}
+    ///@copydoc construct()
+    Chebyshev( const ContainerType& copyable):
+        m_ax(copyable), m_xm1(m_ax){}
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_ax;}
+
+    /**
+     * @brief Allocate memory for the pcg method
+     *
+     * @param copyable A ContainerType must be copy-constructible from this
+     * @param max_iterations Maximum number of iterations to be used
+     */
+    void construct( const ContainerType& copyable) {
+        m_xm1 = m_ax = copyable;
+    }
+    /**
+     * @brief Solve the system A*x = b using Chebyshev iteration
+     *
+     * The iteration stops when the maximum number of iterations is reached
+     * @param A A symmetric, positive definit matrix
+     * @param x Contains an initial value on input and the solution on output.
+     * @param b The right hand side vector. x and b may be the same vector.
+     * @param min_ev the minimum Eigenvalue
+     * @param max_ev the minimum Eigenvalue
+     * @param num_iter the number of iterations k (equals the number of times A is applied)
+     *
+     * @copydoc hide_matrix
+     * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
+     */
+    template< class MatrixType, class ContainerType0, class ContainerType1>
+    void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b,
+        double min_ev, double max_ev, unsigned num_iter)
+    {
+        if( num_iter == 0)
+            return;
+        assert ( min_ev < max_ev);
+        double theta = (min_ev+max_ev)/2., delta = (max_ev-min_ev)/2.;
+        double rhokm1 = delta/theta, rhok=0;
+        dg::blas1::copy( x, m_xm1); //x0
+        dg::blas2::symv( A, x, m_ax);
+        dg::blas1::axpbypgz( 1./theta, b, -1./theta, m_ax, 1., x); //x1
+        for ( unsigned k=1; k<num_iter; k++)
+        {
+            rhok = 1./(2.*theta/delta - rhokm1);
+            dg::blas2::symv( A, x, m_ax);
+            dg::blas1::evaluate( m_xm1, dg::equals(), PairSum(),
+                             1.+rhok*rhokm1,   x,
+                            -rhok*rhokm1,       m_xm1,
+                             2.*rhok/delta,       b,
+                            -2.*rhok/delta,        m_ax
+                            );
+            x.swap(m_xm1);
+            rhokm1 = rhok;
+        }
+    }
+  private:
+    ContainerType m_ax, m_xm1;
+};
+
+} //namespace dg
+
+#endif // _DG_CHEB_
diff --git a/inc/dg/eve.h b/inc/dg/eve.h
new file mode 100644
index 000000000..0b222d6ad
--- /dev/null
+++ b/inc/dg/eve.h
@@ -0,0 +1,84 @@
+/* EVE adds an estimator for the largest Eigenvalue
+   to not-yet-preconditioned CG.
+
+                           */
+
+#ifndef _DG_EVE_
+#define _DG_EVE_
+
+#include <cmath>
+#include "blas.h"
+#include "functors.h"
+
+namespace dg
+{
+
+//MW: please document
+/* EVE (EigenValueEstimator) estimate largest EV using CG */
+template< class Vector>
+class EVE
+{
+public:
+    using value_type  = get_value_type<Vector>;
+    EVE() {}
+    EVE( const Vector& copyable, unsigned max_iter):r( copyable), p( r), ap( r), max_iter( max_iter) {}
+    void set_max( unsigned new_max)
+    {   max_iter = new_max;
+    }
+    unsigned get_max() const
+    {   return max_iter;
+    }
+    void construct( const Vector& copyable, unsigned max_iterations = 100)
+    {   ap = p = r = copyable;
+        max_iter = max_iterations;
+    }
+    template< class Matrix>
+    unsigned operator()( Matrix& A, Vector& x, const Vector& b, value_type& ev_max, value_type eps_ev=1e-16);
+private:
+    Vector r, p, ap;
+    unsigned max_iter;
+};
+
+template< class Vector>
+template< class Matrix>
+unsigned EVE< Vector>::operator()( Matrix& A, Vector& x, const Vector&
+b, value_type& ev_max, value_type eps_ev)
+{
+    blas2::symv( A, x, r);
+    blas1::axpby( 1., b, -1., r);
+    value_type nrm2r_old = blas1::dot( r,r);
+    blas1::copy( r, p);
+    value_type nrm2r_new, nrmAp;
+    value_type alpha = 1., alpha_inv = 1., delta = 0.;
+    value_type evdash, gamma = 0., lambda, omega, beta = 0.;
+    value_type ev_est = 0.;
+    ev_max = 0.;
+    for( unsigned i=1; i<max_iter; i++)
+    {
+        lambda = delta*alpha_inv;       // EVE!
+        blas2::symv( A, p, ap);
+        nrmAp = blas1::dot( p, ap);
+        alpha = nrm2r_old /nrmAp;
+        alpha_inv = nrmAp /nrm2r_old;   // EVE!
+        lambda += alpha_inv;            // EVE!
+        blas1::axpby( alpha, p, 1., x);
+        blas1::axpby( -alpha, ap, 1., r);
+        nrm2r_new = blas1::dot( r, r);
+        delta = nrm2r_new /nrm2r_old;                  // EVE!
+        evdash = ev_est -lambda;                       // EVE!
+        omega = sqrt( evdash*evdash +4.*beta*gamma);   // EVE!
+        gamma = 0.5 *(1. -evdash /omega);              // EVE!
+        ev_max += omega*gamma;                         // EVE!
+        if( fabs(ev_est-ev_max) < eps_ev*ev_max) {
+            return i;
+        }
+        beta = delta*alpha_inv*alpha_inv;              // EVE!
+        blas1::axpby(1., r, delta, p);
+        nrm2r_old=nrm2r_new;
+        ev_est = ev_max;
+    }
+    return max_iter;
+};
+
+} //namespace dg
+#endif //_DG_EVE_
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 9c75597c8..2efd342d3 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -6,6 +6,8 @@
 #include "topology/interpolation.h"
 #include "blas.h"
 #include "cg.h"
+#include "chebyshev.h"
+#include "eve.h"
 #ifdef DG_BENCHMARK
 #include "backend/timer.h"
 #endif //DG_BENCHMARK
@@ -67,6 +69,7 @@ struct MultigridCG2d
 
 		m_grids.resize(stages);
         m_cg.resize(stages);
+        m_cheby.resize(stages);
 
         m_grids[0].reset( grid);
         //m_grids[0].get().display();
@@ -95,8 +98,12 @@ struct MultigridCG2d
         for( unsigned u=0; u<m_stages; u++)
             m_x[u] = dg::construct<Container>( dg::evaluate( dg::zero, *m_grids[u]), std::forward<Params>(ps)...);
         m_r = m_b = m_x;
+        m_p = m_cgr = m_r[0];
         for (unsigned u = 0; u < m_stages; u++)
+        {
             m_cg[u].construct(m_x[u], 1);
+            m_cheby[u].construct(m_x[u]);
+        }
     }
 
     /**
@@ -222,39 +229,88 @@ struct MultigridCG2d
     ///@return A copyable object; what it contains is undefined, its size is important
     const Container& copyable() const {return m_x[0];}
 
-  private:
 
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
-    void full_multigrid( std::vector<SymmetricOp>& op,
-        ContainerType0& x, ContainerType1& b,
-        unsigned nu1, unsigned nu2, unsigned gamma, unsigned mu, double eps)
+    void solve( std::vector<SymmetricOp>& op,
+    ContainerType0& x, const ContainerType1& b, std::vector<double> ev, unsigned nu_pre, unsigned
+    nu_post, unsigned gamma, double eps)
     {
         dg::blas2::symv(op[0].weights(), b, m_b[0]);
-        // project x down to coarse grid
+
+        //FULL MULTIGRID
         dg::blas1::copy( x, m_x[0]);
-        dg::blas1::copy( b, m_b[0]);
-        for( unsigned u=0; u<m_stages-1; u++)
-        {
-            dg::blas2::gemv( m_interT[u], m_x[u], m_x[u+1]);
-            dg::blas2::gemv( m_interT[u], m_b[u], m_b[u+1]);
-        }
-        unsigned s = m_stages-1;
-        unsigned number = m_cg[s]( op[s], m_x[s], m_b[s], op[s].precond(),
-            op[s].inv_weights(), eps);
+        unsigned max_iter = 3;
+        for( unsigned u=0; u<max_iter; u++)
+            full_multigrid( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
+        dg::blas1::copy( m_x[0], x);
+
+        //MULTIGRID CYCLES
+        //unsigned max_iter = 3;
+        //dg::blas1::copy( x, m_x[0]);
+        //for( unsigned u=0; u<max_iter; u++)
+        //{
+        //    multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
+        //}
+        //dg::blas1::copy( m_x[0], x);
+
+        //PCG WITH MULTIGRID CYCLE AS PRECONDITIONER
+        //unsigned max_iter_ = m_grids[0]->size();
+        //value_type nrmb = sqrt( blas2::dot( op[0].inv_weights(), m_b[0]));
+        //if( nrmb == 0)
+        //{
+        //    blas1::copy( m_b[0], x);
+        //    return;
+        //}
+        //blas2::symv( op[0],x,m_cgr);
+        //blas1::axpby( 1., m_b[0], -1., m_cgr);
+        ////if x happens to be the solution
+        //if( sqrt( blas2::dot(op[0].inv_weights(),m_cgr) )
+        //        < eps*(nrmb + 1))
+        //    return;
+        ////blas2::symv( P, r, p );//<-- compute p_0
+        ////dg::blas2::symv( op[0].precond(), m_cgr, m_p);
+        ////dg::blas1::copy( 0, m_p);
+        ////m_cheby[0].solve( op[0], m_p, m_cgr, 0.1*ev[0], 1.1*ev[0], nu_post+nu_pre);
+
+        //dg::blas1::copy( 0, m_x[0]);
+        //dg::blas1::copy( m_cgr, m_b[0]);
+        //full_multigrid( op,m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
+        //dg::blas1::copy( m_x[0], m_p);
+
+        ////and store the scalar product
+        //value_type nrmzr_old = blas1::dot( m_p,m_cgr);
+        //value_type alpha, nrmzr_new;
+        //for( unsigned i=2; i<max_iter_; i++)
+        //{
+        //    blas2::symv( op[0], m_p, m_x[0]);
+        //    alpha =  nrmzr_old/blas1::dot( m_p, m_x[0]);
+        //    blas1::axpby( alpha, m_p, 1., x);
+        //    blas1::axpby( -alpha, m_x[0], 1., m_cgr);
+        //    value_type error = sqrt( blas2::dot(op[0].inv_weights(), m_cgr))/(nrmb+1);
+        //    std::cout << "\t\t\tError at "<<i<<" is "<<error<<"\n";
+        //    if( error < eps)
+        //        return;
+        //    dg::blas2::symv( op[0].precond(), m_cgr, m_x[0]);
+        //    dg::blas1::copy( 0, m_x[0]);
+        //    m_cheby[0].solve( op[0], m_x[0], m_cgr, 0.1*ev[0], 1.1*ev[0], nu_post+nu_pre);
+        ////dg::blas1::copy( 0, m_x[0]);
+        ////dg::blas1::copy( m_cgr, m_b[0]);
+        ////multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
+
+        //    nrmzr_new = blas1::dot( m_x[0], m_cgr);
+        //    blas1::axpby(1., m_x[0], nrmzr_new/nrmzr_old, m_p );
+        //    nrmzr_old=nrmzr_new;
+        //}
 
-		for( int p=m_stages-2; p>=0; p--)
-        {
-            dg::blas2::gemv( m_inter[p], m_x[p+1],  m_x[p]);
-            for( unsigned u=0; u<mu; u++)
-                multigrid_cycle( op, m_x, m_b, nu1, nu2, gamma, p, eps);
-        }
     }
 
     template<class SymmetricOp>
     void multigrid_cycle( std::vector<SymmetricOp>& op,
-        std::vector<Container>& x, std::vector<Container>& b,
+    std::vector<Container>& x, std::vector<Container>& b,
+    std::vector<double> ev,
         unsigned nu1, unsigned nu2, unsigned gamma, unsigned p, double eps)
     {
+        // 1 multigrid cycle beginning on grid p
         // p < m_stages-1
         // x[p]    initial condition on input, solution on output
         // x[p+1]  write only (solution of lower stage)
@@ -264,33 +320,77 @@ struct MultigridCG2d
         //
         // gamma:  typically 1 (V-cycle) or 2 (W-cycle)
         // nu1, nu2: typically in {0,1,2,3}
-        double lmin = 0, lmax = 1; //determine lmin and lmax
 
         // 1. Pre-Smooth nu1 times
-        m_cheby[p].solve( op[p], x[p], b[p], 0.1*lmax, 1.1*lmax, nu1);
-        // 2. Residual
+        //std::cout << "STAGE "<<p<<"\n";
+        //dg::blas2::symv( op[p], x[p], m_r[p]);
+        //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
+        //double norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
+        //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
+        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1);
+        // 2. Residuum
         dg::blas2::symv( op[p], x[p], m_r[p]);
         dg::blas1::axpby( 1., b[p], -1., m_r[p]);
+        //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
+        //std::cout<< " Norm residuum after  "<<norm_res<<"\n";
         // 3. Coarsen
         dg::blas2::symv( m_interT[p], m_r[p], b[p+1]);
         // 4. Solve or recursive call to get x[p+1] with initial guess 0
         dg::blas1::scal( x[p+1], 0.);
         if( p+1 == m_stages-1)
         {
+            m_cg[p+1].set_max(m_grids[p+1]->size());
             m_cg[p+1]( op[p+1], x[p+1], b[p+1], op[p+1].precond(),
-                op[p+1].inv_weights(), eps);
+                op[p+1].inv_weights(), 1e-10);
+            //m_cheby[p+1].solve( op[p+1], x[p+1], b[p+1], 0.1*ev[p+1], 1.1*ev[p+1], nu1+nu2);
+            //dg::blas2::symv( op[p+1], x[p+1], m_r[p+1]);
+            //dg::blas1::axpby( 1., b[p+1], -1., m_r[p+1]);
+            //double norm_res = sqrt(dg::blas1::dot( m_r[p+1], m_r[p+1]));
+            //std::cout<< " Exact solution "<<norm_res<<"\n";
         }
         else
         {
             //update x[p+1] gamma times
             for( unsigned u=0; u<gamma; u++)
-                multigrid_cycle( op, x, b, nu1, nu2, gamma, p+1, eps);
+                multigrid_cycle( op, x, b, ev, nu1, nu2, gamma, p+1, eps);
         }
 
         // 5. Correct
         dg::blas2::symv( 1., m_inter[p], x[p+1], 1., x[p]);
+        //dg::blas2::symv( op[p], x[p], m_r[p]);
+        //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
+        //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
+        //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
         // 6. Post-Smooth nu2 times
-        m_cheby[p].solve( op[p], x[p], b[p], 0.1*lmax, 1.1*lmax, nu2);
+        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2);
+        //dg::blas2::symv( op[p], x[p], m_r[p]);
+        //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
+        //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
+        //std::cout<< " Norm residuum after "<<norm_res<<"\n";
+    }
+
+
+	template<class SymmetricOp>
+    void full_multigrid( std::vector<SymmetricOp>& op,
+        std::vector<Container>& x, std::vector<Container>& b, std::vector<double> ev,
+        unsigned nu1, unsigned nu2, unsigned gamma, unsigned mu, double eps)
+    {
+        //begins on coarsest level and cycles through to highest
+        for( unsigned u=0; u<m_stages-1; u++)
+        {
+            dg::blas2::gemv( m_interT[u], x[u], x[u+1]);
+            dg::blas2::gemv( m_interT[u], b[u], b[u+1]);
+        }
+        unsigned s = m_stages-1;
+        m_cg[s]( op[s], x[s], b[s], op[s].precond(),
+            op[s].inv_weights(), 1e-10);
+
+		for( int p=m_stages-2; p>=0; p--)
+        {
+            dg::blas2::gemv( m_inter[p], x[p+1],  x[p]);
+            for( unsigned u=0; u<mu; u++)
+                multigrid_cycle( op, x, b, ev, nu1, nu2, gamma, p, eps);
+        }
     }
 
   private:
@@ -300,8 +400,9 @@ struct MultigridCG2d
     std::vector< MultiMatrix<Matrix, Container> >  m_interT;
     std::vector< MultiMatrix<Matrix, Container> >  m_project;
     std::vector< CG<Container> > m_cg;
-    std::vector< ChebyshevIteration<Container>> m_cheby;
+    std::vector< Chebyshev<Container>> m_cheby;
     std::vector< Container> m_x, m_r, m_b;
+    Container  m_p, m_cgr;
 
 };
 
diff --git a/inc/dg/multigrid_b.cu b/inc/dg/multigrid_b.cu
new file mode 100644
index 000000000..e1fbd3193
--- /dev/null
+++ b/inc/dg/multigrid_b.cu
@@ -0,0 +1,126 @@
+#include <iostream>
+#include <iomanip>
+
+#include <thrust/device_vector.h>
+#include "backend/timer.h"
+
+#include "blas.h"
+#include "elliptic.h"
+#include "multigrid.h"
+
+const double lx = M_PI;
+const double ly = 2.*M_PI;
+dg::bc bcx = dg::DIR;
+dg::bc bcy = dg::PER;
+
+double initial( double x, double y) {return 0.;}
+double amp = 0.9999;
+double pol( double x, double y) {return 1. + amp*sin(x)*sin(y); } //must be strictly positive
+//double pol( double x, double y) {return 1.; }
+//double pol( double x, double y) {return 1. + sin(x)*sin(y) + x; } //must be strictly positive
+
+double rhs( double x, double y) { return 2.*sin(x)*sin(y)*(amp*sin(x)*sin(y)+1)-amp*sin(x)*sin(x)*cos(y)*cos(y)-amp*cos(x)*cos(x)*sin(y)*sin(y);}
+//double rhs( double x, double y) { return 2.*sin( x)*sin(y);}
+//double rhs( double x, double y) { return 2.*sin(x)*sin(y)*(sin(x)*sin(y)+1)-sin(x)*sin(x)*cos(y)*cos(y)-cos(x)*cos(x)*sin(y)*sin(y)+(x*sin(x)-cos(x))*sin(y) + x*sin(x)*sin(y);}
+double sol(double x, double y)  { return sin( x)*sin(y);}
+double der(double x, double y)  { return cos( x)*sin(y);}
+
+
+int main()
+{
+    unsigned n, Nx, Ny;
+    double eps;
+    double jfactor;
+
+	n = 3;
+	Nx = Ny = 64;
+	eps = 1e-6;
+	jfactor = 1;
+
+	/*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
+    std::cin >> n >> Nx >> Ny; //more N means less iterations for same error
+    std::cin >> eps >> jfactor;*/
+    std::cout << "Computation on: "<< n <<" x "<< Nx <<" x "<< Ny << std::endl;
+    //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
+
+	dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
+    dg::DVec w2d = dg::create::weights( grid);
+    dg::DVec v2d = dg::create::inv_weights( grid);
+    dg::DVec one = dg::evaluate( dg::one, grid);
+    //create functions A(chi) x = b
+    dg::DVec x =    dg::evaluate( initial, grid);
+    dg::DVec b =    dg::evaluate( rhs, grid);
+    dg::DVec chi =  dg::evaluate( pol, grid);
+    dg::DVec chi_inv(chi);
+    dg::blas1::transform( chi, chi_inv, dg::INVERT<double>());
+    dg::blas1::pointwiseDot( chi_inv, v2d, chi_inv);
+    dg::DVec temp0( x), temp1(x), temp2(x), temp3(x);
+
+    dg::Timer t;
+    t.tic();
+
+    //create an Elliptic object without volume form (not normed)
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol( grid, dg::not_normed, dg::centered, jfactor);
+
+    //Set the chi function (chi is a dg::DVec of size grid.size())
+    pol.set_chi( chi);
+
+    //construct an invert object
+    dg::EVE<dg::DVec> eve_cg( x, n*n*Nx*Ny);
+    //dg::CG<dg::DVec> eve_cg( x, n*n*Nx*Ny);
+    double lmax = M_PI*M_PI*(n*n*Nx*Nx/lx/lx + n*n*Ny*Ny/ly/ly); //Eigenvalues of Laplace
+    double hxhy = lx*ly/(n*n*Nx*Ny);
+    double chi_max = dg::blas1::reduce( chi, 0, thrust::maximum<double>());
+    lmax *= chi_max;
+    lmax *= hxhy; //we multiplied the matrix by w2d
+    std::cout << "Estimated Eigenvalue Analytical "<<lmax<<"\n";
+
+    //invert the elliptic equation
+    double ev_max =0, eps_ev = 1e-2;
+    unsigned counter;
+    counter = eve_cg( pol, x, b, ev_max, eps_ev);
+    std::cout << "\nPrecision is "<<eps_ev<<"\n";
+    std::cout << "\nEstimated EigenValue Eve is "<<ev_max<<"\n";
+    std::cout << " with "<<counter<<" iterations\n";
+
+    //  Now test multigrid with estimated eigenvalues
+    unsigned stages = 3;
+    dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid(
+        grid, stages);
+    const std::vector<dg::DVec> multi_chi = multigrid.project( chi);
+    x = dg::evaluate( initial, grid);
+    std::vector<dg::DVec> multi_x = multigrid.project( x);
+    const std::vector<dg::DVec> multi_b = multigrid.project( b);
+    std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
+    std::vector<dg::EVE<dg::DVec> > multi_eve(stages);
+    std::vector<double> multi_ev(stages);
+    for(unsigned u=0; u<stages; u++)
+    {
+        multi_pol[u].construct( multigrid.grid(u), dg::not_normed, dg::centered, jfactor);
+        multi_eve[u].construct( multi_chi[u]);
+        multi_pol[u].set_chi( multi_chi[u]);
+        counter = multi_eve[u]( multi_pol[u], multi_x[u], multi_b[u],
+            multi_ev[u], eps_ev);
+        std::cout << "Eigenvalue estimate eve: "<<multi_ev[u]<<"\n";
+    }
+    std::cout << "Type nu1 (3), nu2 (3) gamma (1)\n";
+    unsigned nu1, nu2, gamma;
+    std::cin >> nu1 >> nu2 >> gamma;
+    x = dg::evaluate( initial, grid);
+    multigrid.solve(multi_pol, x, b, multi_ev, nu1, nu2, gamma, eps);
+    //CURRENTLY BEST METHOD:
+    //multigrid.direct_solve(multi_pol, x, b, eps);
+
+    const dg::DVec solution = dg::evaluate( sol, grid);
+    const double norm = dg::blas2::dot( w2d, solution);
+    dg::DVec error( solution);
+    dg::blas1::axpby( 1.,x,-1., solution, error);
+    double err = dg::blas2::dot( w2d, error);
+    err = sqrt( err/norm);
+    std::cout << " Error of Multigrid iterations "<<err<<"\n";
+    //should converge to ~2e-7
+
+
+    return 0;
+}
+
-- 
GitLab


From e5f82c03413570d54628db92eb7a5605f1bd54de Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 26 Feb 2019 16:09:14 +0100
Subject: [PATCH 032/540] Update Doku on output in feltor writeup

- fix forgotten EXTRACT_STATIC in file Doxyfile
- fix forgotten Doku in blas1 reduce
---
 diag/feltordiag.cu    |  2 +-
 inc/dg/blas1.h        |  1 +
 inc/dg/dg_doc.h       | 76 -------------------------------------------
 inc/file/Doxyfile     |  2 +-
 src/feltor/feltor.tex | 21 ++++++------
 5 files changed, 14 insertions(+), 88 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index 65435c98b..6c418a4bb 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -265,7 +265,7 @@ int main( int argc, char* argv[])
 
     size_t steps;
     err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
-    err = nc_inq_unlimdim( ncid, &timeID);
+    err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
     err = nc_inq_dimlen( ncid, timeID, &steps);
     err = nc_close( ncid); //close 3d file
 
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index a542de172..8ab2148f5 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -91,6 +91,7 @@ double result = dg::blas1::reduce( x, 0., thrust::plus<double>()); // result = 2
 @endcode
  * @param x Left Container
  * @param init initial value of the reduction
+ * @param op an associative and commutative binary operator
  * @return Custom reduction as defined above
  * @note This routine is always executed synchronously due to the
         implicit memcpy of the result. With mpi the result is broadcasted to all processes
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 5a49ac092..96f0efe24 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -269,82 +269,6 @@
  *   -# If the tensor category of the Vectors is \c dg::RecursiveVectorTag and
  *   the tensor category of the Matrix is not, then the \c dg::blas2::symv is recursively called with the Matrix on all elements of the Vectors.
  *
- * @subsection dispatch_examples Examples
- *
- * Let us assume that we have two vectors \f$ v\f$ and \f$ w\f$. In a shared
- memory code these will be declared as
- @code
- dg::DVec v, w;
- // initialize v and w with some meaningful values
- @endcode
- In an MPI implementation we would simply write \c dg::MDVec instead of \c dg::DVec.
- Let us now assume that we want to compute the expression \f$ v_i  \leftarrow v_i^2 + w_i\f$
- with the \c dg::blas1::subroutine. The first step is to write a Functor that
- implements this expression
- @code
- struct Expression{
-    DG_DEVICE
-    void operator() ( double& v, double w){
-       v = v*v + w;
-    }
- };
- @endcode
- Note that we used the Marco \ref DG_DEVICE to enable this code on GPUs.
- The next step is just to apply our struct to the vectors we have.
- @code
- dg::blas1::subroutine( Expression(), v, w);
- @endcode
-
- Now, we want to use an additional parameter in our expresion. Let's assume we have
- @code
- double parameter = 3.;
- @endcode
- and we want to compute \f$ v_i \leftarrow p v_i^2 + w_i\f$. We now have two
- possibilities. We can add a private variable in \c Expression and use it in the
- implementation of the paranthesis operator
- @code
- struct Expression{
-    Expression( double param):m_param(param){}
-    DG_DEVICE
-    void operator() ( double& v, double w)const{
-        v = m_param*v*v + w;
-    }
-    private:
-    double m_param;
- };
- dg::blas1::subroutine( Expression(parameter), v, w);
- @endcode
- The other possibility is to extend the paranthesis operator in \c Expression and call \c dg::blas1::subroutine with a scalar
- @code
- struct Expression{
-    DG_DEVICE
-    void operator() ( double& v, double w, double param){
-        v = param*v*v + w;
-    }
- };
- dg::blas1::subroutine( Expression(), v, w, parameter);
- @endcode
- The result (and runtime) is the same in both cases. However, the second is more versatile,
- when we use recursion. Consider that \f$ v,\ w\f$ and \f$ p\f$ are now arrays, declared as
- @code
- std::array<dg::DVec, 3> array_v, array_w;
- std::array<double,3> array_parameter;
- // initialize array_v, array_w and array_parameter meaningfully
- @endcode
- We now want to compute the expression \f$ v_{ij} \leftarrow p_i v_{ij}^2 + w_{ij}\f$,
- where \c i runs from 0 to 2 and \c j runs over all elements in the shared vectors
- <tt> array_v[i] </tt> and <tt> array_w[i] </tt>.
- In this case we just call
- @code
- dg::blas1::subroutine( Expression(), array_v, array_w, array_parameter);
- @endcode
- and use the fact that <tt> std::array </tt> has the \c dg::RecursiveVectorTag.
-
- In order to compute the sum \f$ \sum_{i=0}^2 p_i\f$ we can use
- @code
- double sum = dg::blas1::dot( 1, array_parameter);
- @endcode
-
  * @section mpi_backend The MPI interface
 @note The mpi backend is activated by including \c mpi.h before any other feltor header file
 @subsection mpi_vector MPI Vectors and the blas functions
diff --git a/inc/file/Doxyfile b/inc/file/Doxyfile
index e2cbab369..95bbaf5db 100644
--- a/inc/file/Doxyfile
+++ b/inc/file/Doxyfile
@@ -444,7 +444,7 @@ EXTRACT_PACKAGE        = NO
 # included in the documentation.
 # The default value is: NO.
 
-EXTRACT_STATIC         = NO
+EXTRACT_STATIC         = YES
 
 # If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined
 # locally in source files will be included in the documentation. If set to NO,
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index fc625a562..bf3b3f6b5 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -870,7 +870,7 @@ the value is not found,
 the program exits with an error message.
 
 \subsection{Output}
-Output file format: netcdf-4/hdf5
+Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Dataset with the same name as a dimension.
 %
 %Name | Type | Dimensionality | Description
 %---|---|---|---|
@@ -879,9 +879,9 @@ Output file format: netcdf-4/hdf5
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
 inputfile  &     text attribute & 1 & verbose input file as a string (valid JSON) \\
 geomfile   &     text attribute & 1 & verbose geometry input file as a string (valid JSON) \\
-x                & Dataset & 1 & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
-y                & Dataset & 1 & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
-z                & Dataset & 1 & $\varphi$-coordinate (computational space, size: $N_z$) \\
+x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
+y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
+z                & Coord. Var. & 1 (z) & $\varphi$-coordinate (computational space, size: $N_z$) \\
 x\_XYZ           & Dataset & 3 (z,y,x) & Cartesian x-coordinate $x=R\sin(\varphi)$ \\
 y\_XYZ           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
 z\_XYZ           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
@@ -892,14 +892,14 @@ Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta(\rho_{s} - \rho
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
-time             & Dataset & 1 & time at which fields are written (size: maxout$+1$ \\
+time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
 electrons        & Dataset & 4 (time, z, y, x) & electron density $n_e$ \\
 ions             & Dataset & 4 (time, z, y, x) & ion density $N_i$ \\
 Ue               & Dataset & 4 (time, z, y, x) & electron velocity $u_e$ \\
 Ui               & Dataset & 4 (time, z, y, x) & ion velocity $U_i$ \\
 potential        & Dataset & 4 (time, z, y, x) & electric potential $\phi$ \\
 induction        & Dataset & 4 (time, z, y, x) & parallel vector potential $A_\parallel$ \\
-energy\_time     & Dataset & 1 & timesteps at which 1d variables are written (size: itstp$\cdot$maxout$+1$ \\
+energy\_time     & Coord. Var. & 1 (energy\_time) & timesteps at which 1d variables are written (variable size: itstp$\cdot$maxout$+1$, dimension size: unlimited ) \\
 mass      & Dataset & 1 (energy\_time) & total mass integral Eq.~\eqref{eq:mass_conservation} \\
 diff      & Dataset & 1 (energy\_time) & total mass integral diffusion Eq.~\eqref{eq:mass_conservation} \\
 energy    & Dataset & 1 (energy\_time) & total energy integral Eq.~\eqref{eq:energy_conservation} \\
@@ -933,16 +933,17 @@ This program reads in the output file above and writes into another output file.
    L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}
 \end{align}
 
-Output file format: netcdf-4/hdf5
+Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Dataset with the same name as a dimension.
 
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
 inputfile  &     text attribute & 1 & verbose input file as a string (valid JSON) \\
 geomfile   &     text attribute & 1 & verbose geometry input file as a string (valid JSON) \\
-x                & Dataset & 1 & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
-y                & Dataset & 1 & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
-psip1d           & Dataset & 1 & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
+x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
+y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
+psip1d           & Coord. Var. & 1 (psip1d) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
+time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
 X\_avg           & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
 X\_fsa\_mp       & Dataset & 3 (time,y,x) & Fluctuation level on midplane ($\varphi\equiv \pi$) $\delta X := X(R,Z,\pi) - \langle X\rangle_{\psi_{p}}$ \\
-- 
GitLab


From 97f850cd626d8b6f353e9987ec2716b2a47cdbc1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 26 Feb 2019 17:06:29 +0100
Subject: [PATCH 033/540] Initial implementation of restart in feltor_hpc

untested yet
---
 src/feltor/feltor.cu     |   2 +
 src/feltor/feltor.tex    |   8 +-
 src/feltor/feltor_hpc.cu | 204 ++++++++++++++++++++++++++++-----------
 3 files changed, 154 insertions(+), 60 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 722893a21..dbb63141c 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -285,6 +285,8 @@ int main( int argc, char* argv[])
                         {
                             failed_counter++;
                             std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
+                            time -= dt; // time has to be reset here
+                            // in case of failure diffusion is applied twice?
                         }
                     }while ( adaptive.failed());
                     dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index bf3b3f6b5..642a8a073 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -877,8 +877,8 @@ Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Data
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
-inputfile  &     text attribute & 1 & verbose input file as a string (valid JSON) \\
-geomfile   &     text attribute & 1 & verbose geometry input file as a string (valid JSON) \\
+inputfile  &     text attribute & - & verbose input file as a string (valid JSON, no comments) \\
+geomfile   &     text attribute & - & verbose geometry input file as a string (valid JSON, no comments) \\
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 z                & Coord. Var. & 1 (z) & $\varphi$-coordinate (computational space, size: $N_z$) \\
@@ -938,8 +938,8 @@ Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Data
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
-inputfile  &     text attribute & 1 & verbose input file as a string (valid JSON) \\
-geomfile   &     text attribute & 1 & verbose geometry input file as a string (valid JSON) \\
+inputfile  &     text attribute & - & verbose input file as a string (valid JSON, no comments) \\
+geomfile   &     text attribute & - & verbose geometry input file as a string (valid JSON, no comments) \\
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 psip1d           & Coord. Var. & 1 (psip1d) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 539be5d6a..e50e2fef1 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -104,10 +104,11 @@ int main( int argc, char* argv[])
     Json::CharReaderBuilder parser;
     parser["collectComments"] = false;
     std::string errs;
-    if( argc != 4)
+    if( argc != 4 && argc != 5)
     {
         MPI_OUT std::cerr << "ERROR: Wrong number of arguments!\nUsage: "
-                << argv[0]<<" [inputfile] [geomfile] [outputfile]\n";
+                << argv[0]<<" [input.json] [geometry.json] [output.nc]\n OR \n"
+                << argv[0]<<" [input.json] [geometry.json] [output.nc] [initial.nc] \n";
         return -1;
     }
     else
@@ -150,7 +151,7 @@ int main( int argc, char* argv[])
     feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
-    /////////////////////The initial field///////////////////////////////////////////
+    //!///////////////////The profiles///////////////////////////////////////////
     //First the profile and the source (on the host since we want to output those)
     HVec profile = dg::pullback( dg::geo::Compose<dg::LinearX>( mag.psip(),
         p.nprofamp/mag.psip()(mag.R0(), 0.), 0.), grid);
@@ -177,68 +178,156 @@ int main( int argc, char* argv[])
 
     feltor.set_source( profile, p.omega_source, source_damping);
 
-    //Now perturbation
-    HVec ntilde = dg::evaluate(dg::zero,grid);
-    if( p.initne == "blob" || p.initne == "straight blob")
+    //!///////////////////The initial field///////////////////////////////////////////
+    double time = 0;
+    std::array<std::array<DVec,2>,2> y0;
+    if( argc == 4)
     {
-        dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-        dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
-        if( p.symmetric)
-            ntilde = dg::pullback( init0, grid);
-        else if( p.initne == "blob")//rounds =3 ->2*3-1
+        //Now perturbation
+        HVec ntilde = dg::evaluate(dg::zero,grid);
+        if( p.initne == "blob" || p.initne == "straight blob")
         {
-            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-            //evaluate should always be used with mx,my > 1
-            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 3);
+            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
+            dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
+            if( p.symmetric)
+                ntilde = dg::pullback( init0, grid);
+            else if( p.initne == "blob")//rounds =3 ->2*3-1
+            {
+                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+                //evaluate should always be used with mx,my > 1
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 3);
+            }
+            else if( p.initne == "straight blob")//rounds =1 ->2*1-1
+            {
+                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+                //evaluate should always be used with mx,my > 1
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+            }
         }
-        else if( p.initne == "straight blob")//rounds =1 ->2*1-1
+        else if( p.initne == "turbulence")
         {
-            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-            //evaluate should always be used with mx,my > 1
-            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
+            dg::BathRZ init0(16,16,Rmin,Zmin, 30.,5.,p.amp);
+            if( p.symmetric)
+                ntilde = dg::pullback( init0, grid);
+            else
+            {
+                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+                //evaluate should always be used with mx,my > 1
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+            }
+            dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
         }
-    }
-    else if( p.initne == "turbulence")
-    {
-        dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-        dg::BathRZ init0(16,16,Rmin,Zmin, 30.,5.,p.amp);
-        if( p.symmetric)
-            ntilde = dg::pullback( init0, grid);
-        else
+        else if( p.initne == "zonal")
         {
-            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-            //evaluate should always be used with mx,my > 1
-            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+            dg::geo::ZonalFlow init0(mag.psip(), p.amp, 0., p.k_psi);
+            ntilde = dg::pullback( init0, grid);
+            dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
         }
-        dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
+        else
+            MPI_OUT std::cerr <<"WARNING: Unknown initial condition!\n";
+        y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile);
+        dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
+        MPI_OUT std::cout << "initialize ni" << std::endl;
+        if( p.initphi == "zero")
+            feltor.initializeni( y0[0][0], y0[0][1]);
+        else if( p.initphi == "balance")
+            dg::blas1::copy( y0[0][0], y0[0][1]); //set N_i = n_e
+        else
+            MPI_OUT std::cerr <<"WARNING: Unknown initial condition for phi!\n";
+
+        dg::blas1::copy( 0., y0[1][0]); //set we = 0
+        dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
     }
-    else if( p.initne == "zonal")
+    if( argc == 5)
     {
-        dg::geo::ZonalFlow init0(mag.psip(), p.amp, 0., p.k_psi);
-        ntilde = dg::pullback( init0, grid);
-        dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
-    }
-    else
-        MPI_OUT std::cerr <<"WARNING: Unknown initial condition!\n";
-    std::array<std::array<DVec,2>,2> y0;
-    y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile);
-    dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-    MPI_OUT std::cout << "initialize ni" << std::endl;
-    if( p.initphi == "zero")
-        feltor.initializeni( y0[0][0], y0[0][1]);
-    else if( p.initphi == "balance")
-        dg::blas1::copy( y0[0][0], y0[0][1]); //set N_i = n_e
-    else
-        MPI_OUT std::cerr <<"WARNING: Unknown initial condition for phi!\n";
+        ///////////////////read in and show inputfile
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( argv[4], NC_NOWRITE, &ncidIN);
+        size_t lengthIN;
+        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
+        std::string inputIN( lengthIN, 'x');
+        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
+
+        Json::Value jsIN;
+        std::stringstream is(inputIN);
+        parseFromStream( parser, is, &jsIN, &errs); //read input without comments
+        const feltor::Parameters pIN(  jsIN);
+        MPI_OUT std::cout << "RESTART from file "<<argv[4]<< std::endl;
+        MPI_OUT std::cout << " file parameters:" << std::endl;
+        MPI_OUT pIN.display( std::cout);
+
+        // Now read in last timestep
+        Geometry grid_IN( Rmin, Rmax, Zmin, Zmax, 0, 2.*M_PI,
+            pIN.n_out, pIN.Nx_out, pIN.Ny_out, pIN.symmetric ? 1 : pIN.Nz_out, pIN.bcxN, pIN.bcyN, dg::PER
+            #ifdef FELTOR_MPI
+            , comm
+            #endif //FELTOR_MPI
+            );
+        IHMatrix interpolateIN = dg::create::interpolation( grid, grid_IN);
+
+        #ifdef FELTOR_MPI
+        int dimsIN[3],  coordsIN[3];
+        MPI_Cart_get( comm, 3, dimsIN, periods, coordsIN);
+        size_t countIN[4] = {1, grid_IN.local().Nz(),
+            grid_out.n()*(grid_IN.local().Ny()),
+            grid_out.n()*(grid_IN.local().Nx())};
+        size_t startIN[4] = {0, coordsIN[2]*countIN[1],
+                                coordsIN[1]*countIN[2],
+                                coordsIN[0]*countIN[3]};
+        #else //FELTOR_MPI
+        size_t startIN[4] = {0, 0, 0, 0};
+        size_t countIN[4] = {1, grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
+            grid_IN.n()*grid_IN.Nx()};
+        #endif //FELTOR_MPI
+        std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid_IN));
+        HVec transferINH( dg::evaluate(dg::zero, grid_IN));
 
-    dg::blas1::copy( 0., y0[1][0]); //set we = 0
-    dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
+        std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
+
+        int dataIDsIN[5];
+        int timeIDIN;
+        /////////////////////Get time length and initial data///////////////////////////
+        errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
+        errIN = nc_inq_dimlen(ncidIN, timeIDIN, &startIN[0]);
+        startIN[0] -= 1;
+        errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
+        errIN = nc_get_vara_double( ncidIN, timeIDIN, startIN, countIN, &time);
+        MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
+        for( unsigned i=0; i<5; i++)
+        {
+            errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataIDsIN[i]);
+            errIN = nc_get_vara_double( ncidIN, dataIDsIN[0], startIN, countIN,
+                #ifdef FELTOR_MPI
+                    transferINH.data().data()
+                #else //FELTOR_MPI
+                    transferINH.data()
+                #endif //FELTOR_MPI
+                );
+            dg::blas2::gemv( interpolateIN, transferINH, transferINHvec[i]);
+        }
+        errIN = nc_close(ncidIN);
+        /// ///////////////Now Construct initial fields
+        //
+        //Convert to N-1 and W
+        dg::blas1::plus( transferINHvec[0], -1.);
+        dg::blas1::plus( transferINHvec[1], -1.);
+        dg::blas1::axpby( 1., transferINHvec[2], p.beta/p.mu[0], transferINHvec[4], transferINHvec[2]);
+        dg::blas1::axpby( 1., transferINHvec[3], p.beta/p.mu[1], transferINHvec[4], transferINHvec[3]);
+
+        dg::assign( transferINHvec[0], y0[0][0]); //ne-1
+        dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
+        dg::assign( transferINHvec[2], y0[1][0]); //We
+        dg::assign( transferINHvec[3], y0[1][1]); //Wi
+
+    }
     ////////////map quantities to output/////////////////
     //since we map pointers we don't need to update those later
     std::map<std::string, const DVec* > v4d;
@@ -335,7 +424,7 @@ int main( int argc, char* argv[])
     }
     err = nc_enddef(ncid);
     ///////////////////////////////////first output/////////////////////////
-    double time = 0, dt_new = p.dt, dt =0;
+    double dt_new = p.dt, dt =0;
     MPI_OUT std::cout << "First output ... \n";
     //first, update quantities in feltor
     {
@@ -426,6 +515,8 @@ int main( int argc, char* argv[])
                         {
                             failed_counter++;
                             MPI_OUT std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
+                            time -= dt; // time has to be reset here
+                            // in case of failure diffusion is applied twice?
                         }
                     }while ( adaptive.failed());
                     dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
@@ -449,6 +540,7 @@ int main( int argc, char* argv[])
             accuracyM = 2.*fabs( (dMdt - *v0d["diff"])/( dMdt + *v0d["diff"]));
             #ifndef FELTOR_MPI
             err = nc_open(argv[3], NC_WRITE, &ncid);
+            //maybe MPI should use nc_open_par ? Or is problem if the ids are still the same?
             #endif //FELTOR_MPI
             Estart[0] = step;
             err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
-- 
GitLab


From a9d59fd29d7ae07730f2e6c70b34a7359e20a5d0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 Feb 2019 16:53:02 +0100
Subject: [PATCH 034/540] Hopeful fix of random bug in nvcc BathRZ

not entirely sure if there even was a bug, but BathRZ had different
results when compiled with nvcc, problem seemingly solved by replacing
thrust random with std::random
Not sure if that solves the infinity bugs in nvcc execution of feltor_hpc
---
 inc/dg/functors.h | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index fb5492965..40c6a09a9 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -6,9 +6,7 @@
 #define M_PI 3.14159265358979323846
 #endif
 #include <vector>
-#include <thrust/random/linear_congruential_engine.h>
-#include <thrust/random/uniform_real_distribution.h>
-#include <thrust/random/normal_distribution.h>
+#include <random>
 #include "blas1.h"
 #include "topology/grid.h"
 #include "topology/evaluation.h"
@@ -1522,7 +1520,7 @@ struct Vortex
 };
 
 /**
-* @brief Makes a random bath in the RZ plane
+* @brief A random bath in the RZ plane
 *
 \f[f(R,Z) = A B \sum_\vec{k} \sqrt{E_k} \alpha_k \cos{\left(k \kappa_k + \theta_k \right)}
 \f]
@@ -1568,9 +1566,9 @@ struct BathRZ{
         double N_kRh = N_kR_/2.;
         double N_kZh = N_kZ_/2.;
 
-        thrust::random::minstd_rand generator;
-        thrust::random::normal_distribution<double> ndistribution;
-        thrust::random::uniform_real_distribution<double> udistribution(0.0,tpi);
+        std::minstd_rand generator;
+        std::normal_distribution<double> ndistribution( 0.0, 1.0); // ( mean, stddev)
+        std::uniform_real_distribution<double> udistribution(0.0,tpi); //between [0 and 2pi)
         for (unsigned j=1;j<=N_kZ_;j++)
         {
             double kZ2=tpi2*(j-N_kZh)*(j-N_kZh)/(N_kZ2);
-- 
GitLab


From cd5f6ffd51f59c49ab8aee1fd58a045ab90bc25a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 Feb 2019 17:43:38 +0100
Subject: [PATCH 035/540] Debug and document restart in feltor_hpc

Also fixed energy_time bug in output file
---
 src/feltor/feltor.cuh    |  6 +--
 src/feltor/feltor.tex    | 89 ++++++++++++++++++++++++++--------------
 src/feltor/feltor_hpc.cu | 10 ++---
 3 files changed, 67 insertions(+), 38 deletions(-)

diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 2790d2cef..4191b4850 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -798,8 +798,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     const std::array<std::array<Container,2>,2>& y,
     std::array<std::array<Container,2>,2>& yp)
 {
-    dg::Timer timer;
-    timer.tic();
+    //dg::Timer timer;
+    //timer.tic();
 
     // set m_phi[0]
     compute_phi( t, y[0]);
@@ -857,7 +857,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
 #endif //DG_MANUFACTURED
-    timer.toc();
+    //timer.toc();
     //#ifdef MPI_VERSION
     //    int rank;
     //    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 642a8a073..30724f414 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -772,14 +772,29 @@ time & Strang Splitting & $2$nd order\\
 \qquad implicit & Trapezoidal & perp. Diffusion and Resistive term. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
 \bottomrule
 \end{longtable}
-\section{Input/Output}
-The programs \texttt{feltor\_hpc.cu} and \texttt{feltor.cu} expect two input
-files. One is for the physical and numerical parameters of the model equations
-and the other one describes the Solov'ev equilibrium. Results are written into
-an output file in the first program and plotted on screen with glfw in the
-second.
+\section{Usage}
+Compilation:\\
+\texttt{make feltor device=\{gpu,omp\}} Compile \texttt{feltor.cu} (only shared memory)\\
+\texttt{make feltor\_hpc device=\{gpu,omp\}} Compile \texttt{feltor\_hpc.cu} for shared memory system.\\
+\texttt{make feltor\_mpi device=\{gpu,omp,skl,knl\}} Compile \texttt{feltor\_hpc.cu} for distributed memory systems.\\
+Usage:\\
+\texttt{./feltor\_hpc input.json geometry.json output.nc [initial.nc]} \\
+\texttt{./feltor\_mpi input.json geometry.json output.nc [initial.nc]} \\
+\texttt{./feltor input.json geometry.json } \\
 
-\subsection{Input file structure}
+The programs \texttt{feltor\_hpc.cu} and \texttt{feltor.cu} expect two input
+files \texttt{input.json} and \texttt{geometry.json}, described in Sections~\ref{sec:input_file} and \ref{sec:geometry_file}.
+The first is for the physical and numerical parameters of the model equations
+while the latter describes the Solov'ev equilibrium.
+ The program \texttt{feltor.cu} plots the results directly to the screen using \texttt{glfw3}.
+The program \texttt{feltor\_hpc.cu} writes results into
+the output file \texttt{output.nc}.
+ The output file is described in Section~\ref{sec:output_file}.
+ The optional file \texttt{initial.nc} can be used to initialize a simulation from an existing file.
+ This behavior is described in Section~\ref{sec:restart_file}.
+
+
+\subsection{Input file structure} \label{sec:input_file}
 Input file format: json
 
 %%This is a booktabs table
@@ -837,20 +852,17 @@ posY       & float &0.0    & - & blob Z-position in units of $a$ \\
 sigma\_z    & float &0.25   & - & variance in units of $R_0$  \\
 k\_psi     & float &0    & - & zonal mode wave number  \\
 nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} \\
-alpha       & float & 0.02 & - & Width $\alpha$ of Heaviside profile in Eq.~\eqref{eq:heaviside_profile} and \eqref{eq:modified_psi} \\
+alpha       & float & 0.02 & - & Width $\alpha$ of Heaviside profile in Eq.~\eqref{eq:approx_heaviside} and \eqref{eq:modified_psi} \\
 source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
 rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source}  \\
 %damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
 rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0$ in Eq.~\eqref{eq:modified_psi}  \\
-%restart    & bool & false & false & If true, all input and geometry parameters except maxout,
-%    itstp and inner\_loop are ignored. Instead, the parameters and all fields are initialized from output.nc
-%    The actual output is then appended to output.nc This effectively continues an existing simulation.
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
 the value is not found,
 the program exits with an error message.
-\subsection{Geometry file structure}
+\subsection{Geometry file structure} \label{sec:geometry_file}
 File format: json
 
 %%This is a booktabs table
@@ -869,7 +881,7 @@ The default value is taken if the value name is not found in the input file. If
 the value is not found,
 the program exits with an error message.
 
-\subsection{Output}
+\subsection{Output} \label{sec:output_file}
 Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Dataset with the same name as a dimension.
 %
 %Name | Type | Dimensionality | Description
@@ -882,6 +894,8 @@ geomfile   &     text attribute & - & verbose geometry input file as a string (v
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 z                & Coord. Var. & 1 (z) & $\varphi$-coordinate (computational space, size: $N_z$) \\
+time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
+energy\_time     & Coord. Var. & 1 (energy\_time) & timesteps at which 1d variables are written (variable size: itstp$\cdot$maxout$+1$, dimension size: unlimited ) \\
 x\_XYZ           & Dataset & 3 (z,y,x) & Cartesian x-coordinate $x=R\sin(\varphi)$ \\
 y\_XYZ           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
 z\_XYZ           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
@@ -892,14 +906,12 @@ Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta(\rho_{s} - \rho
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
-time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
 electrons        & Dataset & 4 (time, z, y, x) & electron density $n_e$ \\
 ions             & Dataset & 4 (time, z, y, x) & ion density $N_i$ \\
 Ue               & Dataset & 4 (time, z, y, x) & electron velocity $u_e$ \\
 Ui               & Dataset & 4 (time, z, y, x) & ion velocity $U_i$ \\
 potential        & Dataset & 4 (time, z, y, x) & electric potential $\phi$ \\
 induction        & Dataset & 4 (time, z, y, x) & parallel vector potential $A_\parallel$ \\
-energy\_time     & Coord. Var. & 1 (energy\_time) & timesteps at which 1d variables are written (variable size: itstp$\cdot$maxout$+1$, dimension size: unlimited ) \\
 mass      & Dataset & 1 (energy\_time) & total mass integral Eq.~\eqref{eq:mass_conservation} \\
 diff      & Dataset & 1 (energy\_time) & total mass integral diffusion Eq.~\eqref{eq:mass_conservation} \\
 energy    & Dataset & 1 (energy\_time) & total energy integral Eq.~\eqref{eq:energy_conservation} \\
@@ -915,23 +927,27 @@ accuracy  & Dataset & 1 (energy\_time) & accuracy of energy theorem in time  \\
 aligned   & Dataset & 1 (energy\_time) & Alignment parameter $\int\dV \ln n_e\Delta_\parallel n_e$\\
 \bottomrule
 \end{longtable}
+\subsection{Restart file} \label{sec:restart_file}
+The program \texttt{feltor\_hpc.cu} has the possibility to initialize time and the fields with
+the results of a previous simulation. This behaviour is enabled by giving an additional file \texttt{initial.nc}
+to the command line. In this case the \texttt{initne} and \texttt{initphi} parameters of the input
+file are ignored. Instead, the fields \texttt{electrons, ions, Ue, Ui, induction} at the latest timestep
+are read from the given file, interpolated to the current grid and used to initialize the simulation.
+Apart from that the behaviour of the program is unchanged i.e. the magnetic field, profiles, resolutions, etc.
+are all taken from the regular input files. This means that the user must take care that these are consistent
+with the paramters in the existing \texttt{initial.nc} file. Also note that we try to discourage
+appending new results to an exisiting file directly,
+because if for some reason the cluster crashes and the file is corrupted
+the whole simulation is lost. It is safer to just merge files afterwards with for example\\
+\texttt{ncrcat output1.nc output2.nc output.nc}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Diagnostics}\label{sec:diagnostics}
+Compilation\\
+\texttt{make feltordiag} \\
+Usage \\
+\texttt{./feltordiag input.nc output.nc} \\
 We have the program \texttt{feltor/diag/feltordiag.cu}.
-This program reads in the output file above and writes into another output file.
-
-\begin{align}
- \vec j_e\cdot \vec \nabla\psi_p =& n_e\left( \vec v_E + \vec v_C + \vec v_{\nabla
- B} + u_e \left(\bhat + \tilde{\vec b}\right)\right) \cdot \vec
- \nabla\psi_p \nonumber\\
- =& \beta n_eu_e\left( A_\parallel \mathcal
- K_\kappa(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \nonumber\\
-  &+ n_e\frac{1}{B}[\phi, \psi_p]_\perp + \left(\tau_e + \mu_e u_e^2\right)
-   n_e\mathcal K_\kappa(\psi_p) + \tau_e n_e \mathcal K_{\nabla B}(\psi_p) \nonumber\\
-   f_e :=& \frac{\vec j_e\cdot\vec \nabla \psi_p}{|\vec \nabla\psi_p|} \quad
-   L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}\quad
-   L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}
-\end{align}
+This program reads a previously generated simulation file \texttt{input.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows.
 
 Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Dataset with the same name as a dimension.
 
@@ -957,7 +973,20 @@ total\_flux      & Dataset & 1 (time) & $\int \dV f_e$\\
 X $\in$ \{ electrons, ions, Ue, Ui, potential,
 induction, vorticity, fluxe, Lperpinv, Lparallelinv\} corresponding to \{
     $n_e$, $N_i$, $u_e$, $U_i$, $\phi$, $A_\parallel$, $-\Delta_\perp \phi$,
-    $f_e$, $L_\perp^{-1}$, $L_\parallel^{-1}$\}
+    $f_e$, $L_\perp^{-1}$, $L_\parallel^{-1}$\} with
+
+\begin{align}
+ \vec j_e\cdot \vec \nabla\psi_p =& n_e\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ B} + u_e \left(\bhat + \tilde{\vec b}\right)\right) \cdot \vec
+ \nabla\psi_p \nonumber\\
+ =& \beta n_eu_e\left( A_\parallel \mathcal
+ K_\kappa(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \nonumber\\
+  &+ n_e\frac{1}{B}[\phi, \psi_p]_\perp + \left(\tau_e + \mu_e u_e^2\right)
+   n_e\mathcal K_\kappa(\psi_p) + \tau_e n_e \mathcal K_{\nabla B}(\psi_p) \nonumber\\
+   f_e :=& \frac{\vec j_e\cdot\vec \nabla \psi_p}{|\vec \nabla\psi_p|} \quad
+   L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}\quad
+   L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}
+\end{align}
 
 
 
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index e50e2fef1..ef3681698 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -292,7 +292,6 @@ int main( int argc, char* argv[])
 
         std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
 
-        int dataIDsIN[5];
         int timeIDIN;
         /////////////////////Get time length and initial data///////////////////////////
         errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
@@ -303,8 +302,9 @@ int main( int argc, char* argv[])
         MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
         for( unsigned i=0; i<5; i++)
         {
-            errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataIDsIN[i]);
-            errIN = nc_get_vara_double( ncidIN, dataIDsIN[0], startIN, countIN,
+            int dataID;
+            errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataID);
+            errIN = nc_get_vara_double( ncidIN, dataID, startIN, countIN,
                 #ifdef FELTOR_MPI
                     transferINH.data().data()
                 #else //FELTOR_MPI
@@ -533,7 +533,7 @@ int main( int argc, char* argv[])
             }
             double deltat = time - previous_time;
             feltor.update_quantities();
-            MPI_OUT std::cout << "Timestep "<<dt_new<<"\n";
+            MPI_OUT std::cout << "Time "<<time<<" Timestep "<<dt_new<<"\n";
             dEdt = (*v0d["energy"] - E0)/deltat, dMdt = (*v0d["mass"] - M0)/deltat;
             E0 = *v0d["energy"], M0 = *v0d["mass"];
             accuracy  = 2.*fabs( (dEdt - *v0d["ediff"])/( dEdt + *v0d["ediff"]));
@@ -542,7 +542,7 @@ int main( int argc, char* argv[])
             err = nc_open(argv[3], NC_WRITE, &ncid);
             //maybe MPI should use nc_open_par ? Or is problem if the ids are still the same?
             #endif //FELTOR_MPI
-            Estart[0] = step;
+            Estart[0]++;
             err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
             for( auto pair : v0d)
                 err = nc_put_vara_double( ncid, id0d.at(pair.first),
-- 
GitLab


From 2c7aaad0c92977dc7ad88afd282bbc165f23749e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 28 Feb 2019 11:59:45 +0100
Subject: [PATCH 036/540] Fix interpolation bug in feltor_hpc restart

---
 src/feltor/feltor_hpc.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index ef3681698..25acdfd45 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -287,7 +287,7 @@ int main( int argc, char* argv[])
         size_t countIN[4] = {1, grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
             grid_IN.n()*grid_IN.Nx()};
         #endif //FELTOR_MPI
-        std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid_IN));
+        std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
         HVec transferINH( dg::evaluate(dg::zero, grid_IN));
 
         std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
-- 
GitLab


From ad327892e0b52b4d60b68880cb6e10329368582a Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <Matthias.Wiesenberger@uibk.ac.at>
Date: Thu, 28 Feb 2019 16:01:23 +0100
Subject: [PATCH 037/540] Tested signal handler in feltor_hpc

and found that only in connection with --signal=SIGINT@30
in submit script it seems to work properly, but not entirely certain
what happens when interrupt happens inside a file output
---
 src/feltor/feltor_hpc.cu | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 25acdfd45..fb0ab4d3e 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -36,15 +36,19 @@ using Geometry = dg::CylindricalGrid3d;
 int ncid = -1; //netcdf id (signal handler can close the file)
 
 #ifdef FELTOR_MPI
+//ATTENTION: in slurm should be used with --signal=SIGINT@30 (<signal>@<time in seconds>)
 void sigterm_handler(int signal)
 {
-    file :: NC_Error_Handle err;
-    std::cout << "sigterm_handler, got signal " << signal << std::endl;
-    std::cout << "ncid = " << ncid << std::endl;
-    if(ncid != -1)
+    int rank, size;
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    std::cout << " pid "<<rank<<" sigterm_handler, got signal " << signal << std::endl;
+    std::cout << " pid "<<rank<<" ncid = " << ncid << std::endl;
+    if( ncid != -1)
     {
+        file :: NC_Error_Handle err;
         err = nc_close(ncid);
-        std::cerr << "SIGTERM caught. Closing NetCDF file with id " << ncid << std::endl;
+        std::cout << " pid "<<rank<<" Closing NetCDF file with id " << ncid << std::endl;
+        ncid = -1;
     }
     MPI_Finalize();
     exit(signal);
-- 
GitLab


From 322cb78ce3cccfa867c0735710ca09bc5c6120a8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 17 Mar 2019 13:33:11 +0100
Subject: [PATCH 038/540] Update feltor.tex with Curl-b and b_perp notation

- cross-checked equations with Markus gyrofluid writeup (should be ok
with 2nd order FLR in electric potential and no FLR in magnetic
potential)
---
 src/feltor/feltor.tex | 96 ++++++++++++++++++++++---------------------
 1 file changed, 50 insertions(+), 46 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 30724f414..b2568c4fe 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -24,7 +24,7 @@ Given a vector field $\vec B(\vec x)$ with unit vector $\bhat(\vec x) := (\vec B
 we can define various differential operations.
 %Let us further assume that $\bhat$ is perturbed by the parallel
 %vector potential $A_\parallel$ via
-%$\tilde{ \vec b } := ({\nabla \times A_\parallel \bhat)}/{B}$
+%$\tilde{ \vec b }_\perp := ({\nabla \times A_\parallel \bhat)}/{B}$
 \rowcolors{2}{gray!25}{white}
 %\begin{longtable}{>{\RaggedRight}p{7cm}>{\RaggedRight}p{7cm}}
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
@@ -48,20 +48,20 @@ we can define various differential operations.
     $ \Delta_\perp f:= \vec \nabla\cdot (\vec \nabla_\perp f)
     = \nabla\cdot( h\cdot\nabla f)$  \\
     Curl-b Curvature &
-    $\mathcal K_\kappa$ &
-    $\mathcal K_\kappa(f) := \vec{ \mathcal K_\kappa }\cdot \vec \nabla f = \frac{1}{B}(\nabla \times \bhat)\cdot \vec \nabla f$ \\
+    $\mathcal K_{\nabla\times\bhat}$ &
+    $\mathcal K_{\nabla\times\bhat}(f) := \vec{ \mathcal K_{\nabla\times\bhat} }\cdot \vec \nabla f = \frac{1}{B}(\nabla \times \bhat)\cdot \vec \nabla f$ \\
     Grad-B Curvature &
     $\mathcal K_{\nabla B} $ &
     $\mathcal K_{\nabla B}(f) := \vec{\mathcal K_{\nabla B}} \cdot \vec \nabla f = \frac{1}{B}(\bhat \times \vec \nabla \ln B)\cdot \vec \nabla f$ \\
     Curvature &
     $\mathcal K$ &
-    $\mathcal{K}(f):=\vec{\mathcal K} \cdot \vec \nabla f = \vec{\nabla} \cdot \left(\frac{ \bhat \times \vec{\nabla} f}{B} \right) = \mathcal K_\kappa(f) + \mathcal K_{\nabla B}(f)$,\\
+    $\mathcal{K}(f):=\vec{\mathcal K} \cdot \vec \nabla f = \vec{\nabla} \cdot \left(\frac{ \bhat \times \vec{\nabla} f}{B} \right) = \mathcal K_{\nabla\times\bhat}(f) + \mathcal K_{\nabla B}(f)$,\\
     Parallel derivative&
     $\nabla_\parallel $&
     $ \nabla_\parallel f := \bhat\cdot\vec{\nabla} f$ \\
     %Perturbed parallel Derivative&
     %$\bar\nabla_\parallel$ &
-    %$\bar\nabla_\parallel f := (\bhat + \tilde{\vec b })\cdot \nabla f = \nabla_\parallel f + A_\parallel \mathcal K_\kappa(f) + \frac{1}{B}[ f, A_\parallel]_\perp$ \\
+    %$\bar\nabla_\parallel f := (\bhat + \tilde{\vec b }_\perp)\cdot \nabla f = \nabla_\parallel f + A_\parallel \mathcal K_{\nabla\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$ \\
     Parallel Laplacian&
     $\Delta_\parallel $&
     $\Delta_\parallel f:= \vec{\nabla} \cdot ( \bhat\bhat\cdot\vec{\nabla} f )$\\
@@ -73,10 +73,10 @@ Explicit expressions for the above expressions
 depend on the choice of the magnetic field and the underlying coordinate system.
 Note that we have
 \begin{align}
-    \vec \nabla \cdot \vec{\mathcal K_\kappa}
-    &= -\vec\nabla \cdot \vec{\mathcal K_{\nabla B}} = -\vec{ \mathcal K_\kappa}\cdot\nabla\ln B, \\
+    \vec \nabla \cdot \vec{\mathcal K_{\nabla\times\bhat}}
+    &= -\vec\nabla \cdot \vec{\mathcal K_{\nabla B}} = -\vec{ \mathcal K_{\nabla\times\bhat}}\cdot\nabla\ln B, \\
     \vec\nabla\cdot\vec{ \mathcal K} &= 0, \\
-    \vec{ \mathcal K_\kappa} - \vec{ \mathcal K_{\nabla B}} &= \frac{1}{B^2} (\vec \nabla \times \vec B), \\
+    \vec{ \mathcal K_{\nabla\times\bhat}} - \vec{ \mathcal K_{\nabla B}} &= \frac{1}{B^2} (\vec \nabla \times \vec B), \\
     \nabla_\parallel \ln B &= -\vec\nabla\cdot\bhat.
     \label{eq:curl_curvature}
 \end{align}
@@ -241,18 +241,18 @@ The curl of $\bhat$ reduces to
 %end{align}
 This simplifies the curvature operators to:
 \begin{align}
-\vec{\mathcal{K}}_{\kappa}  &\approx  -  \frac{1}{B R} \ehat_Z , &
+\vec{\mathcal{K}}_{{\nabla\times\bhat}}  &\approx  -  \frac{1}{B R} \ehat_Z , &
 \vec{ \mathcal{K} }_{\vec{\nabla}  B}  &\approx  -\frac{1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
 %\ehat_\varphi \times \vec{\nabla} B, &
-\vec{ \mathcal{K} } &\approx \vec{ \mathcal{K} }_{\vec{\nabla}  B}  +\vec{ \mathcal{K} }_{\kappa} ,
+\vec{ \mathcal{K} } &\approx \vec{ \mathcal{K} }_{\vec{\nabla}  B}  +\vec{ \mathcal{K} }_{{\nabla\times\bhat}} ,
 %\\
-%\mathcal{K}_{\kappa}(f)   &\approx  -  \frac{1}{B R} \frac{\partial f}{\partial Z},&
+%\mathcal{K}_{{\nabla\times\bhat}}(f)   &\approx  -  \frac{1}{B R} \frac{\partial f}{\partial Z},&
 %\mathcal{K}_{\vec{\nabla}  B} (f)  &= \frac{1}{B} \left[\ln B, f \right]_{RZ},&
 %\mathcal{K} (f) &\approx\frac{1}{B} \left[\ln B, f \right]_{RZ}-  \frac{1}{B R} \frac{\partial f}{\partial Z} ,
 \end{align}
 and
 \begin{align}
- \vec{\nabla} \cdot \vec{\mathcal{K}}_{\kappa} &\approx \frac{1}{R B^2} \frac{\partial B}{\partial Z}.
+ \vec{\nabla} \cdot \vec{\mathcal{K}}_{{\nabla\times\bhat}} &\approx \frac{1}{R B^2} \frac{\partial B}{\partial Z}.
 \end{align}
 which, results in a vanishing divergence of the curvature operators \( \vec{\nabla} \cdot \vec{ \mathcal{K} } = 0\).
 
@@ -264,9 +264,9 @@ Note that in an actual toroidal field we have
 We then have $\bhat = \ehat_\varphi$ and the curvature operators further
 simplify to
 \begin{align}
-  \vec{ \mathcal K_\kappa} = \vec{ \mathcal K_{\nabla B}} = -\frac{1}{R_0} \ehat_Z =
+  \vec{ \mathcal K_{\nabla\times\bhat}} = \vec{ \mathcal K_{\nabla B}} = -\frac{1}{R_0} \ehat_Z =
 \vec{ \mathcal K}/2\\
-  \nabla\cdot\vec{\mathcal K_{\kappa}}=
+  \nabla\cdot\vec{\mathcal K_{{\nabla\times\bhat}}}=
     \nabla_\parallel \ln B = 0
     \label{}
 \end{align}
@@ -275,7 +275,7 @@ simplify to
 In this approximation we apply the toroidal field line approximation
 as in Section
 \ref{sec:torfieldlineapprox}
-but approximate the curvature operator $\mathcal K_\kappa \approx \bhat\times\kappa$
+but approximate the curvature operator $\mathcal K_{\nabla\times\bhat} \approx \bhat\times\vec \kappa$
   with
   $\vec \kappa := \bhat \cdot \vec \nabla\bhat = -\bhat \times(\vec \nabla\times \bhat)$.
 For an isotropic pressure plasma \(\vec{P} = \vec{I} P_\perp + \vec{b} \vec{b} P_\Delta \approx \vec{I} P_\perp\) and with the definition of the plasma beta parameter
@@ -290,7 +290,8 @@ In low beta plasmas \(\beta\ll1\) the curvature reduces to:
 \end{align}
 This simplifies the curvature operators to:
 \begin{align}
-\mathcal{K}_{\kappa}(f)   &\approx  \mathcal{K}_{\vec{\nabla}  B}(f),  &
+\vec{\mathcal{K}_{{\nabla\times\bhat}}}(f) \approx
+\vec{ \mathcal{K} }_{\vec{\nabla}  B}  &\approx  -\frac{1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
 \mathcal{K} (f) &\approx 2\mathcal{K}_{\vec{\nabla}  B} (f) , &
  \vec{\kappa} \cdot \vec{\mathcal{K}}_{\vec{\nabla}  B} &= 0.
 \end{align}
@@ -318,16 +319,16 @@ K_{\nabla B}^\varphi &= \frac{R_0}{B^3R^2}\left(
 \end{align}
 and
 \begin{align}
-K_\kappa^R &= \frac{R_0 }{RB^3}\left( B\frac{\partial I}{\partial Z} -I\frac{\partial B}{\partial Z}\right) \\
-K_\kappa^Z &= \frac{R_0 }{RB^3} \left( I\frac{\partial B}{\partial R} - B\frac{\partial I}{\partial R} \right)\\
-K_\kappa^\varphi &= \frac{R_0}{R^2B^2}\left(
+K_{\nabla\times\bhat}^R &= \frac{R_0 }{RB^3}\left( B\frac{\partial I}{\partial Z} -I\frac{\partial B}{\partial Z}\right) \\
+K_{\nabla\times\bhat}^Z &= \frac{R_0 }{RB^3} \left( I\frac{\partial B}{\partial R} - B\frac{\partial I}{\partial R} \right)\\
+K_{\nabla\times\bhat}^\varphi &= \frac{R_0}{R^2B^2}\left(
 + \frac{1}{B}\frac{\partial\psi}{\partial Z} \frac{\partial B}{\partial Z}
 + \frac{1}{B}\frac{\partial \psi}{\partial R}\frac{\partial B}{\partial R}
 -R\frac{\partial}{\partial R}\left(\frac{1}{R}\frac{\partial\psi}{\partial R}\right) 
 - \frac{\partial^2 \psi}{\partial Z^2}
 \right) \\
-\vec\nabla\cdot\vec{\mathcal K_\kappa} &= -\vec\nabla\cdot\vec{\mathcal K_{\nabla B}}=
-    -\vec{\mathcal K_\kappa}\cdot \vec \nabla\ln B = \frac{R_0}{RB^3}[I,B]_{RZ}
+\vec\nabla\cdot\vec{\mathcal K_{\nabla\times\bhat}} &= -\vec\nabla\cdot\vec{\mathcal K_{\nabla B}}=
+    -\vec{\mathcal K_{\nabla\times\bhat}}\cdot \vec \nabla\ln B = \frac{R_0}{RB^3}[I,B]_{RZ}
 %contravariant phi component
 \label{}
 \end{align}
@@ -410,28 +411,31 @@ Note that $\Theta(0) = 0.5$ and $\theta(0) = 35\alpha/256$.
 % Since it is based on a global geometry, treating the complete poloidal flux surface, it is suited for coupled simulations of core, edge and SOL.
 % However, our isothermal model misses important core physics ingredients, such as the ion temperature gradient (ITG) and trapped electron mode (TEM)~\cite{Wesson07}. Thus the validity the global model is limited to certain
 % parameter regimes.\\
+We set up an isothermal 3d gyro-fluid model with up to 2nd order FLR effects
+on in the electric potential $\phi$ and 0th order FLR effects in the parallel magnetic
+potential $A_\parallel$.
 The continuity equation for the electron density \(n_e\) and the ion gyro-centre
 density \(N_i\) and the momentum conservation equation for
 the parallel electron velocity \(u_e\) and the parallel ion gyro-centre velocity \(U_i\) are
 (omitting species labels)~\cite{WiesenbergerPhD, HeldPhD}
 \begin{align}
 \frac{\partial}{\partial t} N &+ \vec\nabla\cdot\left( N \left(
-    \vec v_E + \vec v_C + \vec v_{\nabla B} + U(\bhat + \tilde{\vec b})\right)\right) = \Lambda_N + S_N \\
+    \vec v_E + \vec v_C + \vec v_{\nabla B} + U\left(\bhat + \tilde{\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
 mN \frac{\partial}{\partial t} U &+ mN \left(
-    \vec v_E + \vec v_C + \vec v_{\nabla B} + U(\bhat + \tilde{\vec b})
+    \vec v_E + \vec v_C + \vec v_{\nabla B} + U\left(\bhat + \tilde{\vec b}_\perp\right)
     \right)\cdot \vec\nabla U  \nonumber \\
-    &+ 2m\vec \nabla \cdot ( NU \vec v_\kappa)
-    -mNU\vec \nabla\cdot \vec v_\kappa
-    + mNU\mathcal K_\kappa(\phi) \nonumber\\
-    &= -T (\bhat + \tilde{\vec b})\cdot \nabla N -qN \left( (\bhat+\tilde{\vec b})\cdot \nabla \psi + \frac{\partial A_\parallel}{\partial t}\right) + mN R_{\eta_\parallel} + mN(\Lambda_U + S_U)
+    &+ 2m\vec \nabla \cdot ( NU \vec v_{\nabla\times\bhat})
+    -mNU\vec \nabla\cdot \vec v_{\nabla\times\bhat}
+    + mNU\mathcal K_{\nabla\times\bhat}(\psi) \nonumber\\
+    &= -T \left(\bhat + \tilde{\vec b}_\perp\right)\cdot \nabla N -qN \left( \left(\bhat+\tilde{\vec b}_\perp\right)\cdot \nabla \psi + \frac{\partial A_\parallel}{\partial t}\right) + mN R_{\eta_\parallel} + mN(\Lambda_U + S_U)
 \label{}
 \end{align}
 with
 \begin{align}
-\tilde{\vec b} = \frac{\nabla\times A_\parallel \bhat}{B}, \quad
+\tilde{\vec b}_\perp = \frac{\nabla\times A_\parallel \bhat}{B}, \quad
 \vec v_E := \frac{\bhat\times\nabla\psi}{B},\quad
-\vec v_C := \frac{T+mU^2}{q}\vec{\mathcal K_\kappa},\quad
-\vec v_\kappa := \frac{T}{q}\vec{\mathcal K_\kappa},\quad
+\vec v_C := \frac{T+mU^2}{q}\vec{\mathcal K_{\nabla\times\bhat}},\quad
+\vec v_{\nabla\times\bhat} := \frac{T}{q}\vec{\mathcal K_{\nabla\times\bhat}},\quad
 \vec v_{\nabla B} := \frac{T}{q}\vec{\mathcal K_{\nabla B}}.
 \label{}
 \end{align}
@@ -482,7 +486,7 @@ Similarly, for the perpendicular dissipation we apply viscous or hyperviscous te
 \end{align}
 Here the mass diffusion coefficient coincides with the viscous coefficient, hence we fixed the Schmidt number \(\mathit{Sc}_\parallel:= \frac{\nu_U}{\nu_N}\) to unity.
 
-\subsection{Boundary and Initial conditions}
+\subsection{Boundary and initial conditions}
 We define the simulation box as
 $[ R_{\min}, R_{\max}]\times [Z_{\min}, Z_{\max}] \times [0,2\pi]$,
 where we define
@@ -519,7 +523,7 @@ and initialize the electron density with
     n_e(R,Z,\varphi, 0)= n_{prof}(R,Z) + \tilde n(R,Z,\varphi)
 \end{align}
 consisting of a toroidally symmetric background profile $n_{\text{prof}}(R,Z)$ and a perturbation
-$\tilde n(R,Z,\varphi)$.
+$\tilde n(R,Z,\varphi)$, which breaks the toroidal symmetry.
 Note that we should take care to intitialize a smooth profile with ideally well-defined $\Delta^2_\perp n_e$.
 %Let us define another approximation to the Heaviside function
 %\begin{align}
@@ -636,9 +640,9 @@ boundary)
     +\frac{1}{2} m_i N_i \left(\frac{\vec\nabla_\perp\phi}{B}\right)^2 \right.\nonumber\\
     &\left.+\frac{1}{2} m_e  n_e u_e^2 +\frac{1}{2} m_i  N_i U_i^2  \right),\\
   \mathcal S = \sum_s \int \vec{\dA} \cdot &\left[ \left(
-  U(\bhat+\tilde{\vec b}) + \vec v_E + \vec v_C + \vec v_{\nabla B} \right)
+  U\left(\bhat+\tilde{\vec b}_\perp\right) + \vec v_E + \vec v_C + \vec v_{\nabla B} \right)
   \left(T N\ln N + \frac{1}{2}m NU^2 + q\psi N \right) \right. \nonumber\\
-  & \left . + \vec v_\kappa m NU^2  + (\bhat + \tilde{\vec b}) UNT\right], \\
+  & \left . + \vec v_{\nabla\times\bhat} m NU^2  + \left(\bhat + \tilde{\vec b}_\perp\right) UNT\right], \\
   \Lambda =  \int \dV & \bigg\{  \left[t_e\left( 1+\ln{(n_e)}\right) -e \phi + \frac{1}{2} m_e u_e^2 \right](\Lambda_{n_e} + S_{n_e})
   \nonumber\\ &
 +\left[T_i\left( 1+\ln{(N_i)}\right) +e \psi_i + \frac{1}{2} m_i U_i^2 \right](\Lambda_{N_i}+S_{N_i})
@@ -671,22 +675,22 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
     \frac{\partial}{\partial t} N =&
         - \frac{1}{B}[\psi, N]_{\perp}%\nonumber\\
         - \bar \nabla_\parallel \left( NU\right)
-        - NU\left(\vec \nabla\cdot\bhat+\vec \nabla\cdot\tilde{\vec b}\right)
+        - NU\left(\vec \nabla\cdot\bhat+\vec \nabla\cdot\tilde{\vec b}_\perp\right)
         - \tau \mathcal K(N) \nonumber \\&
         - N \mathcal K(\psi)
-        -\mu \mathcal K_\kappa(NU^2)
-        -\mu NU^2\nabla\cdot \vec{ \mathcal K_\kappa}
+        -\mu \mathcal K_{\nabla\times\bhat}(NU^2)
+        -\mu NU^2\nabla\cdot \vec{ \mathcal K_{\nabla\times\bhat}}
         + \nu_\perp\Delta_\perp N + \nu_\parallel \Delta_\parallel N + S_N, \\
     \frac{\partial}{\partial t} W =&
         - \frac{1}{B}\left[\psi, U\right]_{\perp}%& \nonumber\\
         - \frac{1}{\mu} \bar \nabla_\parallel \psi% \nonumber\\
         - \frac{1}{2}\bar \nabla_\parallel U^2
         -\frac{\tau}{\mu} \bar \nabla_\parallel \ln N
-        - U\mathcal K_\kappa(\psi)
+        - U\mathcal K_{\nabla\times\bhat}(\psi)
         - \tau \mathcal K(U)
-        -\tau U\nabla\cdot\vec{ \mathcal K_\kappa}\nonumber\\&
-        - \left(2\tau + {\mu}U^2\right) \mathcal K_\kappa (U)
-        -2\tau U\mathcal K_\kappa(\ln N)
+        -\tau U\nabla\cdot\vec{ \mathcal K_{\nabla\times\bhat}}\nonumber\\&
+        - \left(2\tau + {\mu}U^2\right) \mathcal K_{\nabla\times\bhat} (U)
+        -2\tau U\mathcal K_{\nabla\times\bhat}(\ln N)
         - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
         + \nu_\perp\Delta_\perp W
         - \nu_\perp\frac{\beta}{\mu}\Delta_\perp A_\parallel
@@ -697,8 +701,8 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
     \label{eq:Egyrofluid}
 \end{subequations}
 together with
-$\bar\nabla_\parallel f = \nabla_\parallel f + \beta A_\parallel \mathcal K_\kappa(f) + \frac{\beta}{B}[ f, A_\parallel]_\perp$
-and $\vec \nabla \cdot \tilde{ \vec b} = \beta A_\parallel \vec \nabla\cdot\vec{ \mathcal{ K_\kappa}} - \beta \mathcal K_{\nabla B}(A_\parallel) $
+$\bar\nabla_\parallel f = \nabla_\parallel f + \beta A_\parallel \mathcal K_{\nabla\times\bhat}(f) + \frac{\beta}{B}[ f, A_\parallel]_\perp$
+and $\vec \nabla \cdot \tilde{ \vec b}_\perp = \beta A_\parallel \vec \nabla\cdot\vec{ \mathcal{ K}_{\nabla\times\bhat}} - \beta \mathcal K_{\nabla B}(A_\parallel) $
 and
 \begin{subequations} \label{eq:elliptic}
   \begin{align}
@@ -977,12 +981,12 @@ induction, vorticity, fluxe, Lperpinv, Lparallelinv\} corresponding to \{
 
 \begin{align}
  \vec j_e\cdot \vec \nabla\psi_p =& n_e\left( \vec v_E + \vec v_C + \vec v_{\nabla
- B} + u_e \left(\bhat + \tilde{\vec b}\right)\right) \cdot \vec
+ B} + u_e \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
  \nabla\psi_p \nonumber\\
  =& \beta n_eu_e\left( A_\parallel \mathcal
- K_\kappa(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \nonumber\\
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \nonumber\\
   &+ n_e\frac{1}{B}[\phi, \psi_p]_\perp + \left(\tau_e + \mu_e u_e^2\right)
-   n_e\mathcal K_\kappa(\psi_p) + \tau_e n_e \mathcal K_{\nabla B}(\psi_p) \nonumber\\
+   n_e\mathcal K_{\nabla\times\bhat}(\psi_p) + \tau_e n_e \mathcal K_{\nabla B}(\psi_p) \nonumber\\
    f_e :=& \frac{\vec j_e\cdot\vec \nabla \psi_p}{|\vec \nabla\psi_p|} \quad
    L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}\quad
    L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}
-- 
GitLab


From 51385452899789bdbe10414c1599166922948208 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 18 Mar 2019 16:04:16 +0100
Subject: [PATCH 039/540] Update normalization for Psi in feltor.tex

---
 inc/geometries/geometry_diag.cu |  1 +
 src/feltor/feltor.tex           | 41 ++++++++++++++++++++-------------
 2 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index aa0a689ac..d23647743 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -150,6 +150,7 @@ int main( int argc, char* argv[])
     const double N1 = -(1.+alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.+alpha_);
     const double N2 =  (1.-alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.-alpha_);
     const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
+    std::cout << "psip( 1, 0) "<<c.psip()(gp.R_0, 0)<<"\n";
     std::cout << "TEST ACCURACY OF PSI (values must be close to 0!)\n";
     if( gp.hasXpoint())
         std::cout << "    Equilibrium with X-point!\n";
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index b2568c4fe..b530a2a31 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -100,13 +100,13 @@ obtain a right handed system. The parametric representation in Cartesian \((x,y,
 \end{align}
 Note here that the angle $\varphi = 0$ corresponds to the Cartesian $y$-axis.
 The unit
-basis vectors and metric tensor are:
+basis vectors and (covariant) metric tensor are:
 \begin{align}
  \ehat_R      &= (\sin{(\varphi)} ,   \cos{(\varphi)},0)^T, &
  \ehat_Z      &= ( 0 ,0 ,1 )^T, &
  \ehat_{\varphi} &= ( \cos{(\varphi)} , -\sin{(\varphi)} , 0 )^T,
 \\
- g &= \begin{pmatrix}
+    (g_{ij}) &= \begin{pmatrix}
   1 & 0 & 0 \\
   0 & 1 & 0 \\
   0 & 0 & R^2
@@ -119,13 +119,19 @@ With the help of the metric elements we get a well behaved volume element \(\sqr
 The cylindrical coordinate basis vectors are mutually orthogonal to each other.
 
 \subsection{Solov'ev equilbrium}\label{sec:solovev}
-In cylindrical coordinates the general axisymmetric magnetic field normalized with $R_0$ can be written as
+In cylindrical coordinates the general axisymmetric  magnetic field can be written as (dimensionless)
 \begin{align}
  \vec{B} &= \frac{R_0}{R}\left[I(\psi) \ehat_{\varphi} + \frac{\partial
  \psi_p}{\partial Z} \ehat_R -  \frac{\partial \psi_p}{\partial R} \ehat_Z\right] ,
 \end{align}
-which can obviously not be manipulated to be in Clebsch form. Hence we are dealing with a non-flux aligned coordinate system.
-Note that with a typically convex function $\psi$, $I(\psi)>0$ and the previously defined coordinate system the field line winding is a {\bf left handed screw} in the positive $\vec B$-direction.
+which can obviously not be manipulated to be in Clebsch form.
+Hence we are dealing with a non-flux aligned coordinate system.
+We scaled $R$, $Z$ and $R_0$ with $\rho_s$, the magnetic field with $B_0$, the flux with $\psi_{p0} = B_0\rho_s \hat R_0$
+and $I_0 = B_0 \hat R_0$ (with $\hat R_0$ being the dimensional major radius).
+Note that this normalization is in line with the one later chosen for the gyrofluid
+equations but is unnatural for the MHD type equilibrium equations through the introduction
+of $\rho_s$.
+Also note that with a typically convex function $\psi$, $I(\psi)>0$ and the previously defined coordinate system the field line winding is a {\bf left handed screw} in the positive $\vec B$-direction.
 For the sake of clarity we define the poloidal magnetic field \( \vec{B}_p = \frac{R_0}{R}\left( \frac{\partial \psi}{\partial Z}\ehat_R - \frac{\partial \psi}{\partial R}\ehat_Z\right)
 \) and the toroidal magnetic field \(\vec{B}_t =\frac{R_0I}{R} \ehat_{\varphi}\).
 %The unit vectors are denoted by \(\ehat_{R}\), \(\ehat_{Z}\), \(\ehat_{\varphi}\).
@@ -135,7 +141,7 @@ Cartesian coordinates while straightening the field lines for large $R_0$
 we recover the familiar slab geometry.
 We have
 the equilibrium equations in toroidally symmetric, ideal MHD (
-$\vec\nabla p = \vec j\times \vec B$ and $\vec \nabla\times\vec B = \vec j$ )
+$\vec\nabla p = \vec j\times \vec B$ and $\vec \nabla\times\vec B = \vec j$ normalized with $p_0 = B_0^2/\mu_0$, and $j_0 = B_0/\rho_s/\mu_0$ )
 \begin{align}
     \vec\nabla\times \vec B &= \frac{R_0}{R}\left[ -\Delta^*\psi_p\ehat_\varphi + I_Z \ehat_R - I_R\ehat_Z \right]\equiv \vec j\\
  j_\parallel &= \vec j\cdot \bhat = \frac{\d p}{\d\psi_p} \frac{I(\psi_p)}{B} +
@@ -150,10 +156,6 @@ from where we recover the Grad-Shafranov equation
     -\Delta^*_\perp  \psi_p &= \frac{R^2}{R_0^2} \frac{d p}{d  \psi_p } + I \frac{d I}{d  \psi_p } \equiv \frac{R}{R_0} j_{\hat\varphi}
 \end{align}
 with $\Delta^*_\perp \psi_p = R\partial_R (R^{-1}\psi_R) + \psi_{ZZ}$.
-Note that $R$ and $Z$ are normalized
-with $\rho_s$, $\psi_p$ with $\psi_{p0}$, $p$ with
-$p_0 := \psi_{p0}^2/\mu_0\rho_s^4$, $I$ with $I_0:=\psi_{p0}/\rho_s$,
-and $j_\varphi$ with $\j_{\varphi 0} = \psi_{p0}/\rho_s^3\mu_0$.
 The Solov'ev assumptions consist of \(A/R_0 = -I \frac{d I}{d  \psi_p }\) and \((1-A)/R_0 = -\frac{d p}{d  \psi_p }\), where \(A\) is a constant~\cite{Cerfon2010,Cerfon2014}.
 By integration over \(\psi_p\) we find
 \begin{align}\label{eq:solovevassumption}
@@ -282,32 +284,38 @@ For an isotropic pressure plasma \(\vec{P} = \vec{I} P_\perp + \vec{b} \vec{b} P
 \(\beta = \frac{P}{B^2/(2 \mu_0) } \)
 we can rewrite the curvature to
 \begin{align}
- \vec{\kappa} &\approx \frac{\beta}{2} \vec{\nabla} \ln(P) +\vec{\nabla}_\perp \ln{B} .
+    \vec{\kappa} &\approx \frac{\beta}{2} \vec{\nabla} \ln(P) +\vec{\nabla}_\perp \ln{B} .
 \end{align}
 In low beta plasmas \(\beta\ll1\) the curvature reduces to:
 \begin{align}
- \vec{\kappa} & \approx \vec{\nabla}_\perp \ln{B} .
+    \vec{\kappa} & \approx \vec{\nabla}_\perp \ln{B} .
 \end{align}
 This simplifies the curvature operators to:
 \begin{align}
 \vec{\mathcal{K}_{{\nabla\times\bhat}}}(f) \approx
 \vec{ \mathcal{K} }_{\vec{\nabla}  B}  &\approx  -\frac{1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
 \mathcal{K} (f) &\approx 2\mathcal{K}_{\vec{\nabla}  B} (f) , &
- \vec{\kappa} \cdot \vec{\mathcal{K}}_{\vec{\nabla}  B} &= 0.
+    \vec{{\nabla\times\bhat}} \cdot \vec{\mathcal{K}}_{\vec{\nabla}  B} &= 0.
 \end{align}
 The divergence over the curvature vanishes \( \vec{\nabla} \cdot \vec{ \mathcal{K} } = 0\) only if \( \vec{\nabla} \cdot \vec{ \mathcal{K}}_{\vec{\nabla}  B}   = 0\).
 In general, the divergence \( \vec{\nabla} \cdot \vec{ \mathcal{K} } \approx 0\) is only approximately vanishing.
 \subsubsection{True perpendicular terms}
+
 Without any approximations we have
 \begin{align}
 b^R = {\frac{\partial \psi}{\partial Z}}\left(I^2+|\nabla\psi|^2\right)^{-1/2} \quad
 b^Z = -{\frac{\partial \psi}{\partial R}}\left(I^2+|\nabla\psi|^2\right)^{-1/2} \quad 
 b^\varphi = \frac{I}{R}\left(I^2+|\nabla\psi|^2\right)^{-1/2} \\
-\vec\nabla\cdot\bhat = -\nabla_\parallel \ln B = -\frac{R_0}{R B^2}[B,\psi_p]_{RZ}
+\vec\nabla\cdot\bhat = -\nabla_\parallel \ln B = -\frac{R_0}{R B^2}[B,\psi_p]_{RZ} \\
+\left({\nabla\times\bhat}\right) \cdot\bhat =
+    (I'(\nabla\psi_p)^2 - I \Delta_\perp^* \psi_p)\frac{ R_0^2}{R^2B^2} \propto 1/R_0
 \label{}
 \end{align}
-We then need to take the exact definitions for $[.,.]_\perp$, $\nabla_\perp$ and $\Delta_\perp$ from Section~\ref{sec:magnetic}.
-We can explicitly write
+where for the last
+estimate we inserted the Grad-Shafranov equation and the Solov'ev assumptions.
+We can then insert $\bhat$ into the exact definitions for $[.,.]_\perp$, $\nabla_\perp$ and $\Delta_\perp$ from Section~\ref{sec:magnetic}.
+
+For the curvature terms we can explicitly write
 \begin{align}
 K_{\nabla B}^R &= -\frac{R_0 I}{B^3R}\frac{\partial B}{\partial Z} \equiv -\frac{1}{B^2}\frac{\partial B}{\partial Z}b^\varphi \\
 K_{\nabla B}^Z &= \frac{R_0 I}{B^3R}\frac{\partial B}{\partial R}\equiv \frac{1}{B^2}\frac{\partial B}{\partial R}b^\varphi \\
@@ -332,6 +340,7 @@ K_{\nabla\times\bhat}^\varphi &= \frac{R_0}{R^2B^2}\left(
 %contravariant phi component
 \label{}
 \end{align}
+
 \subsection{Flux surface average and Safety factor}
 We define the toroidal average of a function $f(R,Z,\varphi)$ as
 \begin{align} \label{eq:phi_average}
-- 
GitLab


From 2f745406ca8cf35cef0799c50fb64ba54fcb132c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 Mar 2019 15:03:50 +0100
Subject: [PATCH 040/540] Revert feltor back to Karniadakis

- Adaptive just does not seem to work so well
---
 src/feltor/feltor.cu     | 51 ++++++++++++++++++++++-----------------
 src/feltor/feltor.tex    |  9 +++----
 src/feltor/feltor_hpc.cu | 52 ++++++++++++++++++++++------------------
 src/feltor/implicit.h    |  3 +--
 4 files changed, 64 insertions(+), 51 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index dbb63141c..b0c7f64fe 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -146,13 +146,19 @@ int main( int argc, char* argv[])
     dg::Timer t;
     double time = 0, dt_new = p.dt, dt =0;
     unsigned step = 0;
-    dg::Adaptive< dg::ERKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
-        "Bogacki-Shampine-4-2-3", y0);
-    adaptive.stepper().ignore_fsal();//necessary for splitting
-    dg::ImplicitRungeKutta<std::array<std::array<dg::DVec,2>,2>,
+    dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
         feltor::FeltorSpecialSolver<
-        dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>> dirk(
-            "Trapezoidal-2-2", grid, p, mag);
+            dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
+        > karniadakis( grid, p, mag);
+    karniadakis.init( feltor, im, time, y0, p.dt);
+
+    //dg::Adaptive< dg::ERKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
+    //    "Bogacki-Shampine-4-2-3", y0);
+    //adaptive.stepper().ignore_fsal();//necessary for splitting
+    //dg::ImplicitRungeKutta<std::array<std::array<dg::DVec,2>,2>,
+    //    feltor::FeltorSpecialSolver<
+    //    dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>> dirk(
+    //        "Trapezoidal-2-2", grid, p, mag);
     //since we map pointers we don't need to update those later
 
     std::map<std::string, const dg::DVec* > v4d;
@@ -274,22 +280,23 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    do
-                    {
-                        //Strang splitting
-                        dt = dt_new;
-                        dirk.step( im, time, y0, time, y0, dt/2.);
-                        adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
-                            dg::pid_control, dg::l2norm, p.rtol, 1e-10);
-                        if( adaptive.failed())
-                        {
-                            failed_counter++;
-                            std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
-                            time -= dt; // time has to be reset here
-                            // in case of failure diffusion is applied twice?
-                        }
-                    }while ( adaptive.failed());
-                    dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
+                    karniadakis.step( feltor, im, time, y0);
+                    //do
+                    //{
+                    //    //Strang splitting
+                    //    dt = dt_new;
+                    //    dirk.step( im, time, y0, time, y0, dt/2.);
+                    //    adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
+                    //        dg::pid_control, dg::l2norm, p.rtol, 1e-10);
+                    //    if( adaptive.failed())
+                    //    {
+                    //        failed_counter++;
+                    //        std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
+                    //        time -= dt; // time has to be reset here
+                    //        // in case of failure diffusion is applied twice?
+                    //    }
+                    //}while ( adaptive.failed());
+                    //dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
                 }
                 catch( dg::Fail& fail) {
                     std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index b530a2a31..df3c5585f 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -139,8 +139,8 @@ Note that with $\psi_p = I(\psi_p) \equiv 1$ we recover a purely toroidal field.
 With a subsequent identification of $x:=R-R_0$ and the transition to
 Cartesian coordinates while straightening the field lines for large $R_0$
 we recover the familiar slab geometry.
-We have
-the equilibrium equations in toroidally symmetric, ideal MHD (
+
+We have the equilibrium equations in toroidally symmetric, ideal MHD (
 $\vec\nabla p = \vec j\times \vec B$ and $\vec \nabla\times\vec B = \vec j$ normalized with $p_0 = B_0^2/\mu_0$, and $j_0 = B_0/\rho_s/\mu_0$ )
 \begin{align}
     \vec\nabla\times \vec B &= \frac{R_0}{R}\left[ -\Delta^*\psi_p\ehat_\varphi + I_Z \ehat_R - I_R\ehat_Z \right]\equiv \vec j\\
@@ -201,7 +201,8 @@ $\bar{\psi}_{p11}=3 \bar{Z}\bar{R}^4 - 4\bar{Z}^3\bar{R}^2$\\
    & \\
 \bottomrule
 \end{longtable}
-The choice of the coefficients \(c_{i}\) and \(A\) determines the actual form of the magnetic field. It allows axisymmetric equilibria with e.g. single and asymmetric double X-point configurations, force-free states,
+The choice of the coefficients \(c_{i}\) and \(A\) determines the actual form of the magnetic field, while $R_0$ appears as an artificial scaling factor (note here that a change in $\rho_s$ changes $R_0$ but not the form or size of the dimensional equilibrium magnetic field).
+Eq.~\eqref{eq:solovev} allows axisymmetric equilibria with e.g. single and asymmetric double X-point configurations, force-free states,
 field reversed configurations and low and high beta tokamak equilbria. This casts this simple analytical equilibrium to the ideal choice in order to study geometric effects (e.g. inverse aspect ratio, elongation and triangularity) in magnetised plasmas.
 
 Note that
@@ -884,7 +885,7 @@ File format: json
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
     A      & float & 1 &  - & Solovev parameter in Eq.~\eqref{eq:solovev} \\
     C      & float[12] &  - & - & Solovev coefficients in Eq.~\eqref{eq:solovev}  \\
-    R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ in Eq.~\eqref{eq:solovev} \\
+    R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ in Eq.~\eqref{eq:solovev} (This is the only geometry quantity to change if $\rho_s$ changes)\\
     elongation    & float & 1 & - & Elongation $e$ \\
     triangularity & float & 0 & - & Triangularity $\delta$ \\
     inverseaspectratio & float & 0.16667 & - & minor to major radius $a/R_0$ \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index fb0ab4d3e..6e0566e75 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -487,12 +487,17 @@ int main( int argc, char* argv[])
 #endif //FELTOR_MPI
     MPI_OUT std::cout << "First write successful!\n";
     ///////////////////////////////////////Timeloop/////////////////////////////////
-    dg::Adaptive< dg::ERKStep<std::array<std::array<DVec,2>,2>> > adaptive(
-        "Bogacki-Shampine-4-2-3", y0);
-    adaptive.stepper().ignore_fsal();//necessary for splitting
-    dg::ImplicitRungeKutta<std::array<std::array<DVec,2>,2>,
-        feltor::FeltorSpecialSolver< Geometry, IDMatrix, DMatrix, DVec>> dirk(
-            "Trapezoidal-2-2", grid, p, mag);
+    dg::Karniadakis< std::array<std::array<DVec,2>,2 >,
+        feltor::FeltorSpecialSolver<
+            Geometry, IDMatrix, DMatrix, DVec>
+        > karniadakis( grid, p, mag);
+    karniadakis.init( feltor, im, time, y0, p.dt);
+    //dg::Adaptive< dg::ERKStep<std::array<std::array<DVec,2>,2>> > adaptive(
+    //    "Bogacki-Shampine-4-2-3", y0);
+    //adaptive.stepper().ignore_fsal();//necessary for splitting
+    //dg::ImplicitRungeKutta<std::array<std::array<DVec,2>,2>,
+    //    feltor::FeltorSpecialSolver< Geometry, IDMatrix, DMatrix, DVec>> dirk(
+    //        "Trapezoidal-2-2", grid, p, mag);
     dg::Timer t;
     t.tic();
     unsigned step = 0, failed_counter = 0;
@@ -508,22 +513,23 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    do
-                    {
-                        //Strang splitting
-                        dt = dt_new;
-                        dirk.step( im, time, y0, time, y0, dt/2.);
-                        adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
-                            dg::pid_control, dg::l2norm, p.rtol, 1e-10);
-                        if( adaptive.failed())
-                        {
-                            failed_counter++;
-                            MPI_OUT std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
-                            time -= dt; // time has to be reset here
-                            // in case of failure diffusion is applied twice?
-                        }
-                    }while ( adaptive.failed());
-                    dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
+                    karniadakis.step( feltor, im, time, y0);
+                    //do
+                    //{
+                    //    //Strang splitting
+                    //    dt = dt_new;
+                    //    dirk.step( im, time, y0, time, y0, dt/2.);
+                    //    adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
+                    //        dg::pid_control, dg::l2norm, p.rtol, 1e-10);
+                    //    if( adaptive.failed())
+                    //    {
+                    //        failed_counter++;
+                    //        MPI_OUT std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
+                    //        time -= dt; // time has to be reset here
+                    //        // in case of failure diffusion is applied twice?
+                    //    }
+                    //}while ( adaptive.failed());
+                    //dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
                 }
                 catch( dg::Fail& fail) {
                     MPI_OUT std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
@@ -537,7 +543,7 @@ int main( int argc, char* argv[])
             }
             double deltat = time - previous_time;
             feltor.update_quantities();
-            MPI_OUT std::cout << "Time "<<time<<" Timestep "<<dt_new<<"\n";
+            MPI_OUT std::cout << "Time "<<time<<" Current timestep "<<dt_new<<"\n";
             dEdt = (*v0d["energy"] - E0)/deltat, dMdt = (*v0d["mass"] - M0)/deltat;
             E0 = *v0d["energy"], M0 = *v0d["mass"];
             accuracy  = 2.*fabs( (dEdt - *v0d["ediff"])/( dEdt + *v0d["ediff"]));
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 9c78636e6..dd2e7f2f1 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -216,8 +216,7 @@ struct Implicit
     ImplicitVelocity<Geometry, IMatrix, Matrix, Container> m_velo;
 };
 
-/*!@brief Multigrid Solver class for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
-*
+/*!@brief Solver class for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
 */
 template< class Geometry, class IMatrix, class Matrix, class Container>
 struct FeltorSpecialSolver
-- 
GitLab


From 4d6367edbef64116029ee9dfc2f98eba748f906e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 20 Mar 2019 11:49:05 +0100
Subject: [PATCH 041/540] Implement send/recv transfer bracket for cuda+mpi

Motivation is davide, which has no cuda-aware mpi
---
 inc/dg/backend/config.h     |  9 +++++---
 inc/dg/backend/mpi_vector.h | 42 ++++++++++++++++++++++++++++++-------
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/inc/dg/backend/config.h b/inc/dg/backend/config.h
index e2404adaa..607beb9b1 100644
--- a/inc/dg/backend/config.h
+++ b/inc/dg/backend/config.h
@@ -1,5 +1,5 @@
 #pragma once
-//See also exblas/mylibm.hpp for more preprocessor configurations
+//See also exblas/config.h for more preprocessor configurations
 
 //%%%%%%%%%%%%define RESTRICT keyword for host compiler%%%%%%%%%%%%%%%%%%%%%%%%%
 #if defined(__INTEL_COMPILER)
@@ -59,11 +59,14 @@
 
 #include "mpi-ext.h"
 #if defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
+#pragma message( "Using CUDA-aware MPI support!")
 //Has cuda aware MPI support. Everything fine
 #elif defined(MPIX_CUDA_AWARE_SUPPORT) && !MPIX_CUDA_AWARE_SUPPORT
-#warning "CUDA aware MPI installation required!"
+#warning "NO CUDA aware MPI installation! Falling back to regular MPI!"
+#define _DG_CUDA_UNAWARE_MPI
 #else
-#pragma message( "Cannot determine CUDA-aware MPI support!")
+#pragma message( "Cannot determine CUDA-aware MPI support! Falling back to regular MPI!")
+#define _DG_CUDA_UNAWARE_MPI
 #endif //MPIX_CUDA
 
 #endif //THRUST = CUDA
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 289325e77..c5d9d1400 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -9,6 +9,7 @@
 #include "blas1_dispatch_shared.h"
 #include "mpi_communicator.h"
 #include "memory.h"
+#include "config.h"
 
 //TODO: should we catch the cases where outer_size \in {1,2,3} in NearestNeighborComm?
 namespace dg
@@ -262,7 +263,7 @@ struct NearestNeighborComm
     {
         unsigned size = buffer_size();
         //init pointers on host
-        const_pointer_type host_ptr[6];
+        const_pointer_type host_ptr[6]; //the ptr lives on the host but can point to device memory
         if(trivial_)
         {
             host_ptr[0] = thrust::raw_pointer_cast(&internal_buffer_.data()[0*size]);
@@ -281,12 +282,13 @@ struct NearestNeighborComm
             host_ptr[4] = thrust::raw_pointer_cast(&internal_buffer_.data()[4*size]);
             host_ptr[5] = thrust::raw_pointer_cast(&internal_buffer_.data()[5*size]);
         }
-        //copy to device
+        //copy pointers to device
         thrust::copy( host_ptr, host_ptr+6, buffer.begin());
+        //fill internal_buffer if !trivial
         do_global_gather_init( get_execution_policy<Vector>(), input, rqst);
         sendrecv( host_ptr[1], host_ptr[4],
-                  thrust::raw_pointer_cast(&internal_buffer_.data()[0*size]),
-                  thrust::raw_pointer_cast(&internal_buffer_.data()[5*size]),
+                  thrust::raw_pointer_cast(&internal_buffer_.data()[0*size]), //host_ptr is const!
+                  thrust::raw_pointer_cast(&internal_buffer_.data()[5*size]), //host_ptr is const!
                   rqst);
     }
     /**
@@ -300,6 +302,15 @@ struct NearestNeighborComm
     void global_gather_wait(const_pointer_type input, const buffer_type& buffer, MPI_Request rqst[4])const
     {
         MPI_Waitall( 4, rqst, MPI_STATUSES_IGNORE );
+#ifdef _DG_CUDA_UNAWARE_MPI
+        unsigned size = buffer_size();
+        cudaMemcpy( thrust::raw_pointer_cast(&internal_host_buffer_.data()[0*size],
+                    thrust::raw_pointer_cast(&internal_buffer_.data()[0*size],
+                    buffer_size*sizeof(get_value_type<V>), cudaMemcpyHostToDevice);
+        cudaMemcpy( thrust::raw_pointer_cast(&internal_host_buffer_.data()[5*size],
+                    thrust::raw_pointer_cast(&internal_buffer_.data()[5*size],
+                    buffer_size*sizeof(get_value_type<V>), cudaMemcpyHostToDevice);
+#endif
     }
     private:
     void do_global_gather_init( OmpTag, const_pointer_type, MPI_Request rqst[4])const;
@@ -314,6 +325,9 @@ struct NearestNeighborComm
     unsigned outer_size_ = 1; //size of vector in units of buffer_size
     Index gather_map_middle_;
     dg::Buffer<Vector> internal_buffer_;
+#ifdef _DG_CUDA_UNAWARE_MPI
+    dg::Buffer<thrust::host_vector<get_value_type<Vector>> internal_host_buffer_; //a copy of the data on the host
+#endif
 
     void sendrecv(const_pointer_type, const_pointer_type, pointer_type, pointer_type, MPI_Request rqst[4])const;
     int m_source[2], m_dest[2];
@@ -386,6 +400,9 @@ void NearestNeighborComm<I,B,V>::construct( unsigned n, const unsigned dimension
     }
     gather_map_middle_ = mid_gather; //transfer to device
     internal_buffer_.data().resize( 6*buffer_size() );
+#ifdef _DG_CUDA_UNAWARE_MPI
+    internal_host_buffer.data().resize( 6*buffer_size() );
+#endif
     }
 }
 
@@ -446,17 +463,26 @@ void NearestNeighborComm<I,B,V>::do_global_gather_init( CudaTag, const_pointer_t
 template<class I, class B, class V>
 void NearestNeighborComm<I,B,V>::sendrecv( const_pointer_type sb1_ptr, const_pointer_type sb2_ptr, pointer_type rb1_ptr, pointer_type rb2_ptr, MPI_Request rqst[4]) const
 {
-    MPI_Isend( sb1_ptr, buffer_size(),
+    unsigned size = buffer_size();
+#ifdef _DG_CUDA_UNAWARE_MPI
+    cudaMemcpy( thrust::raw_pointer_cast(&internal_host_buffer_.data()[1*size]), sb1_ptr, buffer_size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost);
+    cudaMemcpy( thrust::raw_pointer_cast(&internal_host_buffer_.data()[4*size]), sb2_ptr, buffer_size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost);
+    sb1_ptr = thrust::raw_pointer_cast(&internal_host_buffer_.data()[1*size]);
+    sb2_ptr = thrust::raw_pointer_cast(&internal_host_buffer_.data()[4*size]);
+    rb1_ptr = thrust::raw_pointer_cast(&internal_host_buffer_.data()[0*size]);
+    rb2_ptr = thrust::raw_pointer_cast(&internal_host_buffer_.data()[5*size]);
+#endif
+    MPI_Isend( sb1_ptr, size,
                getMPIDataType<get_value_type<V>>(),  //sender
                m_dest[0], 3, comm_, &rqst[0]); //destination
-    MPI_Irecv( rb2_ptr, buffer_size(),
+    MPI_Irecv( rb2_ptr, size,
                getMPIDataType<get_value_type<V>>(), //receiver
                m_source[0], 3, comm_, &rqst[1]); //source
 
-    MPI_Isend( sb2_ptr, buffer_size(),
+    MPI_Isend( sb2_ptr, size,
                getMPIDataType<get_value_type<V>>(),  //sender
                m_dest[1], 9, comm_, &rqst[2]);  //destination
-    MPI_Irecv( rb1_ptr, buffer_size(),
+    MPI_Irecv( rb1_ptr, size,
                getMPIDataType<get_value_type<V>>(), //receiver
                m_source[1], 9, comm_, &rqst[3]); //source
 }
-- 
GitLab


From 8cf7faadc134e354a1ae99ec0f569126efc773a0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 20 Mar 2019 14:08:27 +0100
Subject: [PATCH 042/540] Fix bug in mpi parallel 3d output in feltor

---
 src/feltor/feltor_hpc.cu | 37 +++++++++++++++++--------------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 6e0566e75..baf5a4664 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -39,7 +39,7 @@ int ncid = -1; //netcdf id (signal handler can close the file)
 //ATTENTION: in slurm should be used with --signal=SIGINT@30 (<signal>@<time in seconds>)
 void sigterm_handler(int signal)
 {
-    int rank, size;
+    int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     std::cout << " pid "<<rank<<" sigterm_handler, got signal " << signal << std::endl;
     std::cout << " pid "<<rank<<" ncid = " << ncid << std::endl;
@@ -363,8 +363,19 @@ int main( int argc, char* argv[])
 #ifdef FELTOR_MPI
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
     err = nc_var_par_access( ncid, tvarID, NC_COLLECTIVE);
+    int dims[3],  coords[3];
+    MPI_Cart_get( comm, 3, dims, periods, coords);
+    size_t count[4] = {1, grid_out.local().Nz(),
+        grid_out.n()*(grid_out.local().Ny()),
+        grid_out.n()*(grid_out.local().Nx())};
+    size_t start[4] = {0, coords[2]*count[1],
+                          coords[1]*count[2],
+                          coords[0]*count[3]};
 #else //FELTOR_MPI
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
+    size_t start[4] = {0, 0, 0, 0};
+    size_t count[4] = {1, grid_out.Nz(), grid_out.n()*grid_out.Ny(),
+        grid_out.n()*grid_out.Nx()};
 #endif //FELTOR_MPI
     {   //output 3d variables into file
         dg::geo::BFieldR fieldR(mag);
@@ -392,12 +403,12 @@ int main( int argc, char* argv[])
             #endif //FELTOR_MPI
             err = nc_enddef( ncid);
             dg::blas2::symv( project, *pair.second, transferH);
-            err = nc_put_var_double( ncid, vecID,
-            #ifdef FELTOR_MPI
+            err = nc_put_vara_double( ncid, vecID, &start[1], &count[1],
+                #ifdef FELTOR_MPI
                 transferH.data().data()
-            #else //FELTOR_MPI
+                #else //FELTOR_MPI
                 transferH.data()
-            #endif //FELTOR_MPI
+                #endif //FELTOR_MPI
             );
             err = nc_redef(ncid);
         }
@@ -428,7 +439,7 @@ int main( int argc, char* argv[])
     }
     err = nc_enddef(ncid);
     ///////////////////////////////////first output/////////////////////////
-    double dt_new = p.dt, dt =0;
+    double dt_new = p.dt;//, dt =0;
     MPI_OUT std::cout << "First output ... \n";
     //first, update quantities in feltor
     {
@@ -445,20 +456,6 @@ int main( int argc, char* argv[])
     }
     MPI_OUT q.display(std::cout);
     double energy0 = q.energy, mass0 = q.mass, E0 = energy0, M0 = mass0;
-#ifdef FELTOR_MPI
-    int dims[3],  coords[3];
-    MPI_Cart_get( comm, 3, dims, periods, coords);
-    size_t count[4] = {1, grid_out.local().Nz(),
-        grid_out.n()*(grid_out.local().Ny()),
-        grid_out.n()*(grid_out.local().Nx())};
-    size_t start[4] = {0, coords[2]*count[1],
-                          coords[1]*count[2],
-                          coords[0]*count[3]};
-#else //FELTOR_MPI
-    size_t start[4] = {0, 0, 0, 0};
-    size_t count[4] = {1, grid_out.Nz(), grid_out.n()*grid_out.Ny(),
-        grid_out.n()*grid_out.Nx()};
-#endif //FELTOR_MPI
     DVec transferD( dg::evaluate(dg::zero, grid_out));
     HVec transferH( dg::evaluate(dg::zero, grid_out));
     IDMatrix project = dg::create::projection( grid_out, grid);
-- 
GitLab


From 3bacf462824f13e5a76d1b46676117b4e960e858 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 20 Mar 2019 15:24:26 +0100
Subject: [PATCH 043/540] Renormalize Apar

This seems to behave better numerically when beta goes to zero
---
 src/feltor/feltor.cu  |  2 +-
 src/feltor/feltor.cuh | 31 ++++++++++++++++---------------
 src/feltor/feltor.tex | 18 ++++++++----------
 src/feltor/implicit.h |  8 ++++----
 4 files changed, 29 insertions(+), 30 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index b0c7f64fe..4751957ca 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -144,7 +144,7 @@ int main( int argc, char* argv[])
     ////////////////////////create timer and timestepper
     //
     dg::Timer t;
-    double time = 0, dt_new = p.dt, dt =0;
+    double time = 0, dt_new = p.dt;//, dt =0;
     unsigned step = 0;
     dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
         feltor::FeltorSpecialSolver<
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 4191b4850..417bd71c1 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -16,8 +16,8 @@ namespace feltor
 
 namespace routines{
 struct ComputePerpDrifts{
-    ComputePerpDrifts( double mu, double tau, double beta):
-        m_mu(mu), m_tau(tau), m_beta(beta){}
+    ComputePerpDrifts( double mu, double tau):
+        m_mu(mu), m_tau(tau){}
     DG_DEVICE
     void operator()(
             double N, double d0N, double d1N, double d2N,
@@ -94,16 +94,16 @@ struct ComputePerpDrifts{
         double PA = b_0*( d1P*d2A-d2P*d1A)+
                     b_1*( d2P*d0A-d0P*d2A)+
                     b_2*( d0P*d1A-d1P*d0A);
-        dtN +=  -m_beta*( A*N*KappaU + A*U*KappaN + N*UA + U*NA)
-                -m_beta*N*U*( A*divCurvKappa - KnablaBA);
-        dtU +=  -m_beta/m_mu*( A*KappaP + PA)
-                -m_beta*U*( A*KappaU + UA)
-                -m_beta*m_tau/m_mu/N*(A*KappaN + NA);
+        dtN +=  -( A*N*KappaU + A*U*KappaN + N*UA + U*NA)
+                -N*U*( A*divCurvKappa - KnablaBA);
+        dtU +=  -1./m_mu*( A*KappaP + PA)
+                -1.*U*( A*KappaU + UA)
+                -1.*m_tau/m_mu/N*(A*KappaN + NA);
 
 
     }
     private:
-    double m_mu, m_tau, m_beta;
+    double m_mu, m_tau;
 };
 struct ComputeChi{
     DG_DEVICE
@@ -575,8 +575,9 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
         m_multi_induction[u].set_chi( m_multi_chi[u]);
 
     //----------Compute right hand side------------------------//
-    dg::blas1::pointwiseDot(  1., fields[0][1], fields[1][1],
-                             -1., fields[0][0], fields[1][0], 0., m_temp0);
+    dg::blas1::pointwiseDot(  m_p.beta, fields[0][1], fields[1][1],
+                             -m_p.beta, fields[0][0], fields[1][0],
+                              0., m_temp0);
     //----------Invert Induction Eq----------------------------//
     m_old_apar.extrapolate( time, m_apar);
 #ifdef DG_MANUFACTURED
@@ -595,8 +596,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
     if(!m_p.symmetric) dg::blas2::symv( m_dz, m_apar, m_dzA);
 
     //----------Compute Velocities-----------------------------//
-    dg::blas1::axpby( 1., fields[1][0], -m_p.beta/m_p.mu[0], m_apar, fields[1][0]);
-    dg::blas1::axpby( 1., fields[1][1], -m_p.beta/m_p.mu[1], m_apar, fields[1][1]);
+    dg::blas1::axpby( 1., fields[1][0], -1./m_p.mu[0], m_apar, fields[1][0]);
+    dg::blas1::axpby( 1., fields[1][1], -1./m_p.mu[1], m_apar, fields[1][1]);
 }
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
@@ -618,7 +619,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
         if(!m_p.symmetric) dg::blas2::symv( m_dz, fields[1][i], m_dzU[i]);
         if( m_p.beta == 0){
             dg::blas1::subroutine( routines::ComputePerpDrifts(
-                m_p.mu[i], m_p.tau[i], m_p.beta),
+                m_p.mu[i], m_p.tau[i]),
                 //species depdendent
                 fields[0][i], m_dxN[i], m_dyN[i], m_dzN[i],
                 fields[1][i], m_dxU[i], m_dyU[i], m_dzU[i],
@@ -632,7 +633,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
         }
         if( m_p.beta != 0){
             dg::blas1::subroutine( routines::ComputePerpDrifts(
-                m_p.mu[i], m_p.tau[i], m_p.beta),
+                m_p.mu[i], m_p.tau[i]),
                 //species depdendent
                 fields[0][i], m_dxN[i], m_dyN[i], m_dzN[i],
                 fields[1][i], m_dxU[i], m_dyU[i], m_dzU[i],
@@ -710,7 +711,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_energies(
         dg::blas1::subroutine( routines::ComputePsi(),
             m_temp0, m_dxA, m_dyA, m_dzA,
             m_temp0, m_temp1, m_temp2);
-        m_q.Apar = 0.5*m_p.beta*dg::blas1::dot( m_vol3d, m_temp0);
+        m_q.Apar = 0.5*dg::blas1::dot( m_vol3d, m_temp0);
     }
     //= 0.5 mu_i N_i u_E^2
     m_q.Tperp = 0.5*m_p.mu[1]*dg::blas2::dot( fields[0][1], m_vol3d, m_UE2);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index df3c5585f..8abec3107 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -666,8 +666,7 @@ In \(\Lambda\) we insert the dissipative terms of Section~\ref{sec:dissres}. \\
 We scale all spatial lengths by $\rho_s = \sqrt{T_e m_i}/(eB_0)$ and time by the ion gyrofrequency $\Omega_0 = eB_0/m_i$.
 The magnetic field is scaled with $B_0$, densities with $n_0$ and the parallel velocity is scaled with $c_s = \sqrt{T_e/m_i}$.
 The potential is scaled with $\hat \phi = e/T_e$ and the vector potential with 
-$\hat A_\parallel = \beta \rho_s B_0$ (this particular choice for $\hat
-A_\parallel$ allows to easily recover the electrostatic limit $\beta = 0$).
+$\hat A_\parallel = \rho_s B_0$.
 We introduce the dimensionless parameters
 \begin{align}
   \tau_a = \frac{T_a}{z_aT_e}~,\quad \mu_a = \frac{m_a}{z_am_i}\text{ and } 
@@ -702,17 +701,16 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
         - \left(2\tau + {\mu}U^2\right) \mathcal K_{\nabla\times\bhat} (U)
         -2\tau U\mathcal K_{\nabla\times\bhat}(\ln N)
         - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
-        + \nu_\perp\Delta_\perp W
-        - \nu_\perp\frac{\beta}{\mu}\Delta_\perp A_\parallel
+        + \nu_\perp\Delta_\perp U
         + \nu_\parallel \Delta_\parallel U + S_U,
         \label{eq:EgyrofluidU} \\
-        W&:= \left( U + \frac{\beta}{\mu}A_\parallel\right)
+        W&:= \left( U + \frac{A_\parallel}{\mu}\right)
     \end{align}
     \label{eq:Egyrofluid}
 \end{subequations}
 together with
-$\bar\nabla_\parallel f = \nabla_\parallel f + \beta A_\parallel \mathcal K_{\nabla\times\bhat}(f) + \frac{\beta}{B}[ f, A_\parallel]_\perp$
-and $\vec \nabla \cdot \tilde{ \vec b}_\perp = \beta A_\parallel \vec \nabla\cdot\vec{ \mathcal{ K}_{\nabla\times\bhat}} - \beta \mathcal K_{\nabla B}(A_\parallel) $
+$\bar\nabla_\parallel f = \nabla_\parallel f + A_\parallel \mathcal K_{\nabla\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$
+and $\vec \nabla \cdot \tilde{ \vec b}_\perp = A_\parallel \vec \nabla\cdot\vec{ \mathcal{ K}_{\nabla\times\bhat}} - \mathcal K_{\nabla B}(A_\parallel) $
 and
 \begin{subequations} \label{eq:elliptic}
   \begin{align}
@@ -720,7 +718,7 @@ and
     \Gamma_{1,i}^{-1} = 1-\frac{1}{2}\tau_i\mu_i \Delta_\perp \\
     \psi_e = \phi, \quad \psi_i &= \Gamma_{1,i}\phi -\frac{\mu_i}{2}\frac{(\nabla_\perp\phi)^2}{B^2} \\
     \left(\frac{\beta}{\mu_i}N_i - \frac{\beta}{\mu_e}n_e-\Delta_\perp\right)
-    A_\parallel &= N_iW_i-n_e w_e
+    A_\parallel &= \beta\left(N_iW_i-n_e w_e\right)
   \end{align}
 \end{subequations}
 Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite. Also note the peculiar notation of the perpendicular viscosity on $U$.
@@ -728,7 +726,7 @@ We indicate that $\Delta_\perp W$ is treated implicitly while the correction $\D
 The energy theorem reads $\partial_t \mathcal E + \mathcal S = \Lambda$ (with $z_e=-1$ and $z_i=+1$)
 \begin{align}
   \mathcal{E}= \int  \dV & \left( z_e\tau_e n_e \ln{(n_e)} +z_i\tau_i N_i\ln{(N_i)}
-  +\frac{\beta}{2}\left(\vec \nabla_\perp A_\parallel\right)^2
+  +\frac{1}{2}\left(\vec \nabla_\perp A_\parallel\right)^2
    +  \frac{1}{2} \mu_i N_i u_E^2  \right .\nonumber\\
    &\left. +\frac{1}{2} z_e\mu_e  n_e u_e^2
   +\frac{1}{2} z_i\mu_i  N_i U_i^2  \right),\\
@@ -993,7 +991,7 @@ induction, vorticity, fluxe, Lperpinv, Lparallelinv\} corresponding to \{
  \vec j_e\cdot \vec \nabla\psi_p =& n_e\left( \vec v_E + \vec v_C + \vec v_{\nabla
  B} + u_e \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
  \nabla\psi_p \nonumber\\
- =& \beta n_eu_e\left( A_\parallel \mathcal
+ =& n_eu_e\left( A_\parallel \mathcal
  K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \nonumber\\
   &+ n_e\frac{1}{B}[\phi, \psi_p]_\perp + \left(\tau_e + \mu_e u_e^2\right)
    n_e\mathcal K_{\nabla\times\bhat}(\psi_p) + \tau_e n_e \mathcal K_{\nabla B}(\psi_p) \nonumber\\
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index dd2e7f2f1..98a8443a1 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -143,8 +143,8 @@ struct ImplicitVelocity
         dg::blas1::copy( w, m_fields[1]);
         if( m_p.beta != 0){
             //let us solve for apar
-            dg::blas1::pointwiseDot(  1., m_fields[0][1], m_fields[1][1],
-                                     -1., m_fields[0][0], m_fields[1][0],
+            dg::blas1::pointwiseDot(  m_p.beta, m_fields[0][1], m_fields[1][1],
+                                     -m_p.beta, m_fields[0][0], m_fields[1][0],
                                       0., m_temp);
             m_invert( m_induction, m_apar, m_temp, weights(),
                 inv_weights(), precond());
@@ -154,8 +154,8 @@ struct ImplicitVelocity
             dg::blas1::axpby( 1., m_fields[1][1], -m_p.beta/m_p.mu[1],
                 m_apar, m_fields[1][1]);
         }
-        /* fields[0] := u_e
-           fields[1] := U_i
+        /* fields[1][0] := u_e
+           fields[1][1] := U_i
         */
         for( unsigned i=0; i<2; i++)
         {
-- 
GitLab


From 9c7ec90a112cd9fc02883effca6d36be7984d2b6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 21 Mar 2019 15:35:07 +0100
Subject: [PATCH 044/540] Update manufactured solution

and already fix first bug in implicit.h feltor
---
 src/feltor/feltor.tex      |    9 +-
 src/feltor/implicit.h      |    6 +-
 src/feltor/manufactured.cu |   53 +-
 src/feltor/manufactured.h  | 1391 +++++++++++++++++-------------------
 4 files changed, 679 insertions(+), 780 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 8abec3107..fad7d8ede 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -603,7 +603,7 @@ This will result in exponential adaption of the core and wall
 density profile of the form $n_e \propto n_{prof}+(n_{prof}-n_{e,0})e^{-\omega_st}$.
 For ions we use
 \begin{align}
-    S_{N_i} = \Gamma_{1,i}^{-1} S_{n_e} = \left(1-\frac{m_i T_i}{2q^2 B_0^2} \nabla_\perp^2\right) S_{n_e}
+    S_{N_i} = \Gamma_{1,i}^{-1} S_{n_e} = \left(1-\frac{m_i T_i}{2q^2 B_0^2} \Delta_\perp\right) S_{n_e}
   \label{eq:ion_source}
 \end{align}
 Note that Eq.~\eqref{eq:ion_source} is explicitly chosen as to avoid vorticity generation
@@ -678,6 +678,7 @@ where $a\in\{e,i\}$ is the species label and $z$ is the charge number. Finally,
   \eta:=\frac{en_0\eta_\parallel}{B_0} = 8.45\cdot 10^{-5}\ln \lambda \left(\frac{n_0}{10^{19}\text{m}^3}\right) \left(\frac{T_e}{\text{eV}}\right)^{-3/2} \left(\frac{B_0}{\text{T}}\right)^{-1}.
     \label{eq:resistivity}
 \end{align}
+with $\ln \lambda \approx 10$.
 Omitting the species label we arrive at (dividing the density equation by $\Omega_0n_0$ and the velocity equation by $\Omega_0 c_s$)
 \begin{subequations}
     \begin{align}
@@ -753,7 +754,7 @@ u_e(R,Z,\varphi, t) &:= \sin(2\pi(R-R_0))\sin(2\pi Z)\sin(2\varphi)\sin(2\pi t)/
 U_i(R,Z,\varphi, t) &:= \sqrt{-\mu_e}u_e(R,Z,\varphi,t) \\
 \phi(R,Z,\varphi,t) &:= \sin(3\pi(R-R_0))\sin(3\pi Z)\sin(3\varphi)\sin(3\pi t)/5; \\
 \psi(R,Z,\varphi,t) &:= \phi(R,Z,\varphi, t) = \Gamma \phi \\
-A_\parallel( R,Z,\varphi,t) &:= \sin(4\pi(R-R_0))\sin(4\pi Z)\sin(4\varphi)\sin(4\pi t)/4;
+A_\parallel( R,Z,\varphi,t) &:= \beta\sin(4\pi(R-R_0))\sin(4\pi Z)\sin(4\varphi)\sin(4\pi t)/4;
 \end{align*}
 We choose circular flux surfaces of the form
 \begin{align*}
@@ -845,7 +846,7 @@ curvmode  & string & "low beta"  & "toroidal"& curvature mode ("low beta", "true
 symmetric & bool & false & false & If true, initialize all quantities symmetric in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used to construct the parallel derivatives and then overwritten to $N_z\equiv 1$. \\
 bc & dict & & & Dictionary of boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
 \qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y for $n_e$ and $N_i$\\
-\qquad velocity  & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $u_e$ and $U_i$\\
+\qquad velocity  & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $u_e$ and $U_i$ and $A_\parallel$\\
 \qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $\phi$ and $\psi$\\
     boxscaleR  & float[2] & [1.1,1.1]     & [1.05,1.05] & $[\varepsilon^{R-}, \varepsilon^{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
     boxscaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon^{Z-}, \varepsilon^{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box} \\
@@ -881,7 +882,7 @@ File format: json
 \begin{longtable}{llll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
-    A      & float & 1 &  - & Solovev parameter in Eq.~\eqref{eq:solovev} \\
+    A      & float & 0 &  - & Solovev parameter in Eq.~\eqref{eq:solovev} \\
     C      & float[12] &  - & - & Solovev coefficients in Eq.~\eqref{eq:solovev}  \\
     R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ in Eq.~\eqref{eq:solovev} (This is the only geometry quantity to change if $\rho_s$ changes)\\
     elongation    & float & 1 & - & Elongation $e$ \\
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 98a8443a1..1982c7569 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -148,10 +148,10 @@ struct ImplicitVelocity
                                       0., m_temp);
             m_invert( m_induction, m_apar, m_temp, weights(),
                 inv_weights(), precond());
-            //compute u_e and U_i
-            dg::blas1::axpby( 1., m_fields[1][0], -m_p.beta/m_p.mu[0],
+            //compute u_e and U_i from w_e, W_i and apar
+            dg::blas1::axpby( 1., m_fields[1][0], -1./m_p.mu[0],
                 m_apar, m_fields[1][0]);
-            dg::blas1::axpby( 1., m_fields[1][1], -m_p.beta/m_p.mu[1],
+            dg::blas1::axpby( 1., m_fields[1][1], -1./m_p.mu[1],
                 m_apar, m_fields[1][1]);
         }
         /* fields[1][0] := u_e
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index db72481c6..2de1f69c3 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -7,11 +7,13 @@
 
 #include "parameters.h"
 #define DG_MANUFACTURED
-#define FELTORPARALLEL 1
+//Change here to selectively test parallel and perp parts
+#define FELTORPARALLEL 0
 #define FELTORPERP 1
-//#include "/mnt/hgfs/shared/manufactured.h"
+
 #include "manufactured.h"
 #include "feltor.cuh"
+#include "implicit.h"
 
 int main( int argc, char* argv[])
 {
@@ -47,19 +49,19 @@ int main( int argc, char* argv[])
     std::cout << "Initialize implicit" << std::endl;
     feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
 
-    feltor::manufactured::Ne ne{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.c,
+    feltor::manufactured::Ne ne{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                  p.beta,p.nu_perp,p.nu_parallel};
-    feltor::manufactured::Ni ni{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.c,
+    feltor::manufactured::Ni ni{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                  p.beta,p.nu_perp,p.nu_parallel};
-    feltor::manufactured::Ue ue{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.c,
+    feltor::manufactured::Ue ue{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                 p.beta,p.nu_perp,p.nu_parallel};
-    feltor::manufactured::Ui ui{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.c,
+    feltor::manufactured::Ui ui{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                  p.beta,p.nu_perp,p.nu_parallel};
-    feltor::manufactured::Phie phie{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.c,
+    feltor::manufactured::Phie phie{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                      p.beta,p.nu_perp,p.nu_parallel};
-    feltor::manufactured::Phii phii{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.c,
+    feltor::manufactured::Phii phii{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                      p.beta,p.nu_perp,p.nu_parallel};
-    feltor::manufactured::A aa{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.c,
+    feltor::manufactured::A aa{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                 p.beta,p.nu_perp,p.nu_parallel};
 
     dg::DVec R = dg::pullback( dg::cooX3d, grid);
@@ -75,26 +77,33 @@ int main( int argc, char* argv[])
     dg::blas1::evaluate( apar, dg::equals(), aa, R,Z,P,0);
     dg::blas1::plus(y0[0][0],-1); //ne-1
     dg::blas1::plus(y0[0][1],-1); //Ni-1
-    dg::blas1::axpby(p.beta/p.mu[0], apar, 1., y0[1][0]); //we=ue+b/mA
-    dg::blas1::axpby(p.beta/p.mu[1], apar, 1., y0[1][1]); //Wi=Ui+b/mA
+    dg::blas1::axpby(1./p.mu[0], apar, 1., y0[1][0]); //we=ue+1/mA
+    dg::blas1::axpby(1./p.mu[1], apar, 1., y0[1][1]); //Wi=Ui+1/mA
 
     //Adaptive solver
-    dg::Adaptive< dg::ARKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
-        "ARK-4-2-3", y0, y0[0][0].size(), p.eps_time);
+    //dg::Adaptive< dg::ARKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
+    //    "ARK-4-2-3", y0, y0[0][0].size(), p.eps_time);
+    //Multistep solver
+    dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
+        feltor::FeltorSpecialSolver<
+            dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
+        > karniadakis( grid, p, mag);
     double time = 0, dt_new = p.dt, TMAX = 1e-3;
+    karniadakis.init( feltor, im, time, y0, p.dt);
     while( time < TMAX)
     {
         if( time + dt_new > TMAX)
             dt_new = TMAX - time;
 
         try{
-            do
-            {
-                adaptive.step( feltor, im, time, y0, time, y0, dt_new,
-                    dg::pid_control, dg::l2norm, p.rtol, 1e-10);
-                if( adaptive.failed())
-                    std::cout << "FAILED STEP! REPEAT!\n";
-            }while ( adaptive.failed());
+            karniadakis.step( feltor, im, time, y0);
+            //do
+            //{
+            //    adaptive.step( feltor, im, time, y0, time, y0, dt_new,
+            //        dg::pid_control, dg::l2norm, p.rtol, 1e-10);
+            //    if( adaptive.failed())
+            //        std::cout << "FAILED STEP! REPEAT!\n";
+            //}while ( adaptive.failed());
         }
         catch( dg::Fail& fail) {
             std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
@@ -131,8 +140,8 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( 1., num_phi[1], -1.,sol_phi[1]);
     dg::blas1::axpby( 1., num_apar, -1.,sol_apar);
     std::cout<<std::scientific;
-    std::cout <<"Error: \n"
-              <<"    Time: "<<time<<"\n"
+    std::cout <<"           rel. Error\tNorm: \n"
+              //<<"    Time: "<<time<<"\n"
               <<"    ne:   "<<sqrt(dg::blas2::dot( w3d, sol[0][0]))/normne<<"\t"<<normne<<"\n"
               <<"    ni:   "<<sqrt(dg::blas2::dot( w3d, sol[0][1]))/normni<<"\t"<<normni<<"\n"
               <<"    ue:   "<<sqrt(dg::blas2::dot( w3d, sol[1][0]))/normue<<"\t"<<normue<<"\n"
diff --git a/src/feltor/manufactured.h b/src/feltor/manufactured.h
index 736799586..0a35a398f 100644
--- a/src/feltor/manufactured.h
+++ b/src/feltor/manufactured.h
@@ -56,7 +56,7 @@ struct GammaNi{
 struct A{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/4.
+    return (beta*Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/4.
 ; }};
 struct SNe{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
@@ -230,19 +230,17 @@ struct SNe{
            (27.925268031909273*(-50. + 1.*R - 0.1*Power(Z,2))*Cos(2*Pi*Z)*
               Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (26.31894506957162*beta*Cos(4*Pi*Z)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   (39.78873577297384 - 0.7957747154594768*R + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*P))*
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
+           (4*beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (R*(-3141.592653589793 + 125.66370614359172*R - 
+                   6.283185307179586*Power(R,2) - 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
                  Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2)) - (80.*Z*(-50. + 1.*R - 0.1*Power(Z,2))*
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2)) - 
+           (80.*Z*(-50. + 1.*R - 0.1*Power(Z,2))*
               (1.*taue - 0.1111111111111111*Power(Sin(2*P),2)*
                  Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
                  Power(Sin(2*Pi*Z),2)))/
@@ -255,37 +253,30 @@ struct SNe{
             (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
            (2.*Z*(3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
                 0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (13.15947253478581*beta*Cos(2*Pi*Z)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   (39.78873577297384 - 0.7957747154594768*R + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*P))*
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
+           (2*beta*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (R*(-3141.592653589793 + 125.66370614359172*R - 
+                   6.283185307179586*Power(R,2) - 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
                  Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2)) - (2.0943951023931953*beta*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*
-              (2.*R*Z*Cos(4*Pi*R)*Sin(4*P) + 
-                (-0.03183098861837907*Power(Z,2)*Cos(4*P) + 
-                   (-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   0.15915494309189535*Z*Sin(4*P))*Sin(4*Pi*R))*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2)) + (8.377580409572781*beta*Z*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   (39.78873577297384 - 0.7957747154594768*R + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*P))*
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2)) + 
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (-12.566370614359172*R*Z*Cos(4*Pi*R)*Sin(4*P) + 
+                (0.2*Power(Z,2)*Cos(4*P) + 
+                   (50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) - 1.*Z*Sin(4*P))*Sin(4*Pi*R))*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2)) - 
+           (1.3333333333333333*beta*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (R*(-3141.592653589793 + 125.66370614359172*R - 
+                   6.283185307179586*Power(R,2) - 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
                  Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
               Sin(4*Pi*Z))/
             (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
@@ -300,20 +291,19 @@ struct SNe{
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
            ((3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
                 0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   (39.78873577297384 - 0.7957747154594768*R + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*P))*
+            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (R*(-3141.592653589793 + 125.66370614359172*R - 
+                   6.283185307179586*Power(R,2) - 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
                  Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
               Sin(4*Pi*Z))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2))) + R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-            Sin(Pi*Z))*((4.*R*(-20. + 2.*R)*taue*Z)/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2))) + 
+        R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+         ((4.*R*(-20. + 2.*R)*taue*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
            (2.*taue*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
@@ -341,52 +331,50 @@ struct SNe{
               (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
                 (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(-20. + 2.*R)*Cos(4*Pi*Z)*Sin(4*P) + 
-                (499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((11.140846016432674 - 0.954929658551372*R + 
-                      0.0477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2)) + (26.31894506957162*beta*Cos(4*Pi*R)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2)) + (13.15947253478581*beta*Cos(2*Pi*(-10 + R))*Sin(2*P)*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2)) - (4.1887902047863905*beta*(-20. + 2.*R)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*
-              Sin(2*Pi*Z)*(R*(499.99999999999994 - 20.*R + 
-                   1.*Power(R,2) + 1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               3))) + 1.5707963267948966*R*Cos(Pi*(-10 + R))*Sin(P)*
-         Sin(Pi*t)*Sin(Pi*Z)*(0. - 
-           (2.*R*taue*Z)/
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(-125.66370614359172 + 12.566370614359172*R)*
+                 Cos(4*Pi*Z)*Sin(4*P) + 
+                (3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((70. - 6.*R + 0.30000000000000004*Power(R,2) + 
+                      0.1*Power(Z,2))*Cos(4*P) - 0.5*Z*Sin(4*P))*
+                 Sin(4*Pi*Z)))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2)) + 
+           (4*beta*Pi*Cos(4*Pi*R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2)) + 
+           (2*beta*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(4*Pi*R)*
+              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2)) - 
+           (2*beta*(-20. + 2.*R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*
+              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),3))) + 
+        1.5707963267948966*R*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z)*
+         (0. - (2.*R*taue*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
            (R*Z*(-2.*taue + 0.2222222222222222*Power(Sin(2*P),2)*
                  Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
@@ -396,19 +384,18 @@ struct SNe{
               (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
                 (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
             (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2))) + (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-            Sin(Pi*Z))*(0. - (2.*R*taue*Z)/
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2))) + 
+        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+         (0. - (2.*R*taue*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
            (R*Z*(-2.*taue + 0.2222222222222222*Power(Sin(2*P),2)*
                  Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
@@ -418,19 +405,18 @@ struct SNe{
               (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
                 (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
             (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2))) + R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-            Sin(Pi*Z))*((0.044444444444444446*
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2))) + 
+        R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+         ((0.044444444444444446*
               (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(2*P)*
               Sin(2*P)*Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
               Power(Sin(2*Pi*Z),2))/
@@ -439,32 +425,36 @@ struct SNe{
               (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
                 (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
                  Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (0.41887902047863906*beta*Cos(4*P)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-                 Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-                ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                      10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*
-                    Cos(4*Pi*R) + 
-                   (71.6197243913529 - 1.5915494309189535*R + 
-                      0.07957747154594767*Power(R,2) + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*
-                 Sin(4*Pi*Z)))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2)) - (0.20943951023931953*beta*Cos(2*P)*Sin(4*P)*
-              Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-                 Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-                ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                      10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*
-                    Cos(4*Pi*R) + 
-                   (71.6197243913529 - 1.5915494309189535*R + 
-                      0.07957747154594767*Power(R,2) + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*
-                 Sin(4*Pi*Z)))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2))) + 0.5*R*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)*
+            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+           (4*beta*Cos(4*P)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                   0.3141592653589793*Power(R,2) - 
+                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) \
++ ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                      0.3141592653589793*Power(R,3) + 
+                      3.141592653589793*Power(Z,2) + 
+                      R*(-219.9114857512855 - 
+                        0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
+                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2)) + 
+           (2*beta*Cos(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                   0.3141592653589793*Power(R,2) - 
+                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) \
++ ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                      0.3141592653589793*Power(R,3) + 
+                      3.141592653589793*Power(Z,2) + 
+                      R*(-219.9114857512855 - 
+                         0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
+                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2))) + 
+        0.5*R*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)*
          ((taue*(250000.00000000006 - 80.*Power(R,3) + 1.*Power(R,4) + 
                 1000.0000000000001*Power(Z,2) + 1.*Power(Z,4) + 
                 R*(-40000. - 80.*Power(Z,2)) + 
@@ -479,20 +469,21 @@ struct SNe{
               (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
                 (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
                  Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (0.10471975511965977*beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-                 Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-                ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                      10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*
-                    Cos(4*Pi*R) + 
-                   (71.6197243913529 - 1.5915494309189535*R + 
-                      0.07957747154594767*Power(R,2) + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*
-                 Sin(4*Pi*Z)))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               2))))/R)
+            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+           (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                   0.3141592653589793*Power(R,2) - 
+                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
+                ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                      0.3141592653589793*Power(R,3) + 
+                      3.141592653589793*Power(Z,2) + 
+                      R*(-219.9114857512855 - 
+                         0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
+                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2))))/R)
 ; }};
 struct SNi{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
@@ -663,18 +654,15 @@ struct SNi{
            (27.925268031909273*mui*(-50. + 1.*R - 0.1*Power(Z,2))*
               Cos(2*Pi*Z)*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
               Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (26.31894506957162*beta*Cos(4*Pi*Z)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   (39.78873577297384 - 0.7957747154594768*R + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*P))*
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
+           (4*beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (R*(-3141.592653589793 + 125.66370614359172*R - 
+                   6.283185307179586*Power(R,2) - 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
                  Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) - 
            (8.88888888888889*Z*(-50. + 1.*R - 0.1*Power(Z,2))*
               (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
                  Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
@@ -687,35 +675,28 @@ struct SNi{
             (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
            (2.*Z*(3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
                 0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (13.15947253478581*beta*Cos(2*Pi*Z)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   (39.78873577297384 - 0.7957747154594768*R + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*P))*
-                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (2.*R*Z*Cos(4*Pi*R)*Sin(4*P) + 
-                (-0.03183098861837907*Power(Z,2)*Cos(4*P) + 
-                   (-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   0.15915494309189535*Z*Sin(4*P))*Sin(4*Pi*R))*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (8.377580409572781*beta*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   (39.78873577297384 - 0.7957747154594768*R + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*P))*
+           (2*beta*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (R*(-3141.592653589793 + 125.66370614359172*R - 
+                   6.283185307179586*Power(R,2) - 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
+                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (-12.566370614359172*R*Z*Cos(4*Pi*R)*Sin(4*P) + 
+                (0.2*Power(Z,2)*Cos(4*P) + 
+                   (50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) - 1.*Z*Sin(4*P))*Sin(4*Pi*R))*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) - 
+           (1.3333333333333333*beta*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (R*(-3141.592653589793 + 125.66370614359172*R - 
+                   6.283185307179586*Power(R,2) - 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
                  Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
               Sin(4*Pi*Z))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3)) + 
@@ -728,18 +709,16 @@ struct SNi{
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
            ((3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
                 0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                      0.015915494309189534*Power(R,2) - 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                   (39.78873577297384 - 0.7957747154594768*R + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*P))*
+            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              (R*(-3141.592653589793 + 125.66370614359172*R - 
+                   6.283185307179586*Power(R,2) - 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
                  Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
               Sin(4*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))) + 
         R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
          ((4.*R*(-20. + 2.*R)*taui*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
@@ -768,49 +747,44 @@ struct SNi{
               (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
                 (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(-20. + 2.*R)*Cos(4*Pi*Z)*Sin(4*P) + 
-                (499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((11.140846016432674 - 0.954929658551372*R + 
-                      0.0477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (26.31894506957162*beta*Cos(4*Pi*R)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (13.15947253478581*beta*Cos(2*Pi*(-10 + R))*Sin(2*P)*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (4.1887902047863905*beta*(-20. + 2.*R)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*
-              Sin(2*Pi*Z)*(R*(499.99999999999994 - 20.*R + 
-                   1.*Power(R,2) + 1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3)) + 
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(-125.66370614359172 + 12.566370614359172*R)*
+                 Cos(4*Pi*Z)*Sin(4*P) + 
+                (3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((70. - 6.*R + 0.30000000000000004*Power(R,2) + 
+                      0.1*Power(Z,2))*Cos(4*P) - 0.5*Z*Sin(4*P))*
+                 Sin(4*Pi*Z)))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
+           (4*beta*Pi*Cos(4*Pi*R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
+           (2*beta*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(4*Pi*R)*
+              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) - 
+           (2*beta*(-20. + 2.*R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*
+              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3))) + 
         1.5707963267948966*R*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z)*
          (0. - (2.*R*taui*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
@@ -822,17 +796,15 @@ struct SNi{
               (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
                 (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
             (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))) + 
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
          (0. - (2.*R*taui*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
@@ -844,17 +816,15 @@ struct SNi{
               (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
                 (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
             (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                   1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-79.57747154594767 + 11.140846016432674*R - 
-                      0.477464829275686*Power(R,2) + 
-                      0.015915494309189534*Power(R,3) - 
-                      0.15915494309189535*Power(Z,2) + 
-                      0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                   0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
+           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (R*(3141.592653589793 - 125.66370614359172*R + 
+                   6.283185307179586*Power(R,2) + 
+                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))) + 
         R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
          ((-0.044444444444444446*mui*
               (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(2*P)*
@@ -865,31 +835,33 @@ struct SNi{
               (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
                 (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
                  Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (0.41887902047863906*beta*Cos(4*P)*Sin(2*P)*
-              Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-                 Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-                ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                      10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*
-                    Cos(4*Pi*R) + 
-                   (71.6197243913529 - 1.5915494309189535*R + 
-                      0.07957747154594767*Power(R,2) + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*
-                 Sin(4*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (0.20943951023931953*beta*Cos(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-                 Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-                ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                      10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*
-                    Cos(4*Pi*R) + 
-                   (71.6197243913529 - 1.5915494309189535*R + 
-                      0.07957747154594767*Power(R,2) + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*
-                 Sin(4*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
+            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+           (4*beta*Cos(4*P)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                   0.3141592653589793*Power(R,2) - 
+                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) \
++ ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                      0.3141592653589793*Power(R,3) + 
+                      3.141592653589793*Power(Z,2) + 
+                      R*(-219.9114857512855 - 
+                        0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
+                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
+           (2*beta*Cos(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                   0.3141592653589793*Power(R,2) - 
+                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) \
++ ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                      0.3141592653589793*Power(R,3) + 
+                      3.141592653589793*Power(Z,2) + 
+                      R*(-219.9114857512855 - 
+                         0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
+                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))) + 
         0.5*R*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)*
          ((taui*(250000.00000000006 - 80.*Power(R,3) + 1.*Power(R,4) + 
                 1000.0000000000001*Power(Z,2) + 1.*Power(Z,4) + 
@@ -905,19 +877,20 @@ struct SNi{
               (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
                 (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
                  Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (0.10471975511965977*beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-                 Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-                ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                      10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*
-                    Cos(4*Pi*R) + 
-                   (71.6197243913529 - 1.5915494309189535*R + 
-                      0.07957747154594767*Power(R,2) + 
-                      0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*
-                 Sin(4*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))/R)
+            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+           (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+              Sin(4*Pi*t)*Sin(2*Pi*Z)*
+              (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                   0.3141592653589793*Power(R,2) - 
+                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
+                ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                      0.3141592653589793*Power(R,3) + 
+                      3.141592653589793*Power(Z,2) + 
+                      R*(-219.9114857512855 - 
+                         0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
+                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))))/R)
 ; }};
 struct SUe{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
@@ -1069,8 +1042,80 @@ struct SUe{
 - (2.6666666666666665*R*taue*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
            (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
          (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))/
-      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z)*
+      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) - 
+     (nuperp*((2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*Z)*Sin(2*P)*
+             Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+           (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+          (2*Pi*(1 - (1.*Power(Z,2))/
+                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
+             Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (3.*Sqrt(-mue)) - (13.333333333333332*Z*Cos(2*P)*
+             Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+          R*((13.15947253478581*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
+                Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
+              (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+             (83.7758040957278*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+             (4.1887902047863905*Power(-10 + R,2)*Z*Cos(2*Pi*Z)*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
+                 2)) - (4.1887902047863905*(-10 + R)*Power(Z,2)*
+                Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
+                 2)) + (2.0943951023931953*(-10 + R)*Cos(2*Pi*(-10 + R))*
+                Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+             (26.666666666666664*(-10 + R)*Z*Cos(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*Power(400 + 1.*Power(-10 + R,2) + 
+                  1.*Power(Z,2),2)) - 
+             (4*Power(Pi,2)*(1 - 
+                  (1.*Power(-10 + R,2))/
+                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (3.*Sqrt(-mue))) + 
+          R*((83.7758040957278*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+             (83.7758040957278*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*
+                Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+             (4*(Power(R,-2) - 
+                  400/
+                   (Power(R,2)*
+                     (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
+                Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (3.*Sqrt(-mue))) + 
+          R*((13.15947253478581*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
+                Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
+              (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+             (4.1887902047863905*Power(-10 + R,2)*Z*Cos(2*Pi*Z)*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
+                 2)) + (2.0943951023931953*Z*Cos(2*Pi*Z)*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+             (83.7758040957278*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+                Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+             (4.1887902047863905*(-10 + R)*Power(Z,2)*Cos(2*Pi*(-10 + R))*
+                Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
+                 2)) + (26.666666666666664*(-10 + R)*Z*Cos(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*Power(400 + 1.*Power(-10 + R,2) + 
+                  1.*Power(Z,2),2)) + 
+             (13.333333333333332*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
+                Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*Power(R,2)*
+                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+             (4*Power(Pi,2)*(1 - 
+                  (1.*Power(Z,2))/
+                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue))\
+)))/R + (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z)*
         ((-1884.9555921538758 + 37.69911184307752*R - 
              3.7699111843077526*Power(Z,2))*Cos(3*Pi*Z)*Sin(3*P)*
            Sin(3*Pi*R) + (-3.7699111843077517*R*Z*Cos(3*Pi*R)*Sin(3*P) + 
@@ -1089,18 +1134,17 @@ struct SUe{
            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
           ((3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
                0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-          (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-             (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                  1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-               (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                     0.015915494309189534*Power(R,2) - 
-                     0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                  (39.78873577297384 - 0.7957747154594768*R + 
-                     0.07957747154594767*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R)\
-)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-           (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-              2))))/(3.*Sqrt(-mue)) + 
+           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+          (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+             (R*(-3141.592653589793 + 125.66370614359172*R - 
+                  6.283185307179586*Power(R,2) - 
+                  6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+               (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                   Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
+                Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
+             Sin(4*Pi*Z))/
+           (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+               1.*Power(Z,2),2))))/(3.*Sqrt(-mue)) + 
      (2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z)*
         (0. - (2.*R*taue*Z)/
            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
@@ -1112,44 +1156,42 @@ struct SUe{
              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-          (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-             Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                  1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-               ((-79.57747154594767 + 11.140846016432674*R - 
-                     0.477464829275686*Power(R,2) + 
-                     0.015915494309189534*Power(R,3) - 
-                     0.15915494309189535*Power(Z,2) + 
-                     0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                  0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-           (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-              2))))/(3.*Sqrt(-mue)) + 
+          (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
+             Sin(4*Pi*t)*Sin(2*Pi*Z)*
+             (R*(3141.592653589793 - 125.66370614359172*R + 
+                  6.283185307179586*Power(R,2) + 
+                  6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+               ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                     1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                  0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+           (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+               1.*Power(Z,2),2))))/(3.*Sqrt(-mue)) + 
      (beta*taue*Sin(Pi*t)*Sin(4*Pi*t)*
-        (-9.869604401089358*Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*
-           (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-             (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                   0.015915494309189534*Power(R,2) - 
-                   0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                (39.78873577297384 - 0.7957747154594768*R + 
-                   0.07957747154594767*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R))*
-           Sin(4*Pi*Z) + 9.869604401089358*Cos(Pi*R)*Sin(P)*Sin(4*Pi*R)*
-           Sin(Pi*Z)*(R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-             ((-79.57747154594767 + 11.140846016432674*R - 
-                   0.477464829275686*Power(R,2) + 
-                   0.015915494309189534*Power(R,3) - 
-                   0.15915494309189535*Power(Z,2) + 
-                   0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)) - 
-          0.15707963267948966*Cos(P)*Sin(4*P)*Sin(Pi*R)*Sin(Pi*Z)*
-           (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(4*Pi*Z)*
-              Sin(4*Pi*R) + ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                   10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*Cos(4*Pi*R) \
-+ (71.6197243913529 - 1.5915494309189535*R + 
-                   0.07957747154594767*Power(R,2) + 
-                   0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*Sin(4*Pi*Z))\
-))/(mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)*
+        (1.5707963267948966*Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*
+           (R*(-3141.592653589793 + 125.66370614359172*R - 
+                6.283185307179586*Power(R,2) - 
+                6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+             (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                 Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
+              Sin(4*Pi*R))*Sin(4*Pi*Z) + 
+          1.5707963267948966*Cos(Pi*R)*Sin(P)*Sin(4*Pi*R)*Sin(Pi*Z)*
+           (R*(3141.592653589793 - 125.66370614359172*R + 
+                6.283185307179586*Power(R,2) + 
+                6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+             ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                   1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)) + 
+          0.5*Cos(P)*Sin(4*P)*Sin(Pi*R)*Sin(Pi*Z)*
+           (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                0.3141592653589793*Power(R,2) - 
+                0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
+             ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                   0.3141592653589793*Power(R,3) + 
+                   3.141592653589793*Power(Z,2) + 
+                   R*(-219.9114857512855 - 0.3141592653589793*Power(Z,2))\
+)*Cos(4*Pi*R) + (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                 Sin(4*Pi*R))*Sin(4*Pi*Z))))/
+      (mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)*
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
      (2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
         ((taue*(250000.00000000006 - 80.*Power(R,3) + 1.*Power(R,4) + 
@@ -1166,158 +1208,47 @@ struct SUe{
              (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
                (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
                 Sin(3*Pi*Z)))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-          (0.10471975511965977*beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-             Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-                Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-               ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                     10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*
-                   Cos(4*Pi*R) + 
-                  (71.6197243913529 - 1.5915494309189535*R + 
-                     0.07957747154594767*Power(R,2) + 
-                     0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*
-                Sin(4*Pi*Z)))/
-           (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-              2))))/(3.*Sqrt(-mue)) + 
-     (beta*Sin(3*Pi*t)*Sin(4*Pi*t)*
-        (-11.84352528130723*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
-           (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-             (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                   0.015915494309189534*Power(R,2) - 
-                   0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                (39.78873577297384 - 0.7957747154594768*R + 
-                   0.07957747154594767*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R))*
-           Sin(4*Pi*Z) + 11.84352528130723*Cos(3*Pi*R)*Sin(3*P)*
-           Sin(4*Pi*R)*Sin(3*Pi*Z)*
-           (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-             ((-79.57747154594767 + 11.140846016432674*R - 
-                   0.477464829275686*Power(R,2) + 
-                   0.015915494309189534*Power(R,3) - 
-                   0.15915494309189535*Power(Z,2) + 
-                   0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)) - 
-          0.18849555921538758*Cos(3*P)*Sin(4*P)*Sin(3*Pi*R)*Sin(3*Pi*Z)*
-           (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(4*Pi*Z)*
-              Sin(4*Pi*R) + ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                   10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*Cos(4*Pi*R) \
-+ (71.6197243913529 - 1.5915494309189535*R + 
-                   0.07957747154594767*Power(R,2) + 
-                   0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*Sin(4*Pi*Z))\
-))/(mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) - 
-     (nuperp*((1.*(-10 + R)*Z*((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/(3.*Sqrt(-mue)) + 
-               (beta*Pi*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                  Sin(4*Pi*t))/mue))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (1 - (1.*Power(Z,2))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-           ((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (3.*Sqrt(-mue)) + 
-             (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-              mue) - (20.*Z*((2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                  Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-               (beta*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-                mue))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-          R*((20.*(-10 + R)*((4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
-                     Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/(3.*Sqrt(-mue)) + 
-                  (4*beta*Pi*Cos(4*P)*Cos(4*Pi*Z)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mue))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (20.*Z*((4*Pi*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-                  (4*beta*Pi*Cos(4*P)*Cos(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mue))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (Power(R,-2) - 400/
-                 (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
-              ((-4*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                 (3.*Sqrt(-mue)) - 
-                (4*beta*Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                   Sin(4*Pi*Z))/mue)) + 
-          R*((1.*(-10 + R)*Z*((4*Power(Pi,2)*Cos(2*Pi*(-10 + R))*
-                     Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/(3.*Sqrt(-mue)) + 
-                  (4*beta*Power(Pi,2)*Cos(4*Pi*(-10 + R))*Cos(4*Pi*Z)*
-                     Sin(4*P)*Sin(4*Pi*t))/mue))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-             (20.*(-10 + R)*((4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
-                     Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/(3.*Sqrt(-mue)) + 
-                  (4*beta*Pi*Cos(4*P)*Cos(4*Pi*Z)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mue))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (2.*Power(-10 + R,2)*Z*
-                ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-                     Sin(2*Pi*t))/(3.*Sqrt(-mue)) + 
-                  (beta*Pi*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mue))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-             (2.*(-10 + R)*Power(Z,2)*
-                ((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-                  (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mue))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-             (1.*(-10 + R)*((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*
-                     Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-                  (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mue))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-             (40.*(-10 + R)*Z*
-                ((2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-                  (beta*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mue))/
-              (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-             (1 - (1.*Power(-10 + R,2))/
-                 (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-              ((-4*Power(Pi,2)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                   Sin(2*Pi*Z))/(3.*Sqrt(-mue)) - 
-                (4*beta*Power(Pi,2)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                   Sin(4*Pi*t)*Sin(4*Pi*Z))/mue)) + 
-          R*((1.*(-10 + R)*Z*((4*Power(Pi,2)*Cos(2*Pi*(-10 + R))*
-                     Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/(3.*Sqrt(-mue)) + 
-                  (4*beta*Power(Pi,2)*Cos(4*Pi*(-10 + R))*Cos(4*Pi*Z)*
-                     Sin(4*P)*Sin(4*Pi*t))/mue))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-             (2.*Power(-10 + R,2)*Z*
-                ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-                     Sin(2*Pi*t))/(3.*Sqrt(-mue)) + 
-                  (beta*Pi*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mue))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-             (1.*Z*((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-                     Sin(2*Pi*t))/(3.*Sqrt(-mue)) + 
-                  (beta*Pi*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mue))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-             (20.*Z*((4*Pi*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-                  (4*beta*Pi*Cos(4*P)*Cos(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mue))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (2.*(-10 + R)*Power(Z,2)*
-                ((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-                  (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mue))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-             (40.*(-10 + R)*Z*((2*Cos(2*P)*Sin(2*Pi*(-10 + R))*
-                     Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-                  (beta*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mue))/
-              (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-             (20.*Z*((2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-                  (beta*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mue))/
-              (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (1 - (1.*Power(Z,2))/
-                 (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-              ((-4*Power(Pi,2)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                   Sin(2*Pi*Z))/(3.*Sqrt(-mue)) - 
-                (4*beta*Power(Pi,2)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                   Sin(4*Pi*t)*Sin(4*Pi*Z))/mue))))/R)
+           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+          (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+             Sin(4*Pi*t)*Sin(2*Pi*Z)*
+             (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                  0.3141592653589793*Power(R,2) - 
+                  0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
+               ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                     0.3141592653589793*Power(R,3) + 
+                     3.141592653589793*Power(Z,2) + 
+                     R*(-219.9114857512855 - 
+                        0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
+                  (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                   Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+           (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+               1.*Power(Z,2),2))))/(3.*Sqrt(-mue)) + 
+     (3*beta*Sin(3*Pi*t)*Sin(4*Pi*t)*
+        (Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
+           (R*(-3141.592653589793 + 125.66370614359172*R - 
+                6.283185307179586*Power(R,2) - 
+                6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+             (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*Cos(4*P) + 
+                (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R))*
+           Sin(4*Pi*Z) + Pi*Cos(3*Pi*R)*Sin(3*P)*Sin(4*Pi*R)*Sin(3*Pi*Z)*
+           (R*(3141.592653589793 - 125.66370614359172*R + 
+                6.283185307179586*Power(R,2) + 
+                6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+             ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                   1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)) + 
+          Cos(3*P)*Sin(4*P)*Sin(3*Pi*R)*Sin(3*Pi*Z)*
+           (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                0.3141592653589793*Power(R,2) - 
+                0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
+             ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                   0.3141592653589793*Power(R,3) + 
+                   3.141592653589793*Power(Z,2) + 
+                   R*(-219.9114857512855 - 0.3141592653589793*Power(Z,2)))*
+                 Cos(4*Pi*R) + 
+                (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                 Sin(4*Pi*R))*Sin(4*Pi*Z))))/
+      (5.*mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))
 ; }};
 struct SUi{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
@@ -1450,7 +1381,73 @@ struct SUi{
         (2.6666666666666665*R*taui*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
            (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
          Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))/
-      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
+      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) - 
+     (nuperp*((2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*Z)*Sin(2*P)*
+             Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+          (2*Pi*(1 - (1.*Power(Z,2))/
+                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
+             Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/3. - 
+          (13.333333333333332*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+             Sin(2*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+          R*((13.15947253478581*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
+                Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
+              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+             (83.7758040957278*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+             (4.1887902047863905*Power(-10 + R,2)*Z*Cos(2*Pi*Z)*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
+             (4.1887902047863905*(-10 + R)*Power(Z,2)*
+                Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
+             (2.0943951023931953*(-10 + R)*Cos(2*Pi*(-10 + R))*Sin(2*P)*
+                Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+             (26.666666666666664*(-10 + R)*Z*Cos(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) - 
+             (4*Power(Pi,2)*(1 - 
+                  (1.*Power(-10 + R,2))/
+                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3.) + 
+          R*((83.7758040957278*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+             (83.7758040957278*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*
+                Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+             (4*(Power(R,-2) - 
+                  400/
+                   (Power(R,2)*
+                     (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
+                Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3.) + 
+          R*((13.15947253478581*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
+                Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
+              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+             (4.1887902047863905*Power(-10 + R,2)*Z*Cos(2*Pi*Z)*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
+             (2.0943951023931953*Z*Cos(2*Pi*Z)*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+             (83.7758040957278*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+                Sin(2*Pi*Z))/
+              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+             (4.1887902047863905*(-10 + R)*Power(Z,2)*Cos(2*Pi*(-10 + R))*
+                Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
+             (26.666666666666664*(-10 + R)*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
+                Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
+             (13.333333333333332*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
+                Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+             (4*Power(Pi,2)*(1 - 
+                  (1.*Power(Z,2))/
+                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(2*P)*
+                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3.)))/R + 
      (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z)*
         ((-1884.9555921538758 + 37.69911184307752*R - 
              3.7699111843077526*Power(Z,2))*Cos(3*Pi*Z)*Sin(3*P)*
@@ -1469,17 +1466,16 @@ struct SUi{
            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
           ((3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
                0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-          (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-             (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                  1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-               (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                     0.015915494309189534*Power(R,2) - 
-                     0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                  (39.78873577297384 - 0.7957747154594768*R + 
-                     0.07957747154594767*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R)\
-)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))/3. + 
+           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+          (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+             (R*(-3141.592653589793 + 125.66370614359172*R - 
+                  6.283185307179586*Power(R,2) - 
+                  6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+               (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                   Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
+                Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
+             Sin(4*Pi*Z))/
+           (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))))/3. + 
      (2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z)*
         (0. - (2.*R*taui*Z)/
            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
@@ -1491,43 +1487,41 @@ struct SUi{
              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-          (2.0943951023931953*beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-             Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                  1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-               ((-79.57747154594767 + 11.140846016432674*R - 
-                     0.477464829275686*Power(R,2) + 
-                     0.015915494309189534*Power(R,3) - 
-                     0.15915494309189535*Power(Z,2) + 
-                     0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                  0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))/3. + 
+          (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
+             Sin(4*Pi*t)*Sin(2*Pi*Z)*
+             (R*(3141.592653589793 - 125.66370614359172*R + 
+                  6.283185307179586*Power(R,2) + 
+                  6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+               ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                     1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                  0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
+           (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))))/3. + 
      (beta*taui*Sin(Pi*t)*Sin(4*Pi*t)*
-        (-9.869604401089358*Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*
-           (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-             (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                   0.015915494309189534*Power(R,2) - 
-                   0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                (39.78873577297384 - 0.7957747154594768*R + 
-                   0.07957747154594767*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R))*
-           Sin(4*Pi*Z) + 9.869604401089358*Cos(Pi*R)*Sin(P)*Sin(4*Pi*R)*
-           Sin(Pi*Z)*(R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-             ((-79.57747154594767 + 11.140846016432674*R - 
-                   0.477464829275686*Power(R,2) + 
-                   0.015915494309189534*Power(R,3) - 
-                   0.15915494309189535*Power(Z,2) + 
-                   0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)) - 
-          0.15707963267948966*Cos(P)*Sin(4*P)*Sin(Pi*R)*Sin(Pi*Z)*
-           (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(4*Pi*Z)*
-              Sin(4*Pi*R) + ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                   10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*Cos(4*Pi*R) \
-+ (71.6197243913529 - 1.5915494309189535*R + 
-                   0.07957747154594767*Power(R,2) + 
-                   0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*Sin(4*Pi*Z))\
-))/(mui*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)*
+        (1.5707963267948966*Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*
+           (R*(-3141.592653589793 + 125.66370614359172*R - 
+                6.283185307179586*Power(R,2) - 
+                6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+             (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
+                 Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
+              Sin(4*Pi*R))*Sin(4*Pi*Z) + 
+          1.5707963267948966*Cos(Pi*R)*Sin(P)*Sin(4*Pi*R)*Sin(Pi*Z)*
+           (R*(3141.592653589793 - 125.66370614359172*R + 
+                6.283185307179586*Power(R,2) + 
+                6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+             ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                   1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)) + 
+          0.5*Cos(P)*Sin(4*P)*Sin(Pi*R)*Sin(Pi*Z)*
+           (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                0.3141592653589793*Power(R,2) - 
+                0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
+             ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                   0.3141592653589793*Power(R,3) + 
+                   3.141592653589793*Power(Z,2) + 
+                   R*(-219.9114857512855 - 0.3141592653589793*Power(Z,2))\
+)*Cos(4*Pi*R) + (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                 Sin(4*Pi*R))*Sin(4*Pi*Z))))/
+      (mui*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)*
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
      (2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
         ((taui*(250000.00000000006 - 80.*Power(R,3) + 1.*Power(R,4) + 
@@ -1544,156 +1538,46 @@ struct SUi{
              (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
                (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
                 Sin(3*Pi*Z)))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-          (0.10471975511965977*beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-             Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-                Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-               ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                     10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*
-                   Cos(4*Pi*R) + 
-                  (71.6197243913529 - 1.5915494309189535*R + 
-                     0.07957747154594767*Power(R,2) + 
-                     0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*
-                Sin(4*Pi*Z)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))/3. + 
-     (beta*Sin(3*Pi*t)*Sin(4*Pi*t)*
-        (-11.84352528130723*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
-           (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-             (Z*(-7.957747154594767 + 0.3183098861837907*R - 
-                   0.015915494309189534*Power(R,2) - 
-                   0.015915494309189534*Power(Z,2))*Cos(4*P) + 
-                (39.78873577297384 - 0.7957747154594768*R + 
-                   0.07957747154594767*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R))*
-           Sin(4*Pi*Z) + 11.84352528130723*Cos(3*Pi*R)*Sin(3*P)*
-           Sin(4*Pi*R)*Sin(3*Pi*Z)*
-           (R*(499.99999999999994 - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-             ((-79.57747154594767 + 11.140846016432674*R - 
-                   0.477464829275686*Power(R,2) + 
-                   0.015915494309189534*Power(R,3) - 
-                   0.15915494309189535*Power(Z,2) + 
-                   0.015915494309189534*R*Power(Z,2))*Cos(4*P) - 
-                0.07957747154594767*R*Z*Sin(4*P))*Sin(4*Pi*Z)) - 
-          0.18849555921538758*Cos(3*P)*Sin(4*P)*Sin(3*Pi*R)*Sin(3*Pi*Z)*
-           (Z*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(4*Pi*Z)*
-              Sin(4*Pi*R) + ((-5000. - 30.*Power(R,2) + 1.*Power(R,3) - 
-                   10.*Power(Z,2) + R*(700. + 1.*Power(Z,2)))*Cos(4*Pi*R) \
-+ (71.6197243913529 - 1.5915494309189535*R + 
-                   0.07957747154594767*Power(R,2) + 
-                   0.07957747154594767*Power(Z,2))*Sin(4*Pi*R))*Sin(4*Pi*Z))\
-))/(mui*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) - 
-     (nuperp*((1.*(-10 + R)*Z*((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/3. + 
-               (beta*Pi*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                  Sin(4*Pi*t))/mui))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (1 - (1.*Power(Z,2))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-           ((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              3. + (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*
-                Sin(4*Pi*Z))/mui) - 
-          (20.*Z*((2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                  Sin(2*Pi*Z))/3. + 
-               (beta*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-                mui))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-          R*((20.*(-10 + R)*((4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
-                     Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/3. + 
-                  (4*beta*Pi*Cos(4*P)*Cos(4*Pi*Z)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mui))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (20.*Z*((4*Pi*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/3. + 
-                  (4*beta*Pi*Cos(4*P)*Cos(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mui))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (Power(R,-2) - 400/
-                 (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
-              ((-4*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                 3. - (4*beta*Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                   Sin(4*Pi*Z))/mui)) + 
-          R*((1.*(-10 + R)*Z*((4*Power(Pi,2)*Cos(2*Pi*(-10 + R))*
-                     Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/3. + 
-                  (4*beta*Power(Pi,2)*Cos(4*Pi*(-10 + R))*Cos(4*Pi*Z)*
-                     Sin(4*P)*Sin(4*Pi*t))/mui))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-             (20.*(-10 + R)*((4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
-                     Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/3. + 
-                  (4*beta*Pi*Cos(4*P)*Cos(4*Pi*Z)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mui))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (2.*Power(-10 + R,2)*Z*
-                ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-                     Sin(2*Pi*t))/3. + 
-                  (beta*Pi*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mui))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-             (2.*(-10 + R)*Power(Z,2)*
-                ((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/3. + 
-                  (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mui))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-             (1.*(-10 + R)*((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*
-                     Sin(2*Pi*t)*Sin(2*Pi*Z))/3. + 
-                  (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mui))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-             (40.*(-10 + R)*Z*
-                ((2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/3. + 
-                  (beta*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mui))/
-              (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-             (1 - (1.*Power(-10 + R,2))/
-                 (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-              ((-4*Power(Pi,2)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                   Sin(2*Pi*Z))/3. - 
-                (4*beta*Power(Pi,2)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                   Sin(4*Pi*t)*Sin(4*Pi*Z))/mui)) + 
-          R*((1.*(-10 + R)*Z*((4*Power(Pi,2)*Cos(2*Pi*(-10 + R))*
-                     Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/3. + 
-                  (4*beta*Power(Pi,2)*Cos(4*Pi*(-10 + R))*Cos(4*Pi*Z)*
-                     Sin(4*P)*Sin(4*Pi*t))/mui))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-             (2.*Power(-10 + R,2)*Z*
-                ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-                     Sin(2*Pi*t))/3. + 
-                  (beta*Pi*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mui))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-             (1.*Z*((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-                     Sin(2*Pi*t))/3. + 
-                  (beta*Pi*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                     Sin(4*Pi*t))/mui))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-             (20.*Z*((4*Pi*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/3. + 
-                  (4*beta*Pi*Cos(4*P)*Cos(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mui))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (2.*(-10 + R)*Power(Z,2)*
-                ((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/3. + 
-                  (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mui))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-             (40.*(-10 + R)*Z*((2*Cos(2*P)*Sin(2*Pi*(-10 + R))*
-                     Sin(2*Pi*t)*Sin(2*Pi*Z))/3. + 
-                  (beta*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mui))/
-              (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-             (20.*Z*((2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                     Sin(2*Pi*Z))/3. + 
-                  (beta*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-                     Sin(4*Pi*Z))/mui))/
-              (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (1 - (1.*Power(Z,2))/
-                 (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-              ((-4*Power(Pi,2)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                   Sin(2*Pi*Z))/3. - 
-                (4*beta*Power(Pi,2)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-                   Sin(4*Pi*t)*Sin(4*Pi*Z))/mui))))/R)
+           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+          (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+             Sin(4*Pi*t)*Sin(2*Pi*Z)*
+             (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                  0.3141592653589793*Power(R,2) - 
+                  0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
+               ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                     0.3141592653589793*Power(R,3) + 
+                     3.141592653589793*Power(Z,2) + 
+                     R*(-219.9114857512855 - 
+                        0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
+                  (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                   Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+           (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))))/3. + 
+     (3*beta*Sin(3*Pi*t)*Sin(4*Pi*t)*
+        (Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
+           (R*(-3141.592653589793 + 125.66370614359172*R - 
+                6.283185307179586*Power(R,2) - 
+                6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
+             (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*Cos(4*P) + 
+                (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R))*
+           Sin(4*Pi*Z) + Pi*Cos(3*Pi*R)*Sin(3*P)*Sin(4*Pi*R)*Sin(3*Pi*Z)*
+           (R*(3141.592653589793 - 125.66370614359172*R + 
+                6.283185307179586*Power(R,2) + 
+                6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
+             ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
+                   1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
+                0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)) + 
+          Cos(3*P)*Sin(4*P)*Sin(3*Pi*R)*Sin(3*Pi*Z)*
+           (Z*(-157.07963267948966 + 6.283185307179586*R - 
+                0.3141592653589793*Power(R,2) - 
+                0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
+             ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
+                   0.3141592653589793*Power(R,3) + 
+                   3.141592653589793*Power(Z,2) + 
+                   R*(-219.9114857512855 - 0.3141592653589793*Power(Z,2)))*
+                 Cos(4*Pi*R) + 
+                (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
+                 Sin(4*Pi*R))*Sin(4*Pi*Z))))/
+      (5.*mui*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))
 ; }};
 struct SPhie{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
@@ -1963,65 +1847,70 @@ struct SGammaNi{
 struct SA{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return -(Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+    return -(beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
       (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/3. \
-+ (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
++ (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
      (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-   (3.*Sqrt(-mue)) - ((3.141592653589793*(-10 + R)*Z*Cos(4*Pi*Z)*Sin(4*P)*
-        Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
+   (3.*Sqrt(-mue)) - ((3.141592653589793*beta*(-10 + R)*Z*Cos(4*Pi*Z)*
+        Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
       (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-     Pi*(1 - (1.*Power(Z,2))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-      Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*Sin(4*Pi*Z) - 
-     (20.*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/
+     beta*Pi*(1 - (1.*Power(Z,2))/
+         (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Cos(4*Pi*(-10 + R))*
+      Sin(4*P)*Sin(4*Pi*t)*Sin(4*Pi*Z) - 
+     (20.*beta*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/
       (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-     R*((39.47841760435743*(-10 + R)*Z*Cos(4*Pi*(-10 + R))*Cos(4*Pi*Z)*
-           Sin(4*P)*Sin(4*Pi*t))/
+     R*((39.47841760435743*beta*(-10 + R)*Z*Cos(4*Pi*(-10 + R))*
+           Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*t))/
          (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-        (251.32741228718345*(-10 + R)*Cos(4*P)*Cos(4*Pi*Z)*
+        (251.32741228718345*beta*(-10 + R)*Cos(4*P)*Cos(4*Pi*Z)*
            Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
          (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (6.283185307179586*Power(-10 + R,2)*Z*Cos(4*Pi*Z)*Sin(4*P)*
+        (6.283185307179586*beta*Power(-10 + R,2)*Z*Cos(4*Pi*Z)*Sin(4*P)*
            Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
          Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-        (6.283185307179586*(-10 + R)*Power(Z,2)*Cos(4*Pi*(-10 + R))*
+        (6.283185307179586*beta*(-10 + R)*Power(Z,2)*Cos(4*Pi*(-10 + R))*
            Sin(4*P)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
          Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-        (3.141592653589793*(-10 + R)*Cos(4*Pi*(-10 + R))*Sin(4*P)*
+        (3.141592653589793*beta*(-10 + R)*Cos(4*Pi*(-10 + R))*Sin(4*P)*
            Sin(4*Pi*t)*Sin(4*Pi*Z))/
          (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-        (40.*(-10 + R)*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
+        (40.*beta*(-10 + R)*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
            Sin(4*Pi*Z))/
          (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) - 
-        4*Power(Pi,2)*(1 - (1.*Power(-10 + R,2))/
+        4*beta*Power(Pi,2)*(1 - 
+           (1.*Power(-10 + R,2))/
             (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(4*P)*
          Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z)) + 
-     R*((251.32741228718345*(-10 + R)*Cos(4*P)*Cos(4*Pi*Z)*
+     R*((251.32741228718345*beta*(-10 + R)*Cos(4*P)*Cos(4*Pi*Z)*
            Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
          (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-        (251.32741228718345*Z*Cos(4*P)*Cos(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-           Sin(4*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-        4*(Power(R,-2) - 400/
+        (251.32741228718345*beta*Z*Cos(4*P)*Cos(4*Pi*(-10 + R))*
+           Sin(4*Pi*t)*Sin(4*Pi*Z))/
+         (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+        4*beta*(Power(R,-2) - 400/
             (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
          Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z)) + 
-     R*((39.47841760435743*(-10 + R)*Z*Cos(4*Pi*(-10 + R))*Cos(4*Pi*Z)*
-           Sin(4*P)*Sin(4*Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) \
-- (6.283185307179586*Power(-10 + R,2)*Z*Cos(4*Pi*Z)*Sin(4*P)*
+     R*((39.47841760435743*beta*(-10 + R)*Z*Cos(4*Pi*(-10 + R))*
+           Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*t))/
+         (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+        (6.283185307179586*beta*Power(-10 + R,2)*Z*Cos(4*Pi*Z)*Sin(4*P)*
            Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
          Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-        (3.141592653589793*Z*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
+        (3.141592653589793*beta*Z*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
            Sin(4*Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-        (251.32741228718345*Z*Cos(4*P)*Cos(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-           Sin(4*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (6.283185307179586*(-10 + R)*Power(Z,2)*Cos(4*Pi*(-10 + R))*
+        (251.32741228718345*beta*Z*Cos(4*P)*Cos(4*Pi*(-10 + R))*
+           Sin(4*Pi*t)*Sin(4*Pi*Z))/
+         (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+        (6.283185307179586*beta*(-10 + R)*Power(Z,2)*Cos(4*Pi*(-10 + R))*
            Sin(4*P)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
          Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-        (40.*(-10 + R)*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
+        (40.*beta*(-10 + R)*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
            Sin(4*Pi*Z))/
          (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-        (20.*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/
+        (20.*beta*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/
          (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-        4*Power(Pi,2)*(1 - (1.*Power(Z,2))/
-            (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(4*P)*
-         Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z)))/R
+        4*beta*Power(Pi,2)*(1 - 
+           (1.*Power(Z,2))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
+         Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z)))/R
 ; }};
 }}//namespace feltor namespace manufactured
-- 
GitLab


From cc09aa04b7fe8d693216de1517c7bc2a1e44ab29 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 21 Mar 2019 17:18:53 +0100
Subject: [PATCH 045/540] Insert Multigrid in Apar solution again

since the inner loop seems to take most of the time
---
 src/feltor/implicit.h | 45 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 1982c7569..d3149f8e1 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -119,16 +119,32 @@ struct ImplicitVelocity
             = dg::geo::createProjectionTensor( bhat, g);
         //set perpendicular projection tensor h
         m_lapM_perpU.set_chi( hh);
-        m_induction.construct(  g,
-            p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
-        m_induction.elliptic().set_chi( hh);
-        m_invert.construct( m_temp, g.size(), p.eps_pol,1 );
+        //m_induction.construct(  g,
+        //    p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
+        //m_induction.elliptic().set_chi( hh);
+        //m_invert.construct( m_temp, g.size(), p.eps_pol,1 );
+        //Multigrid setup
+        m_multi_induction.resize(p.stages);
+        m_multigrid.construct( g, p.stages);
+        for( unsigned u=0; u<p.stages; u++)
+        {
+            dg::SparseTensor<Container> hh = dg::geo::createProjectionTensor(
+                bhat, m_multigrid.grid(u));
+            m_multi_induction[u].construct(  m_multigrid.grid(u),
+                p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
+            m_multi_induction[u].elliptic().set_chi( hh);
+        }
+        m_multi_chi = m_multigrid.project( m_temp);
+        m_old_apar = dg::Extrapolation<Container>( 1, dg::evaluate( dg::zero, g));
     }
     void set_density( const std::array<Container, 2>& dens){
         dg::blas1::transform( dens, m_fields[0], dg::PLUS<double>(+1));
         dg::blas1::axpby(  m_p.beta/m_p.mu[1], m_fields[0][1],
                           -m_p.beta/m_p.mu[0], m_fields[0][0], m_temp);
-        m_induction.set_chi( m_temp);
+        //m_induction.set_chi( m_temp);
+        m_multigrid.project( m_temp, m_multi_chi);
+        for( unsigned u=0; u<m_p.stages; u++)
+            m_multi_induction[u].set_chi( m_multi_chi[u]);
     }
 
     void operator()( double t, const std::array<Container,2>& w,
@@ -146,8 +162,15 @@ struct ImplicitVelocity
             dg::blas1::pointwiseDot(  m_p.beta, m_fields[0][1], m_fields[1][1],
                                      -m_p.beta, m_fields[0][0], m_fields[1][0],
                                       0., m_temp);
-            m_invert( m_induction, m_apar, m_temp, weights(),
-                inv_weights(), precond());
+            //m_invert( m_induction, m_apar, m_temp, weights(),
+            //    inv_weights(), precond());
+            m_old_apar.extrapolate( m_apar);
+            std::vector<unsigned> number = m_multigrid.direct_solve(
+                m_multi_induction, m_apar, m_temp, m_p.eps_pol);
+            m_old_apar.update( m_apar);
+            if(  number[0] == m_multigrid.max_iter())
+                throw dg::Fail( m_p.eps_pol);
+
             //compute u_e and U_i from w_e, W_i and apar
             dg::blas1::axpby( 1., m_fields[1][0], -1./m_p.mu[0],
                 m_apar, m_fields[1][0]);
@@ -190,8 +213,12 @@ struct ImplicitVelocity
   private:
     feltor::Parameters m_p;
     Container m_temp, m_apar;
-    dg::Invert<Container> m_invert;
-    dg::Helmholtz3d<Geometry, Matrix, Container> m_induction;
+    //dg::Invert<Container> m_invert;
+    //dg::Helmholtz3d<Geometry, Matrix, Container> m_induction;
+    dg::MultigridCG2d<Geometry, Matrix, Container> m_multigrid;
+    std::vector<dg::Helmholtz3d<Geometry, Matrix, Container>> m_multi_induction;
+    dg::Extrapolation<Container> m_old_apar;
+    std::vector<Container> m_multi_chi;
     std::array<std::array<Container,2>,2> m_fields;
     dg::Elliptic3d<Geometry, Matrix, Container> m_lapM_perpU;
 };
-- 
GitLab


From 966c4739f60ec02bc823e8d988561f5362b54d9a Mon Sep 17 00:00:00 2001
From: mattwi <mattwi@fysik.dtu.dk>
Date: Fri, 22 Mar 2019 21:23:30 +0100
Subject: [PATCH 046/540] Correct Doku of nrmb_correction in CG

---
 inc/dg/cg.h          | 14 +++++++-------
 src/feltor/feltor.cu |  4 ++++
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index f5890f9bd..60dd3a7b6 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -65,13 +65,13 @@ class CG
      * @brief Solve the system A*x = b using a preconditioned conjugate gradient method
      *
      * The iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + C) \f$ where \f$C\f$ is
-     * a correction factor to the absolute error
+     * the absolute error in units of \f$ \epsilon\f$
      * @param A A symmetric, positive definit matrix
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
      * @param P The preconditioner to be used
      * @param eps The relative error to be respected
-     * @param nrmb_correction Correction factor C for norm of b
+     * @param nrmb_correction the absolute error \c C in units of \c eps to be respected
      * @attention This version uses the Preconditioner to compute the norm for the error condition (this safes one scalar product)
      *
      * @return Number of iterations used to achieve desired precision
@@ -90,14 +90,14 @@ class CG
      * @brief Solve \f$ Ax = b\f$ using a preconditioned conjugate gradient method
      *
      * The iteration stops if \f$ ||Ax||_S < \epsilon( ||b||_S + C) \f$ where \f$C\f$ is
-     * a correction factor to the absolute error and \f$ S \f$ defines a square norm
+     * the absolute error in units of \f$ \epsilon\f$ and \f$ S \f$ defines a square norm
      * @param A A symmetric positive definit matrix
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
      * @param P The preconditioner to be used
      * @param S Weights used to compute the norm for the error condition
      * @param eps The relative error to be respected
-     * @param nrmb_correction Correction factor C for norm of b
+     * @param nrmb_correction the absolute error \c C in units of \c eps to be respected
      *
      * @return Number of iterations used to achieve desired precision
      * @note Required memops per iteration (\c P and \c S are assumed vectors):
@@ -576,7 +576,7 @@ struct Invert
      * @param eps relative error in conjugate gradient
      * @param extrapolationType number of last values to use for extrapolation of the current guess
      * @param multiplyWeights if true the rhs shall be multiplied by the weights before cg is applied
-     * @param nrmb_correction Correction factor for norm of b (cf. CG)
+     * @param nrmb_correction the absolute error \c C in units of \c eps in conjugate gradient
      */
     void construct( const ContainerType& copyable, unsigned max_iter, value_type eps, int extrapolationType = 2, bool multiplyWeights = true, value_type nrmb_correction = 1.)
     {
@@ -601,8 +601,8 @@ struct Invert
     /**
      * @brief Set accuracy parameters for following inversions
      *
-     * @param eps
-     * @param nrmb_correction
+     * @param eps the relative error in conjugate gradient
+     * @param nrmb_correction the absolute error \c C in units of \c eps in conjugate gradient
      */
     void set_accuracy( value_type eps, value_type nrmb_correction = 1.) {
         eps_ = eps;
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 4751957ca..9cb0bddf9 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -132,7 +132,9 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( 1., dg::construct<dg::DVec>(ntilde), 1., y0[0][0]);
     std::cout << "initialize ni" << std::endl;
     if( p.initphi == "zero")
+    {
         feltor.initializeni( y0[0][0], y0[0][1]);
+    }
     else if( p.initphi == "balance")
         dg::blas1::copy( y0[0][0], y0[0][1]); //set N_i = n_e
     else
@@ -140,6 +142,7 @@ int main( int argc, char* argv[])
 
     dg::blas1::copy( 0., y0[1][0]); //set we = 0
     dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
+    std::cout << "Initialize Timestepper" << std::endl;
 
     ////////////////////////create timer and timestepper
     //
@@ -151,6 +154,7 @@ int main( int argc, char* argv[])
             dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
         > karniadakis( grid, p, mag);
     karniadakis.init( feltor, im, time, y0, p.dt);
+    std::cout << "Done!" << std::endl;
 
     //dg::Adaptive< dg::ERKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
     //    "Bogacki-Shampine-4-2-3", y0);
-- 
GitLab


From f26d5283f4ce5a7081ea177bed6c20f020197697 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 22 Mar 2019 14:22:37 +0100
Subject: [PATCH 047/540] Store manufactured.json paramters

instead of the old verification.json
---
 src/feltor/input/manufactured.json | 50 ++++++++++++++++++++++++++++
 src/feltor/input/verification.json | 53 ------------------------------
 2 files changed, 50 insertions(+), 53 deletions(-)
 create mode 100644 src/feltor/input/manufactured.json
 delete mode 100644 src/feltor/input/verification.json

diff --git a/src/feltor/input/manufactured.json b/src/feltor/input/manufactured.json
new file mode 100644
index 000000000..b76e031bc
--- /dev/null
+++ b/src/feltor/input/manufactured.json
@@ -0,0 +1,50 @@
+{
+    "n"  : 3,
+    "Nx" : 64,
+    "Ny" : 64,
+    "Nz" : 32,
+    "dt" : 1e-4,
+    "compression" : [2,2],
+    "refineDS" : [1,1],
+    "rk4eps" : 1e-6,
+    "inner_loop" : 2,
+    "itstp"  : 2,
+    "maxout" : 5,
+    "eps_pol"    : 1e-7,
+    "jumpfactor" : 1,
+    "eps_gamma"  : 1e-5,
+    "stages"     : 3,
+    "eps_time"   : 1e-7,
+    "rtol"       : 1e-4,
+    "mu"          : -0.000272121,
+    "tau"         : 0,
+    "beta"        : 10,
+    "nu_perp"     : 0.1,
+    "perp_diff"   : "viscous",
+    "nu_parallel" : 0,
+    "resistivity" : 0,
+    "amplitude" : 0.01,
+    "sigma"     : 2.0,
+    "posX"      : 0.6,
+    "posY"      : 0,
+    "sigma_z"   : 0.25,
+    "k_psi"     : 0,
+    "nprofileamp" : 4,
+    "bgprofamp"   : 1,
+    "bc" :
+    {
+        "density" : ["DIR", "DIR"],
+        "velocity": ["DIR", "DIR"],
+        "potential":["DIR", "DIR"]
+    },
+    "boxscaleR" :  [1.15,1.2],
+    "boxscaleZ" :  [1.2,1.15],
+    "initne"     : "turbulence",
+    "initphi"    : "zero",
+    "curvmode"   : "true",
+    "symmetric"  : false,
+    "alpha"      : 0.05,
+    "source"     : 0,
+    "rho_source" : 0.2,
+    "rho_damping" : 1.2
+}
diff --git a/src/feltor/input/verification.json b/src/feltor/input/verification.json
deleted file mode 100644
index 2cedcc5b5..000000000
--- a/src/feltor/input/verification.json
+++ /dev/null
@@ -1,53 +0,0 @@
-
-//                * Input-File for "FELTOR" *
-//                -------------------------
-{
-    //----------------------------Space and Time discretization------------
-    "n"  : 3,   //(# of x,y-polynomials)
-    "Nx" : 52,  //(grid points in x)
-    "Ny" : 52,  //(grid points in y)
-    "Nz" : 16,  //(grid points in z)
-    "dt" : 1e-2,//(time step in units c_s/rho_s)
-    //-------------------------------Output parameters--------------------
-    "n_out" : 3,    //(# of x-y polynomials in output)
-    "Nx_out" : 52,  //(# grid points in output field)
-    "Ny_out" : 52,  //(# grid points in output field)
-    "Nz_out" : 16,  //(# grid points in output field)
-    "itstp"  : 2,   //(steps between outputs)
-    "maxout" : 10,  //total # of outputs (excluding first)
-    //-------------------------------Algorithmic parameters---------------------
-    "eps_pol"    : 1e-5, //( stop for polarisation)   
-    "jumpfactor" : 1, //jumpfactor € [0.01,1]
-    "eps_gamma"  : 1e-6, //( stop for Gamme operator) 
-    "eps_time"   : 1e-9, //( stop for time inversion) 
-    //----------------------Physical parameters----------------------------
-    "mu"          : -0.00272121, //(-m_e/m_i) -0.000544617, -0.000272121, -0.000181372
-    "tau"         : 0,      //T_i/T_e
-    "nu_perp"     : 1e-2,   //  (perpendicular viscosity)
-    "nu_parallel" : 0.1,     //  (parallel viscosity)
-    "resistivity" : 1e-4,   //parallel resistivity
-    //---------------------Initial perturbation parameters---------------------
-    "amplitude" : 0.01, // (blob amplitude)
-    "sigma"     : 2.0,  // (blob variance in units of rho_s)
-    "posX"      : 0.3,  //x-position ( in units of a)
-    "posY"      : 0,    //y-position ( in units of a)
-    "sigma_z"   : 0.25, // (variance in units of R_0)
-    "k_psi"     : 0,    // (zonal modes)
-    "nprofileamp" : 4,  //Profile peak amplitude
-    "bgprofamp"   : 1,  //Background Prof amplitude (density on the boundary)
-    //-------------------------------Boundary condition----------------------------
-    "bc"          : "DIR", //GLOBAL BC for parallel derivative
-    //-------------------------------Miscellaneous----------------------------
-    "source"     : 0,     //profile source rate
-    "boxscaleRp" :  1.1, // (a little larger than 1)
-    "boxscaleRm" :  1.1, // (a little larger than 1)
-    "boxscaleZp" :  1.1, // (a little larger than 1)
-    "boxscaleZm" :  1.2, // (a little larger than 1)
-    //-------------------------------Sim Setup-----------------------------
-    "pollim"     : 0,    //poloidal limiter (0/1) 
-    "pardiss"    : 0,    //Parallel dissipation(adj (0), nadj(1))
-    "mode"       : 2,    //initial condition blob(0), straight blob(1), turbulence(2)
-    "initial"    : 0,    //init. phi cond. (stand(0), Force Balance(1)
-    "curvmode"    : 1    //curvature (low beta (0), tfl (1))
-}
-//@ ------------------------------------------------------------
-- 
GitLab


From 5e3de1a09e73f8600b201a85fb8c3e49fa676dfe Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 25 Mar 2019 15:33:17 +0100
Subject: [PATCH 048/540] Change initial profile damping to 4 alpha

- should avoid too many oscillations in ue in the beginning
---
 src/feltor/feltor.cu     |  2 +-
 src/feltor/feltor.tex    | 34 ++++++++++++++++++----------------
 src/feltor/feltor_hpc.cu |  2 +-
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 9cb0bddf9..48fc129a2 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -71,7 +71,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
     dg::HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
-        mag.psip(), -p.alpha, p.alpha, -1), grid);
+        mag.psip(), -4*p.alpha, 4*p.alpha, -1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index fad7d8ede..223846fe1 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -382,7 +382,7 @@ field component vanishes in the corners of the domain.
 
 We define an approximation to the step function with width $\alpha$
 \begin{align}
-\Theta(\psi) := \begin{cases}
+\Theta_\alpha(\psi) := \begin{cases}
     1 & \text{ for } \psi < - \alpha  \\
     \frac{1}{32 \alpha^7}  \left(16 \alpha^3+29 \alpha^2 \psi+20 \alpha \psi^2+5 \psi^3\right) (\alpha-\psi)^4
     &\text{ for } -\alpha<\psi<+\alpha \\
@@ -394,7 +394,7 @@ We define an approximation to the step function with width $\alpha$
 if $H(\psi)$ is the Heaviside step function.
 An integral of this function is
 \begin{align}
-\theta(\psi) := \begin{cases}
+\theta_\alpha(\psi) := \begin{cases}
 \psi &\text{ for } \psi < -\alpha \\
     - \frac{1}{256 \alpha^7} \left(35 \alpha^3+47 \alpha^2 \psi+25 \alpha \psi^2+5 \psi^3\right) (\alpha-\psi)^5
      &\text{ for } -\alpha<\psi<+\alpha \\
@@ -404,10 +404,10 @@ An integral of this function is
 \label{eq:modified_psi}
 \end{align}
 
-We now use $\theta(\psi-\psi_0)+\psi_0$ instead of $\psi$ for the computation of the
+We now use $\theta_\alpha(\psi-\psi_0)+\psi_0$ instead of $\psi$ for the computation of the
 magnetic field, which introduces a shear layer around $\psi_0$ where the
 fieldlines are straightened to match $\ehat_\varphi$.
-Note that $\Theta(0) = 0.5$ and $\theta(0) = 35\alpha/256$.
+Note that $\Theta_\alpha(0) = 0.5$ and $\theta_\alpha(0) = 35\alpha/256$.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
@@ -546,11 +546,13 @@ Note that we should take care to intitialize a smooth profile with ideally well-
 Let us define a flux-aligned density profile as
 \begin{align} \label{eq:density_profile}
   n_{\text{prof}}(R,Z)=
-      n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) \Theta(\psi_p(R, Z)+\alpha)}{\psi_p(R_0,0)} H(Z-Z_X)
+      n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_p(R_0,0)}\Theta_{4\alpha}(\psi_p(R, Z)+4\alpha) H(Z-Z_X)
 \end{align}
 %NOTE: this profile is slightly below n_0 outside the LCFS! (not sure whether this does anything for nabla_parallel)
 The second Heaviside is multiplied only if the equilibrium $\psi_p$ has an
-X-point and avoids a profile in the private flux region.
+X-point and avoids a profile in the private flux region. The factor $4\alpha$ provides a smooth transition
+zone that avoids numerical oscillations.
+
 
 We have two possibilities to initialize the ion density
 \begin{align} \label{eq:initphi}
@@ -577,14 +579,14 @@ We either follow fieldlines around the torus several times (``blob'') or only on
 We can initialize the R-Z plane with a turbulent bath with a certain amplitude $A$.
 Again, we transform this to all poloidal planes along the magnetic field lines and multiply the bath with
 \begin{align} \label{eq:initial_turbulent}
-\tilde n_e(R,Z,\varphi) = \tilde n_{\text{bath}}(R,Z,\varphi)\Theta(-\psi_p)H(Z-Z_X)
+\tilde n_e(R,Z,\varphi) = \tilde n_{\text{bath}}(R,Z,\varphi)\Theta_{4\alpha}(\psi_p(R, Z)+4\alpha) H(Z-Z_X)
 \end{align}
 \subsubsection{Zonal flows}
 We can initialize the R-Z plane with zonal flows of amplitude $A$ and
 wavelength $k_\psi$ aligned with the magnetic flux surfaces.
 \begin{align} \label{eq:initial_zonal_flow}
     \tilde n_{\text{zonal}}(R,Z) &= A \sin (2\pi k_\psi \psi_p(R,Z)) \nonumber\\
-\tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta(-\psi_p)H(Z-Z_X)
+\tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta_{4\alpha}(\psi_p(R, Z)+4\alpha) H(Z-Z_X)
 \end{align}
 
 \subsection{Sinks and sources} \label{sec:sources}
@@ -773,16 +775,16 @@ Unfortunately, we were unable to find a closed solution for the energy integrals
 \section{Numerical methods}
 discontinuous Galerkin on structured grid
 \rowcolors{2}{gray!25}{white} %%% Use this line in front of longtable
-\begin{longtable}{p{3cm}l>{\RaggedRight}p{7cm}}
+\begin{longtable}{p{3cm}l>{\RaggedRight}p{8cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Term} &  \textbf{Method} & \textbf{Description}  \\ \midrule
     coordinate system & Cylindrical & equidistant discretization of $[R_{\min},R_{\max}] \times [Z_{\min},Z_{\max}] \times [0,2\pi]$ (Eq.~\eqref{eq:box}, equal number of Gaussian nodes in $R$ and $Z$, equidistant planes in $\varphi$ with one Gaussian node \\
 Helmholtz and Elliptic matrix inversions & multigrid/ conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
 Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017} \\
 Curvature terms & direct dG & regular dG approximation of derivatives \\
-time & Strang Splitting & $2$nd order\\
-\qquad explicit & Bogacki-Shampine embedded & $3$rd order explicit\\
-\qquad implicit & Trapezoidal & perp. Diffusion and Resistive term. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
+time & Multistep "Karniadakis" & \\
+\qquad explicit & Multistep "Karniadakis" & $3$rd order explicit\\
+\qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion and Resistive terms. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
 \bottomrule
 \end{longtable}
 \section{Usage}
@@ -817,7 +819,7 @@ Input file format: json
 n      & integer & 3 & - &Number of Gaussian nodes in R and Z \\
 Nx     & integer &52& - &Number of grid points in R \\
 Ny     & integer &52& - &Number of grid points in Z \\
-Nz     & integer &16& - &Number of grid points in $\varphi$ \\
+Nz     & integer &16& - &Number of grid points in $\varphi$ (determines dt since parallel velocity dominates timestep) \\
 dt     & integer &1e-2& - &initial time stepsize in units of $c_s/\rho_s$ \\
 compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing points in x and y: output contains n*Nx/c[0] points in x,
     (has to divde Nx evenly), and n*Ny/c[1] points in y,
@@ -825,9 +827,9 @@ compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing poin
 inner\_loop & integer & 2  & 1 & Number of time steps between each energy output \\
 itstp       & integer & 2  & - & Number of energy outputs for each fields outputs \\
 maxout      & integer & 10 & - & Total Number of fields outputs excluding first (The total number of time steps is maxout$\cdot$itstp$\cdot$inner\_loop) \\
-eps\_time   & float & 1e-9  & - & Accuracy of solver for implicit part in time-stepper \\
+eps\_time   & float & 1e-9  & - & Accuracy of solver for implicit part in time-stepper (if too low, you'll see oscillations in ue) \\
 rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper \\
-eps\_pol    & float & 1e-5  & - &  Accuracy of residual of the inversion of polarisation and induction Eq. \\
+eps\_pol    & float & 1e-5  & - &  Accuracy of residual of the inversion of polarisation and induction Eq. (should not be too far from eps\_time for $\beta\neq  0$ ) \\
 jumpfactor  & float & 1 & 1 & Jumpfactor $\in \left[0.01,1\right]$ in Elliptic\\
 eps\_gamma  & float & 1e-6  & - & Accuracy of $\Gamma_1$  \\
 stages      & integer & 3 & 3 & number of stages in multigrid, $2^{\text{stages-1}}$
@@ -914,7 +916,7 @@ y\_XYZ           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi
 z\_XYZ           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 Psip             & Dataset & 3 (z,y,x) & Flux function $\psi_p(R,Z)$ \\
 Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ \\
-Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta(\rho_{s} - \rho(R,Z)) H(Z-Z_X)$\\
+Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta_\alpha(\rho_{s} - \rho(R,Z)) H(Z-Z_X)$\\
 %Damping          & Dataset & 3 (z,y,x) & Damping profile $\Theta(\rho(R,Z) - \rho_{d} )$\\
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index baf5a4664..6a30b35e3 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -176,7 +176,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
     HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
-        mag.psip(), -p.alpha, p.alpha, -1), grid);
+        mag.psip(), -4.*p.alpha, 4.*p.alpha, -1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
-- 
GitLab


From bdfb7cfc07741d247d76cb8de59f8f180dcae0c5 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 Mar 2019 15:30:08 +0100
Subject: [PATCH 049/540] Revert alpha back to one

Problem is that the core does not become unstable so we need to
destabilize the edge region
---
 src/feltor/feltor.cu     |  2 +-
 src/feltor/feltor.tex    | 14 +++++++-------
 src/feltor/feltor_hpc.cu |  2 +-
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 48fc129a2..e255908f0 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -71,7 +71,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
     dg::HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
-        mag.psip(), -4*p.alpha, 4*p.alpha, -1), grid);
+        mag.psip(), -1*p.alpha, 1*p.alpha, -1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 223846fe1..e22c4d76e 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -361,7 +361,7 @@ A numerically tractable approximation to the $\delta$-function reads
 \delta(\psi_p-\psi_{p0}) = \frac{1}{2\pi \epsilon^2}
 \exp\left( - \frac{\left(\psi_p-\psi_{p0}\right)^2}{2\epsilon^2}\right)
 \end{align}
-where $\epsilon$ is a small, free parameter. In the dG framework the integrals
+where $\epsilon$ is a small, free parameter. In the DG framework the integrals
 in Eq.~\eqref{eq:fsa} can be computed via Gauss-Legendre quadrature.
 
 If we write $B_p:=R_0|\nabla\psi_p|/R$, we can define the safety factor (in two dimensions) as
@@ -546,11 +546,11 @@ Note that we should take care to intitialize a smooth profile with ideally well-
 Let us define a flux-aligned density profile as
 \begin{align} \label{eq:density_profile}
   n_{\text{prof}}(R,Z)=
-      n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_p(R_0,0)}\Theta_{4\alpha}(\psi_p(R, Z)+4\alpha) H(Z-Z_X)
+      n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_p(R_0,0)}\Theta_{\alpha}(\psi_p(R, Z)+\alpha) H(Z-Z_X)
 \end{align}
 %NOTE: this profile is slightly below n_0 outside the LCFS! (not sure whether this does anything for nabla_parallel)
 The second Heaviside is multiplied only if the equilibrium $\psi_p$ has an
-X-point and avoids a profile in the private flux region. The factor $4\alpha$ provides a smooth transition
+X-point and avoids a profile in the private flux region. The factor $\alpha$ provides a smooth transition
 zone that avoids numerical oscillations.
 
 
@@ -579,14 +579,14 @@ We either follow fieldlines around the torus several times (``blob'') or only on
 We can initialize the R-Z plane with a turbulent bath with a certain amplitude $A$.
 Again, we transform this to all poloidal planes along the magnetic field lines and multiply the bath with
 \begin{align} \label{eq:initial_turbulent}
-\tilde n_e(R,Z,\varphi) = \tilde n_{\text{bath}}(R,Z,\varphi)\Theta_{4\alpha}(\psi_p(R, Z)+4\alpha) H(Z-Z_X)
+\tilde n_e(R,Z,\varphi) = \tilde n_{\text{bath}}(R,Z,\varphi)\Theta_{\alpha}(\psi_p(R, Z)+\alpha) H(Z-Z_X)
 \end{align}
 \subsubsection{Zonal flows}
 We can initialize the R-Z plane with zonal flows of amplitude $A$ and
 wavelength $k_\psi$ aligned with the magnetic flux surfaces.
 \begin{align} \label{eq:initial_zonal_flow}
     \tilde n_{\text{zonal}}(R,Z) &= A \sin (2\pi k_\psi \psi_p(R,Z)) \nonumber\\
-\tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta_{4\alpha}(\psi_p(R, Z)+4\alpha) H(Z-Z_X)
+\tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta_{\alpha}(\psi_p(R, Z)+\alpha) H(Z-Z_X)
 \end{align}
 
 \subsection{Sinks and sources} \label{sec:sources}
@@ -767,7 +767,7 @@ with $R_0=10$ and $I_0=20$ and a simulation box $[R_0-a,R_0+a]\times[-a,a]\times
 We then symbolically compute (with the help of Mathematica) source terms that we insert to the right hand side of
 the corresponding equation in code (\texttt{manufactured.h}) and simulate from $t=0...10^{-3}$.
 By comparing the numerical solution to the manufactured one we can observe the convergence of our numerical methods. Note that in order to better distinguish
-the convergence of the dG discretized terms from our parallel derivative
+the convergence of the DG discretized terms from our parallel derivative
 we can selectively choose to only activate perpendicular (including $A_\parallel$ terms) or parallel terms (those that involve derivatives along $\bhat$).
 
 Unfortunately, we were unable to find a closed solution for the energy integrals with the above fields.
@@ -781,7 +781,7 @@ discontinuous Galerkin on structured grid
     coordinate system & Cylindrical & equidistant discretization of $[R_{\min},R_{\max}] \times [Z_{\min},Z_{\max}] \times [0,2\pi]$ (Eq.~\eqref{eq:box}, equal number of Gaussian nodes in $R$ and $Z$, equidistant planes in $\varphi$ with one Gaussian node \\
 Helmholtz and Elliptic matrix inversions & multigrid/ conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
 Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017} \\
-Curvature terms & direct dG & regular dG approximation of derivatives \\
+Curvature terms & direct DG & regular DG approximation of derivatives \\
 time & Multistep "Karniadakis" & \\
 \qquad explicit & Multistep "Karniadakis" & $3$rd order explicit\\
 \qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion and Resistive terms. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 6a30b35e3..baf5a4664 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -176,7 +176,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
 
     HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
-        mag.psip(), -4.*p.alpha, 4.*p.alpha, -1), grid);
+        mag.psip(), -p.alpha, p.alpha, -1), grid);
     dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
     dg::blas1::pointwiseDot( profile_damping, profile, profile);
 
-- 
GitLab


From 0c0504856ec0eb43d5d60e1236528bad53de32a8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 Mar 2019 19:10:24 +0100
Subject: [PATCH 050/540] Repair Flux-Surface Average and Safety Factor

- remove beta from j_e in feltordiag
- improve q and FSA with introduction of H(R,Z)
- improve FSA with optional multiplication of gradpsi
- update geometry_diag and feltordiag with these changes and rho
- update feltor.tex about diagnostics
---
 diag/feltordiag.cu              |  64 +++++++++-------
 inc/geometries/average.h        | 127 ++++++++++++++++++++------------
 inc/geometries/geometry_diag.cu |  24 +++---
 src/feltor/feltor.tex           | 101 +++++++++++++------------
 4 files changed, 189 insertions(+), 127 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index 6c418a4bb..75c786460 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -12,8 +12,8 @@
 #include "feltor/parameters.h"
 
 struct RadialElectricFlux{
-    RadialElectricFlux( double beta, double tau, double mu):
-        m_beta(beta), m_tau(tau), m_mu(mu){
+    RadialElectricFlux( double tau, double mu):
+        m_tau(tau), m_mu(mu){
     }
 
     DG_DEVICE double operator()( double ne, double ue, double A,
@@ -32,17 +32,16 @@ struct RadialElectricFlux{
         double PS = b_0*( d1P*d2S-d2P*d1S)+
                     b_1*( d2P*d0S-d0P*d2S)+
                     b_2*( d0P*d1S-d1P*d0S);
-        double sqrtPsi = sqrt( d0S*d0S + d1S*d1S );
         double JPsi =
-            ne*ue*m_beta * (A*curvKappaS + SA )
+            ne*ue* (A*curvKappaS + SA )
             + ne * PS
             + ne * (m_tau + m_mu*ue*ue)*curvKappaS
             + ne * m_tau*curvNablaS;
-        return JPsi  / sqrtPsi;
+        return JPsi;
 
     }
     private:
-    double m_beta, m_tau, m_mu;
+    double m_tau, m_mu;
 };
 
 int main( int argc, char* argv[])
@@ -83,10 +82,11 @@ int main( int argc, char* argv[])
     gp.display();
     //-------------------Construct grids-------------------------------------//
 
-    double Rmin=gp.R_0-p.boxscaleRm*gp.a;
-    double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
-    double Rmax=gp.R_0+p.boxscaleRp*gp.a;
-    double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    const double Rmin=gp.R_0-p.boxscaleRm*gp.a;
+    const double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
+    const double Rmax=gp.R_0+p.boxscaleRp*gp.a;
+    const double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    const double Z_X = -1.1*gp.elongation*gp.a;
 
     dg::CylindricalGrid3d g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bcxN, p.bcyN, dg::PER);
@@ -94,11 +94,13 @@ int main( int argc, char* argv[])
         p.n_out, p.Nx_out, p.Ny_out, p.bcxN, p.bcyN);
 
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    const double psip0 = mag.psip()(gp.R_0, 0);
+    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*psip0, p.alpha);
     dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
-    double psipmin = (double)thrust::reduce(
-        psipog2d.begin(), psipog2d.end(), 0.0,thrust::minimum<double>() );
-    double psipmax = (double)thrust::reduce(
-        psipog2d.begin(), psipog2d.end(), psipmin,thrust::maximum<double>() );
+    //double psipmin = (double)thrust::reduce(
+    //    psipog2d.begin(), psipog2d.end(), 0.0,thrust::minimum<double>() );
+    double psipmin = dg::blas1::reduce( psipog2d, 0,thrust::minimum<double>() );
+    double psipmax = dg::blas1::reduce( psipog2d, -1000,thrust::maximum<double>() );
 
     unsigned Npsi = 50;//set number of psivalues
     dg::Grid1d g1d_out(psipmin, psipmax, 3, Npsi, dg::NEU);
@@ -178,8 +180,8 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDivide( binv, vol, vol); //1/vol/B
     for( int i=0; i<3; i++)
         dg::blas1::pointwiseDot( vol, bhat[i], bhat[i]); //b_i/vol/B
-    dg::DMatrix dxA = dg::create::dx( g3d_out, p.bcxA);
-    dg::DMatrix dyA = dg::create::dy( g3d_out, p.bcyA);
+    dg::DMatrix dxA = dg::create::dx( g3d_out, p.bcxU);
+    dg::DMatrix dyA = dg::create::dy( g3d_out, p.bcyU);
     dg::DMatrix dxP = dg::create::dx( g3d_out, p.bcxP);
     dg::DMatrix dyP = dg::create::dy( g3d_out, p.bcyP);
     dg::DMatrix dxN = dg::create::dx( g3d_out, p.bcxN);
@@ -218,8 +220,8 @@ int main( int argc, char* argv[])
     // define 2d and 1d and 0d dimensions and variables
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
-    int dim_ids1d[2] = {dim_ids[0],dim_ids[0]};
-    err = file::define_dimension( ncid_out, "psip1d", &dim_ids1d[1], g1d_out);
+    int dim_ids1d[2] = {dim_ids[0],dim_ids[1]}; //time , psi
+    err = file::define_dimension( ncid_out, "psi", &dim_ids1d[1], g1d_out);
 
     std::map<std::string, int> id0d, id1d, id2d;
     for( auto pair : v3d)
@@ -249,14 +251,21 @@ int main( int argc, char* argv[])
     size_t count3d[4] = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(), g3d_out.n()*g3d_out.Nx()};
     size_t start3d[4] = {0, 0, 0, 0};
 
-    //put safety factor into file
+    //put safety factor and rho into file
     std::cout << "Compute safety factor   "<< "\n";
-    dg::geo::SafetyFactor qprofile(g2d_out, mag);
+    dg::HVec xpoint_damping = dg::evaluate( dg::one, g2d_out);
+    if( gp.hasXpoint())
+        xpoint_damping = dg::evaluate( dg::geo::ZCutter(Z_X), g2d_out);
+    dg::geo::SafetyFactor qprofile(g2d_out, mag, xpoint_damping);
     dg::HVec sf = dg::evaluate(qprofile, g1d_out);
-    int qID;
+    int qID, rhoID;
     err = nc_def_var( ncid_out, "q", NC_DOUBLE, 1, &dim_ids1d[1], &qID);
+    err = nc_def_var( ncid_out, "rho", NC_DOUBLE, 1, &dim_ids1d[1], &rhoID);
     err = nc_enddef(ncid_out);
     err = nc_put_vara_double( ncid_out, qID, &start1d[1], &count1d[1], sf.data());
+    dg::HVec rho = dg::evaluate( dg::cooX1d, g1d_out);//evaluate psi
+    dg::blas1::axpby( -1./psip0, rho, +1., 1., rho); //transform psi to rho
+    err = nc_put_vara_double( ncid_out, rhoID, &start1d[1], &count1d[1], rho.data());
     err = nc_close(ncid_out);
 
     /////////////////////////////////////////////////////////////////////////
@@ -270,6 +279,7 @@ int main( int argc, char* argv[])
     err = nc_close( ncid); //close 3d file
 
     dg::Average<dg::DVec> toroidal_average( g3d_out, dg::coo3d::z);
+    dg::geo::FluxSurfaceAverage<dg::DVec> fsa(g2d_out, mag, t2d, dg::DVec(xpoint_damping) );
 
     for( unsigned i=0; i<steps; i++)//timestepping
     {
@@ -303,7 +313,7 @@ int main( int argc, char* argv[])
         dg::blas2::symv( dyP, v3d["potential"], dy_P);
         dg::blas2::symv( dz , v3d["potential"], dz_P);
         dg::blas1::evaluate( v3d["fluxe"], dg::equals(),
-            RadialElectricFlux( p.beta, p.tau[0], p.mu[0]),
+            RadialElectricFlux( p.tau[0], p.mu[0]),
             v3d["electrons"], v3d["Ue"], v3d["induction"],
             dx_A, dy_A, dz_A, dx_P, dy_P, dz_P, psipR, psipZ, psipP,
             bhat[0], bhat[1], bhat[2],
@@ -345,6 +355,7 @@ int main( int argc, char* argv[])
         v0d["correlationNPhi"] = dg::blas2::dot( t3d, w3d, v3d["electrons"])
             /norm1/norm2;  //<e^phi, N>/||e^phi||/||N||
 
+
         //now write out 2d and 1d quantities
         err = nc_open(argv[2], NC_WRITE, &ncid_out);
         for( auto pair : v3d)// {name, DVec}
@@ -355,9 +366,11 @@ int main( int argc, char* argv[])
                 start2d, count2d, transfer2d.data());
 
             //computa fsa of quantities
-            dg::geo::FluxSurfaceAverage<dg::DVec> fsa(g2d_out, mag, t2d );
-            t1d = dg::evaluate(fsa, g1d_out);
-            dg::assign( t1d, transfer1d);
+            if(pair.first == "fluxe")
+                fsa.set_container(t2d, false);
+            else
+                fsa.set_container(t2d, true);
+            transfer1d = dg::evaluate(fsa, g1d_out);
             err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_fsa"),
                 start1d, count1d, transfer1d.data());
 
@@ -367,6 +380,7 @@ int main( int argc, char* argv[])
                 pair.second.begin() + (kmp+1)*g2d_out.size() );
 
             //compute delta f on midplane : df = f_mp - <f>
+            dg::assign( transfer1d, t1d);
             dg::blas2::gemv(fsaonrzmatrix, t1d, t2d); //fsa on RZ grid
             dg::blas1::axpby( 1.0, t2d_mp, -1.0, t2d);
             dg::assign( t2d, transfer2d);
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 9669909b9..0cf428d3f 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -14,16 +14,13 @@ namespace geo
 {
 /**
  * @brief Delta function for poloidal flux \f$ B_Z\f$
-     \f[ |\nabla \psi_p|\delta(\psi_p(R,Z)-\psi_0) = \frac{\sqrt{ (\nabla \psi_p)^2}}{\sqrt{2\pi\varepsilon}} \exp\left(-\frac{(\psi_p(R,Z) - \psi_{0})^2}{2\varepsilon} \right)  \f]
+     \f[ \delta(\psi_p(R,Z)-\psi_0) = \frac{ 1}{\sqrt{2\pi\varepsilon}} \exp\left(-\frac{(\psi_p(R,Z) - \psi_{0})^2}{2\varepsilon} \right)  \f]
      @ingroup profiles
  */
 struct DeltaFunction
 {
     DeltaFunction(const TokamakMagneticField& c, double epsilon,double psivalue) :
-        c_(c),
-        epsilon_(epsilon),
-        psivalue_(psivalue){
-    }
+        c_(c), epsilon_(epsilon), psivalue_(psivalue){ }
     /**
     * @brief Set a new \f$ \varepsilon\f$
     *
@@ -38,13 +35,13 @@ struct DeltaFunction
     void setpsi(double psi_0 ){psivalue_ = psi_0;}
 
     /**
-     *@brief \f[ \frac{\sqrt{ (\nabla \psi_p)^2}}{\sqrt{2\pi\varepsilon}} \exp\left(-\frac{(\psi_p(R,Z) - \psi_{0})^2}{2\varepsilon} \right)  \f]
+     *@brief \f[ \frac{1}{\sqrt{2\pi\varepsilon}} \exp\left(-\frac{(\psi_p(R,Z) - \psi_{0})^2}{2\varepsilon} \right)  \f]
      */
     double operator()( double R, double Z) const
     {
-        double psip = c_.psip()(R,Z), psipR = c_.psipR()(R,Z), psipZ = c_.psipZ()(R,Z);
+        double psip = c_.psip()(R,Z);
         return 1./sqrt(2.*M_PI*epsilon_)*
-               exp(-( (psip-psivalue_)* (psip-psivalue_))/2./epsilon_)*sqrt(psipR*psipR +psipZ*psipZ);
+               exp(-( (psip-psivalue_)* (psip-psivalue_))/2./epsilon_);
     }
     /**
      * @brief == operator()(R,Z)
@@ -61,20 +58,35 @@ struct DeltaFunction
 
 /**
  * @brief Global safety factor
-\f[ \alpha(R,Z) = \frac{|B^\varphi|}{R|B^\eta|} = \frac{I_{pol}(R,Z)}{R|\nabla\psi_p|} \f]
+\f[ \alpha(R,Z) = \frac{I_{pol}(R,Z)}{R|\nabla\psi_p|} \frac{1}{\sqrt{2\pi\varepsilon}} \exp\left(-\frac{(\psi_p(R,Z) - \psi_{0})^2}{2\varepsilon} \right)  \f]
      @ingroup profiles
  */
 struct Alpha
 {
     Alpha( const TokamakMagneticField& c):c_(c){}
+    Alpha(const TokamakMagneticField& c, double epsilon,double psivalue) :
+        c_(c), m_eps(epsilon), m_psi(psivalue){ }
+    /**
+    * @brief Set a new \f$ \varepsilon\f$
+    *
+    * @param eps new value
+    */
+    void setepsilon(double eps ){m_eps = eps;}
+    /**
+    * @brief Set a new \f$ \psi_0\f$
+    *
+    * @param psi_0 new value
+    */
+    void setpsi(double psi_0 ){m_psi = psi_0;}
 
     /**
     * @brief \f[ \frac{ I_{pol}(R,Z)}{R \sqrt{\nabla\psi_p}} \f]
     */
     double operator()( double R, double Z) const
     {
-        double psipR = c_.psipR()(R,Z), psipZ = c_.psipZ()(R,Z);
-        return (1./R)*(c_.ipol()(R,Z)/sqrt(psipR*psipR + psipZ*psipZ )) ;
+        double psip = c_.psip()(R,Z);
+        return c_.ipol()(R,Z)/R /sqrt(2.*M_PI*m_eps)*
+               exp(-( (psip-m_psi)* (psip-m_psi))/2./m_eps);
     }
     /**
      * @brief == operator()(R,Z)
@@ -85,13 +97,15 @@ struct Alpha
     }
     private:
     TokamakMagneticField c_;
+    double m_eps, m_psi;
 };
 
 /**
  * @brief Flux surface average over quantity
- \f[ \langle f\rangle(\psi_0) = \frac{1}{A} \int dV \delta(\psi_p(R,Z)-\psi_0) |\nabla\psi_p|f(R,Z) \f]
+ \f[ \langle f\rangle(\psi_0) = \frac{1}{A} \int dV \delta(\psi_p(R,Z)-\psi_0) |\nabla\psi_p|f(R,Z)H(R,Z) \f]
 
- with \f$ A = \int dV \delta(\psi_p(R,Z)-\psi_0)|\nabla\psi_p|\f$
+ with \f$ A = \int dV \delta(\psi_p(R,Z)-\psi_0)|\nabla\psi_p|H(R,Z)\f$
+ where \c H is a weight function that can be used to e.g. cut away parts of the domain below the X-point
  * @ingroup misc_geo
  */
 template <class container = thrust::host_vector<double> >
@@ -102,12 +116,15 @@ struct FluxSurfaceAverage
      * @param g2d 2d grid
      * @param c contains psip, psipR and psipZ
      * @param f container for global safety factor
+     * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point)
      */
-    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f) :
-    g2d_(g2d),
-    f_(f),
-    deltaf_(c, 0.,0.),
-    w2d_ ( dg::create::weights( g2d))
+    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, const container& weights) :
+    m_f(f), m_deltafog2d(f),
+    m_deltaf(c, 0.,0.),
+    m_w2d ( dg::create::weights( g2d)),
+    m_x ( dg::evaluate( dg::cooX2d, g2d)),
+    m_y ( dg::evaluate( dg::cooY2d, g2d)),
+    m_weights(weights)
     {
         thrust::host_vector<double> psipRog2d  = dg::evaluate( c.psipR(), g2d);
         thrust::host_vector<double> psipZog2d  = dg::evaluate( c.psipZ(), g2d);
@@ -116,8 +133,25 @@ struct FluxSurfaceAverage
         double psipZmax = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), 0.,      thrust::maximum<double>()  );
         //double psipZmin = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), psipZmax,thrust::minimum<double>()  );
         double deltapsi = fabs(psipZmax/g2d.Ny()/g2d.n() +psipRmax/g2d.Nx()/g2d.n());
-        //deltaf_.setepsilon(deltapsi/4.);
-        deltaf_.setepsilon(deltapsi); //macht weniger Zacken
+        m_deltaf.setepsilon(deltapsi/4);
+        //m_deltaf.setepsilon(deltapsi); //macht weniger Zacken
+        dg::blas1::pointwiseDot( 1., psipRog2d, psipRog2d, 1., psipZog2d, psipZog2d, 0., psipRog2d);
+        dg::blas1::transform( psipRog2d, psipRog2d, dg::SQRT<double>());
+        dg::assign( psipRog2d, m_gradpsi);
+    }
+
+    /**
+     * @brief Set the function to average
+     *
+     * @param f the container containing the discretized function
+     * @param multiplyByGradPsi if true multiply with GradPsi, else not
+     */
+    void set_container( const container& f, bool multiplyByGradPsi=true){
+        dg::blas1::copy( f, m_f);
+        if(multiplyByGradPsi)
+        {
+            dg::blas1::pointwiseDot( m_f, m_gradpsi, m_f);
+        }
     }
     /**
      * @brief Calculate the Flux Surface Average
@@ -127,23 +161,23 @@ struct FluxSurfaceAverage
      */
     double operator()(double psip0)
     {
-        deltaf_.setpsi( psip0);
-        deltafog2d_ = dg::evaluate( deltaf_, g2d_);
-        double psipcut = dg::blas2::dot( f_, w2d_, deltafog2d_); //int deltaf psip
-        double vol     = dg::blas1::dot( w2d_, deltafog2d_); //int deltaf
+        m_deltaf.setpsi( psip0);
+        dg::blas1::evaluate( m_deltafog2d, dg::equals(), m_deltaf, m_x, m_y);
+        dg::blas1::pointwiseDot( m_deltafog2d, m_weights, m_deltafog2d);
+        double psipcut = dg::blas2::dot( m_f, m_w2d, m_deltafog2d); //int deltaf psip
+        double vol     = dg::blas2::dot( m_gradpsi, m_w2d, m_deltafog2d); //int deltaf
         return psipcut/vol;
     }
     private:
-    Grid2d g2d_;
-    container f_, deltafog2d_;
-    geo::DeltaFunction deltaf_;
-    const container w2d_;
+    container m_f, m_deltafog2d, m_gradpsi;
+    geo::DeltaFunction m_deltaf;
+    const container m_w2d, m_x, m_y, m_weights;
 };
 /**
  * @brief Class for the evaluation of the safety factor q
- * \f[ q(\psi_0) = \frac{1}{2\pi} \int dV |\nabla\psi_p| \delta(\psi_p-\psi_0) \alpha( R,Z) \f]
+ * \f[ q(\psi_0) = \frac{1}{2\pi} \int dV \alpha( R,Z) H(R,Z) \f]
 
-where \f$ \alpha\f$ is the dg::geo::Alpha functor.
+where \f$ \alpha\f$ is the \c dg::geo::Alpha functor and \c H is a weights function.
  * @copydoc hide_container
  * @ingroup misc_geo
  *
@@ -154,13 +188,13 @@ struct SafetyFactor
      * @brief Construct from a field and a grid
      * @param g2d 2d grid
      * @param c contains psip, psipR and psipZ
+     * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point)
      */
-    SafetyFactor(const dg::Grid2d& g2d, const TokamakMagneticField& c) :
-    g2d_(g2d),
-    deltaf_(geo::DeltaFunction(c,0.0,0.0)),
-    w2d_ ( dg::create::weights( g2d)),
-    alphaog2d_(dg::evaluate( dg::geo::Alpha(c), g2d)),
-    deltafog2d_(dg::evaluate(dg::one,g2d))
+    SafetyFactor(const dg::Grid2d& g2d, const TokamakMagneticField& c, const thrust::host_vector<double>& weights) :
+    m_g2d(g2d),
+    m_alpha(c,0.0,0.0),
+    m_w2d ( dg::create::weights( g2d)),
+    m_weights( weights)
     {
         thrust::host_vector<double> psipRog2d  = dg::evaluate( c.psipR(), g2d);
         thrust::host_vector<double> psipZog2d  = dg::evaluate( c.psipZ(), g2d);
@@ -169,26 +203,27 @@ struct SafetyFactor
         double psipZmax = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), 0.,      thrust::maximum<double>()  );
         //double psipZmin = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), psipZmax,thrust::minimum<double>()  );
         double deltapsi = fabs(psipZmax/g2d.Ny() +psipRmax/g2d.Nx());
-        //deltaf_.setepsilon(deltapsi/4.);
-        deltaf_.setepsilon(4.*deltapsi); //macht weniger Zacken
+        //m_alpha.setepsilon(deltapsi/4.);
+        m_alpha.setepsilon(deltapsi/10.);
     }
     /**
      * @brief Calculate the q profile over the function f which has to be the global safety factor
-     * \f[ q(\psi_0) = \frac{1}{2\pi} \int dV |\nabla\psi_p| \delta(\psi_p-\psi_0) \alpha( R,Z) \f]
+     * \f[ q(\psi_0) = \frac{1}{2\pi} \int dV \alpha( R,Z) H(R,Z) \f]
      *
      * @param psip0 the actual psi value for q(psi)
      */
     double operator()(double psip0)
     {
-        deltaf_.setpsi( psip0);
-        deltafog2d_ = dg::evaluate( deltaf_, g2d_);
-        return dg::blas2::dot( alphaog2d_,w2d_,deltafog2d_)/(2.*M_PI);
+        m_alpha.setpsi( psip0);
+        m_alphaog2d = dg::evaluate( m_alpha, m_g2d);
+        return dg::blas2::dot( m_alphaog2d, m_w2d, m_weights)/(2.*M_PI);
     }
     private:
-    dg::Grid2d g2d_;
-    geo::DeltaFunction deltaf_;
-    const thrust::host_vector<double> w2d_, alphaog2d_;
-    thrust::host_vector<double> deltafog2d_;
+    dg::Grid2d m_g2d;
+    geo::Alpha m_alpha;
+    const thrust::host_vector<double> m_w2d;
+    thrust::host_vector<double> m_alphaog2d;
+    thrust::host_vector<double> m_weights;
 };
 
 }//namespace geo
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index d23647743..bf438cfa7 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -150,7 +150,8 @@ int main( int argc, char* argv[])
     const double N1 = -(1.+alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.+alpha_);
     const double N2 =  (1.-alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.-alpha_);
     const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
-    std::cout << "psip( 1, 0) "<<c.psip()(gp.R_0, 0)<<"\n";
+    const double psip0 = c.psip()(gp.R_0, 0);
+    std::cout << "psip( 1, 0) "<<psip0<<"\n";
     std::cout << "TEST ACCURACY OF PSI (values must be close to 0!)\n";
     if( gp.hasXpoint())
         std::cout << "    Equilibrium with X-point!\n";
@@ -174,7 +175,6 @@ int main( int argc, char* argv[])
     std::cout << "psipZZ( 1-e,0)         "<<c.psipZZ()(gp.R_0-gp.a,0.)+N2*c.psipR()(gp.R_0-gp.a,0)<<"\n";
     std::cout << "psipRR( 1-de,ke)       "<<c.psipRR()(R_H,Z_H)+N3*c.psipZ()(R_H,Z_H)<<"\n";
 
-    dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
 
     dg::HVec hvisual;
     //allocate mem for visual
@@ -226,16 +226,20 @@ int main( int argc, char* argv[])
             M_PI, p.sigma, p.sigma, p.sigma, p.amp)}
     };
     //Compute flux average
+    dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
     dg::DVec psipog2d   = dg::evaluate( c.psip(), grid2d);
-    double psipmin = (float)thrust::reduce( psipog2d .begin(), psipog2d .end(), 0.0,thrust::minimum<double>()  );
+    dg::HVec xpoint_damping = dg::evaluate( dg::one, grid2d);
+    if( gp.hasXpoint() )
+        xpoint_damping = dg::evaluate( dg::geo::ZCutter(Z_X), grid2d);
+    double psipmin = psip0;//(float)thrust::reduce( psipog2d.begin(), psipog2d.end(), 0.0,thrust::minimum<double>()  );
+    double psipmax = 0;//(float)thrust::reduce( psipog2d.begin(), psipog2d.end(), 0.0,thrust::maximum<double>()  );
     unsigned npsi = 3, Npsi = 150;//set number of psivalues
-    psipmin += (gp.psipmax - psipmin)/(double)Npsi; //the inner value is not good
-    dg::Grid1d grid1d(psipmin , gp.psipmax, npsi ,Npsi,dg::DIR);
-    dg::geo::SafetyFactor     qprof(grid2d, c);
-    dg::HVec sf         = dg::evaluate( qprof,    grid1d);
+    //psipmin += (gp.psipmax - psipmin)/(double)Npsi; //the inner value is not good
+    dg::Grid1d grid1d(psipmin + 0.05 , psipmax, npsi ,Npsi,dg::NEU);
+    dg::geo::SafetyFactor     qprof(grid2d, c, xpoint_damping);
+    dg::HVec sf         = dg::evaluate( qprof,      grid1d);
     dg::HVec abs        = dg::evaluate( dg::cooX1d, grid1d);
-
-
+    dg::blas1::axpby( -1./psip0, abs, +1., 1., abs); //transform psi to rho
 
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
@@ -252,7 +256,7 @@ int main( int argc, char* argv[])
     //write 1d vectors
     int avgID[2];
     err = nc_def_var( ncid, "q-profile", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[0]);
-    err = nc_def_var( ncid, "psip1d", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[1]);
+    err = nc_def_var( ncid, "rho", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[1]);
     err = nc_enddef( ncid);
     err = nc_put_var_double( ncid, avgID[0], sf.data());
     err = nc_put_var_double( ncid, avgID[1], abs.data());
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index e22c4d76e..0fd4c2f66 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -342,34 +342,6 @@ K_{\nabla\times\bhat}^\varphi &= \frac{R_0}{R^2B^2}\left(
 \label{}
 \end{align}
 
-\subsection{Flux surface average and Safety factor}
-We define the toroidal average of a function $f(R,Z,\varphi)$ as
-\begin{align} \label{eq:phi_average}
-\langle f\rangle_\varphi := \frac{1}{2\pi}\oint f\d \varphi
-\end{align}
-The flux surface average
-of a function $f(R,Z,\varphi)$ is given by the formula
-\begin{align}\label{eq:fsa}
-\langle f \rangle_{\psi_{p0}} :=
-\frac{ \oint_{\psi_p = \psi_{p0} } f(R,Z,\varphi)\dA}{\oint_{\psi_p = \psi_{p0} } \dA} =
-\frac{\int_\Omega f(R,Z,\varphi) \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p| \dV}
-{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p|\dV}
-\end{align}
-with $\dV := \d R\d Z\d \varphi$ (we define the average in computational space and omit one $R$).
-A numerically tractable approximation to the $\delta$-function reads
-\begin{align}\label{eq:delta}
-\delta(\psi_p-\psi_{p0}) = \frac{1}{2\pi \epsilon^2}
-\exp\left( - \frac{\left(\psi_p-\psi_{p0}\right)^2}{2\epsilon^2}\right)
-\end{align}
-where $\epsilon$ is a small, free parameter. In the DG framework the integrals
-in Eq.~\eqref{eq:fsa} can be computed via Gauss-Legendre quadrature.
-
-If we write $B_p:=R_0|\nabla\psi_p|/R$, we can define the safety factor (in two dimensions) as
-\begin{align}
-q:=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vec\nabla\psi_p|} \d s
-= \frac{1}{2\pi}\int \frac{I(\psi_p)}{R}\delta(\psi_p-\psi_{p0}) \d R\d Z
-\end{align}
-
 \subsection{ Modified $\psi_p$}
 Our computational domain is a box and in particular not aligned with the
 magnetic flux surfaces. This means that particularly in the corners of
@@ -957,6 +929,56 @@ the whole simulation is lost. It is safer to just merge files afterwards with fo
 \texttt{ncrcat output1.nc output2.nc output.nc}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Diagnostics}\label{sec:diagnostics}
+\subsection{Flux surface average and Safety factor}
+We define the toroidal average of a function $f(R,Z,\varphi)$ as
+\begin{align} \label{eq:phi_average}
+\langle f\rangle_\varphi := \frac{1}{2\pi}\oint f\d \varphi
+\end{align}
+The flux surface average
+of a function $f(R,Z,\varphi)$ is given by the formula
+\begin{align}\label{eq:fsa}
+\langle f \rangle_{\psi_{p0}} :=
+\frac{ \oint_{\psi_p = \psi_{p0} } f(R,Z,\varphi)\dA}{\oint_{\psi_p = \psi_{p0} } \dA} =
+\frac{\int_\Omega f(R,Z,\varphi) \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p| H(R,Z)\dV}
+{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p|H(R,Z)\dV}
+\end{align}
+with $\dV := \d R\d Z\d \varphi$ (we define the average in computational space and omit one $R$)
+and we use the Heaviside function $H(R,Z) = H(Z-Z_X)$ to cut away contributions from below the X-point
+in our domain.
+
+A numerically tractable approximation to the $\delta$-function reads
+\begin{align}\label{eq:delta}
+\delta(\psi_p-\psi_{p0}) = \frac{1}{2\pi \epsilon^2}
+\exp\left( - \frac{\left(\psi_p-\psi_{p0}\right)^2}{2\epsilon^2}\right)
+\end{align}
+where $\epsilon$ is a small, free parameter. In the DG framework the integrals
+in Eq.~\eqref{eq:fsa} can be computed via Gauss-Legendre quadrature.
+
+If we write $B_p:=R_0|\nabla\psi_p|/R$, we can define the safety factor (in two dimensions) as
+\begin{align}
+q:=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vec\nabla\psi_p|} \d s
+= \frac{1}{2\pi}\int \frac{I(\psi_p)}{R}\delta(\psi_p-\psi_{p0}) H(Z-Z_X) \d R\d Z
+\end{align}
+\subsection{ Radial flux}
+Let us define
+
+\begin{align}
+ \vec j_e\cdot \vec \nabla\psi_p =& n_e\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ B} + u_e \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ \nabla\psi_p \nonumber\\
+ =& n_eu_e\left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \nonumber\\
+  &+ n_e\frac{1}{B}[\phi, \psi_p]_\perp + \left(\tau_e + \mu_e u_e^2\right)
+   n_e\mathcal K_{\nabla\times\bhat}(\psi_p) + \tau_e n_e \mathcal K_{\nabla B}(\psi_p) \nonumber\\
+   f_e :=& \frac{\vec j_e\cdot\vec \nabla \psi_p}{|\vec \nabla\psi_p|} \quad
+   L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}\quad
+   L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}
+\end{align}
+
+Note that the toroidal average of $f_e$ is ill-defined since $\nabla\psi$ vanishes at the X-point.
+Instead we output the average of $\vec j_e\cdot\nabla\psi_p$.
+
+\subsection{Program and files}
 Compilation\\
 \texttt{make feltordiag} \\
 Usage \\
@@ -973,12 +995,14 @@ inputfile  &     text attribute & - & verbose input file as a string (valid JSON
 geomfile   &     text attribute & - & verbose geometry input file as a string (valid JSON, no comments) \\
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
-psip1d           & Coord. Var. & 1 (psip1d) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
+psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
+q                & Dataset & 1 (psi) & The safety factor \\
+$\rho$           & Dataset & 1 (psi) & The flux label $\rho:= (\psi_{p,0} - \psi_p)/\psi_{p,0}$ \\
 X\_avg           & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
 X\_fsa\_mp       & Dataset & 3 (time,y,x) & Fluctuation level on midplane ($\varphi\equiv \pi$) $\delta X := X(R,Z,\pi) - \langle X\rangle_{\psi_{p}}$ \\
-X\_fsa           & Dataset & 2 (time, psip1d) & Flux surface average $\langle X\rangle_\psi$ Eq.~\eqref{eq:fsa} \\
+X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_\psi$ Eq.~\eqref{eq:fsa} \\
 aligned          & Dataset & 1 (time) & $\int \dV (\nabla_\parallel n_e)^2 /n_e$ \\
 perp\_aligned    & Dataset & 1 (time) & $\int \dV (\vec\nabla_\perp n_e)^2 /n_e$ \\
 correlationNPhi  & Dataset & 1 (time) & $\left( \int\dV e^\phi n_e \right)/ ||e^\phi||||n_e||$\\
@@ -988,22 +1012,7 @@ total\_flux      & Dataset & 1 (time) & $\int \dV f_e$\\
 X $\in$ \{ electrons, ions, Ue, Ui, potential,
 induction, vorticity, fluxe, Lperpinv, Lparallelinv\} corresponding to \{
     $n_e$, $N_i$, $u_e$, $U_i$, $\phi$, $A_\parallel$, $-\Delta_\perp \phi$,
-    $f_e$, $L_\perp^{-1}$, $L_\parallel^{-1}$\} with
-
-\begin{align}
- \vec j_e\cdot \vec \nabla\psi_p =& n_e\left( \vec v_E + \vec v_C + \vec v_{\nabla
- B} + u_e \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- \nabla\psi_p \nonumber\\
- =& n_eu_e\left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \nonumber\\
-  &+ n_e\frac{1}{B}[\phi, \psi_p]_\perp + \left(\tau_e + \mu_e u_e^2\right)
-   n_e\mathcal K_{\nabla\times\bhat}(\psi_p) + \tau_e n_e \mathcal K_{\nabla B}(\psi_p) \nonumber\\
-   f_e :=& \frac{\vec j_e\cdot\vec \nabla \psi_p}{|\vec \nabla\psi_p|} \quad
-   L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}\quad
-   L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}
-\end{align}
-
-
+    $f_e$, $L_\perp^{-1}$, $L_\parallel^{-1}$\}.
 
 
 
-- 
GitLab


From 789b73917ab27fdb9bded12e2095c379770dd238 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 29 Mar 2019 09:54:04 +0100
Subject: [PATCH 051/540] Debug and test FSA in geometry_diag

---
 inc/geometries/average.h        |  7 ++++---
 inc/geometries/geometry_diag.cu | 14 ++++++++++----
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 0cf428d3f..df8ea0ad9 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -117,8 +117,9 @@ struct FluxSurfaceAverage
      * @param c contains psip, psipR and psipZ
      * @param f container for global safety factor
      * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point)
+     * @param multiplyByGradPsi if true multiply f with GradPsi, else not
      */
-    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, const container& weights) :
+    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, const container& weights, bool multiplyByGradPsi = true) :
     m_f(f), m_deltafog2d(f),
     m_deltaf(c, 0.,0.),
     m_w2d ( dg::create::weights( g2d)),
@@ -138,6 +139,8 @@ struct FluxSurfaceAverage
         dg::blas1::pointwiseDot( 1., psipRog2d, psipRog2d, 1., psipZog2d, psipZog2d, 0., psipRog2d);
         dg::blas1::transform( psipRog2d, psipRog2d, dg::SQRT<double>());
         dg::assign( psipRog2d, m_gradpsi);
+        if(multiplyByGradPsi)
+            dg::blas1::pointwiseDot( m_f, m_gradpsi, m_f);
     }
 
     /**
@@ -149,9 +152,7 @@ struct FluxSurfaceAverage
     void set_container( const container& f, bool multiplyByGradPsi=true){
         dg::blas1::copy( f, m_f);
         if(multiplyByGradPsi)
-        {
             dg::blas1::pointwiseDot( m_f, m_gradpsi, m_f);
-        }
     }
     /**
      * @brief Calculate the Flux Surface Average
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index bf438cfa7..4135a0514 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -237,9 +237,11 @@ int main( int argc, char* argv[])
     //psipmin += (gp.psipmax - psipmin)/(double)Npsi; //the inner value is not good
     dg::Grid1d grid1d(psipmin + 0.05 , psipmax, npsi ,Npsi,dg::NEU);
     dg::geo::SafetyFactor     qprof(grid2d, c, xpoint_damping);
+    dg::geo::FluxSurfaceAverage<dg::HVec>  fsa( grid2d, c, psipog2d, xpoint_damping);
+    dg::HVec psi_fsa    = dg::evaluate( fsa,        grid1d);
     dg::HVec sf         = dg::evaluate( qprof,      grid1d);
-    dg::HVec abs        = dg::evaluate( dg::cooX1d, grid1d);
-    dg::blas1::axpby( -1./psip0, abs, +1., 1., abs); //transform psi to rho
+    dg::HVec psi        = dg::evaluate( dg::cooX1d, grid1d), rho(psi);
+    dg::blas1::axpby( -1./psip0, rho, +1., 1., rho); //transform psi to rho
 
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
@@ -254,12 +256,16 @@ int main( int argc, char* argv[])
     dim2d_ids[0] = dim3d_ids[1], dim2d_ids[1] = dim3d_ids[2];
 
     //write 1d vectors
-    int avgID[2];
+    int avgID[4];
     err = nc_def_var( ncid, "q-profile", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[0]);
     err = nc_def_var( ncid, "rho", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[1]);
+    err = nc_def_var( ncid, "psip1d", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[2]);
+    err = nc_def_var( ncid, "psi_fsa", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[3]);
     err = nc_enddef( ncid);
     err = nc_put_var_double( ncid, avgID[0], sf.data());
-    err = nc_put_var_double( ncid, avgID[1], abs.data());
+    err = nc_put_var_double( ncid, avgID[1], rho.data());
+    err = nc_put_var_double( ncid, avgID[2], psi.data());
+    err = nc_put_var_double( ncid, avgID[3], psi_fsa.data());
     err = nc_redef(ncid);
 
     //write 2d vectors
-- 
GitLab


From 5266b17ccc693b90eb6145dcf67697277437737d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 29 Mar 2019 11:15:29 +0100
Subject: [PATCH 052/540] Fix BUG in collective mpi for zero message size

Zero message size on one process is not synonymous with no
communication!! Took the opportunity to rename private
data members of involved classes.
---
 inc/dg/backend/mpi_collective.h   | 229 ++++++++++++++++--------------
 inc/dg/backend/mpi_communicator.h |  15 +-
 inc/dg/backend/mpi_matrix.h       |   6 +-
 inc/dg/backend/mpi_vector.h       | 199 +++++++++++++-------------
 inc/dg/topology/mpi_projection.h  |   4 +-
 5 files changed, 236 insertions(+), 217 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 414f8a1b7..fcc963556 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -28,7 +28,9 @@ namespace dg{
 template<class Index, class Vector>
 struct Collective
 {
-    Collective(){}
+    Collective(){
+        m_comm = MPI_COMM_NULL;
+    }
     /**
      * @brief Construct from a map: PID -> howmanyToSend
      *
@@ -37,26 +39,26 @@ struct Collective
      * @param comm Communicator
      */
     Collective( const thrust::host_vector<int>& sendTo, MPI_Comm comm) {
-        construct( sendTo, comm);}
+        construct( sendTo, comm);
+    }
 
-    void construct( const thrust::host_vector<int>& map, MPI_Comm comm){
+    void construct( thrust::host_vector<int> sendTo, MPI_Comm comm){
         //sollte schnell sein
-        thrust::host_vector<int> sendTo=map, recvFrom=sendTo;
-        comm_=comm;
-        thrust::host_vector<int> accS = sendTo, accR = recvFrom;
+        thrust::host_vector<int> recvFrom(sendTo), accS(sendTo), accR(sendTo);
+        m_comm=comm;
         int rank, size;
-        MPI_Comm_rank( comm_, &rank);
-        MPI_Comm_size( comm_, &size);
+        MPI_Comm_rank( m_comm, &rank);
+        MPI_Comm_size( m_comm, &size);
         assert( sendTo.size() == (unsigned)size);
         thrust::host_vector<unsigned> global( size*size);
         MPI_Allgather( sendTo.data(), size, MPI_UNSIGNED,
                        global.data(), size, MPI_UNSIGNED,
-                       comm_);
+                       m_comm);
         for( unsigned i=0; i<(unsigned)size; i++)
             recvFrom[i] = global[i*size+rank];
         thrust::exclusive_scan( sendTo.begin(),   sendTo.end(),   accS.begin());
         thrust::exclusive_scan( recvFrom.begin(), recvFrom.end(), accR.begin());
-        sendTo_=sendTo, recvFrom_=recvFrom, accS_=accS, accR_=accR;
+        m_sendTo=sendTo, m_recvFrom=recvFrom, m_accS=accS, m_accR=accR;
     }
     /**
      * @brief Number of processes in the communicator
@@ -66,26 +68,30 @@ struct Collective
      * @return Total number of processes
      */
     unsigned size() const {return values_size();}
-    MPI_Comm comm() const {return comm_;}
+    MPI_Comm comm() const {return m_comm;}
 
-    void transpose(){ sendTo_.swap( recvFrom_);}
-    void invert(){ sendTo_.swap( recvFrom_);}
+    void transpose(){ m_sendTo.swap( m_recvFrom);}
+    void invert(){ m_sendTo.swap( m_recvFrom);}
 
     void scatter( const Vector& values, Vector& store) const;
     void gather( const Vector& store, Vector& values) const;
     unsigned store_size() const{
-        if( recvFrom_.empty()) return 0;
-        return thrust::reduce( recvFrom_.begin(), recvFrom_.end() );}
+        if( m_recvFrom.empty())
+            return 0;
+        return thrust::reduce( m_recvFrom.begin(), m_recvFrom.end() );
+    }
     unsigned values_size() const{
-        if( sendTo_.empty()) return 0;
-        return thrust::reduce( sendTo_.begin(), sendTo_.end() );}
-    MPI_Comm communicator() const{return comm_;}
+        if( m_sendTo.empty())
+            return 0;
+        return thrust::reduce( m_sendTo.begin(), m_sendTo.end() );
+    }
+    MPI_Comm communicator() const{return m_comm;}
     private:
-    unsigned sendTo( unsigned pid) const {return sendTo_[pid];}
-    unsigned recvFrom( unsigned pid) const {return recvFrom_[pid];}
-    Index sendTo_,   accS_; //accumulated send
-    Index recvFrom_, accR_; //accumulated recv
-    MPI_Comm comm_;
+    unsigned sendTo( unsigned pid) const {return m_sendTo[pid];}
+    unsigned recvFrom( unsigned pid) const {return m_recvFrom[pid];}
+    Index m_sendTo,   m_accS; //accumulated send
+    Index m_recvFrom, m_accR; //accumulated recv
+    MPI_Comm m_comm;
 };
 
 template< class Index, class Device>
@@ -97,11 +103,11 @@ void Collective<Index, Device>::scatter( const Device& values, Device& store) co
 #endif //THRUST_DEVICE_SYSTEM
     MPI_Alltoallv(
             thrust::raw_pointer_cast( values.data()),
-            thrust::raw_pointer_cast( sendTo_.data()),
-            thrust::raw_pointer_cast( accS_.data()), getMPIDataType<get_value_type<Device> >(),
+            thrust::raw_pointer_cast( m_sendTo.data()),
+            thrust::raw_pointer_cast( m_accS.data()), getMPIDataType<get_value_type<Device> >(),
             thrust::raw_pointer_cast( store.data()),
-            thrust::raw_pointer_cast( recvFrom_.data()),
-            thrust::raw_pointer_cast( accR_.data()), getMPIDataType<get_value_type<Device> >(), comm_);
+            thrust::raw_pointer_cast( m_recvFrom.data()),
+            thrust::raw_pointer_cast( m_accR.data()), getMPIDataType<get_value_type<Device> >(), m_comm);
 }
 
 template< class Index, class Device>
@@ -114,11 +120,11 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
 #endif //THRUST_DEVICE_SYSTEM
     MPI_Alltoallv(
             thrust::raw_pointer_cast( gatherFrom.data()),
-            thrust::raw_pointer_cast( recvFrom_.data()),
-            thrust::raw_pointer_cast( accR_.data()), getMPIDataType<get_value_type<Device> >(),
+            thrust::raw_pointer_cast( m_recvFrom.data()),
+            thrust::raw_pointer_cast( m_accR.data()), getMPIDataType<get_value_type<Device> >(),
             thrust::raw_pointer_cast( values.data()),
-            thrust::raw_pointer_cast( sendTo_.data()),
-            thrust::raw_pointer_cast( accS_.data()), getMPIDataType<get_value_type<Device> >(), comm_);
+            thrust::raw_pointer_cast( m_sendTo.data()),
+            thrust::raw_pointer_cast( m_accS.data()), getMPIDataType<get_value_type<Device> >(), m_comm);
 }
 //BijectiveComm ist der Spezialfall, dass jedes Element nur ein einziges Mal gebraucht wird.
 ///@endcond
@@ -170,7 +176,7 @@ struct BijectiveComm : public aCommunicator<Vector>
     BijectiveComm( unsigned local_size, thrust::host_vector<int> localGatherMap, thrust::host_vector<int> pidGatherMap, MPI_Comm comm)
     {
         construct( pidGatherMap, comm);
-        p_.transpose();
+        m_p.transpose();
     }
     ///@copydoc GeneralComm::GeneralComm(const thrust::host_vector<int>&,const ConversionPolicy&)
     ///@note we assume that the gather map is bijective
@@ -183,7 +189,7 @@ struct BijectiveComm : public aCommunicator<Vector>
             if( !p.global2localIdx(globalGatherMap[i], local[i], pids[i]) ) success = false;
         assert( success);
         construct( pids, p.communicator());
-        p_.transpose();
+        m_p.transpose();
     }
 
     ///@copydoc GeneralComm::GeneralComm(const GeneralComm<OtherIndex,OtherVector>&)
@@ -196,20 +202,22 @@ struct BijectiveComm : public aCommunicator<Vector>
     * @brief These are the pids that were given in the Constructor
     * @return the vector given in the constructor
     */
-    const thrust::host_vector<int>& get_pids()const{return pids_;}
+    const thrust::host_vector<int>& get_pids()const{return m_pids;}
     virtual BijectiveComm* clone() const override final {return new BijectiveComm(*this);}
     private:
     virtual bool do_isCommunicating() const override final{
         int rank;
         MPI_Comm_rank( do_communicator(), &rank);
-        bool communicating = false;
-        for( unsigned i=0; i<pids_.size(); i++)
-            if( pids_[i] != rank)
-                communicating = true;
-        return communicating;
+        bool local_communicating = false, global_communicating=false;
+        for( unsigned i=0; i<m_pids.size(); i++)
+            if( m_pids[i] != rank)
+                local_communicating = true;
+        MPI_Allreduce( &local_communicating, &global_communicating, 1,
+                       MPI_C_BOOL, MPI_LOR, m_p.communicator());
+        return global_communicating;
     }
-    virtual MPI_Comm do_communicator() const override final {return p_.communicator();}
-    virtual unsigned do_size() const override final { return p_.store_size();}
+    virtual MPI_Comm do_communicator() const override final {return m_p.communicator();}
+    virtual unsigned do_size() const override final { return m_p.store_size();}
     virtual Vector do_make_buffer()const override final{
         Vector tmp( do_size() );
         return tmp;
@@ -217,17 +225,16 @@ struct BijectiveComm : public aCommunicator<Vector>
     void construct( thrust::host_vector<int> pids, MPI_Comm comm)
     {
         this->set_local_size( pids.size());
-        pids_ = pids;
-        dg::assign( pids, idx_);
-        int rank, size;
+        m_pids = pids;
+        dg::assign( pids, m_idx);
+        int size;
         MPI_Comm_size( comm, &size);
-        MPI_Comm_rank( comm, &rank);
         for( unsigned i=0; i<pids.size(); i++)
             assert( 0 <= pids[i] && pids[i] < size);
         thrust::host_vector<int> index(pids);
         thrust::sequence( index.begin(), index.end());
         thrust::stable_sort_by_key( pids.begin(), pids.end(), index.begin());//note: this also sorts the pids
-        idx_=index;
+        m_idx=index;
         //now we can repeat/invert the sort by a gather/scatter operation with index as map
         //i.e. we could sort pids by a gather
 
@@ -241,33 +248,33 @@ struct BijectiveComm : public aCommunicator<Vector>
         thrust::host_vector<int> sendTo( size, 0 );
         for( unsigned i=0; i<distance; i++)
             sendTo[keys[i]] = number[i];
-        p_.construct( sendTo, comm);
-        values_.data().resize( idx_.size());
+        m_p.construct( sendTo, comm);
+        m_values.data().resize( m_idx.size());
     }
     virtual void do_global_gather( const get_value_type<Vector>* values, Vector& store)const override final
     {
         //actually this is a scatter but we constructed it invertedly
         //we could maybe transpose the Collective object!?
-        //assert( values.size() == idx_.size());
+        //assert( values.size() == m_idx.size());
         //nach PID ordnen
         typename Vector::const_pointer values_ptr(values);
-        thrust::gather( idx_.begin(), idx_.end(), values_ptr, values_.data().begin());
+        thrust::gather( m_idx.begin(), m_idx.end(), values_ptr, m_values.data().begin());
         //senden
-        p_.scatter( values_.data(), store);
+        m_p.scatter( m_values.data(), store);
     }
 
     virtual void do_global_scatter_reduce( const Vector& toScatter, get_value_type<Vector>* values) const override final
     {
         //actually this is a gather but we constructed it invertedly
-        p_.gather( toScatter, values_.data());
+        m_p.gather( toScatter, m_values.data());
         typename Vector::pointer values_ptr(values);
         //nach PID geordnete Werte wieder umsortieren
-        thrust::scatter( values_.data().begin(), values_.data().end(), idx_.begin(), values_ptr);
+        thrust::scatter( m_values.data().begin(), m_values.data().end(), m_idx.begin(), values_ptr);
     }
-    Buffer<Vector> values_;
-    Index idx_;
-    Collective<Index, Vector> p_;
-    thrust::host_vector<int> pids_;
+    Buffer<Vector> m_values;
+    Index m_idx;
+    Collective<Index, Vector> m_p;
+    thrust::host_vector<int> m_pids;
 };
 
 /**
@@ -291,7 +298,7 @@ struct SurjectiveComm : public aCommunicator<Vector>
 {
     ///@copydoc GeneralComm::GeneralComm()
     SurjectiveComm(){
-        buffer_size_ = store_size_ = 0;
+        m_buffer_size = m_store_size = 0;
     }
     ///@copydoc GeneralComm::GeneralComm(unsigned,const thrust::host_vector<int>&,const thrust::host_vector<int>&,MPI_Comm)
     ///@note we assume that the gather map is surjective
@@ -322,13 +329,15 @@ struct SurjectiveComm : public aCommunicator<Vector>
     }
 
     ///@copydoc GeneralComm::getLocalGatherMap
-    const thrust::host_vector<int>& getLocalGatherMap() const {return localGatherMap_;}
+    const thrust::host_vector<int>& getLocalGatherMap() const {return m_localGatherMap;}
     ///@copydoc GeneralComm::getPidGatherMap
-    const thrust::host_vector<int>& getPidGatherMap() const {return pidGatherMap_;}
-    const Index& getSortedGatherMap() const {return sortedGatherMap_;}
+    const thrust::host_vector<int>& getPidGatherMap() const {return m_pidGatherMap;}
+    const Index& getSortedGatherMap() const {return m_sortedGatherMap;}
     virtual SurjectiveComm* clone() const override final {return new SurjectiveComm(*this);}
     private:
-    virtual bool do_isCommunicating() const override final{ return bijectiveComm_.isCommunicating();}
+    virtual bool do_isCommunicating() const override final{
+        return m_bijectiveComm.isCommunicating();
+    }
     virtual Vector do_make_buffer()const override final{
         Vector tmp(do_size());
         return tmp;
@@ -337,52 +346,52 @@ struct SurjectiveComm : public aCommunicator<Vector>
     {
         //gather values to store
         typename Vector::const_pointer values_ptr(values);
-        thrust::gather( gatherMap_.begin(), gatherMap_.end(), values_ptr, store_.data().begin());
-        bijectiveComm_.global_scatter_reduce( store_.data(), thrust::raw_pointer_cast(buffer.data()));
+        thrust::gather( m_gatherMap.begin(), m_gatherMap.end(), values_ptr, m_store.data().begin());
+        m_bijectiveComm.global_scatter_reduce( m_store.data(), thrust::raw_pointer_cast(buffer.data()));
     }
     virtual void do_global_scatter_reduce( const Vector& toScatter, get_value_type<Vector>* values)const override final
     {
         //first gather values into store
-        Vector store_t = bijectiveComm_.global_gather( thrust::raw_pointer_cast(toScatter.data()));
+        Vector m_storet = m_bijectiveComm.global_gather( thrust::raw_pointer_cast(toScatter.data()));
         //now perform a local sort, reduce and scatter operation
-        thrust::gather( sortMap_.begin(), sortMap_.end(), store_t.begin(), store_.data().begin());
+        thrust::gather( m_sortMap.begin(), m_sortMap.end(), m_storet.begin(), m_store.data().begin());
         typename Vector::pointer values_ptr(values);
-        thrust::reduce_by_key( sortedGatherMap_.begin(), sortedGatherMap_.end(), store_.data().begin(), keys_.data().begin(), values_ptr);
+        thrust::reduce_by_key( m_sortedGatherMap.begin(), m_sortedGatherMap.end(), m_store.data().begin(), m_keys.data().begin(), values_ptr);
     }
-    virtual MPI_Comm do_communicator()const override final{return bijectiveComm_.communicator();}
-    virtual unsigned do_size() const override final {return buffer_size_;}
+    virtual MPI_Comm do_communicator()const override final{return m_bijectiveComm.communicator();}
+    virtual unsigned do_size() const override final {return m_buffer_size;}
     void construct( unsigned local_size, thrust::host_vector<int> localGatherMap, thrust::host_vector<int> pidGatherMap, MPI_Comm comm)
     {
         this->set_local_size(local_size);
-        bijectiveComm_ = BijectiveComm<Index, Vector>( pidGatherMap, comm);
-        localGatherMap_ = localGatherMap, pidGatherMap_ = pidGatherMap;
-        buffer_size_ = localGatherMap.size();
-        assert( buffer_size_ == pidGatherMap.size());
+        m_bijectiveComm = BijectiveComm<Index, Vector>( pidGatherMap, comm);
+        m_localGatherMap = localGatherMap, m_pidGatherMap = pidGatherMap;
+        m_buffer_size = localGatherMap.size();
+        assert( m_buffer_size == pidGatherMap.size());
         //the bijectiveComm behaves as if we had given the gather map for the store
         //now gather the localGatherMap from the buffer to the store to get the final gather map
-        Vector localGatherMap_d = dg::construct<Vector>( localGatherMap);
-        const typename aCommunicator<Vector>::value_type * v_ptr = thrust::raw_pointer_cast(localGatherMap_d.data());
-        Vector gatherMap_V = bijectiveComm_.global_gather( v_ptr);
-        Index gatherMap_I = dg::construct<Index>(gatherMap_V);
-        gatherMap_ = gatherMap_I;
-        store_size_ = gatherMap_.size();
-        store_.data().resize( store_size_);
-        keys_.data().resize( store_size_);
+        Vector m_localGatherMapd = dg::construct<Vector>( localGatherMap);
+        const typename aCommunicator<Vector>::value_type * v_ptr = thrust::raw_pointer_cast(m_localGatherMapd.data());
+        Vector m_gatherMapV = m_bijectiveComm.global_gather( v_ptr);
+        Index m_gatherMapI = dg::construct<Index>(m_gatherMapV);
+        m_gatherMap = m_gatherMapI;
+        m_store_size = m_gatherMap.size();
+        m_store.data().resize( m_store_size);
+        m_keys.data().resize( m_store_size);
 
         //now prepare a reduction map and a scatter map
-        sortMap_ = gatherMap_I;
-        thrust::sequence( sortMap_.begin(), sortMap_.end());
-        thrust::stable_sort_by_key( gatherMap_I.begin(), gatherMap_I.end(), sortMap_.begin());//note: this also sorts the gatherMap
-        dg::assign( gatherMap_I, sortedGatherMap_);
+        m_sortMap = m_gatherMapI;
+        thrust::sequence( m_sortMap.begin(), m_sortMap.end());
+        thrust::stable_sort_by_key( m_gatherMapI.begin(), m_gatherMapI.end(), m_sortMap.begin());//note: this also sorts the gatherMap
+        dg::assign( m_gatherMapI, m_sortedGatherMap);
         //now we can repeat/invert the sort by a gather/scatter operation with sortMap as map
     }
-    unsigned buffer_size_, store_size_;
-    BijectiveComm<Index, Vector> bijectiveComm_;
-    Index gatherMap_;
-    Index sortMap_, sortedGatherMap_;
-    Buffer<Index> keys_;
-    Buffer<Vector> store_;
-    thrust::host_vector<int> localGatherMap_, pidGatherMap_;
+    unsigned m_buffer_size, m_store_size;
+    BijectiveComm<Index, Vector> m_bijectiveComm;
+    Index m_gatherMap;
+    Index m_sortMap, m_sortedGatherMap;
+    Buffer<Index> m_keys;
+    Buffer<Vector> m_store;
+    thrust::host_vector<int> m_localGatherMap, m_pidGatherMap;
 };
 
 /**
@@ -453,22 +462,24 @@ struct GeneralComm : public aCommunicator<Vector>
     }
 
     ///@brief read access to the local index gather map
-    const thrust::host_vector<int>& getLocalGatherMap() const {return surjectiveComm_.getLocalGatherMap();}
+    const thrust::host_vector<int>& getLocalGatherMap() const {return m_surjectiveComm.getLocalGatherMap();}
     ///@brief read access to the pid gather map
-    const thrust::host_vector<int>& getPidGatherMap() const {return surjectiveComm_.getPidGatherMap();}
+    const thrust::host_vector<int>& getPidGatherMap() const {return m_surjectiveComm.getPidGatherMap();}
     virtual GeneralComm* clone() const override final {return new GeneralComm(*this);}
     private:
-    virtual bool do_isCommunicating() const override final{ return surjectiveComm_.isCommunicating();}
+    virtual bool do_isCommunicating() const override final{
+        return m_surjectiveComm.isCommunicating();
+    }
     virtual Vector do_make_buffer() const override final{
         Vector tmp(do_size());
         return tmp;
     }
-    virtual MPI_Comm do_communicator()const override final{return surjectiveComm_.communicator();}
+    virtual MPI_Comm do_communicator()const override final{return m_surjectiveComm.communicator();}
     virtual void do_global_gather( const get_value_type<Vector>* values, Vector& sink)const override final {
-        surjectiveComm_.global_gather( values, sink);
+        m_surjectiveComm.global_gather( values, sink);
     }
     virtual void do_global_scatter_reduce( const Vector& toScatter, get_value_type<Vector>* values)const override final {
-        surjectiveComm_.global_scatter_reduce( toScatter, thrust::raw_pointer_cast(store_.data().data()));
+        m_surjectiveComm.global_scatter_reduce( toScatter, thrust::raw_pointer_cast(m_store.data().data()));
         typename Vector::pointer values_ptr(values);
         dg::blas1::detail::doSubroutine_dispatch(
             get_execution_policy<Vector>(),
@@ -477,30 +488,30 @@ struct GeneralComm : public aCommunicator<Vector>
             values,
             0
         );
-        thrust::scatter( store_.data().begin(), store_.data().end(), scatterMap_.begin(), values_ptr);
+        thrust::scatter( m_store.data().begin(), m_store.data().end(), m_scatterMap.begin(), values_ptr);
     }
 
-    virtual unsigned do_size() const override final{return surjectiveComm_.buffer_size();}
+    virtual unsigned do_size() const override final{return m_surjectiveComm.buffer_size();}
     void construct( unsigned local_size, const thrust::host_vector<int>& localGatherMap, const thrust::host_vector<int>& pidGatherMap, MPI_Comm comm)
     {
         this->set_local_size( local_size);
-        surjectiveComm_ = SurjectiveComm<Index,Vector>(local_size, localGatherMap, pidGatherMap, comm);
+        m_surjectiveComm = SurjectiveComm<Index,Vector>(local_size, localGatherMap, pidGatherMap, comm);
 
-        const Index& sortedGatherMap_ = surjectiveComm_.getSortedGatherMap();
-        thrust::host_vector<int> gatherMap = dg::construct<thrust::host_vector<int>>( sortedGatherMap_);
+        const Index& m_sortedGatherMap = m_surjectiveComm.getSortedGatherMap();
+        thrust::host_vector<int> gatherMap = dg::construct<thrust::host_vector<int>>( m_sortedGatherMap);
         thrust::host_vector<int> one( gatherMap.size(), 1), keys(one), number(one);
         typedef thrust::host_vector<int>::iterator iterator;
         thrust::pair< iterator, iterator> new_end =
             thrust::reduce_by_key( gatherMap.begin(), gatherMap.end(), //sorted!
                 one.begin(), keys.begin(), number.begin() );
         unsigned distance = thrust::distance( keys.begin(), new_end.first);
-        store_.data().resize( distance);
-        scatterMap_.resize(distance);
-        thrust::copy( keys.begin(), keys.begin() + distance, scatterMap_.begin());
+        m_store.data().resize( distance);
+        m_scatterMap.resize(distance);
+        thrust::copy( keys.begin(), keys.begin() + distance, m_scatterMap.begin());
     }
-    SurjectiveComm<Index, Vector> surjectiveComm_;
-    Buffer<Vector> store_;
-    Index scatterMap_;
+    SurjectiveComm<Index, Vector> m_surjectiveComm;
+    Buffer<Vector> m_store;
+    Index m_scatterMap;
 };
 
 }//namespace dg
diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
index 0d27b7094..3ba056db2 100644
--- a/inc/dg/backend/mpi_communicator.h
+++ b/inc/dg/backend/mpi_communicator.h
@@ -144,7 +144,9 @@ struct aCommunicator
     * In Feltor the vector v is distributed equally among processes and the local size
     * of v is the same for all processes. However, the buffer size might be different for each process.
     * @return buffer size (may be different for each process)
-    * @note may return 0 to indicate that no MPI communication is needed
+    * @note may return 0
+    * @attention it is NOT enough to check for zero buffer size if you want to find out whether a given process
+    * needs to send MPI messages or not. See \c isCommunicating() for an explanation
     * @sa local_size() isCommunicating()
     */
     unsigned buffer_size() const{return do_size();}
@@ -163,18 +165,25 @@ struct aCommunicator
     /**
      * @brief True if the gather/scatter operation involves actual MPI communication
      *
+     * This is more than just a test for zero message size.
+     * This is because even if a process has zero message size indicating that it
+     * technically does not need to send any data at all it might still need to participate in an MPI communication (sending an empty message to
+     * indicate that a certain point in execution has been reached). Only if NONE of the processes in the process group has anything to send will
+     * this function return false.
      * This test can be used to avoid the gather operation alltogether in e.g. the construction of a MPI distributed matrix.
+     * @note this check involves MPI communication itself, because a process needs to check if itself or any other process in its
+     * group is communicating.
+     *
      * @return False, if the global gather can be done without MPI communication (i.e. the indices are all local to each calling process). True else.
      * @sa buffer_size()
      */
     bool isCommunicating() const{
-        if( do_size() == 0) return false;
         return do_isCommunicating();
     }
     /**
     * @brief The internal MPI communicator used
     *
-    * e.g. used to assert that communicators of matrix and vector are the same
+    * can be e.g. used to assert that communicators of matrix and vector are the same
     * @return MPI Communicator
     */
     MPI_Comm communicator() const{return do_communicator();}
diff --git a/inc/dg/backend/mpi_matrix.h b/inc/dg/backend/mpi_matrix.h
index 9b54f9b58..37bb35659 100644
--- a/inc/dg/backend/mpi_matrix.h
+++ b/inc/dg/backend/mpi_matrix.h
@@ -215,7 +215,7 @@ enum dist_type
 * @tparam Collective models aCommunicator The Communication class needs to scatter and gather values across processes.
 Gather all points (including the ones that the process already has) necessary for the local matrix-vector
 product into one vector, such that the local matrix can be applied.
-If \c buffer_size()==0 the global_gather and global_scatter_reduce functions won't be called and
+If \c !isCommunicating() the global_gather and global_scatter_reduce functions won't be called and
 only the local matrix is applied.
 */
 template<class LocalMatrix, class Collective >
@@ -264,7 +264,7 @@ struct MPIDistMat
     void symv( value_type alpha, const ContainerType1& x, value_type beta, ContainerType2& y) const
     {
         //the blas2 functions should make enough static assertions on tpyes
-        if( m_c->buffer_size() == 0) //no communication needed
+        if( !m_c->isCommunicating()) //no communication needed
         {
             dg::blas2::symv( alpha, m_m, x.data(), beta, y.data());
             return;
@@ -290,7 +290,7 @@ struct MPIDistMat
     void symv( const ContainerType1& x, ContainerType2& y) const
     {
         //the blas2 functions should make enough static assertions on tpyes
-        if( m_c->buffer_size() == 0) //no communication needed
+        if( !m_c->isCommunicating()) //no communication needed
         {
             dg::blas2::symv( m_m, x.data(), y.data());
             return;
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 289325e77..09084e7fc 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -32,7 +32,7 @@ struct MPI_Vector
     typedef container container_type;//!< typedef to acces underlying container
     ///no data is allocated, communicators are \c MPI_COMM_NULL
     MPI_Vector(){
-        comm_ = comm128_ = comm128Reduce_ = MPI_COMM_NULL;
+        m_comm = m_comm128 = m_comm128Reduce = MPI_COMM_NULL;
     }
     /**
      * @brief construct a vector
@@ -41,8 +41,8 @@ struct MPI_Vector
      * @param data internal data copy
      * @param comm MPI communicator (may not be \c MPI_COMM_NULL)
      */
-    MPI_Vector( const container& data, MPI_Comm comm): data_( data), comm_(comm) {
-        exblas::mpi_reduce_communicator( comm, &comm128_, &comm128Reduce_);
+    MPI_Vector( const container& data, MPI_Comm comm): m_data( data), m_comm(comm) {
+        exblas::mpi_reduce_communicator( comm, &m_comm128, &m_comm128Reduce);
     }
 
     /**
@@ -54,55 +54,55 @@ struct MPI_Vector
     */
     template<class OtherContainer>
     MPI_Vector( const MPI_Vector<OtherContainer>& src){
-        data_ = src.data();
-        comm_ = src.communicator();
-        comm128_ = src.communicator_mod();
-        comm128Reduce_ = src.communicator_mod_reduce();
+        m_data = src.data();
+        m_comm = src.communicator();
+        m_comm128 = src.communicator_mod();
+        m_comm128Reduce = src.communicator_mod_reduce();
     }
 
     ///@brief Get underlying data
     ///@return read access to data
-    const container& data() const {return data_;}
+    const container& data() const {return m_data;}
     ///@brief Set underlying data
     ///@return write access to data
-    container& data() {return data_;}
+    container& data() {return m_data;}
 
     ///@brief Get the communicator to which this vector belongs
     ///@return read access to MPI communicator
-    MPI_Comm communicator() const{return comm_;}
+    MPI_Comm communicator() const{return m_comm;}
     ///@brief Returns a communicator of fixed size 128
-    MPI_Comm communicator_mod() const{return comm128_;}
+    MPI_Comm communicator_mod() const{return m_comm128;}
 
     /**
      * @brief Returns a communicator consisting of all processes with rank 0 in \c communicator_mod()
      *
      * @return returns MPI_COMM_NULL to processes not part of that group
      */
-    MPI_Comm communicator_mod_reduce() const{return comm128Reduce_;}
+    MPI_Comm communicator_mod_reduce() const{return m_comm128Reduce;}
     /**
     * @brief Set the communicators with \c exblas::mpi_reduce_communicator
     */
     void set_communicator(MPI_Comm comm, MPI_Comm comm_mod, MPI_Comm comm_mod_reduce){
-        comm_ = comm;
-        comm128_ = comm_mod;
-        comm128Reduce_ = comm_mod_reduce;
+        m_comm = comm;
+        m_comm128 = comm_mod;
+        m_comm128Reduce = comm_mod_reduce;
     }
 
     ///@brief Return the size of the data object
     ///@return local size
-    unsigned size() const{return data_.size();}
+    unsigned size() const{return m_data.size();}
 
     ///@brief Swap data  and communicator
     ///@param src communicator and data is swapped
     void swap( MPI_Vector& src){
-        data_.swap(src.data_);
-        std::swap( comm_ , src.comm_);
-        std::swap( comm128_ , src.comm128_);
-        std::swap( comm128Reduce_ , src.comm128Reduce_);
+        m_data.swap(src.m_data);
+        std::swap( m_comm , src.m_comm);
+        std::swap( m_comm128 , src.m_comm128);
+        std::swap( m_comm128Reduce , src.m_comm128Reduce);
     }
   private:
-    container data_;
-    MPI_Comm comm_, comm128_, comm128Reduce_;
+    container m_data;
+    MPI_Comm m_comm, m_comm128, m_comm128Reduce;
 
 };
 
@@ -165,7 +165,8 @@ struct NearestNeighborComm
     using const_pointer_type = get_value_type<Vector> const *;
     ///@brief no communication
     NearestNeighborComm(){
-        silent_ = true;
+        m_comm = MPI_COMM_NULL;
+        m_silent = true;
     }
     /**
     * @brief Construct
@@ -192,7 +193,7 @@ struct NearestNeighborComm
     */
     template< class OtherIndex, class OtherBuffer, class OtherVector>
     NearestNeighborComm( const NearestNeighborComm<OtherIndex, OtherBuffer, OtherVector>& src){
-        if( src.buffer_size() == 0)  silent_=true;
+        if( src.buffer_size() == 0)  m_silent=true;
         else
             construct( src.n(), src.dims(), src.communicator(), src.direction());
     }
@@ -201,20 +202,20 @@ struct NearestNeighborComm
     * @brief halo size
     * @return  halo size
     */
-    unsigned n() const{return n_;}
+    unsigned n() const{return m_n;}
     /**
     * @brief  The dimensionality of the input vector
     * @return dimensions ( 3)
     */
-    const unsigned* dims() const{return dim_;}
+    const unsigned* dims() const{return m_dim;}
     /**
     * @brief The direction of communication
     *
     * @return direction
     */
-    unsigned direction() const {return direction_;}
-    ///@copydoc aCommunicator::isCommunicating()
-    MPI_Comm communicator() const{return comm_;}
+    unsigned direction() const {return m_direction;}
+    ///@copydoc aCommunicator::communicator()
+    MPI_Comm communicator() const{return m_comm;}
 
     /**
      * @brief Allocate a buffer object
@@ -245,9 +246,9 @@ struct NearestNeighborComm
         if( i==-1) return 0;
         if( i== 0) return 1;
         if( i==+1) return 2;
-        if( i==(int)outer_size_-0) return 5;
-        if( i==(int)outer_size_-1) return 4;
-        if( i==(int)outer_size_-2) return 3;
+        if( i==(int)m_outer_size-0) return 5;
+        if( i==(int)m_outer_size-1) return 4;
+        if( i==(int)m_outer_size-2) return 3;
         throw Error( Message(_ping_)<<"Index not mappable!");
         return -1;
     }
@@ -263,30 +264,30 @@ struct NearestNeighborComm
         unsigned size = buffer_size();
         //init pointers on host
         const_pointer_type host_ptr[6];
-        if(trivial_)
+        if(m_trivial)
         {
-            host_ptr[0] = thrust::raw_pointer_cast(&internal_buffer_.data()[0*size]);
+            host_ptr[0] = thrust::raw_pointer_cast(&m_internal_buffer.data()[0*size]);
             host_ptr[1] = input;
             host_ptr[2] = input+size;
-            host_ptr[3] = input+(outer_size_-2)*size;
-            host_ptr[4] = input+(outer_size_-1)*size;
-            host_ptr[5] = thrust::raw_pointer_cast(&internal_buffer_.data()[5*size]);
+            host_ptr[3] = input+(m_outer_size-2)*size;
+            host_ptr[4] = input+(m_outer_size-1)*size;
+            host_ptr[5] = thrust::raw_pointer_cast(&m_internal_buffer.data()[5*size]);
         }
         else
         {
-            host_ptr[0] = thrust::raw_pointer_cast(&internal_buffer_.data()[0*size]);
-            host_ptr[1] = thrust::raw_pointer_cast(&internal_buffer_.data()[1*size]);
-            host_ptr[2] = thrust::raw_pointer_cast(&internal_buffer_.data()[2*size]);
-            host_ptr[3] = thrust::raw_pointer_cast(&internal_buffer_.data()[3*size]);
-            host_ptr[4] = thrust::raw_pointer_cast(&internal_buffer_.data()[4*size]);
-            host_ptr[5] = thrust::raw_pointer_cast(&internal_buffer_.data()[5*size]);
+            host_ptr[0] = thrust::raw_pointer_cast(&m_internal_buffer.data()[0*size]);
+            host_ptr[1] = thrust::raw_pointer_cast(&m_internal_buffer.data()[1*size]);
+            host_ptr[2] = thrust::raw_pointer_cast(&m_internal_buffer.data()[2*size]);
+            host_ptr[3] = thrust::raw_pointer_cast(&m_internal_buffer.data()[3*size]);
+            host_ptr[4] = thrust::raw_pointer_cast(&m_internal_buffer.data()[4*size]);
+            host_ptr[5] = thrust::raw_pointer_cast(&m_internal_buffer.data()[5*size]);
         }
         //copy to device
         thrust::copy( host_ptr, host_ptr+6, buffer.begin());
         do_global_gather_init( get_execution_policy<Vector>(), input, rqst);
         sendrecv( host_ptr[1], host_ptr[4],
-                  thrust::raw_pointer_cast(&internal_buffer_.data()[0*size]),
-                  thrust::raw_pointer_cast(&internal_buffer_.data()[5*size]),
+                  thrust::raw_pointer_cast(&m_internal_buffer.data()[0*size]),
+                  thrust::raw_pointer_cast(&m_internal_buffer.data()[5*size]),
                   rqst);
     }
     /**
@@ -307,13 +308,13 @@ struct NearestNeighborComm
     void do_global_gather_init( CudaTag, const_pointer_type, MPI_Request rqst[4])const;
     void construct( unsigned n, const unsigned vector_dimensions[3], MPI_Comm comm, unsigned direction);
 
-    unsigned n_, dim_[3]; //deepness, dimensions
-    MPI_Comm comm_;
-    unsigned direction_;
-    bool silent_, trivial_=false; //silent -> no comm, trivial -> comm in last dim
-    unsigned outer_size_ = 1; //size of vector in units of buffer_size
-    Index gather_map_middle_;
-    dg::Buffer<Vector> internal_buffer_;
+    unsigned m_n, m_dim[3]; //deepness, dimensions
+    MPI_Comm m_comm;
+    unsigned m_direction;
+    bool m_silent, m_trivial=false; //silent -> no comm, m_trivial-> comm in last dim
+    unsigned m_outer_size = 1; //size of vector in units of buffer_size
+    Index m_gather_map_middle;
+    dg::Buffer<Vector> m_internal_buffer;
 
     void sendrecv(const_pointer_type, const_pointer_type, pointer_type, pointer_type, MPI_Request rqst[4])const;
     int m_source[2], m_dest[2];
@@ -326,81 +327,81 @@ void NearestNeighborComm<I,B,V>::construct( unsigned n, const unsigned dimension
 {
     static_assert( std::is_base_of<SharedVectorTag, get_tensor_category<V>>::value,
                "Only Shared vectors allowed");
-    silent_=false;
-    n_=n;
-    dim_[0] = dimensions[0], dim_[1] = dimensions[1], dim_[2] = dimensions[2];
-    direction_ = direction;
-    if( dimensions[2] == 1 && direction == 1) trivial_ = true;
-    else if( direction == 2) trivial_ = true;
-    else trivial_ = false;
+    m_silent=false;
+    m_n=n;
+    m_dim[0] = dimensions[0], m_dim[1] = dimensions[1], m_dim[2] = dimensions[2];
+    m_direction = direction;
+    if( dimensions[2] == 1 && direction == 1) m_trivial = true;
+    else if( direction == 2) m_trivial = true;
+    else m_trivial = false;
     assert( direction <3);
-    comm_ = comm;
+    m_comm = comm;
     //mpi_cart_shift may return MPI_PROC_NULL then the receive buffer is not modified
-    MPI_Cart_shift( comm_, direction_, -1, &m_source[0], &m_dest[0]);
-    MPI_Cart_shift( comm_, direction_, +1, &m_source[1], &m_dest[1]);
+    MPI_Cart_shift( m_comm, m_direction, -1, &m_source[0], &m_dest[0]);
+    MPI_Cart_shift( m_comm, m_direction, +1, &m_source[1], &m_dest[1]);
     {
         int ndims;
         MPI_Cartdim_get( comm, &ndims);
         int dims[ndims], periods[ndims], coords[ndims];
         MPI_Cart_get( comm, ndims, dims, periods, coords);
-        if( dims[direction] == 1) silent_ = true;
+        if( dims[direction] == 1) m_silent = true;
     }
-    if( !silent_)
+    if( !m_silent)
     {
-    outer_size_ = dimensions[0]*dimensions[1]*dimensions[2]/buffer_size();
-    assert( outer_size_ > 1 && "Parallelization too fine grained!"); //right now we cannot have that
+    m_outer_size = dimensions[0]*dimensions[1]*dimensions[2]/buffer_size();
+    assert( m_outer_size > 1 && "Parallelization too fine grained!"); //right now we cannot have that
     thrust::host_vector<int> mid_gather( 4*buffer_size());
     switch( direction)
     {
         case( 0):
-        for( unsigned i=0; i<dim_[2]*dim_[1]; i++)
+        for( unsigned i=0; i<m_dim[2]*m_dim[1]; i++)
             for( unsigned j=0; j<n; j++)
             {
-                mid_gather[(0*n+j)*dim_[2]*dim_[1]+i] = i*dim_[0]               + j;
-                mid_gather[(1*n+j)*dim_[2]*dim_[1]+i] = i*dim_[0] + n           + j;
-                mid_gather[(2*n+j)*dim_[2]*dim_[1]+i] = i*dim_[0] + dim_[0]-2*n + j;
-                mid_gather[(3*n+j)*dim_[2]*dim_[1]+i] = i*dim_[0] + dim_[0]-  n + j;
+                mid_gather[(0*n+j)*m_dim[2]*m_dim[1]+i] = i*m_dim[0]                + j;
+                mid_gather[(1*n+j)*m_dim[2]*m_dim[1]+i] = i*m_dim[0] + n            + j;
+                mid_gather[(2*n+j)*m_dim[2]*m_dim[1]+i] = i*m_dim[0] + m_dim[0]-2*n + j;
+                mid_gather[(3*n+j)*m_dim[2]*m_dim[1]+i] = i*m_dim[0] + m_dim[0]-  n + j;
             }
         break;
         case( 1):
-        for( unsigned i=0; i<dim_[2]; i++)
+        for( unsigned i=0; i<m_dim[2]; i++)
             for( unsigned j=0; j<n; j++)
-                for( unsigned k=0; k<dim_[0]; k++)
+                for( unsigned k=0; k<m_dim[0]; k++)
                 {
-                    mid_gather[((0*n+j)*dim_[2]+i)*dim_[0] + k] = (i*dim_[1]               + j)*dim_[0] + k;
-                    mid_gather[((1*n+j)*dim_[2]+i)*dim_[0] + k] = (i*dim_[1] + n           + j)*dim_[0] + k;
-                    mid_gather[((2*n+j)*dim_[2]+i)*dim_[0] + k] = (i*dim_[1] + dim_[1]-2*n + j)*dim_[0] + k;
-                    mid_gather[((3*n+j)*dim_[2]+i)*dim_[0] + k] = (i*dim_[1] + dim_[1]-  n + j)*dim_[0] + k;
+                    mid_gather[((0*n+j)*m_dim[2]+i)*m_dim[0] + k] = (i*m_dim[1]                + j)*m_dim[0] + k;
+                    mid_gather[((1*n+j)*m_dim[2]+i)*m_dim[0] + k] = (i*m_dim[1] + n            + j)*m_dim[0] + k;
+                    mid_gather[((2*n+j)*m_dim[2]+i)*m_dim[0] + k] = (i*m_dim[1] + m_dim[1]-2*n + j)*m_dim[0] + k;
+                    mid_gather[((3*n+j)*m_dim[2]+i)*m_dim[0] + k] = (i*m_dim[1] + m_dim[1]-  n + j)*m_dim[0] + k;
                 }
         break;
         case( 2):
         for( unsigned i=0; i<n; i++)
-            for( unsigned j=0; j<dim_[0]*dim_[1]; j++)
+            for( unsigned j=0; j<m_dim[0]*m_dim[1]; j++)
             {
-                mid_gather[(0*n+i)*dim_[0]*dim_[1]+j] = (i               )*dim_[0]*dim_[1] + j;
-                mid_gather[(1*n+i)*dim_[0]*dim_[1]+j] = (i + n           )*dim_[0]*dim_[1] + j;
-                mid_gather[(2*n+i)*dim_[0]*dim_[1]+j] = (i + dim_[2]-2*n )*dim_[0]*dim_[1] + j;
-                mid_gather[(3*n+i)*dim_[0]*dim_[1]+j] = (i + dim_[2]-  n )*dim_[0]*dim_[1] + j;
+                mid_gather[(0*n+i)*m_dim[0]*m_dim[1]+j] = (i                )*m_dim[0]*m_dim[1] + j;
+                mid_gather[(1*n+i)*m_dim[0]*m_dim[1]+j] = (i + n            )*m_dim[0]*m_dim[1] + j;
+                mid_gather[(2*n+i)*m_dim[0]*m_dim[1]+j] = (i + m_dim[2]-2*n )*m_dim[0]*m_dim[1] + j;
+                mid_gather[(3*n+i)*m_dim[0]*m_dim[1]+j] = (i + m_dim[2]-  n )*m_dim[0]*m_dim[1] + j;
             }
         break;
     }
-    gather_map_middle_ = mid_gather; //transfer to device
-    internal_buffer_.data().resize( 6*buffer_size() );
+    m_gather_map_middle = mid_gather; //transfer to device
+    m_internal_buffer.data().resize( 6*buffer_size() );
     }
 }
 
 template<class I, class B, class V>
 unsigned NearestNeighborComm<I,B,V>::buffer_size() const
 {
-    if( silent_) return 0;
-    switch( direction_)
+    if( m_silent) return 0;
+    switch( m_direction)
     {
         case( 0): //x-direction
-            return n_*dim_[1]*dim_[2];
+            return m_n*m_dim[1]*m_dim[2];
         case( 1): //y-direction
-            return n_*dim_[0]*dim_[2];
+            return m_n*m_dim[0]*m_dim[2];
         case( 2): //z-direction
-            return n_*dim_[0]*dim_[1]; //no further n_ (hide in dim_)
+            return m_n*m_dim[0]*m_dim[1]; //no further m_n (hide in m_dim)
         default:
             return 0;
     }
@@ -409,23 +410,23 @@ unsigned NearestNeighborComm<I,B,V>::buffer_size() const
 template<class I, class B, class V>
 void NearestNeighborComm<I,B,V>::do_global_gather_init( SerialTag, const_pointer_type input, MPI_Request rqst[4]) const
 {
-    if( !trivial_)
+    if( !m_trivial)
     {
         unsigned size = buffer_size();
         for( unsigned i=0; i<4*size; i++)
-            internal_buffer_.data()[i+size] = input[gather_map_middle_[i]];
+            m_internal_buffer.data()[i+size] = input[m_gather_map_middle[i]];
     }
 }
 #ifdef _OPENMP
 template<class I, class B, class V>
 void NearestNeighborComm<I,B,V>::do_global_gather_init( OmpTag, const_pointer_type input, MPI_Request rqst[4]) const
 {
-    if(!trivial_)
+    if(!m_trivial)
     {
         unsigned size = buffer_size();
         #pragma omp parallel for
         for( unsigned i=0; i<4*size; i++)
-            internal_buffer_.data()[size+i] = input[gather_map_middle_[i]];
+            m_internal_buffer.data()[size+i] = input[m_gather_map_middle[i]];
     }
 }
 #endif
@@ -434,10 +435,10 @@ template<class I, class B, class V>
 void NearestNeighborComm<I,B,V>::do_global_gather_init( CudaTag, const_pointer_type input, MPI_Request rqst[4]) const
 {
     //gather values from input into sendbuffer
-    if(!trivial_)
+    if(!m_trivial)
     {
         unsigned size = buffer_size();
-        thrust::gather( thrust::cuda::tag(), gather_map_middle_.begin(), gather_map_middle_.end(), input, internal_buffer_.data().begin()+size);
+        thrust::gather( thrust::cuda::tag(), m_gather_map_middle.begin(), m_gather_map_middle.end(), input, m_internal_buffer.data().begin()+size);
     }
     cudaDeviceSynchronize(); //wait until device functions are finished before sending data
 }
@@ -448,17 +449,17 @@ void NearestNeighborComm<I,B,V>::sendrecv( const_pointer_type sb1_ptr, const_poi
 {
     MPI_Isend( sb1_ptr, buffer_size(),
                getMPIDataType<get_value_type<V>>(),  //sender
-               m_dest[0], 3, comm_, &rqst[0]); //destination
+               m_dest[0], 3, m_comm, &rqst[0]); //destination
     MPI_Irecv( rb2_ptr, buffer_size(),
                getMPIDataType<get_value_type<V>>(), //receiver
-               m_source[0], 3, comm_, &rqst[1]); //source
+               m_source[0], 3, m_comm, &rqst[1]); //source
 
     MPI_Isend( sb2_ptr, buffer_size(),
                getMPIDataType<get_value_type<V>>(),  //sender
-               m_dest[1], 9, comm_, &rqst[2]);  //destination
+               m_dest[1], 9, m_comm, &rqst[2]);  //destination
     MPI_Irecv( rb1_ptr, buffer_size(),
                getMPIDataType<get_value_type<V>>(), //receiver
-               m_source[1], 9, comm_, &rqst[3]); //source
+               m_source[1], 9, m_comm, &rqst[3]); //source
 }
 
 
diff --git a/inc/dg/topology/mpi_projection.h b/inc/dg/topology/mpi_projection.h
index bf2b79d20..e18ef1164 100644
--- a/inc/dg/topology/mpi_projection.h
+++ b/inc/dg/topology/mpi_projection.h
@@ -59,7 +59,7 @@ static void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& globa
 /**
  * @brief Convert a matrix with local row and global column indices to a row distributed MPI matrix
  *
- * @tparam ConversionPolicy has to have the members:
+ * @tparam ConversionPolicy (can be one of the MPI %grids ) has to have the members:
  *  - \c bool\c global2localIdx(unsigned,unsigned&,unsigned&) \c const;
  * where the first parameter is the global index and the
  * other two are the output pair (localIdx, rank).
@@ -76,8 +76,6 @@ static void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& globa
 template<class ConversionPolicy, class real_type>
 dg::tMIHMatrix<real_type> convert( const dg::tIHMatrix<real_type>& global, const ConversionPolicy& policy)
 {
-    int rank;
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     dg::iHVec unique_global_idx;
     cusp::array1d<int, cusp::host_memory> buffer_idx;
     dg::detail::global2bufferIdx( global.column_indices, buffer_idx, unique_global_idx);
-- 
GitLab


From 4607c08e9dbc7d4a5abae047ff98ab1583a8b5d8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 29 Mar 2019 11:53:00 +0100
Subject: [PATCH 053/540] Fix BUG beta in feltor3d restart

---
 src/feltor/feltor_hpc.cu | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index baf5a4664..f0ffe628e 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -323,8 +323,8 @@ int main( int argc, char* argv[])
         //Convert to N-1 and W
         dg::blas1::plus( transferINHvec[0], -1.);
         dg::blas1::plus( transferINHvec[1], -1.);
-        dg::blas1::axpby( 1., transferINHvec[2], p.beta/p.mu[0], transferINHvec[4], transferINHvec[2]);
-        dg::blas1::axpby( 1., transferINHvec[3], p.beta/p.mu[1], transferINHvec[4], transferINHvec[3]);
+        dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
+        dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
 
         dg::assign( transferINHvec[0], y0[0][0]); //ne-1
         dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
-- 
GitLab


From 6be6919285b55b192504b1f2e6b946f390fd890b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 29 Mar 2019 13:29:08 +0100
Subject: [PATCH 054/540] Fix BUG mpi_comm_null in isCommunicating

---
 inc/dg/backend/mpi_collective.h   | 3 ++-
 inc/dg/backend/mpi_communicator.h | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index fcc963556..42e344453 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -206,8 +206,9 @@ struct BijectiveComm : public aCommunicator<Vector>
     virtual BijectiveComm* clone() const override final {return new BijectiveComm(*this);}
     private:
     virtual bool do_isCommunicating() const override final{
+        if( m_p.communicator()  == MPI_COMM_NULL) return false;
         int rank;
-        MPI_Comm_rank( do_communicator(), &rank);
+        MPI_Comm_rank( m_p.communicator(), &rank);
         bool local_communicating = false, global_communicating=false;
         for( unsigned i=0; i<m_pids.size(); i++)
             if( m_pids[i] != rank)
diff --git a/inc/dg/backend/mpi_communicator.h b/inc/dg/backend/mpi_communicator.h
index 3ba056db2..b157045ee 100644
--- a/inc/dg/backend/mpi_communicator.h
+++ b/inc/dg/backend/mpi_communicator.h
@@ -174,7 +174,7 @@ struct aCommunicator
      * @note this check involves MPI communication itself, because a process needs to check if itself or any other process in its
      * group is communicating.
      *
-     * @return False, if the global gather can be done without MPI communication (i.e. the indices are all local to each calling process). True else.
+     * @return False, if the global gather can be done without MPI communication (i.e. the indices are all local to each calling process), or if the communicator is \c MPI_COMM_NULL. True else.
      * @sa buffer_size()
      */
     bool isCommunicating() const{
-- 
GitLab


From 5b1808b1ada312e2adb4903145c5266a0cab255e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 30 Mar 2019 23:54:50 +0100
Subject: [PATCH 055/540] Refine definitions around flux-surface average

---
 src/feltor/feltor.tex | 99 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 78 insertions(+), 21 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 0fd4c2f66..3556f000a 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -345,12 +345,15 @@ K_{\nabla\times\bhat}^\varphi &= \frac{R_0}{R^2B^2}\left(
 \subsection{ Modified $\psi_p$}
 Our computational domain is a box and in particular not aligned with the
 magnetic flux surfaces. This means that particularly in the corners of
-the domain the field lines enter and leave the box very quickly.
-It turns out that this behaviour is numerically disadvantageous in the
+the domain the field lines inside the domain are very short (in the
+sense that the distance between the enter point and leave point is short).
+It turns out that this behaviour is numerically disadvantageous (may
+blow up the simulation in the worst case) in the
 computation of parallel derivatives. In order to remedy this situation
 we propose to modify the flux surfaces $\psi_p$ to a constant value
 if $\psi_p$ exceeds a certain critical value. In this way the poloidal
-field component vanishes in the corners of the domain.
+field component vanishes in the corners of the domain at the cost
+of introducing a strong shear layer limiting the scrape-off layer width.
 
 We define an approximation to the step function with width $\alpha$
 \begin{align}
@@ -549,6 +552,9 @@ We either follow fieldlines around the torus several times (``blob'') or only on
 (``straight blob'').
 \subsubsection{Turbulent bath}
 We can initialize the R-Z plane with a turbulent bath with a certain amplitude $A$.
+This especially has the goal to destabilize the edge region right inside the
+last closed flux surface. Notice that the core region is rather stable
+and quickly damps away fluctuations.
 Again, we transform this to all poloidal planes along the magnetic field lines and multiply the bath with
 \begin{align} \label{eq:initial_turbulent}
 \tilde n_e(R,Z,\varphi) = \tilde n_{\text{bath}}(R,Z,\varphi)\Theta_{\alpha}(\psi_p(R, Z)+\alpha) H(Z-Z_X)
@@ -929,36 +935,87 @@ the whole simulation is lost. It is safer to just merge files afterwards with fo
 \texttt{ncrcat output1.nc output2.nc output.nc}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Diagnostics}\label{sec:diagnostics}
-\subsection{Flux surface average and Safety factor}
-We define the toroidal average of a function $f(R,Z,\varphi)$ as
+\subsection{Preliminary}
+Recall that the {\bf Dirac delta-function} has the property (in any dimension):
+\begin{align} \label{eq:dirac_delta}
+\int_V f(\vec x) \delta(h(\vec x) - h') \dV = \int_{h=h'} \frac{f(\vec x)}{|\nabla h|} \dA
+\end{align}
+which means that the delta-function can be used to express area integrals of the
+submanifold given as a contour of the function $h(\vec x)$.
+A numerically tractable approximation to the delta-function reads
+\begin{align}\label{eq:delta}
+\delta(h(\vec x)-h') = \frac{1}{2\pi \epsilon^2}
+\exp\left( - \frac{\left(h(\vec x)-h'\right)^2}{2\epsilon^2}\right)
+\end{align}
+where $\epsilon$ is a small, free parameter.
+In the DG framework the left-hand side
+of Eq.~\eqref{eq:dirac_delta} can thus readily be computed
+via Gauss-Legendre quadrature, which gives us an easy means to compute area
+integrals even if our coordinate system is not aligned to the area.
+
+Furthermore, recall the {\bf coarea formula}
+\begin{align} \label{eq:coarea}
+\int_{\Omega_0} f(\vec x) \dV =
+\int_0^{h_0} \left( \int_{h=h'} \frac{f(\vec x)}{|\nabla h|}  \dA  \right) \d h'
+\end{align}
+where $\Omega_0$ is the volume enclosed by the contour $h=h_0$.
+The coarea formula can be viewd as a change of variables in the
+volume integral.
+
+We define the {\bf toroidal average} of a function $f(R,Z,\varphi)$ as
 \begin{align} \label{eq:phi_average}
-\langle f\rangle_\varphi := \frac{1}{2\pi}\oint f\d \varphi
+\langle f\rangle_\varphi(R,Z) := \frac{1}{2\pi}\oint f(R,Z,\varphi)\d \varphi
 \end{align}
-The flux surface average
+
+\subsection{Flux surface average}
+
+The {\bf flux surface average}
 of a function $f(R,Z,\varphi)$ is given by the formula
 \begin{align}\label{eq:fsa}
-\langle f \rangle_{\psi_{p0}} :=
-\frac{ \oint_{\psi_p = \psi_{p0} } f(R,Z,\varphi)\dA}{\oint_{\psi_p = \psi_{p0} } \dA} =
-\frac{\int_\Omega f(R,Z,\varphi) \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p| H(R,Z)\dV}
-{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p|H(R,Z)\dV}
+\langle f \rangle_{\psi_{p0}} :=&
+\frac{ \oint_{\psi_p = \psi_{p0} } f(R,Z,\varphi)\dA}{\oint_{\psi_p = \psi_{p0} } \dA} \nonumber \\
+=& \frac{\int_\Omega f(R,Z,\varphi) \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p| H(Z-Z_X)\ R \d R \d Z\d\varphi}
+{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p|H(Z-Z_X)\ R \d R \d Z\d\varphi}
 \end{align}
-with $\dV := \d R\d Z\d \varphi$ (we define the average in computational space and omit one $R$)
-and we use the Heaviside function $H(R,Z) = H(Z-Z_X)$ to cut away contributions from below the X-point
+%with $\dV := R\d R\d Z\d \varphi$ %(we define the average in computational space and omit one $R$)
+and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
 in our domain.
 
-A numerically tractable approximation to the $\delta$-function reads
-\begin{align}\label{eq:delta}
-\delta(\psi_p-\psi_{p0}) = \frac{1}{2\pi \epsilon^2}
-\exp\left( - \frac{\left(\psi_p-\psi_{p0}\right)^2}{2\epsilon^2}\right)
+The {\bf total flux} of a given flux density $\vec j$ though the
+flux surface $\psi_p = \psi_{p0}$ is given by
+\begin{align}
+\oint_{\psi_p=\psi_{p0}} \vec j\cdot \vec{\dA} =
+\int \vec j\cdot \vec\nabla\psi_p \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z\d\varphi
+\label{eq:total_flux}
 \end{align}
-where $\epsilon$ is a small, free parameter. In the DG framework the integrals
-in Eq.~\eqref{eq:fsa} can be computed via Gauss-Legendre quadrature.
 
-If we write $B_p:=R_0|\nabla\psi_p|/R$, we can define the safety factor (in two dimensions) as
+\subsection{The safety factor}
+Assume that we pick a random field line and follow it (integrate it) for exactly one
+poloidal turn. The {\bf safety factor} is defined as the ratio between
+the resulting toroidal angle to the poloidal angle ($2\pi$)
 \begin{align}
-q:=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vec\nabla\psi_p|} \d s
+q := \frac{\Delta\varphi}{2\pi}
+\label{}
+\end{align}
+Since our magnetic field is symmetric in $\varphi$ and we used one
+full poloidal turn this definition is independent of which
+fieldline we pick inside a given flux surface.
+
+Let us define the poloidal length $s$ as the fieldline following
+parameter i.e. $\vec B\cdot \nabla s \equiv B_p = R_0|\nabla \psi_p|/R$
+and $\d\varphi/\d s = B^\varphi(R(s), Z(s)) / B_p(R(s),Z(s))$.
+We can then express the safety factor as the line integral
+\begin{align}
+q=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vec\nabla\psi_p|} \d s
 = \frac{1}{2\pi}\int \frac{I(\psi_p)}{R}\delta(\psi_p-\psi_{p0}) H(Z-Z_X) \d R\d Z
 \end{align}
+where we made use of Eq.~\eqref{eq:dirac_delta} in two dimensions in the
+last equality and thus arrive at a numerical tractable expression
+to evaluate the safety factor.
+
+Notice that the safety factor diverges on the last closed flux
+surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa}
+remain finite due to the $\nabla\psi$ factor.
 \subsection{ Radial flux}
 Let us define
 
-- 
GitLab


From 840191838f778051077130a8c738b00d894472bf Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 31 Mar 2019 19:00:58 +0200
Subject: [PATCH 056/540] Update energy and particle fluxes

---
 src/feltor/feltor.tex | 144 ++++++++++++++++++++++++++----------------
 1 file changed, 88 insertions(+), 56 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 3556f000a..d067b4a39 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -603,13 +603,19 @@ so that $\nabla_\perp^2 S_{n_e}$ is well defined.
 \subsubsection{Mass conservation}
 Integrating the density equation we directly get
 \begin{align} \label{eq:mass_conservation}
-  \frac{\partial}{\partial t} \int \dV n_e =  \int dV (\Lambda_{n_e}+S_{n_e})
+  \frac{\partial}{\partial t} \int_\Omega \dV n_e
+  + \int_{\partial\Omega} \vec\dA\cdot\vec{ j_{n_e}}
+  =  \int_\Omega \dV (\Lambda_{n_e}+S_{n_e})
+\end{align}
+with the particle flux
+\begin{align} \label{eq:mass_flux}
+\vec{ j_{n_e}} := n_e\left(\vec v_E + \vec v_C + \vec v_{\nabla B} + U (\bhat + \vec{\tilde b}_\perp)\right)
 \end{align}
 \subsubsection{Charge/vorticity conservation}
 Integrating the polarisation equation~\eqref{eq:polarisation_dimensional}
 and assuming $\Gamma_{1,i}S_{N_i} = S_{n_e}$ we find
 \begin{align} \label{eq:charge_conservation}
-  \frac{\partial}{\partial t} \int \dV \vec{\nabla} \cdot\left(\frac{m_iN_i}{e B^2} \vec{\nabla}_\perp \phi\right) =  - \int \dV (\Lambda_{n_e} - \Gamma_{1,i}\Lambda_{N_i})
+  \frac{\partial}{\partial t} \int_\Omega \dV \vec{\nabla} \cdot\left(\frac{m_iN_i}{e B^2} \vec{\nabla}_\perp \phi\right) =  - \int_\Omega \dV (\Lambda_{n_e} - \Gamma_{1,i}\Lambda_{N_i})
 \end{align}
 Note that if the integrand on the left hand side is interpreted as the \ExB vorticity
 density
@@ -617,35 +623,46 @@ $\zeta := \vec \nabla\cdot( m_iN_i\vec \nabla_\perp\phi/e B^2)$
 and if we further assume that $\Gamma_{1,i} \Lambda_{N_i} \equiv \Lambda_{\Gamma_{1,i}N_i}$,
 we get the vorticity conservation
 \begin{align} \label{eq:vorticity_conservation}
-  \frac{\partial}{\partial t} \int \dV \zeta =  - \int \dV\Lambda_{\zeta}
+  \frac{\partial}{\partial t} \int_\Omega \dV \zeta =  - \int_\Omega \dV\Lambda_{\zeta}
 \end{align}
 
 \subsubsection{Energy theorem}
-The terms of the energy theorem $\partial_t \mathcal E + \mathcal S =
-\Lambda$ are derived to (if either $\psi$ or $\nabla\psi$ vanishes at the
-boundary)
+The terms of the energy theorem are
+\begin{align} \label{eq:energy_theorem}
+\partial_t \int_\Omega \dV \mathcal E +
+\oint_{\partial\Omega} \vec\dA \cdot \vec j_{\mathcal E}
+= \int_\Omega \dV \left( \Lambda_{\mathcal E}
++  S_{\mathcal E}
+-  R_{\mathcal E} \right)
+\end{align}
+with
 \begin{align} \label{eq:energy_conservation}
-  \mathcal{E} = \int  \dV & \left( t_e n_e \ln{(n_e)} +T_i N_i\ln{(N_i)}+
-    \frac{(\vec \nabla_\perp A_\parallel)^2}{2\mu_0} 
-    +\frac{1}{2} m_i N_i \left(\frac{\vec\nabla_\perp\phi}{B}\right)^2 \right.\nonumber\\
-    &\left.+\frac{1}{2} m_e  n_e u_e^2 +\frac{1}{2} m_i  N_i U_i^2  \right),\\
-  \mathcal S = \sum_s \int \vec{\dA} \cdot &\left[ \left(
-  U\left(\bhat+\tilde{\vec b}_\perp\right) + \vec v_E + \vec v_C + \vec v_{\nabla B} \right)
-  \left(T N\ln N + \frac{1}{2}m NU^2 + q\psi N \right) \right. \nonumber\\
-  & \left . + \vec v_{\nabla\times\bhat} m NU^2  + \left(\bhat + \tilde{\vec b}_\perp\right) UNT\right], \\
-  \Lambda =  \int \dV & \bigg\{  \left[t_e\left( 1+\ln{(n_e)}\right) -e \phi + \frac{1}{2} m_e u_e^2 \right](\Lambda_{n_e} + S_{n_e})
-  \nonumber\\ &
-+\left[T_i\left( 1+\ln{(N_i)}\right) +e \psi_i + \frac{1}{2} m_i U_i^2 \right](\Lambda_{N_i}+S_{N_i})
-\nonumber \\ &
-+ m_e u_e n_e (\Lambda_{u_e}+S_{u_e})+m_iU_i N_i (\Lambda_{U_i}+S_{U_i})- \eta_\parallel J_{\parallel,s}^2\bigg\}.
-\end{align}
-The energy consists of the Helmholtz free energy density for electrons and ions, the \(\vec{E} \times \vec{B}\) energy density, the parallel energy densities for electrons and ions and the perturbed magnetic field energy density.
+  \mathcal E = &
+  t_e n_e \ln{(n_e)} +T_i N_i\ln{(N_i)}
+    +\frac{1}{2} m_i N_i \left(\frac{\vec\nabla_\perp\phi}{B}\right)^2 \nonumber\\
+    &+\frac{1}{2} m_e  n_e u_e^2 +\frac{1}{2} m_i  N_i U_i^2
+    + \frac{(\vec \nabla_\perp A_\parallel)^2}{2\mu_0}, \\
+  \vec j_{\mathcal E} =& \sum_s \left[ N\left(
+  \vec v_E + \vec v_C + \vec v_{\nabla B} +U\left(\bhat+\tilde{\vec b}_\perp\right)  \right)
+  \left(T \ln N + \frac{1}{2}m U^2 + q\psi  \right) \right. \nonumber\\
+  &\qquad \left . + mNU^2\vec v_{\nabla\times\bhat} + \left(\bhat + \tilde{\vec b}_\perp\right) NUT\right], \\
+  \Lambda_{\mathcal E} =&  \sum_s \left[\left(T\left( 1+\ln{N}\right) +q \psi + \frac{1}{2} m U^2 \right)\Lambda_{N}  + mNU \Lambda_U\right]
+\nonumber \\
+  S_{\mathcal E} =&  \sum_s  \left[\left(T\left( 1+\ln{N}\right) +q \psi + \frac{1}{2} m U^2 \right)S_{N}  + mNU S_U\right]
+\nonumber \\
+  R_{\mathcal E} =&  \eta_\parallel  \left[e n_e(U_i-u_e)\right]^2.
+\end{align}
+where in the energy flux $\vec j_{\mathcal E}$
+we neglect terms  containing time derivatives
+of the eletric and magnetic potentials and we sum over all species.
+The energy density $\mathcal E$ consists of the Helmholtz free energy density for electrons and ions,
+the \(\vec{E} \times \vec{B}\) energy density, the parallel energy densities for electrons and ions and the perturbed magnetic field energy density.
 In \(\Lambda\) we insert the dissipative terms of Section~\ref{sec:dissres}. \\
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsection{Dimensionless form}
-We scale all spatial lengths by $\rho_s = \sqrt{T_e m_i}/(eB_0)$ and time by the ion gyrofrequency $\Omega_0 = eB_0/m_i$.
+We scale all spatial lengths by $\rho_s = \sqrt{T_e m_i}/(eB_0)$ and time by the ion gyro-frequency $\Omega_0 = eB_0/m_i$.
 The magnetic field is scaled with $B_0$, densities with $n_0$ and the parallel velocity is scaled with $c_s = \sqrt{T_e/m_i}$.
-The potential is scaled with $\hat \phi = e/T_e$ and the vector potential with 
+The potential is scaled with $\hat \phi = e/T_e$ and the vector potential with
 $\hat A_\parallel = \rho_s B_0$.
 We introduce the dimensionless parameters
 \begin{align}
@@ -702,29 +719,55 @@ and
     A_\parallel &= \beta\left(N_iW_i-n_e w_e\right)
   \end{align}
 \end{subequations}
-Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite. Also note the peculiar notation of the perpendicular viscosity on $U$.
-We indicate that $\Delta_\perp W$ is treated implicitly while the correction $\Delta_\perp A_\parallel$ is treated explicitly.
-The energy theorem reads $\partial_t \mathcal E + \mathcal S = \Lambda$ (with $z_e=-1$ and $z_i=+1$)
+Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite.
+
+The terms of the energy theorem read (with $z_e=-1$ and $z_i=+1$)
 \begin{align}
-  \mathcal{E}= \int  \dV & \left( z_e\tau_e n_e \ln{(n_e)} +z_i\tau_i N_i\ln{(N_i)}
+  \mathcal{E}= & z_e\tau_e n_e \ln{(n_e)} +z_i\tau_i N_i\ln{(N_i)}
   +\frac{1}{2}\left(\vec \nabla_\perp A_\parallel\right)^2
-   +  \frac{1}{2} \mu_i N_i u_E^2  \right .\nonumber\\
-   &\left. +\frac{1}{2} z_e\mu_e  n_e u_e^2
-  +\frac{1}{2} z_i\mu_i  N_i U_i^2  \right),\\
-  %\mathcal S = \sum_s \int \vec{\dA} & \cdot\left[ \left( U\bhat + \vec v_\perp )
-  %(z\tau N\ln N + \frac{1}{2}z\mu NU^2 + \psi N \right) + \vec v_C' z\mu NU^2 + \bhat UNT \right], \\
-  \Lambda =  \int \dV & \bigg\{  z_e\left[\tau_e\left( 1+\ln{(n_e)}\right) + \phi + \frac{1}{2} \mu_e u_e^2 \right]
-    \left(\nu_\perp\Delta_\perp n_e + \nu_\parallel\Delta_\parallel n_e + S_{n_e}\right)
-  \nonumber\\ &
-+z_i\left[\tau_i\left( 1+\ln{(N_i)}\right) + \psi_i + \frac{1}{2} \mu_i U_i^2 \right]
-    \left(\nu_\perp\Delta_\perp N_i + \nu_\parallel\Delta_\parallel N_i +S_{N_i}\right)
-\nonumber \\ &
-    +z_e\mu_e u_e n_e \left(\nu_\perp\Delta_\perp u_e + \nu_\parallel \Delta_\parallel u_e + S_{u_e}\right)\nonumber \\ &
-    +z_i\mu_i U_i N_i \left(\nu_\perp\Delta_\perp U_i + \nu_\parallel \Delta_\parallel U_i + S_{U_i}\right)% \nonumber \\ &
-    - \eta \left(n_e(U_i-u_e)\right)^2\bigg\}.
+   +  \frac{1}{2} \mu_i N_i u_E^2  \nonumber\\
+   & +\frac{1}{2} z_e\mu_e  n_e u_e^2
+  +\frac{1}{2} z_i\mu_i  N_i U_i^2,\\
+  \vec j_{\mathcal E} =& \sum_s z\left[ N\left(
+  \vec v_E + \vec v_C + \vec v_{\nabla B} +U\left(\bhat+\tilde{\vec b}_\perp\right)  \right)
+  \left(\tau \ln N + \frac{1}{2}\mu U^2 + \psi \right) \right. \nonumber\\
+  &\qquad \left . + \mu NU^2\vec v_{\nabla\times\bhat} + \tau NU \left(\bhat + \tilde{\vec b}_\perp\right)\right], \\
+  \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U^2 \right)
+  \left(\nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N\right)  +  \mu NU\left(\nu_\perp\Delta_\perp U + \nu_\parallel\Delta_\parallel U\right) \right]
+\nonumber \\
+  S_{\mathcal E} =&  \sum_s  z\left[ \left(\tau\left( 1+\ln{N}\right) +\psi + \frac{1}{2} \mu U^2 \right)S_{N}  + \mu NU S_U\right]
+\nonumber \\
+  R_{\mathcal E} =&  \eta_\parallel  \left[ n_e(U_i-u_e)\right]^2.
 \end{align}
 Replace $\Delta_\perp$ with $-\Delta_\perp^2$ when hyperviscous diffusion is chosen
 for the diffusion terms in the above equations.
+
+Let us here also derive the particle and energy fluxes through a flux surface
+\begin{align} \label{eq:particle_flux}
+ \vec j_{N}\cdot \vec \nabla\psi_p =& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ \nabla\psi_p \nonumber\\
+ =&
+  N\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
+   N\mathcal K_{\nabla\times\bhat}(\psi_p) + \tau N \mathcal K_{\nabla B}(\psi_p) \nonumber\\
+ &+ NU\left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
+\end{align}
+\begin{align} \label{eq:energy_flux}
+ \vec j_{\mathcal E}\cdot \vec \nabla\psi_p =&
+ %\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right)
+ %N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ %\vec j_N \cdot \vec\nabla\psi_p
+ %\nonumber\\
+ %+ z\left(\mu NU^2 \vec v_{\nabla\times \bhat} + \tau NU(\bhat+\tilde{\vec b}_\perp )\right) \cdot\vec \nabla\psi_p \nonumber\\
+ %=
+\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
++ z \mu\tau NU^2 \mathcal K_{\nabla\times\bhat}(\psi_p) \nonumber\\
+&+ z \tau NU
+ \left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
+\end{align}
 \subsection{Manufactured Solution}
 In order to test the implementation we manufacture a solution to Eqs.~\eqref{eq:Egyrofluid} and \eqref{eq:elliptic} of the form
 \begin{align*}
@@ -1016,32 +1059,21 @@ to evaluate the safety factor.
 Notice that the safety factor diverges on the last closed flux
 surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa}
 remain finite due to the $\nabla\psi$ factor.
-\subsection{ Radial flux}
+\subsection{ Length scales}
 Let us define
 
 \begin{align}
- \vec j_e\cdot \vec \nabla\psi_p =& n_e\left( \vec v_E + \vec v_C + \vec v_{\nabla
- B} + u_e \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- \nabla\psi_p \nonumber\\
- =& n_eu_e\left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \nonumber\\
-  &+ n_e\frac{1}{B}[\phi, \psi_p]_\perp + \left(\tau_e + \mu_e u_e^2\right)
-   n_e\mathcal K_{\nabla\times\bhat}(\psi_p) + \tau_e n_e \mathcal K_{\nabla B}(\psi_p) \nonumber\\
-   f_e :=& \frac{\vec j_e\cdot\vec \nabla \psi_p}{|\vec \nabla\psi_p|} \quad
    L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}\quad
    L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}
 \end{align}
 
-Note that the toroidal average of $f_e$ is ill-defined since $\nabla\psi$ vanishes at the X-point.
-Instead we output the average of $\vec j_e\cdot\nabla\psi_p$.
-
 \subsection{Program and files}
+We have the program \texttt{feltor/diag/feltordiag.cu}.
+This program reads a previously generated simulation file \texttt{input.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows. \\
 Compilation\\
 \texttt{make feltordiag} \\
 Usage \\
 \texttt{./feltordiag input.nc output.nc} \\
-We have the program \texttt{feltor/diag/feltordiag.cu}.
-This program reads a previously generated simulation file \texttt{input.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows.
 
 Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Dataset with the same name as a dimension.
 
@@ -1058,7 +1090,7 @@ q                & Dataset & 1 (psi) & The safety factor \\
 $\rho$           & Dataset & 1 (psi) & The flux label $\rho:= (\psi_{p,0} - \psi_p)/\psi_{p,0}$ \\
 X\_avg           & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
-X\_fsa\_mp       & Dataset & 3 (time,y,x) & Fluctuation level on midplane ($\varphi\equiv \pi$) $\delta X := X(R,Z,\pi) - \langle X\rangle_{\psi_{p}}$ \\
+X\_fsa\_mp       & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi\equiv 0$) $\delta X := X(R,Z,\pi) - \langle X\rangle_{\psi_{p}}$ \\
 X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_\psi$ Eq.~\eqref{eq:fsa} \\
 aligned          & Dataset & 1 (time) & $\int \dV (\nabla_\parallel n_e)^2 /n_e$ \\
 perp\_aligned    & Dataset & 1 (time) & $\int \dV (\vec\nabla_\perp n_e)^2 /n_e$ \\
-- 
GitLab


From 8e328ab1697277b00ce68cc39309716061692634 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 1 Apr 2019 17:01:27 +0200
Subject: [PATCH 057/540] More nice flux notation in feltor.tex

---
 src/feltor/feltor.tex | 85 +++++++++++++++++++++++++------------------
 1 file changed, 49 insertions(+), 36 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index d067b4a39..97516c3f1 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -602,7 +602,7 @@ so that $\nabla_\perp^2 S_{n_e}$ is well defined.
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
 Integrating the density equation we directly get
-\begin{align} \label{eq:mass_conservation}
+\begin{align} \label{eq:mass_theorem}
   \frac{\partial}{\partial t} \int_\Omega \dV n_e
   + \int_{\partial\Omega} \vec\dA\cdot\vec{ j_{n_e}}
   =  \int_\Omega \dV (\Lambda_{n_e}+S_{n_e})
@@ -636,7 +636,7 @@ The terms of the energy theorem are
 -  R_{\mathcal E} \right)
 \end{align}
 with
-\begin{align} \label{eq:energy_conservation}
+\begin{align} \label{eq:energy_conservation_dimensional}
   \mathcal E = &
   t_e n_e \ln{(n_e)} +T_i N_i\ln{(N_i)}
     +\frac{1}{2} m_i N_i \left(\frac{\vec\nabla_\perp\phi}{B}\right)^2 \nonumber\\
@@ -721,17 +721,30 @@ and
 \end{subequations}
 Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite.
 
+The terms of the particle conservation read
+\begin{align} \label{eq:mass_conservation}
+  n_e= & n_e,\\
+  \vec j_{n_e} =& n_e\left(
+  \vec v_E + \vec v_C + \vec v_{\nabla B} +u_e\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \nonumber\\
+  =& n_e \left(\frac{\bhat\times \nabla\phi}{B} 
+  + (\tau_e +\mu_e u_e^2)\vec K_{\nabla\times\bhat} 
+  + \tau_e \vec K_{\vec \nabla B} + u_e(\bhat + \tilde{\vec b}_\perp) \right), \\
+  \Lambda_{n_e} =&
+  \nu_\perp\Delta_\perp n_e + \nu_\parallel\Delta_\parallel n_e
+\\
+  S_{n_e} =&  S_{n_e}
+\end{align}
+
 The terms of the energy theorem read (with $z_e=-1$ and $z_i=+1$)
-\begin{align}
+\begin{align} \label{eq:energy_conservation}
   \mathcal{E}= & z_e\tau_e n_e \ln{(n_e)} +z_i\tau_i N_i\ln{(N_i)}
   +\frac{1}{2}\left(\vec \nabla_\perp A_\parallel\right)^2
    +  \frac{1}{2} \mu_i N_i u_E^2  \nonumber\\
    & +\frac{1}{2} z_e\mu_e  n_e u_e^2
   +\frac{1}{2} z_i\mu_i  N_i U_i^2,\\
-  \vec j_{\mathcal E} =& \sum_s z\left[ N\left(
-  \vec v_E + \vec v_C + \vec v_{\nabla B} +U\left(\bhat+\tilde{\vec b}_\perp\right)  \right)
-  \left(\tau \ln N + \frac{1}{2}\mu U^2 + \psi \right) \right. \nonumber\\
-  &\qquad \left . + \mu NU^2\vec v_{\nabla\times\bhat} + \tau NU \left(\bhat + \tilde{\vec b}_\perp\right)\right], \\
+  \vec j_{\mathcal E} =& \sum_s z\left[
+  \left(\tau \ln N + \frac{1}{2}\mu U^2 + \psi \right) \vec j_N 
+  + \mu \tau NU^2\vec K_{\nabla\times\bhat} + \tau NU \left(\bhat + \tilde{\vec b}_\perp\right)\right], \\
   \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U^2 \right)
   \left(\nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N\right)  +  \mu NU\left(\nu_\perp\Delta_\perp U + \nu_\parallel\Delta_\parallel U\right) \right]
 \nonumber \\
@@ -742,32 +755,6 @@ The terms of the energy theorem read (with $z_e=-1$ and $z_i=+1$)
 Replace $\Delta_\perp$ with $-\Delta_\perp^2$ when hyperviscous diffusion is chosen
 for the diffusion terms in the above equations.
 
-Let us here also derive the particle and energy fluxes through a flux surface
-\begin{align} \label{eq:particle_flux}
- \vec j_{N}\cdot \vec \nabla\psi_p =& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- \nabla\psi_p \nonumber\\
- =&
-  N\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
-   N\mathcal K_{\nabla\times\bhat}(\psi_p) + \tau N \mathcal K_{\nabla B}(\psi_p) \nonumber\\
- &+ NU\left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
-\end{align}
-\begin{align} \label{eq:energy_flux}
- \vec j_{\mathcal E}\cdot \vec \nabla\psi_p =&
- %\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right)
- %N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- %\vec j_N \cdot \vec\nabla\psi_p
- %\nonumber\\
- %+ z\left(\mu NU^2 \vec v_{\nabla\times \bhat} + \tau NU(\bhat+\tilde{\vec b}_\perp )\right) \cdot\vec \nabla\psi_p \nonumber\\
- %=
-\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
-+ z \mu\tau NU^2 \mathcal K_{\nabla\times\bhat}(\psi_p) \nonumber\\
-&+ z \tau NU
- \left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
-\end{align}
 \subsection{Manufactured Solution}
 In order to test the implementation we manufacture a solution to Eqs.~\eqref{eq:Egyrofluid} and \eqref{eq:elliptic} of the form
 \begin{align*}
@@ -1035,14 +1022,14 @@ flux surface $\psi_p = \psi_{p0}$ is given by
 \subsection{The safety factor}
 Assume that we pick a random field line and follow it (integrate it) for exactly one
 poloidal turn. The {\bf safety factor} is defined as the ratio between
-the resulting toroidal angle to the poloidal angle ($2\pi$)
+the resulting toroidal angle ($\Delta\varphi$) to the poloidal angle ($2\pi$)
 \begin{align}
 q := \frac{\Delta\varphi}{2\pi}
 \label{}
 \end{align}
 Since our magnetic field is symmetric in $\varphi$ and we used one
 full poloidal turn this definition is independent of which
-fieldline we pick inside a given flux surface.
+fieldline we pick on a given flux surface.
 
 Let us define the poloidal length $s$ as the fieldline following
 parameter i.e. $\vec B\cdot \nabla s \equiv B_p = R_0|\nabla \psi_p|/R$
@@ -1059,7 +1046,33 @@ to evaluate the safety factor.
 Notice that the safety factor diverges on the last closed flux
 surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa}
 remain finite due to the $\nabla\psi$ factor.
-\subsection{ Length scales}
+\subsection{ Fluxes and length scales}
+Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservation} and \eqref{eq:energy_conservation} through a flux surface
+\begin{align} \label{eq:particle_flux}
+ \vec j_{N}\cdot \vec \nabla\psi_p =& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ \nabla\psi_p \nonumber\\
+ =&
+  N\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
+   N\mathcal K_{\nabla\times\bhat}(\psi_p) + \tau N \mathcal K_{\nabla B}(\psi_p) \nonumber\\
+ &+ NU\left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
+\end{align}
+\begin{align} \label{eq:energy_flux}
+ \vec j_{\mathcal E}\cdot \vec \nabla\psi_p =&
+ %\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right)
+ %N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ %\vec j_N \cdot \vec\nabla\psi_p
+ %\nonumber\\
+ %+ z\left(\mu NU^2 \vec v_{\nabla\times \bhat} + \tau NU(\bhat+\tilde{\vec b}_\perp )\right) \cdot\vec \nabla\psi_p \nonumber\\
+ %=
+\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
++ z \mu\tau NU^2 \mathcal K_{\nabla\times\bhat}(\psi_p) \nonumber\\
+&+ z \tau NU
+ \left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
+\end{align}
 Let us define
 
 \begin{align}
-- 
GitLab


From 606ccb5b10d9768e3152674b0a548849ad799f9e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Apr 2019 01:36:14 +0200
Subject: [PATCH 058/540] Redesign Flux surface Average and Safety factor

introducing the FluxSurfaceIntegral class for very general flux
surface computations
---
 diag/feltordiag.cu              |   8 +-
 inc/geometries/average.h        | 190 +++++++++++++++-----------------
 inc/geometries/geometry_diag.cu |   3 +-
 inc/geometries/magnetic_field.h |  14 +++
 src/feltor/feltor.tex           |  28 ++---
 5 files changed, 122 insertions(+), 121 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index 75c786460..f120b2e15 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -256,7 +256,8 @@ int main( int argc, char* argv[])
     dg::HVec xpoint_damping = dg::evaluate( dg::one, g2d_out);
     if( gp.hasXpoint())
         xpoint_damping = dg::evaluate( dg::geo::ZCutter(Z_X), g2d_out);
-    dg::geo::SafetyFactor qprofile(g2d_out, mag, xpoint_damping);
+    dg::geo::SafetyFactor qprofile(g2d_out, mag);
+    qprofile.set_weights( xpoint_damping);
     dg::HVec sf = dg::evaluate(qprofile, g1d_out);
     int qID, rhoID;
     err = nc_def_var( ncid_out, "q", NC_DOUBLE, 1, &dim_ids1d[1], &qID);
@@ -366,10 +367,7 @@ int main( int argc, char* argv[])
                 start2d, count2d, transfer2d.data());
 
             //computa fsa of quantities
-            if(pair.first == "fluxe")
-                fsa.set_container(t2d, false);
-            else
-                fsa.set_container(t2d, true);
+            fsa.set_container(t2d);
             transfer1d = dg::evaluate(fsa, g1d_out);
             err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_fsa"),
                 start1d, count1d, transfer1d.data());
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index df8ea0ad9..439ade454 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -57,58 +57,80 @@ struct DeltaFunction
 };
 
 /**
- * @brief Global safety factor
-\f[ \alpha(R,Z) = \frac{I_{pol}(R,Z)}{R|\nabla\psi_p|} \frac{1}{\sqrt{2\pi\varepsilon}} \exp\left(-\frac{(\psi_p(R,Z) - \psi_{0})^2}{2\varepsilon} \right)  \f]
-     @ingroup profiles
+ * @brief Flux surface integral of the form
+ \f[ \int dR dZ f(R,Z) \delta(\psi_p(R,Z)-\psi_0) g(R,Z) \f]
+
+ where for the \c epsilon in \c DeltaFunction we use the maximum of \c h*GradPsip
+     where \c h is the cell size in the grid
+ * @ingroup misc_geo
  */
-struct Alpha
+template <class container >
+struct FluxSurfaceIntegral
 {
-    Alpha( const TokamakMagneticField& c):c_(c){}
-    Alpha(const TokamakMagneticField& c, double epsilon,double psivalue) :
-        c_(c), m_eps(epsilon), m_psi(psivalue){ }
-    /**
-    * @brief Set a new \f$ \varepsilon\f$
-    *
-    * @param eps new value
-    */
-    void setepsilon(double eps ){m_eps = eps;}
-    /**
-    * @brief Set a new \f$ \psi_0\f$
-    *
-    * @param psi_0 new value
-    */
-    void setpsi(double psi_0 ){m_psi = psi_0;}
+     /**
+     * @brief Construct from a grid and a magnetic field
+     * f and g are default initialized to 1
+     * @param g2d grid
+     * @param c contains psip, psipR and psipZ
+     * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
+     */
+    FluxSurfaceIntegral(const dg::Grid2d& g2d, const TokamakMagneticField& c, double width_factor = 1.):
+            m_f(dg::evaluate(dg::one, g2d)), m_g(m_f), m_delta(m_f),
+            m_psi( dg::evaluate( c.psip(), g2d)),
+            m_w2d ( dg::create::weights( g2d))
+    {
+        thrust::host_vector<double> psipR  = dg::evaluate( c.psipR(), g2d);
+        thrust::host_vector<double> psipZ  = dg::evaluate( c.psipZ(), g2d);
+        double psipRmax = dg::blas1::reduce( psipR, 0., dg::AbsMax<double>()  );
+        double psipZmax = dg::blas1::reduce( psipZ, 0., dg::AbsMax<double>()  );
+        double deltapsi = (psipZmax*g2d.hy() +psipRmax*g2d.hx())/2.;
+        m_eps = deltapsi*width_factor;
+    }
 
     /**
-    * @brief \f[ \frac{ I_{pol}(R,Z)}{R \sqrt{\nabla\psi_p}} \f]
-    */
-    double operator()( double R, double Z) const
-    {
-        double psip = c_.psip()(R,Z);
-        return c_.ipol()(R,Z)/R /sqrt(2.*M_PI*m_eps)*
-               exp(-( (psip-m_psi)* (psip-m_psi))/2./m_eps);
+     * @brief Set the left function to integrate
+     *
+     * @param f the container containing the discretized function
+     */
+    void set_left( const container& f){
+        dg::blas1::copy( f, m_f);
     }
     /**
-     * @brief == operator()(R,Z)
+     * @brief Set the right function to integrate
+     *
+     * @param f the container containing the discretized function
      */
-    double operator()( double R, double Z, double phi) const
+    void set_right( const container& g){
+        dg::blas1::copy( g, m_g);
+    }
+    /**
+     * @brief Calculate the Flux Surface Integral
+     *
+     * @param psip0 the actual psi value of the flux surface
+     * @return Int_psip0 (f,g)
+     */
+    double operator()(double psip0)
     {
-        return operator()(R,Z);
+        dg::GaussianX delta( psip0, m_eps, 1./(sqrt(2.*M_PI)*m_eps));
+        dg::blas1::evaluate( m_delta, dg::equals(), delta, m_psi);
+        dg::blas1::pointwiseDot( 1., m_delta, m_f, m_g, 0., m_delta);
+        return dg::blas1::dot( m_delta, m_w2d);
     }
     private:
-    TokamakMagneticField c_;
-    double m_eps, m_psi;
+    double m_eps;
+    container m_f, m_g, m_delta, m_psi;
+    const container m_w2d;
 };
 
 /**
  * @brief Flux surface average over quantity
- \f[ \langle f\rangle(\psi_0) = \frac{1}{A} \int dV \delta(\psi_p(R,Z)-\psi_0) |\nabla\psi_p|f(R,Z)H(R,Z) \f]
+ \f[ \langle f\rangle(\psi_0) = \frac{1}{A} \int dR dZ \delta(\psi_p(R,Z)-\psi_0) |\nabla\psi_p|f(R,Z)H(R,Z) \f]
 
- with \f$ A = \int dV \delta(\psi_p(R,Z)-\psi_0)|\nabla\psi_p|H(R,Z)\f$
- where \c H is a weight function that can be used to e.g. cut away parts of the domain below the X-point
+ with \f$ A = \int dRdZ \delta(\psi_p(R,Z)-\psi_0)|\nabla\psi_p|H(R,Z)\f$
+ where \c H is a weight function that can be used to e.g. cut away parts of the domain below the X-point or contain a volume form
  * @ingroup misc_geo
  */
-template <class container = thrust::host_vector<double> >
+template <class container >
 struct FluxSurfaceAverage
 {
      /**
@@ -116,43 +138,26 @@ struct FluxSurfaceAverage
      * @param g2d 2d grid
      * @param c contains psip, psipR and psipZ
      * @param f container for global safety factor
-     * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point)
-     * @param multiplyByGradPsi if true multiply f with GradPsi, else not
+     * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point and/or contain a volume form without dg weights)
+     * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
      */
-    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, const container& weights, bool multiplyByGradPsi = true) :
-    m_f(f), m_deltafog2d(f),
-    m_deltaf(c, 0.,0.),
-    m_w2d ( dg::create::weights( g2d)),
-    m_x ( dg::evaluate( dg::cooX2d, g2d)),
-    m_y ( dg::evaluate( dg::cooY2d, g2d)),
-    m_weights(weights)
+    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, const container& weights, double width_factor = 1.) :
+    m_avg( g2d,c, width_factor), m_area( g2d, c, width_factor)
     {
-        thrust::host_vector<double> psipRog2d  = dg::evaluate( c.psipR(), g2d);
-        thrust::host_vector<double> psipZog2d  = dg::evaluate( c.psipZ(), g2d);
-        double psipRmax = (double)thrust::reduce( psipRog2d.begin(), psipRog2d.end(),  0.,     thrust::maximum<double>()  );
-        //double psipRmin = (double)thrust::reduce( psipRog2d.begin(), psipRog2d.end(),  psipRmax,thrust::minimum<double>()  );
-        double psipZmax = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), 0.,      thrust::maximum<double>()  );
-        //double psipZmin = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), psipZmax,thrust::minimum<double>()  );
-        double deltapsi = fabs(psipZmax/g2d.Ny()/g2d.n() +psipRmax/g2d.Nx()/g2d.n());
-        m_deltaf.setepsilon(deltapsi/4);
-        //m_deltaf.setepsilon(deltapsi); //macht weniger Zacken
-        dg::blas1::pointwiseDot( 1., psipRog2d, psipRog2d, 1., psipZog2d, psipZog2d, 0., psipRog2d);
-        dg::blas1::transform( psipRog2d, psipRog2d, dg::SQRT<double>());
-        dg::assign( psipRog2d, m_gradpsi);
-        if(multiplyByGradPsi)
-            dg::blas1::pointwiseDot( m_f, m_gradpsi, m_f);
+        container gradpsi  = dg::evaluate( dg::geo::GradPsip( c), g2d);
+        container weights_ = weights;
+        dg::blas1::pointwiseDot( weights_, gradpsi, weights_);
+        m_avg.set_right( weights_);
+        m_area.set_right( weights_);
     }
 
     /**
-     * @brief Set the function to average
+     * @brief Reset the function to average
      *
      * @param f the container containing the discretized function
-     * @param multiplyByGradPsi if true multiply with GradPsi, else not
      */
-    void set_container( const container& f, bool multiplyByGradPsi=true){
-        dg::blas1::copy( f, m_f);
-        if(multiplyByGradPsi)
-            dg::blas1::pointwiseDot( m_f, m_gradpsi, m_f);
+    void set_container( const container& f){
+        m_avg.set_left( f);
     }
     /**
      * @brief Calculate the Flux Surface Average
@@ -162,23 +167,17 @@ struct FluxSurfaceAverage
      */
     double operator()(double psip0)
     {
-        m_deltaf.setpsi( psip0);
-        dg::blas1::evaluate( m_deltafog2d, dg::equals(), m_deltaf, m_x, m_y);
-        dg::blas1::pointwiseDot( m_deltafog2d, m_weights, m_deltafog2d);
-        double psipcut = dg::blas2::dot( m_f, m_w2d, m_deltafog2d); //int deltaf psip
-        double vol     = dg::blas2::dot( m_gradpsi, m_w2d, m_deltafog2d); //int deltaf
-        return psipcut/vol;
+        return m_avg(psip0)/m_area(psip0);
     }
     private:
-    container m_f, m_deltafog2d, m_gradpsi;
-    geo::DeltaFunction m_deltaf;
-    const container m_w2d, m_x, m_y, m_weights;
+    FluxSurfaceIntegral<container> m_avg, m_area;
 };
+
 /**
  * @brief Class for the evaluation of the safety factor q
- * \f[ q(\psi_0) = \frac{1}{2\pi} \int dV \alpha( R,Z) H(R,Z) \f]
+ * \f[ q(\psi_0) = \frac{1}{2\pi} \int dRdZ \frac{I(\psi_p)}{R} \delta(\psi_p - \psi_0)H(R,Z) \f]
 
-where \f$ \alpha\f$ is the \c dg::geo::Alpha functor and \c H is a weights function.
+where \c H is a weights function that can optionally be used to cut away parts of the domain e.g. below the X-point.
  * @copydoc hide_container
  * @ingroup misc_geo
  *
@@ -190,41 +189,30 @@ struct SafetyFactor
      * @param g2d 2d grid
      * @param c contains psip, psipR and psipZ
      * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point)
+     * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
      */
-    SafetyFactor(const dg::Grid2d& g2d, const TokamakMagneticField& c, const thrust::host_vector<double>& weights) :
-    m_g2d(g2d),
-    m_alpha(c,0.0,0.0),
-    m_w2d ( dg::create::weights( g2d)),
-    m_weights( weights)
+    SafetyFactor(const dg::Grid2d& g2d, const TokamakMagneticField& c, double width_factor = 1.) :
+        m_fsi( g2d, c, width_factor)
     {
-        thrust::host_vector<double> psipRog2d  = dg::evaluate( c.psipR(), g2d);
-        thrust::host_vector<double> psipZog2d  = dg::evaluate( c.psipZ(), g2d);
-        double psipRmax = (double)thrust::reduce( psipRog2d.begin(), psipRog2d.end(), 0.,     thrust::maximum<double>()  );
-        //double psipRmin = (double)thrust::reduce( psipRog2d.begin(), psipRog2d.end(),  psipRmax,thrust::minimum<double>()  );
-        double psipZmax = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), 0.,      thrust::maximum<double>()  );
-        //double psipZmin = (double)thrust::reduce( psipZog2d.begin(), psipZog2d.end(), psipZmax,thrust::minimum<double>()  );
-        double deltapsi = fabs(psipZmax/g2d.Ny() +psipRmax/g2d.Nx());
-        //m_alpha.setepsilon(deltapsi/4.);
-        m_alpha.setepsilon(deltapsi/10.);
+        thrust::host_vector<double> alpha = dg::evaluate( c.ipol(), g2d);
+        thrust::host_vector<double> R = dg::evaluate( dg::cooX2d, g2d);
+        dg::blas1::pointwiseDivide( alpha, R, alpha);
+        m_fsi.set_left( alpha);
+    }
+    void set_weights( const thrust::host_vector<double>& weights){
+        m_fsi.set_right( weights);
     }
     /**
-     * @brief Calculate the q profile over the function f which has to be the global safety factor
-     * \f[ q(\psi_0) = \frac{1}{2\pi} \int dV \alpha( R,Z) H(R,Z) \f]
-     *
-     * @param psip0 the actual psi value for q(psi)
+     * @brief Calculate the q(psip0)
+     * @param psip0 the flux surface
+     * @return q(psip0)
      */
     double operator()(double psip0)
     {
-        m_alpha.setpsi( psip0);
-        m_alphaog2d = dg::evaluate( m_alpha, m_g2d);
-        return dg::blas2::dot( m_alphaog2d, m_w2d, m_weights)/(2.*M_PI);
+        return m_fsi( psip0)/(2.*M_PI);
     }
     private:
-    dg::Grid2d m_g2d;
-    geo::Alpha m_alpha;
-    const thrust::host_vector<double> m_w2d;
-    thrust::host_vector<double> m_alphaog2d;
-    thrust::host_vector<double> m_weights;
+    FluxSurfaceIntegral<thrust::host_vector<double> > m_fsi;
 };
 
 }//namespace geo
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 4135a0514..64ddd37d5 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -236,7 +236,8 @@ int main( int argc, char* argv[])
     unsigned npsi = 3, Npsi = 150;//set number of psivalues
     //psipmin += (gp.psipmax - psipmin)/(double)Npsi; //the inner value is not good
     dg::Grid1d grid1d(psipmin + 0.05 , psipmax, npsi ,Npsi,dg::NEU);
-    dg::geo::SafetyFactor     qprof(grid2d, c, xpoint_damping);
+    dg::geo::SafetyFactor     qprof(grid2d, c);
+    qprof.set_weights( xpoint_damping);
     dg::geo::FluxSurfaceAverage<dg::HVec>  fsa( grid2d, c, psipog2d, xpoint_damping);
     dg::HVec psi_fsa    = dg::evaluate( fsa,        grid1d);
     dg::HVec sf         = dg::evaluate( qprof,      grid1d);
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 440c8394c..146aca54d 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -638,6 +638,20 @@ struct GradBHatP: public aCylindricalFunctor<GradBHatP>
     Divb divb_;
     TokamakMagneticField mag_;
 };
+
+///@brief \f$ |\nabla\psi_p| \f$
+struct GradPsip: public aCylindricalFunctor<GradPsip>
+{
+    GradPsip( const TokamakMagneticField& mag): m_mag(mag){}
+    double do_compute( double R, double Z) const
+    {
+        double psipR = m_mag.psipR()(R,Z), psipZ = m_mag.psipZ()(R,Z);
+        return sqrt(psipR*psipR +psipZ*psipZ);
+    }
+    private:
+    TokamakMagneticField m_mag;
+
+};
 ///@}
 
 } //namespace geo
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 97516c3f1..b921ab6ff 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -989,7 +989,7 @@ Furthermore, recall the {\bf coarea formula}
 \int_0^{h_0} \left( \int_{h=h'} \frac{f(\vec x)}{|\nabla h|}  \dA  \right) \d h'
 \end{align}
 where $\Omega_0$ is the volume enclosed by the contour $h=h_0$.
-The coarea formula can be viewd as a change of variables in the
+The coarea formula can be viewed as a change of variables in the
 volume integral.
 
 We define the {\bf toroidal average} of a function $f(R,Z,\varphi)$ as
@@ -1004,20 +1004,21 @@ of a function $f(R,Z,\varphi)$ is given by the formula
 \begin{align}\label{eq:fsa}
 \langle f \rangle_{\psi_{p0}} :=&
 \frac{ \oint_{\psi_p = \psi_{p0} } f(R,Z,\varphi)\dA}{\oint_{\psi_p = \psi_{p0} } \dA} \nonumber \\
-=& \frac{\int_\Omega f(R,Z,\varphi) \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p| H(Z-Z_X)\ R \d R \d Z\d\varphi}
-{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p0})|\vec\nabla\psi_p|H(Z-Z_X)\ R \d R \d Z\d\varphi}
+=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vec\nabla\psi_p| \delta(\psi_p(R,Z)-\psi_{p0})H(Z-Z_X)\ R \d R \d Z}
+{\int_\Omega |\vec\nabla\psi_p|\delta(\psi_p(R,Z)-\psi_{p0})H(Z-Z_X)\ R \d R \d Z}
 \end{align}
 %with $\dV := R\d R\d Z\d \varphi$ %(we define the average in computational space and omit one $R$)
 and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
-in our domain.
+in our domain $\Omega$.
 
 The {\bf total flux} of a given flux density $\vec j$ though the
 flux surface $\psi_p = \psi_{p0}$ is given by
 \begin{align}
 \oint_{\psi_p=\psi_{p0}} \vec j\cdot \vec{\dA} =
-\int \vec j\cdot \vec\nabla\psi_p \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z\d\varphi
+2\pi\int_\Omega \vec \langle \vec j\cdot \vec\nabla\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
 \label{eq:total_flux}
 \end{align}
+where we used $\vec \dA = (\vec\nabla\psi_p/|\nabla \psi_p|) \dA$.
 
 \subsection{The safety factor}
 Assume that we pick a random field line and follow it (integrate it) for exactly one
@@ -1049,16 +1050,14 @@ remain finite due to the $\nabla\psi$ factor.
 \subsection{ Fluxes and length scales}
 Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservation} and \eqref{eq:energy_conservation} through a flux surface
 \begin{align} \label{eq:particle_flux}
- \vec j_{N}\cdot \vec \nabla\psi_p =& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- \nabla\psi_p \nonumber\\
+ \vec j_{N}\cdot \vec \nabla\psi_p %=& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ %\nabla\psi_p \nonumber\\
  =&
-  N\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
-   N\mathcal K_{\nabla\times\bhat}(\psi_p) + \tau N \mathcal K_{\nabla B}(\psi_p) \nonumber\\
- &+ NU\left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
-\end{align}
-\begin{align} \label{eq:energy_flux}
+  N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
+   \mathcal K_{\nabla\times\bhat}(\psi_p) + \tau  \mathcal K_{\nabla B}(\psi_p) \right] \nonumber\\
+ &+ NU\left [\left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
  \vec j_{\mathcal E}\cdot \vec \nabla\psi_p =&
  %\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right)
  %N\left( \vec v_E + \vec v_C + \vec v_{\nabla
@@ -1072,6 +1071,7 @@ Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservati
 &+ z \tau NU
  \left( A_\parallel \mathcal
  K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
+\label{eq:energy_flux}
 \end{align}
 Let us define
 
-- 
GitLab


From ee2fb8a239c568b02cc6ca6d536ce0fa2e125dda Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Apr 2019 13:53:43 +0200
Subject: [PATCH 059/540] Make new direct SafetyFactor class and test coarea

in geometry_diag and average.h Coarea formula good
with 0.01 relative error in volume.
---
 diag/feltordiag.cu              |  3 +-
 inc/geometries/average.h        | 53 ++++++++++++++++++++++++++-------
 inc/geometries/flux.h           |  4 ++-
 inc/geometries/geometry_diag.cu | 41 ++++++++++++++++++++-----
 src/feltor/feltor.tex           | 27 +++++++++++------
 5 files changed, 97 insertions(+), 31 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index f120b2e15..6e91d76ab 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -256,8 +256,7 @@ int main( int argc, char* argv[])
     dg::HVec xpoint_damping = dg::evaluate( dg::one, g2d_out);
     if( gp.hasXpoint())
         xpoint_damping = dg::evaluate( dg::geo::ZCutter(Z_X), g2d_out);
-    dg::geo::SafetyFactor qprofile(g2d_out, mag);
-    qprofile.set_weights( xpoint_damping);
+    dg::geo::SafetyFactor qprofile(mag);
     dg::HVec sf = dg::evaluate(qprofile, g1d_out);
     int qID, rhoID;
     err = nc_def_var( ncid_out, "q", NC_DOUBLE, 1, &dim_ids1d[1], &qID);
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 439ade454..6daa2f26b 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -3,6 +3,7 @@
 #include <thrust/host_vector.h>
 #include "dg/topology/weights.h"
 #include "magnetic_field.h"
+#include "flux.h"
 
 /*!@file
  *
@@ -72,7 +73,7 @@ struct FluxSurfaceIntegral
      * f and g are default initialized to 1
      * @param g2d grid
      * @param c contains psip, psipR and psipZ
-     * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
+     * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c 0.15*h*GradPsi*width_factor)
      */
     FluxSurfaceIntegral(const dg::Grid2d& g2d, const TokamakMagneticField& c, double width_factor = 1.):
             m_f(dg::evaluate(dg::one, g2d)), m_g(m_f), m_delta(m_f),
@@ -83,7 +84,7 @@ struct FluxSurfaceIntegral
         thrust::host_vector<double> psipZ  = dg::evaluate( c.psipZ(), g2d);
         double psipRmax = dg::blas1::reduce( psipR, 0., dg::AbsMax<double>()  );
         double psipZmax = dg::blas1::reduce( psipZ, 0., dg::AbsMax<double>()  );
-        double deltapsi = (psipZmax*g2d.hy() +psipRmax*g2d.hx())/2.;
+        double deltapsi = 0.15*(psipZmax*g2d.hy() +psipRmax*g2d.hx());
         m_eps = deltapsi*width_factor;
     }
 
@@ -141,14 +142,14 @@ struct FluxSurfaceAverage
      * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point and/or contain a volume form without dg weights)
      * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
      */
-    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, const container& weights, double width_factor = 1.) :
+    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, container weights, double width_factor = 1.) :
     m_avg( g2d,c, width_factor), m_area( g2d, c, width_factor)
     {
+        m_avg.set_left( f);
         container gradpsi  = dg::evaluate( dg::geo::GradPsip( c), g2d);
-        container weights_ = weights;
-        dg::blas1::pointwiseDot( weights_, gradpsi, weights_);
-        m_avg.set_right( weights_);
-        m_area.set_right( weights_);
+        dg::blas1::pointwiseDot( weights, gradpsi, weights);
+        m_avg.set_right( weights);
+        m_area.set_right( weights);
     }
 
     /**
@@ -174,7 +175,7 @@ struct FluxSurfaceAverage
 };
 
 /**
- * @brief Class for the evaluation of the safety factor q
+ * @brief Class for the evaluation of the safety factor q based on a flux-surface integral
  * \f[ q(\psi_0) = \frac{1}{2\pi} \int dRdZ \frac{I(\psi_p)}{R} \delta(\psi_p - \psi_0)H(R,Z) \f]
 
 where \c H is a weights function that can optionally be used to cut away parts of the domain e.g. below the X-point.
@@ -182,16 +183,16 @@ where \c H is a weights function that can optionally be used to cut away parts o
  * @ingroup misc_geo
  *
  */
-struct SafetyFactor
+struct SafetyFactorAverage
 {
      /**
      * @brief Construct from a field and a grid
      * @param g2d 2d grid
-     * @param c contains psip, psipR and psipZ
+     * @param c contains psip, psipR and psipZ and Ipol
      * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point)
      * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
      */
-    SafetyFactor(const dg::Grid2d& g2d, const TokamakMagneticField& c, double width_factor = 1.) :
+    SafetyFactorAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, double width_factor = 1.) :
         m_fsi( g2d, c, width_factor)
     {
         thrust::host_vector<double> alpha = dg::evaluate( c.ipol(), g2d);
@@ -215,6 +216,36 @@ struct SafetyFactor
     FluxSurfaceIntegral<thrust::host_vector<double> > m_fsi;
 };
 
+
+
+/**
+ * @brief Evaluation of the safety factor q based on direct integration of
+ * \f[ q(\psi_0) = \frac{1}{2\pi} \int d\Theta \frac{B^\varphi}{B^\Theta} \f]
+
+ * @attention Return value undefined if evaluated outside the closed fieldline region, but the function always returns, it won't throw an error or something
+ * @copydoc hide_container
+ * @ingroup misc_geo
+ *
+ */
+struct SafetyFactor
+{
+    SafetyFactor( const TokamakMagneticField& c):
+        m_fpsi( c.get_psip(), c.get_ipol(), c.R0(), 0.,false){}
+
+    /**
+     * @brief Calculate q(psip0)
+     * @param psip0 the flux surface
+     * @return q(psip0)
+     */
+    double operator()( double psip0)
+    {
+        return 1./m_fpsi( psip0);
+    }
+private:
+    dg::geo::flux::detail::Fpsi m_fpsi;
+
+};
+
 }//namespace geo
 
 }//namespace dg
diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index 96246b9c9..bded4e9f6 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -25,6 +25,7 @@ namespace detail
 
 //This leightweights struct and its methods finds the initial R and Z values and the coresponding f(\psi) as
 //good as it can, i.e. until machine precision is reached
+//Note that f(psi) = 1/q(psi) (The safety factor)
 struct Fpsi
 {
 
@@ -68,10 +69,11 @@ struct Fpsi
             eps_old = eps, end_old = end; N*=2;
             dg::stepperRK( "Feagin-17-8-10",  fieldRZYT_, 0., begin, 2*M_PI, end, N);
             eps = sqrt( (end[0]-begin[0])*(end[0]-begin[0]) + (end[1]-begin[1])*(end[1]-begin[1]));
+            //Attention: if this succeeds on the first attempt end_old won't be updated
         }
         if(m_verbose)std::cout << "\t error "<<eps<<" with "<<N<<" steps\t";
         if(m_verbose)std::cout <<end_old[2] << " "<<end[2] <<"\n";
-        double f_psi = 2.*M_PI/end_old[2];
+        double f_psi = 2.*M_PI/end[2]; //this actually is 1/q the safety factor
         return f_psi;
     }
 
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 64ddd37d5..c4e73f7ee 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -17,6 +17,7 @@
 #include "magnetic_field.h"
 #include "average.h"
 #include "testfunctors.h"
+#include "flux.h" //to compute Safety factor
 
 struct Parameters
 {
@@ -219,6 +220,7 @@ int main( int argc, char* argv[])
         {"ZonalFlow", dg::geo::ZonalFlow(c.psip(), p.amp, 0., 2.*M_PI*p.k_psi )},
         {"PsiLimiter", dg::geo::PsiLimiter(c.psip(), gp.psipmaxlim)},
         {"Nprofile", dg::geo::Nprofile(c.psip(), p.nprofileamp/c.psip()(c.R0(),0.), p.bgprofamp )},
+        {"Delta", dg::geo::DeltaFunction( c, gp.alpha*gp.alpha, psip0*0.2)},
         {"TanhDamping", dg::geo::TanhDamping(c.psip(), -3*gp.alpha, gp.alpha, -1)},
         ////
         {"BathRZ", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,5., p.amp)},
@@ -228,22 +230,43 @@ int main( int argc, char* argv[])
     //Compute flux average
     dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
     dg::DVec psipog2d   = dg::evaluate( c.psip(), grid2d);
-    dg::HVec xpoint_damping = dg::evaluate( dg::one, grid2d);
+    dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
     if( gp.hasXpoint() )
-        xpoint_damping = dg::evaluate( dg::geo::ZCutter(Z_X), grid2d);
+        dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
     double psipmin = psip0;//(float)thrust::reduce( psipog2d.begin(), psipog2d.end(), 0.0,thrust::minimum<double>()  );
-    double psipmax = 0;//(float)thrust::reduce( psipog2d.begin(), psipog2d.end(), 0.0,thrust::maximum<double>()  );
+    double psipmax = 0.;//dg::blas1::reduce( psipog2d, 0., thrust::maximum<double>()  );
     unsigned npsi = 3, Npsi = 150;//set number of psivalues
     //psipmin += (gp.psipmax - psipmin)/(double)Npsi; //the inner value is not good
-    dg::Grid1d grid1d(psipmin + 0.05 , psipmax, npsi ,Npsi,dg::NEU);
-    dg::geo::SafetyFactor     qprof(grid2d, c);
-    qprof.set_weights( xpoint_damping);
-    dg::geo::FluxSurfaceAverage<dg::HVec>  fsa( grid2d, c, psipog2d, xpoint_damping);
+    dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
+    dg::geo::SafetyFactor     qprof( c);
+    dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, c, psipog2d, xpoint_weights);
     dg::HVec psi_fsa    = dg::evaluate( fsa,        grid1d);
     dg::HVec sf         = dg::evaluate( qprof,      grid1d);
     dg::HVec psi        = dg::evaluate( dg::cooX1d, grid1d), rho(psi);
     dg::blas1::axpby( -1./psip0, rho, +1., 1., rho); //transform psi to rho
 
+    //other flux labels
+    dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, c);
+    fsi.set_right( xpoint_weights);
+    dg::HVec vol_psip = dg::evaluate( fsi, grid1d);
+    dg::HVec w1d = dg::create::weights( grid1d);
+    double volumeRZ = 2.*M_PI*dg::blas1::dot( vol_psip, w1d);
+
+    //area
+    fsi.set_left( dg::evaluate( dg::geo::GradPsip(c), grid2d));
+    dg::HVec area_psip = dg::evaluate( fsi, grid1d);
+
+    dg::geo::Pupil pupil( c.psip(), 0.);
+    dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,1, 2e3, 2e3, dg::PER, dg::PER);
+    xpoint_weights = dg::evaluate( dg::cooX2d, g2dC);
+    if( gp.hasXpoint() )
+        dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), g2dC), xpoint_weights);
+    dg::HVec vec  = dg::evaluate( pupil, g2dC);
+    dg::HVec g2d_weights = dg::create::weights( g2dC);
+    double volumeRZP = 2.*M_PI*dg::blas2::dot( vec, g2d_weights, xpoint_weights);
+    std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeRZ<<" "<<volumeRZP
+              <<" rel error = "<<fabs(volumeRZ-volumeRZP)/volumeRZP<<"\n";
+
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
     int ncid;
@@ -257,16 +280,18 @@ int main( int argc, char* argv[])
     dim2d_ids[0] = dim3d_ids[1], dim2d_ids[1] = dim3d_ids[2];
 
     //write 1d vectors
-    int avgID[4];
+    int avgID[5];
     err = nc_def_var( ncid, "q-profile", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[0]);
     err = nc_def_var( ncid, "rho", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[1]);
     err = nc_def_var( ncid, "psip1d", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[2]);
     err = nc_def_var( ncid, "psi_fsa", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[3]);
+    err = nc_def_var( ncid, "area", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[4]);
     err = nc_enddef( ncid);
     err = nc_put_var_double( ncid, avgID[0], sf.data());
     err = nc_put_var_double( ncid, avgID[1], rho.data());
     err = nc_put_var_double( ncid, avgID[2], psi.data());
     err = nc_put_var_double( ncid, avgID[3], psi_fsa.data());
+    err = nc_put_var_double( ncid, avgID[4], area_psip.data());
     err = nc_redef(ncid);
 
     //write 2d vectors
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index b921ab6ff..d44a2ac72 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1032,17 +1032,26 @@ Since our magnetic field is symmetric in $\varphi$ and we used one
 full poloidal turn this definition is independent of which
 fieldline we pick on a given flux surface.
 
-Let us define the poloidal length $s$ as the fieldline following
-parameter i.e. $\vec B\cdot \nabla s \equiv B_p = R_0|\nabla \psi_p|/R$
-and $\d\varphi/\d s = B^\varphi(R(s), Z(s)) / B_p(R(s),Z(s))$.
-We can then express the safety factor as the line integral
+%Let us define the poloidal length $s$ as the fieldline following
+%parameter i.e. $\vec B\cdot \nabla s \equiv B_p = R_0|\nabla \psi_p|/R$
+%and $\d\varphi/\d s = B^\varphi(R(s), Z(s)) / B_p(R(s),Z(s))$.
+%We can then express the safety factor as the line integral
+%\begin{align}
+%q=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vec\nabla\psi_p|} \d s
+%= \frac{1}{2\pi}\int \frac{I(\psi_p)}{R}\delta(\psi_p-\psi_{p0}) H(Z-Z_X) \d R\d Z
+%\end{align}
+%where we made use of Eq.~\eqref{eq:dirac_delta} in two dimensions in the
+%last equality and thus arrive at a numerical tractable expression
+%to evaluate the safety factor.
+Let us define the geometric poloidal angle $\Theta$ as the fieldline following
+parameter i.e. $\vec B\cdot\nabla\Theta = R_0(\psi_R (R-R_0) + \psi_Z Z)/r^2R$.
+We can then directly integrate the safety factor as
 \begin{align}
-q=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vec\nabla\psi_p|} \d s
-= \frac{1}{2\pi}\int \frac{I(\psi_p)}{R}\delta(\psi_p-\psi_{p0}) H(Z-Z_X) \d R\d Z
+\frac{\d R}{\d\Theta} = \frac{B^R}{B^\Theta}\quad 
+\frac{\d Z}{\d\Theta} = \frac{B^Z}{B^\Theta}\quad 
+\frac{\d \varphi}{\d\Theta} = \frac{B^\varphi}{B^\Theta}\\
+q\equiv\frac{1}{2\pi}\oint \frac{B^\varphi}{B^\Theta} \d\Theta
 \end{align}
-where we made use of Eq.~\eqref{eq:dirac_delta} in two dimensions in the
-last equality and thus arrive at a numerical tractable expression
-to evaluate the safety factor.
 
 Notice that the safety factor diverges on the last closed flux
 surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa}
-- 
GitLab


From 01af29a210a55828990475cf7b24d93252cd837c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Apr 2019 21:43:05 +0200
Subject: [PATCH 060/540] Update output documentation in feltor3d

now we just need to implement all of that
---
 diag/feltordiag.cu    | 39 +++++++++++++++++++++++++++---
 src/feltor/feltor.tex | 55 +++++++++++++++++++++++++++++--------------
 2 files changed, 73 insertions(+), 21 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index 6e91d76ab..cf33a1819 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -11,8 +11,8 @@
 #include "feltor/feltor.cuh"
 #include "feltor/parameters.h"
 
-struct RadialElectricFlux{
-    RadialElectricFlux( double tau, double mu):
+struct RadialParticleFlux{
+    RadialParticleFlux( double tau, double mu):
         m_tau(tau), m_mu(mu){
     }
 
@@ -38,7 +38,40 @@ struct RadialElectricFlux{
             + ne * (m_tau + m_mu*ue*ue)*curvKappaS
             + ne * m_tau*curvNablaS;
         return JPsi;
+    }
+    private:
+    double m_tau, m_mu;
+};
+struct RadialEnergyFlux{
+    RadialEnergyFlux( double tau, double mu):
+        m_tau(tau), m_mu(mu){
+    }
 
+    DG_DEVICE double operator()( double ne, double ue, double A, double P,
+        double d0A, double d1A, double d2A,
+        double d0P, double d1P, double d2P, //Phi
+        double d0S, double d1S, double d2S, //Psip
+        double b_0,         double b_1,         double b_2,
+        double curvNabla0,  double curvNabla1,  double curvNabla2,
+        double curvKappa0,  double curvKappa1,  double curvKappa2
+        ){
+        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
+        double curvNablaS = curvNabla0*d0S+curvNabla1*d1S+curvNabla2*d2S;
+        double SA = b_0*( d1S*d2A-d2S*d1A)+
+                    b_1*( d2S*d0A-d0S*d2A)+
+                    b_2*( d0S*d1A-d1S*d0A);
+        double PS = b_0*( d1P*d2S-d2P*d1S)+
+                    b_1*( d2P*d0S-d0P*d2S)+
+                    b_2*( d0P*d1S-d1P*d0S);
+        double JN =
+            ne*ue* (A*curvKappaS + SA )
+            + ne * PS
+            + ne * (m_tau + m_mu*ue*ue)*curvKappaS
+            + ne * m_tau*curvNablaS;
+        double Je = (m_tau * log(ne) + 0.5*m_mu*ue*ue + P)*JN
+            + m_mu*m_tau*ne*ue*ue*curvKappaS
+            + m_tau*ne*ue* (A*curvKappaS + SA );
+        return Je;
     }
     private:
     double m_tau, m_mu;
@@ -313,7 +346,7 @@ int main( int argc, char* argv[])
         dg::blas2::symv( dyP, v3d["potential"], dy_P);
         dg::blas2::symv( dz , v3d["potential"], dz_P);
         dg::blas1::evaluate( v3d["fluxe"], dg::equals(),
-            RadialElectricFlux( p.tau[0], p.mu[0]),
+            RadialParticleFlux( p.tau[0], p.mu[0]),
             v3d["electrons"], v3d["Ue"], v3d["induction"],
             dx_A, dy_A, dz_A, dx_P, dy_P, dz_P, psipR, psipZ, psipP,
             bhat[0], bhat[1], bhat[2],
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index d44a2ac72..ca34b6d8a 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1046,7 +1046,7 @@ fieldline we pick on a given flux surface.
 Let us define the geometric poloidal angle $\Theta$ as the fieldline following
 parameter i.e. $\vec B\cdot\nabla\Theta = R_0(\psi_R (R-R_0) + \psi_Z Z)/r^2R$.
 We can then directly integrate the safety factor as
-\begin{align}
+\begin{align}\label{eq:safety_factor}
 \frac{\d R}{\d\Theta} = \frac{B^R}{B^\Theta}\quad 
 \frac{\d Z}{\d\Theta} = \frac{B^Z}{B^\Theta}\quad 
 \frac{\d \varphi}{\d\Theta} = \frac{B^\varphi}{B^\Theta}\\
@@ -1082,12 +1082,6 @@ Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservati
  K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
 \label{eq:energy_flux}
 \end{align}
-Let us define
-
-\begin{align}
-   L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}\quad
-   L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}
-\end{align}
 
 \subsection{Program and files}
 We have the program \texttt{feltor/diag/feltordiag.cu}.
@@ -1108,22 +1102,47 @@ x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, co
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
-q                & Dataset & 1 (psi) & The safety factor \\
-$\rho$           & Dataset & 1 (psi) & The flux label $\rho:= (\psi_{p,0} - \psi_p)/\psi_{p,0}$ \\
-X\_avg           & Dataset & 3 (time,y,x) & Toroidal average $\langle X
-    \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
-X\_fsa\_mp       & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi\equiv 0$) $\delta X := X(R,Z,\pi) - \langle X\rangle_{\psi_{p}}$ \\
-X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_\psi$ Eq.~\eqref{eq:fsa} \\
+mass             & Dataset & 1 (time) & Mass inside LCFS $\int_\Omega \dV n_e$ \\
+mass\_loss       & Dataset & 1 (time) & Mass flux through LCFS $\int_{\psi_p = 0} \vec \dA \cdot \vec j_{n_e}$\\
+mass\_diffusion  & Dataset & 1 (time) & Mass diffusion inside LCFS $\int_\Omega \dV \Lambda_{n_e}$ \\
+energy           & Dataset & 1 (time) & Energy inside LCFS $\int_\Omega \dV \mathcal E$ \\
+energy\_loss     & Dataset & 1 (time) & Energy flux through LCFS $\int_{\psi_p = 0} \vec \dA \cdot \vec j_{\mathcal E}$ \\
+energy\_diffusion& Dataset & 1 (time) & Energy diffusion inside LCFS $\int_\Omega \dV \Lambda_{\mathcal E}$ \\
+energy\_dissipation& Dataset & 1 (time) & Energy dissipation inside LCFS$\int_\Omega \dV R_{\mathcal E}$ \\
 aligned          & Dataset & 1 (time) & $\int \dV (\nabla_\parallel n_e)^2 /n_e$ \\
 perp\_aligned    & Dataset & 1 (time) & $\int \dV (\vec\nabla_\perp n_e)^2 /n_e$ \\
 correlationNPhi  & Dataset & 1 (time) & $\left( \int\dV e^\phi n_e \right)/ ||e^\phi||||n_e||$\\
-total\_flux      & Dataset & 1 (time) & $\int \dV f_e$\\
+correlationNTildePhi  & Dataset & 1 (time) & $\left( \int\dV \phi \tilde n_e \right)/ ||\phi||||\tilde n_e||$ with $n_e \equiv \langle n_e \rangle_{\psi_p} ( 1 + \tilde n_e)$\\
+q                & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} \\
+psip1d           & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
+rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,0} - \psi_p)/\psi_{p,0}$ \\
+area             & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\nabla\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
+X                & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
+X\_avg           & Dataset & 3 (time,y,x) & Toroidal average $\langle X
+    \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
+X\_fluc       & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,\pi) - \langle X\rangle_{\psi_{p}}$ \\
+X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa} \\
+particle\_flux   & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{n_e} \cdot \vec \dA$ Eq.~\eqref{eq:particle_flux} \\
+energy\_flux     & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{\mathcal E} \cdot \vec \dA$ Eq.~\eqref{eq:energy_flux} \\
 \bottomrule
 \end{longtable}
-X $\in$ \{ electrons, ions, Ue, Ui, potential,
-induction, vorticity, fluxe, Lperpinv, Lparallelinv\} corresponding to \{
-    $n_e$, $N_i$, $u_e$, $U_i$, $\phi$, $A_\parallel$, $-\Delta_\perp \phi$,
-    $f_e$, $L_\perp^{-1}$, $L_\parallel^{-1}$\}.
+where X $\in$ \{ electrons, ions, Ue, Ui, potential,
+induction, vorticity, maxwell, Lperpinv, Lparallelinv,
+current, momentum
+\} corresponding to \{
+    $n_e$, $N_i$, $u_e$, $U_i$, $\phi$, $A_\parallel$,
+    $-\Delta_\perp \phi$, $-\Delta_\perp A_\parallel$,
+    $L_\perp^{-1}$, $L_\parallel^{-1}$,
+    $j_\parallel$, $j_\rho$
+    \}
+and we define
+
+\begin{align}
+   L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}\quad
+   L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e} \\
+   j_\parallel := N_iU_i-n_e u_e \quad
+   j_{\parallel,\rho} := \mu_i N_iU_i + \mu_e n_e u_e
+\end{align}
 
 
 
-- 
GitLab


From 843bcd5b071cae11a7ad427a8327a9fab0809b6b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 7 Apr 2019 22:01:59 +0200
Subject: [PATCH 061/540] Update flux surface average definitions in feltor

---
 src/feltor/feltor.tex | 69 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 58 insertions(+), 11 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index ca34b6d8a..9bb819ec6 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -997,28 +997,75 @@ We define the {\bf toroidal average} of a function $f(R,Z,\varphi)$ as
 \langle f\rangle_\varphi(R,Z) := \frac{1}{2\pi}\oint f(R,Z,\varphi)\d \varphi
 \end{align}
 
+In arbitrary coordinates the area integral is defined by the pull back
+of the flux 2-form and the metric
+\begin{align}
+\label{}
+\dA^2 = i_{\hat \psi_p} vol^3 \quad \hat \psi_p = \frac{\nabla \psi_p}{|\nabla \psi_p|}
+\end{align}
+to a parameterization of the flux-surface.
+In a flux-aligned coordinate system $\{\zeta, \eta, \varphi\}$ the pull-back is trivial ($\zeta=const$) and we have with $\hat\zeta = \hat \psi_p$
+\begin{align}
+\dA = \sqrt{g} |\nabla \zeta| \d\eta\d\varphi, \quad \vec\dA := \hat\zeta \dA
+\label{}
+\end{align}
+where we used that $g^{\zeta\zeta} = (\nabla\zeta)^2$
+
 \subsection{Flux surface average}
 
-The {\bf flux surface average}
+There is two possible ways to introduce a flux-surface average.
+The first one is the straightforward average on the actual area of the
+flux-surface.
+The {\bf area average}
 of a function $f(R,Z,\varphi)$ is given by the formula
-\begin{align}\label{eq:fsa}
-\langle f \rangle_{\psi_{p0}} :=&
-\frac{ \oint_{\psi_p = \psi_{p0} } f(R,Z,\varphi)\dA}{\oint_{\psi_p = \psi_{p0} } \dA} \nonumber \\
-=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vec\nabla\psi_p| \delta(\psi_p(R,Z)-\psi_{p0})H(Z-Z_X)\ R \d R \d Z}
-{\int_\Omega |\vec\nabla\psi_p|\delta(\psi_p(R,Z)-\psi_{p0})H(Z-Z_X)\ R \d R \d Z}
+\begin{align}\label{eq:fsa_area}
+\langle f \rangle^{area}_{\psi_{p}} :=&
+\frac{ \oint_{\psi_p  } f(R,Z,\varphi)\dA}{\oint_{\psi_p } \dA} \nonumber \\
+=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vec\nabla\psi_p| \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
+{\int_\Omega |\vec\nabla\psi_p|\delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
 \end{align}
 %with $\dV := R\d R\d Z\d \varphi$ %(we define the average in computational space and omit one $R$)
 and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
 in our domain $\Omega$.
 
-The {\bf total flux} of a given flux density $\vec j$ though the
+The second one (the {\bf volume average} after \cite{haeseleer}) defines an average on a
+small volume - a shell centered around the flux-surface.
+If we define $v(\psi) := \int_0^\psi \dV$ as the volume
+flux label, we define
+\begin{align} \label{eq:fsa_vol}
+\langle f \rangle^{vol}_\psi :=& \frac{\partial}{\partial v} \int \dV f
+ = \frac{1}{\int \dA |\nabla\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\nabla\psi_p|} \dA \nonumber\\
+=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
+{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
+\end{align}
+where we used the coarea formula Eq.~\eqref{eq:coarea} for the second
+identiy. We immediately see that this definition differs from the first
+Eq.~\eqref{eq:fsa_area} by the weight factor $|\nabla\psi_p|$.
+
+Both averages fullfill the basic identities
+\begin{align}
+\label{eq:fsa_identities}
+\langle \mu f + \lambda g\rangle &= \mu\langle f\rangle + \lambda \langle g\rangle \\
+\langle f(\psi_p) \rangle &= f(\psi_p)
+\end{align}
+
+
+The volume average is better suited for density-like quantities
+than the area average as we can see with the following identiy.
+Assume we have a quantity $X$ with $\partial_t X + \nabla \vec j_X = \Lambda_X$. Then we can use the volume average to write
+\begin{align}
+\frac{\partial}{\partial t} \langle X \rangle^{vol} + \frac{\partial}{
+  \partial v} J_X  = \langle \Lambda_X\rangle^{vol}
+\label{eq:fsa_balance}
+\end{align}
+where again $v=v(\psi_p)$ is the volume flux label.
+The {\bf total flux} of a given flux density $\vec j_X$ though the
 flux surface $\psi_p = \psi_{p0}$ is given by
 \begin{align}
-\oint_{\psi_p=\psi_{p0}} \vec j\cdot \vec{\dA} =
+J_X:=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
 2\pi\int_\Omega \vec \langle \vec j\cdot \vec\nabla\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
 \label{eq:total_flux}
 \end{align}
-where we used $\vec \dA = (\vec\nabla\psi_p/|\nabla \psi_p|) \dA$.
 
 \subsection{The safety factor}
 Assume that we pick a random field line and follow it (integrate it) for exactly one
@@ -1054,7 +1101,7 @@ q\equiv\frac{1}{2\pi}\oint \frac{B^\varphi}{B^\Theta} \d\Theta
 \end{align}
 
 Notice that the safety factor diverges on the last closed flux
-surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa}
+surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa_area}
 remain finite due to the $\nabla\psi$ factor.
 \subsection{ Fluxes and length scales}
 Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservation} and \eqref{eq:energy_conservation} through a flux surface
@@ -1121,7 +1168,7 @@ X                & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
 X\_avg           & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
 X\_fluc       & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,\pi) - \langle X\rangle_{\psi_{p}}$ \\
-X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa} \\
+X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
 particle\_flux   & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{n_e} \cdot \vec \dA$ Eq.~\eqref{eq:particle_flux} \\
 energy\_flux     & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{\mathcal E} \cdot \vec \dA$ Eq.~\eqref{eq:energy_flux} \\
 \bottomrule
-- 
GitLab


From 21381ee4ea8b343fd7d5faa853635f9cb96fdbcf Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 8 Apr 2019 17:13:44 +0200
Subject: [PATCH 062/540] Update poloidal flux generation

through Lorentz force
---
 src/feltor/feltor.tex | 159 +++++++++++++++++++++++++++++-------------
 1 file changed, 112 insertions(+), 47 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 9bb819ec6..74510d787 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -405,9 +405,9 @@ the parallel electron velocity \(u_e\) and the parallel ion gyro-centre velocity
 (omitting species labels)~\cite{WiesenbergerPhD, HeldPhD}
 \begin{align}
 \frac{\partial}{\partial t} N &+ \vec\nabla\cdot\left( N \left(
-    \vec v_E + \vec v_C + \vec v_{\nabla B} + U\left(\bhat + \tilde{\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
+    \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + \tilde{\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
 mN \frac{\partial}{\partial t} U &+ mN \left(
-    \vec v_E + \vec v_C + \vec v_{\nabla B} + U\left(\bhat + \tilde{\vec b}_\perp\right)
+    \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + \tilde{\vec b}_\perp\right)
     \right)\cdot \vec\nabla U  \nonumber \\
     &+ 2m\vec \nabla \cdot ( NU \vec v_{\nabla\times\bhat})
     -mNU\vec \nabla\cdot \vec v_{\nabla\times\bhat}
@@ -417,11 +417,11 @@ mN \frac{\partial}{\partial t} U &+ mN \left(
 \end{align}
 with
 \begin{align}
-\tilde{\vec b}_\perp = \frac{\nabla\times A_\parallel \bhat}{B}, \quad
 \vec v_E := \frac{\bhat\times\nabla\psi}{B},\quad
-\vec v_C := \frac{T+mU^2}{q}\vec{\mathcal K_{\nabla\times\bhat}},\quad
+\vec v_{K} := \frac{T}{q}\left(\vec{\mathcal K_{\nabla B}} + \vec{\mathcal K_{\nabla\times\bhat}}\right)=\frac{T}{q}\vec{\mathcal K}  ,\nonumber\\
+\vec v_C := \frac{mU^2}{q}\vec{\mathcal K_{\nabla\times\bhat}},\quad
 \vec v_{\nabla\times\bhat} := \frac{T}{q}\vec{\mathcal K_{\nabla\times\bhat}},\quad
-\vec v_{\nabla B} := \frac{T}{q}\vec{\mathcal K_{\nabla B}}.
+\tilde{\vec b}_\perp = \frac{\nabla\times A_\parallel \bhat}{B}.
 \label{}
 \end{align}
 
@@ -436,6 +436,13 @@ Given $\phi$ we define the generalised electric potential
 \begin{align}
     \psi_e := \phi,\quad \psi_i&:= \Gamma_{1,i} \phi - \frac{m_i }{2 e}\left(\frac{\vec \nabla_\perp\phi}{B}\right)^2
 \end{align}
+\subsection{Relation to diamagnetic drift}
+Notice that the term $\nabla (n_e \vec v_K ) \equiv -\nabla (T_e n_e \vec K /e) $
+forms the familiar form of the divergence of the diamagnetic flux.
+\begin{align}
+\nabla\cdot\left( n_e \frac{\bhat \times \nabla n_e T_e}{en_e B}\right) \equiv  \nabla\cdot \left(\frac{T_e n_e}{e} \vec K\right)
+\end{align}
+which is evident with the identities in Section~\ref{sec:magnetic}.
 
 \subsection{Parallel Resistivity and diffusion}\label{sec:dissres}
 The terms $R_{e/i,\eta_\parallel}$ account for resistive friction.
@@ -609,22 +616,25 @@ Integrating the density equation we directly get
 \end{align}
 with the particle flux
 \begin{align} \label{eq:mass_flux}
-\vec{ j_{n_e}} := n_e\left(\vec v_E + \vec v_C + \vec v_{\nabla B} + U (\bhat + \vec{\tilde b}_\perp)\right)
+\vec{ j_{n_e}} := n_e\left(\vec v_E + \vec v_C + \vec v_{K} + U (\bhat + \vec{\tilde b}_\perp)\right)
 \end{align}
 \subsubsection{Charge/vorticity conservation}
 Integrating the polarisation equation~\eqref{eq:polarisation_dimensional}
 and assuming $\Gamma_{1,i}S_{N_i} = S_{n_e}$ we find
 \begin{align} \label{eq:charge_conservation}
-  \frac{\partial}{\partial t} \int_\Omega \dV \vec{\nabla} \cdot\left(\frac{m_iN_i}{e B^2} \vec{\nabla}_\perp \phi\right) =  - \int_\Omega \dV (\Lambda_{n_e} - \Gamma_{1,i}\Lambda_{N_i})
+  \frac{\partial}{\partial t} \int_\Omega \dV \mathcal W
+  + \int_{\partial\Omega} \vec\dA\cdot\vec{ j_{\mathcal W}}
+  =  \int_\Omega \dV \Lambda_{\mathcal W} \nonumber\\
 \end{align}
-Note that if the integrand on the left hand side is interpreted as the \ExB vorticity
-density
-$\zeta := \vec \nabla\cdot( m_iN_i\vec \nabla_\perp\phi/e B^2)$
-and if we further assume that $\Gamma_{1,i} \Lambda_{N_i} \equiv \Lambda_{\Gamma_{1,i}N_i}$,
-we get the vorticity conservation
-\begin{align} \label{eq:vorticity_conservation}
-  \frac{\partial}{\partial t} \int_\Omega \dV \zeta =  - \int_\Omega \dV\Lambda_{\zeta}
+with the definitions
+\begin{align}
+\mathcal W &:= \vec{\nabla} \cdot\left(\frac{m_iN_i}{e B^2} \vec{\nabla}_\perp \phi\right) \nonumber\\
+\vec{j_{\mathcal W}} &:= \vec j_{n_e} - \vec j_{N_i} \nonumber\\
+\Lambda_{\mathcal W} &:= \Lambda_{n_e} - \Lambda_{N_i}
 \end{align}
+where the polarization charge density is approximately related to the \ExB vorticity through
+$\mathcal W = -\vec{\nabla} \cdot\left(\frac{m_iN_i}{e B} \bhat \times \vec u_E\right) \approx \bhat\cdot\left(\vec\nabla \times\frac{m_iN_i}{e B}  \vec u_E\right)$
+and we used that $\Gamma_{1,i}^\dagger 1 = 1$.
 
 \subsubsection{Energy theorem}
 The terms of the energy theorem are
@@ -643,7 +653,7 @@ with
     &+\frac{1}{2} m_e  n_e u_e^2 +\frac{1}{2} m_i  N_i U_i^2
     + \frac{(\vec \nabla_\perp A_\parallel)^2}{2\mu_0}, \\
   \vec j_{\mathcal E} =& \sum_s \left[ N\left(
-  \vec v_E + \vec v_C + \vec v_{\nabla B} +U\left(\bhat+\tilde{\vec b}_\perp\right)  \right)
+  \vec v_E + \vec v_C + \vec v_{K} +U\left(\bhat+\tilde{\vec b}_\perp\right)  \right)
   \left(T \ln N + \frac{1}{2}m U^2 + q\psi  \right) \right. \nonumber\\
   &\qquad \left . + mNU^2\vec v_{\nabla\times\bhat} + \left(\bhat + \tilde{\vec b}_\perp\right) NUT\right], \\
   \Lambda_{\mathcal E} =&  \sum_s \left[\left(T\left( 1+\ln{N}\right) +q \psi + \frac{1}{2} m U^2 \right)\Lambda_{N}  + mNU \Lambda_U\right]
@@ -725,7 +735,7 @@ The terms of the particle conservation read
 \begin{align} \label{eq:mass_conservation}
   n_e= & n_e,\\
   \vec j_{n_e} =& n_e\left(
-  \vec v_E + \vec v_C + \vec v_{\nabla B} +u_e\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \nonumber\\
+  \vec v_E + \vec v_C + \vec v_{K} +u_e\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \nonumber\\
   =& n_e \left(\frac{\bhat\times \nabla\phi}{B} 
   + (\tau_e +\mu_e u_e^2)\vec K_{\nabla\times\bhat} 
   + \tau_e \vec K_{\vec \nabla B} + u_e(\bhat + \tilde{\vec b}_\perp) \right), \\
@@ -734,6 +744,12 @@ The terms of the particle conservation read
 \\
   S_{n_e} =&  S_{n_e}
 \end{align}
+and the vorticity conservation
+\begin{align} \label{eq:vorticity_conservation}
+\mathcal W &= \vec{\nabla} \cdot\left(\frac{N_i}{B^2} \vec{\nabla}_\perp \phi\right) \nonumber\\
+\vec{j_{\mathcal W}} &= \vec j_{n_e} - \vec j_{N_i} \nonumber\\
+\Lambda_{\mathcal W} &= \Lambda_{n_e} - \Lambda_{N_i}
+\end{align}
 
 The terms of the energy theorem read (with $z_e=-1$ and $z_i=+1$)
 \begin{align} \label{eq:energy_conservation}
@@ -1022,7 +1038,7 @@ of a function $f(R,Z,\varphi)$ is given by the formula
 \langle f \rangle^{area}_{\psi_{p}} :=&
 \frac{ \oint_{\psi_p  } f(R,Z,\varphi)\dA}{\oint_{\psi_p } \dA} \nonumber \\
 =& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vec\nabla\psi_p| \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
-{\int_\Omega |\vec\nabla\psi_p|\delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
+{\int_\Omega |\vec\nabla\psi_p|\delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z} \nonumber\\
 \end{align}
 %with $\dV := R\d R\d Z\d \varphi$ %(we define the average in computational space and omit one $R$)
 and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
@@ -1036,11 +1052,13 @@ flux label, we define
 \langle f \rangle^{vol}_\psi :=& \frac{\partial}{\partial v} \int \dV f
  = \frac{1}{\int \dA |\nabla\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\nabla\psi_p|} \dA \nonumber\\
 =& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
-{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
+{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}\nonumber\\
+ =& \frac{1}{\int \sqrt{g}\d\eta\d\varphi } \int_0^{2\pi} f(\zeta,\eta, \varphi) \sqrt{g}\d\eta\d\varphi
 \end{align}
 where we used the coarea formula Eq.~\eqref{eq:coarea} for the second
 identiy. We immediately see that this definition differs from the first
-Eq.~\eqref{eq:fsa_area} by the weight factor $|\nabla\psi_p|$.
+Eq.~\eqref{eq:fsa_area} by the weight factor $|\nabla\psi_p|$ and that it is particularly easy to compute
+in a flux-aligned coordinate system. Notice however that the volume element does appear (unlike e.g. Tokam3X papers)
 
 Both averages fullfill the basic identities
 \begin{align}
@@ -1052,7 +1070,7 @@ Both averages fullfill the basic identities
 
 The volume average is better suited for density-like quantities
 than the area average as we can see with the following identiy.
-Assume we have a quantity $X$ with $\partial_t X + \nabla \vec j_X = \Lambda_X$. Then we can use the volume average to write
+Assume we have a quantity $X$ with $\partial_t X + \nabla \cdot \vec j_X = \Lambda_X$. Then we can use the volume average to write
 \begin{align}
 \frac{\partial}{\partial t} \langle X \rangle^{vol} + \frac{\partial}{
   \partial v} J_X  = \langle \Lambda_X\rangle^{vol}
@@ -1066,6 +1084,80 @@ J_X:=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
 2\pi\int_\Omega \vec \langle \vec j\cdot \vec\nabla\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
 \label{eq:total_flux}
 \end{align}
+\subsection{ Particle, energy and vorticity fluxes}
+Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservation} and \eqref{eq:energy_conservation} through a flux surface
+\begin{align} \label{eq:particle_flux}
+ \vec j_{N}\cdot \vec \nabla\psi_p %=& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ %\nabla\psi_p \nonumber\\
+ =&
+  N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
+   \mathcal K_{\nabla\times\bhat}(\psi_p) + \tau  \mathcal K_{\nabla B}(\psi_p) \right] \nonumber\\
+ &+ NU\left [\left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
+ \vec j_{\mathcal E}\cdot \vec \nabla\psi_p =&
+ %\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right)
+ %N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ %\vec j_N \cdot \vec\nabla\psi_p
+ %\nonumber\\
+ %+ z\left(\mu NU^2 \vec v_{\nabla\times \bhat} + \tau NU(\bhat+\tilde{\vec b}_\perp )\right) \cdot\vec \nabla\psi_p \nonumber\\
+ %=
+\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
++ z \mu\tau NU^2 \mathcal K_{\nabla\times\bhat}(\psi_p) \nonumber\\
+&+ z \tau NU
+ \left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
+\label{eq:energy_flux}
+\end{align}
+
+In order to discuss poloidal flows let us first define orthogonal unit vectors $\{\hat\zeta, \hat\eta,\bhat\}$
+\begin{align}
+\hat \eta = \bhat \times\hat \zeta
+\end{align}
+which points in the anti-clockwise poloidal direction with our choice of the magnetic field direction.
+From where we get $\nabla\phi = \hat \zeta\partial_{\hat\zeta}\phi + \hat\eta\partial_{\hat\eta} \phi$
+with $\partial_{\hat\zeta}:= \zeta\cdot\nabla$.
+Notice that $\hat \eta$ in general has a (small) toroidal component in addition to the dominant poloidal component.
+Furthermore, from $\vec u_E = \bhat\times \vec\nabla\phi/B$ we can derive
+\begin{align}
+u^{\hat\eta} &= \frac{\partial_{\hat\zeta} \phi}{B} \\
+u^{\hat\zeta} &= -\frac{\partial_{\hat\eta} \phi}{B}
+\end{align}
+where $u^{\hat\zeta} := \hat\zeta\cdot\vec u_E$
+
+Now, let us integrate the vorticity density over a flux aligned volume to get
+\begin{align}
+\int \dV \mathcal W = \int \dA u^{\hat \eta} \frac{N_i}{B}
+\end{align}
+from where we get the flux-surface (area) integrated vorticity conservation
+\begin{align}
+\frac{\partial}{\partial t}\int\frac{N_iu^{\hat \eta}}{B}\dA
+ = \int( \vec j \times\bhat)\cdot\hat\eta \dA
+ + \int\dV \Lambda_{n_e} - \Lambda_{N_i}
+ \end{align}
+ with
+\begin{align}
+ \left(\vec j \times \bhat\right)\cdot\hat \eta =
+\hat\zeta\cdot (\vec j_{N_i} - \vec j_{n_e})  =
+   \left(N_i u_{E,i}^{\hat\zeta} - n_e u_{E,e}^{\hat\zeta}\right) \nonumber\\
+ + \left( \mu_iN_i U_i^2 - \mu_e n_e u_e^2\right) K_{\nabla\times\bhat}^{\hat\zeta}
+ + \left(\tau_i N_i - \tau_e n_e  \right) \left(K_{\nabla B}^{\hat\zeta} + K^{\hat\zeta}_{\nabla\times\bhat} \right)
+ + \left(N_i U_i - n_eu_e )\right)\tilde b^{\hat\zeta}_\perp
+\end{align}
+This equation describes that a current through a flux surface feels the Lorentz force pointing
+in the perpendicular direction and thus drives a poloidal flux.
+In the first term of the Lorentz force density integral we find the terms related to the Reynolds/Favre stress.
+The remaining terms describe the currents related to the curvature drifts. Notice
+that the prefactors are always negative since $\tau_e$ and $\mu_e$ are negative.
+Also note that the curvature vectors align with $\hat \zeta$ on the top and bottom of the tokamak.
+(Is this related to the X-point orbit loss mechanism? Chang, Ku, et al. )
+
+A surplus of electrons in the domain will lead to counter-clockwise poloidal acceleration, while positive
+charge leads to clockwise poloidal acceleration.
+
+
+
 
 \subsection{The safety factor}
 Assume that we pick a random field line and follow it (integrate it) for exactly one
@@ -1103,33 +1195,6 @@ q\equiv\frac{1}{2\pi}\oint \frac{B^\varphi}{B^\Theta} \d\Theta
 Notice that the safety factor diverges on the last closed flux
 surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa_area}
 remain finite due to the $\nabla\psi$ factor.
-\subsection{ Fluxes and length scales}
-Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservation} and \eqref{eq:energy_conservation} through a flux surface
-\begin{align} \label{eq:particle_flux}
- \vec j_{N}\cdot \vec \nabla\psi_p %=& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- %\nabla\psi_p \nonumber\\
- =&
-  N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
-   \mathcal K_{\nabla\times\bhat}(\psi_p) + \tau  \mathcal K_{\nabla B}(\psi_p) \right] \nonumber\\
- &+ NU\left [\left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
- \vec j_{\mathcal E}\cdot \vec \nabla\psi_p =&
- %\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right)
- %N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- %\vec j_N \cdot \vec\nabla\psi_p
- %\nonumber\\
- %+ z\left(\mu NU^2 \vec v_{\nabla\times \bhat} + \tau NU(\bhat+\tilde{\vec b}_\perp )\right) \cdot\vec \nabla\psi_p \nonumber\\
- %=
-\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
-+ z \mu\tau NU^2 \mathcal K_{\nabla\times\bhat}(\psi_p) \nonumber\\
-&+ z \tau NU
- \left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
-\label{eq:energy_flux}
-\end{align}
-
 \subsection{Program and files}
 We have the program \texttt{feltor/diag/feltordiag.cu}.
 This program reads a previously generated simulation file \texttt{input.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows. \\
-- 
GitLab


From f44a95ad7afcb5cd42ff9a4609ee688568708e05 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 9 Apr 2019 16:47:50 +0200
Subject: [PATCH 063/540] Small corrections in feltor.tex

---
 src/feltor/feltor.tex | 62 +++++++++++++++++++++++--------------------
 1 file changed, 33 insertions(+), 29 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 74510d787..02b122837 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -440,7 +440,7 @@ Given $\phi$ we define the generalised electric potential
 Notice that the term $\nabla (n_e \vec v_K ) \equiv -\nabla (T_e n_e \vec K /e) $
 forms the familiar form of the divergence of the diamagnetic flux.
 \begin{align}
-\nabla\cdot\left( n_e \frac{\bhat \times \nabla n_e T_e}{en_e B}\right) \equiv  \nabla\cdot \left(\frac{T_e n_e}{e} \vec K\right)
+\nabla\cdot\left( n_e \frac{\bhat \times \nabla n_e T_e}{en_e B}\right) \equiv  \nabla\cdot \left(\frac{T_e n_e}{e} \vec{ \mathcal K}\right)
 \end{align}
 which is evident with the identities in Section~\ref{sec:magnetic}.
 
@@ -999,13 +999,13 @@ of Eq.~\eqref{eq:dirac_delta} can thus readily be computed
 via Gauss-Legendre quadrature, which gives us an easy means to compute area
 integrals even if our coordinate system is not aligned to the area.
 
-Furthermore, recall the {\bf coarea formula}
+Furthermore, recall the {\bf co-area formula}
 \begin{align} \label{eq:coarea}
 \int_{\Omega_0} f(\vec x) \dV =
 \int_0^{h_0} \left( \int_{h=h'} \frac{f(\vec x)}{|\nabla h|}  \dA  \right) \d h'
 \end{align}
 where $\Omega_0$ is the volume enclosed by the contour $h=h_0$.
-The coarea formula can be viewed as a change of variables in the
+The co-area formula can be viewed as a change of variables in the
 volume integral.
 
 We define the {\bf toroidal average} of a function $f(R,Z,\varphi)$ as
@@ -1045,7 +1045,7 @@ and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from belo
 in our domain $\Omega$.
 
 The second one (the {\bf volume average} after \cite{haeseleer}) defines an average on a
-small volume - a shell centered around the flux-surface.
+small volume - a shell centered around the flux-surface - defined by two neighboring flux-surfaces.
 If we define $v(\psi) := \int_0^\psi \dV$ as the volume
 flux label, we define
 \begin{align} \label{eq:fsa_vol}
@@ -1053,14 +1053,14 @@ flux label, we define
  = \frac{1}{\int \dA |\nabla\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\nabla\psi_p|} \dA \nonumber\\
 =& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
 {\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}\nonumber\\
- =& \frac{1}{\int \sqrt{g}\d\eta\d\varphi } \int_0^{2\pi} f(\zeta,\eta, \varphi) \sqrt{g}\d\eta\d\varphi
+ =& \frac{1}{\int \sqrt{g}\d\eta } \int_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
 \end{align}
-where we used the coarea formula Eq.~\eqref{eq:coarea} for the second
-identiy. We immediately see that this definition differs from the first
+where we used the co-area formula Eq.~\eqref{eq:coarea} for the second
+identity. We immediately see that this definition differs from the first
 Eq.~\eqref{eq:fsa_area} by the weight factor $|\nabla\psi_p|$ and that it is particularly easy to compute
 in a flux-aligned coordinate system. Notice however that the volume element does appear (unlike e.g. Tokam3X papers)
 
-Both averages fullfill the basic identities
+Both averages fulfill the basic identities
 \begin{align}
 \label{eq:fsa_identities}
 \langle \mu f + \lambda g\rangle &= \mu\langle f\rangle + \lambda \langle g\rangle \\
@@ -1069,7 +1069,7 @@ Both averages fullfill the basic identities
 
 
 The volume average is better suited for density-like quantities
-than the area average as we can see with the following identiy.
+than the area average as we can see with the following identity.
 Assume we have a quantity $X$ with $\partial_t X + \nabla \cdot \vec j_X = \Lambda_X$. Then we can use the volume average to write
 \begin{align}
 \frac{\partial}{\partial t} \langle X \rangle^{vol} + \frac{\partial}{
@@ -1113,44 +1113,48 @@ Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservati
 
 In order to discuss poloidal flows let us first define orthogonal unit vectors $\{\hat\zeta, \hat\eta,\bhat\}$
 \begin{align}
-\hat \eta = \bhat \times\hat \zeta
+\hat\zeta := \frac{\nabla\psi_p}{|\nabla\psi_p|} \quad
+\hat \eta := \bhat \times\hat \zeta
 \end{align}
-which points in the anti-clockwise poloidal direction with our choice of the magnetic field direction.
-From where we get $\nabla\phi = \hat \zeta\partial_{\hat\zeta}\phi + \hat\eta\partial_{\hat\eta} \phi$
-with $\partial_{\hat\zeta}:= \zeta\cdot\nabla$.
+where $\hat\eta$
+points in the counter-clockwise poloidal direction with our choice of the magnetic field direction.
+We then have $\nabla\phi = \hat \zeta\partial_{\hat\zeta}\phi + \hat\eta\partial_{\hat\eta} \phi$
+with $\partial_{\hat\zeta}:= \hat\zeta\cdot\nabla$ and $\partial_{\hat\eta}:=\hat\eta\cdot\nabla$.
 Notice that $\hat \eta$ in general has a (small) toroidal component in addition to the dominant poloidal component.
 Furthermore, from $\vec u_E = \bhat\times \vec\nabla\phi/B$ we can derive
 \begin{align}
 u^{\hat\eta} &= \frac{\partial_{\hat\zeta} \phi}{B} \\
 u^{\hat\zeta} &= -\frac{\partial_{\hat\eta} \phi}{B}
 \end{align}
-where $u^{\hat\zeta} := \hat\zeta\cdot\vec u_E$
+where $u^{\hat\zeta} := \hat\zeta\cdot\vec u_E$ and $u^{\hat\eta}:=\hat\eta\cdot\vec u_E$.
+Again, we emphasize that $u^{\hat\eta}$ has a toroidal component and is not a purely poloidal flow.
 
 Now, let us integrate the vorticity density over a flux aligned volume to get
 \begin{align}
-\int \dV \mathcal W = \int \dA u^{\hat \eta} \frac{N_i}{B}
+\int \dV \mathcal W =\int \dV \nabla\cdot\left(\frac{N_i\nabla_\perp\phi}{B^2}\right)  =  \int \dA u^{\hat \eta} \frac{N_i}{B}
 \end{align}
 from where we get the flux-surface (area) integrated vorticity conservation
 \begin{align}
 \frac{\partial}{\partial t}\int\frac{N_iu^{\hat \eta}}{B}\dA
- = \int( \vec j \times\bhat)\cdot\hat\eta \dA
+ = \int\frac{( \vec j \times\vec B)\cdot\hat\eta}{B} \dA
  + \int\dV \Lambda_{n_e} - \Lambda_{N_i}
  \end{align}
- with
+ with $(\vec j\times \vec B)\cdot\hat \eta/B =(\vec j_{N_i} - \vec j_{n_e})\cdot \hat\zeta $ and
 \begin{align}
- \left(\vec j \times \bhat\right)\cdot\hat \eta =
-\hat\zeta\cdot (\vec j_{N_i} - \vec j_{n_e})  =
-   \left(N_i u_{E,i}^{\hat\zeta} - n_e u_{E,e}^{\hat\zeta}\right) \nonumber\\
- + \left( \mu_iN_i U_i^2 - \mu_e n_e u_e^2\right) K_{\nabla\times\bhat}^{\hat\zeta}
- + \left(\tau_i N_i - \tau_e n_e  \right) \left(K_{\nabla B}^{\hat\zeta} + K^{\hat\zeta}_{\nabla\times\bhat} \right)
- + \left(N_i U_i - n_eu_e )\right)\tilde b^{\hat\zeta}_\perp
-\end{align}
-This equation describes that a current through a flux surface feels the Lorentz force pointing
-in the perpendicular direction and thus drives a poloidal flux.
-In the first term of the Lorentz force density integral we find the terms related to the Reynolds/Favre stress.
+ (\vec j_{N_i} - \vec j_{n_e})\cdot\vec \nabla\psi_p  =
+ \frac{N_i}{B}[\psi,\psi_p]_\perp - \frac{n_e}{B}[\phi,\psi_p]_\perp
+ \nonumber\\
+ + \left( \mu_iN_i U_i^2 - \mu_e n_e u_e^2\right) \mathcal K_{\nabla\times\bhat}(\psi_p)
+ + \left(\tau_i N_i - \tau_e n_e  \right) \mathcal K(\psi_p)
+ + \left(N_i U_i - n_eu_e \right)\vec{\tilde b_\perp}\cdot\vec\nabla\psi_p
+\end{align}
+This equation describes that a current through a flux surface feels the Lorentz force
+and thus drives a poloidal flux.
+In the first two terms of the Lorentz force density integral we find the terms related to the Reynolds/Favre stress.
 The remaining terms describe the currents related to the curvature drifts. Notice
-that the prefactors are always negative since $\tau_e$ and $\mu_e$ are negative.
-Also note that the curvature vectors align with $\hat \zeta$ on the top and bottom of the tokamak.
+that the prefactors are always positive since $\tau_e$ and $\mu_e$ are negative.
+The curvature vectors counter-align with $\hat \zeta$ ($\mathcal K(\psi_p) < 0$, decelerate) on the top 
+and align ($\mathcal K(\psi_p) > 0$, accelerate) on the bottom of the tokamak.
 (Is this related to the X-point orbit loss mechanism? Chang, Ku, et al. )
 
 A surplus of electrons in the domain will lead to counter-clockwise poloidal acceleration, while positive
-- 
GitLab


From e699fece423123f714118454bf15fcd980c6efac Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Apr 2019 18:32:18 +0200
Subject: [PATCH 064/540] Work on improving the FSA and FVA evaluations

- DG_DEVICE in some functors in functors.h
- improve deltapsi of Delta- function
- implement and test FluxVolume integral
- improve definition of rho (psip 0 != psipmin)
- implement X-point grid in geometry-diag
- improve findXpoint function
---
 inc/dg/functors.h                         | 11 ++-
 inc/geometries/average.h                  | 85 ++++++++++++++++++++---
 inc/geometries/flux_t.cu                  |  2 +-
 inc/geometries/geometry_diag.cu           | 52 +++++++++-----
 inc/geometries/separatrix_orthogonal_t.cu | 11 +--
 inc/geometries/utilitiesX.h               |  5 +-
 src/feltor/feltor.tex                     |  4 +-
 7 files changed, 132 insertions(+), 38 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 40c6a09a9..9673bc450 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -710,6 +710,7 @@ struct Iris
 {
     Iris( double psi_min, double psi_max ):
         m_psimin(psi_min), m_psimax(psi_max) { }
+    DG_DEVICE
     double operator()(double psi)const
     {
         if( psi > m_psimax) return 0.;
@@ -730,6 +731,7 @@ struct Pupil
 {
     Pupil( double psimax):
         psimax_(psimax) { }
+    DG_DEVICE
     double operator()(double psi)const
     {
         if( psi > psimax_) return 0.;
@@ -749,6 +751,7 @@ struct PsiPupil
 {
     PsiPupil(double psimax):
         psimax_(psimax){ }
+    DG_DEVICE
     double operator()(double psi)const
     {
         if( psi > psimax_) return psimax_;
@@ -771,12 +774,13 @@ struct Heaviside
      * @brief Construct with xb and sign
      *
      * @param xb boundary value
-     * @param sign either +1 or -1, If -1, \c x<x_b is replaced with \c x>x_b
-     * thus approximating H(xb-x) if H(x-xb) is the regular Heaviside function
+     * @param sign either +1 or -1, If -1, we swap 1 and 0 in the definition,
+     * thus mirror the Heaviside at the \c x=x_b axis
      */
     Heaviside( double xb, int sign = +1):
-        m_xb(xb){ }
+        m_xb(xb), m_s(sign){ }
 
+    DG_DEVICE
     double operator()(double x)const
     {
         if( (x < m_xb && m_s == 1) || (x > m_xb && m_s == -1)) return 0.;
@@ -800,6 +804,7 @@ struct GaussianDamping
 {
     GaussianDamping( double psimax, double alpha):
         m_psimax(psimax), m_alpha(alpha) { }
+    DG_DEVICE
     double operator()(double psi)const
     {
         if( psi > m_psimax + 4.*m_alpha) return 0.;
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 6daa2f26b..6509fa6dc 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -61,7 +61,7 @@ struct DeltaFunction
  * @brief Flux surface integral of the form
  \f[ \int dR dZ f(R,Z) \delta(\psi_p(R,Z)-\psi_0) g(R,Z) \f]
 
- where for the \c epsilon in \c DeltaFunction we use the maximum of \c h*GradPsip
+ where for the width of the Gaussian shaped delta function we use the maximum of \c 0.5*h*GradPsip
      where \c h is the cell size in the grid
  * @ingroup misc_geo
  */
@@ -73,7 +73,7 @@ struct FluxSurfaceIntegral
      * f and g are default initialized to 1
      * @param g2d grid
      * @param c contains psip, psipR and psipZ
-     * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c 0.15*h*GradPsi*width_factor)
+     * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c 0.5*h*GradPsi*width_factor)
      */
     FluxSurfaceIntegral(const dg::Grid2d& g2d, const TokamakMagneticField& c, double width_factor = 1.):
             m_f(dg::evaluate(dg::one, g2d)), m_g(m_f), m_delta(m_f),
@@ -84,9 +84,10 @@ struct FluxSurfaceIntegral
         thrust::host_vector<double> psipZ  = dg::evaluate( c.psipZ(), g2d);
         double psipRmax = dg::blas1::reduce( psipR, 0., dg::AbsMax<double>()  );
         double psipZmax = dg::blas1::reduce( psipZ, 0., dg::AbsMax<double>()  );
-        double deltapsi = 0.15*(psipZmax*g2d.hy() +psipRmax*g2d.hx());
+        double deltapsi = 0.5*(psipZmax*g2d.hy() +psipRmax*g2d.hx())/g2d.n();
         m_eps = deltapsi*width_factor;
     }
+    double get_deltapsi() const{return m_eps;}
 
     /**
      * @brief Set the left function to integrate
@@ -123,11 +124,71 @@ struct FluxSurfaceIntegral
     const container m_w2d;
 };
 
+//This method for computing volumes is tested against flux-aligned grids in e.g. flux_t.cu
 /**
- * @brief Flux surface average over quantity
- \f[ \langle f\rangle(\psi_0) = \frac{1}{A} \int dR dZ \delta(\psi_p(R,Z)-\psi_0) |\nabla\psi_p|f(R,Z)H(R,Z) \f]
+ * @brief Flux volume integral of the form
+ \f[ \int dR dZ f(R,Z) \Theta(\psi_p(R,Z)-\psi_0) g(R,Z) \f]
 
- with \f$ A = \int dRdZ \delta(\psi_p(R,Z)-\psi_0)|\nabla\psi_p|H(R,Z)\f$
+ where \c Theta is the Heaviside function
+ * @ingroup misc_geo
+ */
+template<class container>
+struct FluxVolumeIntegral
+{
+     /**
+     * @brief Construct from a grid and a magnetic field
+     * f and g are default initialized to 1
+     * @param g2d grid
+     * @param c contains psip
+     */
+    FluxVolumeIntegral(const dg::Grid2d& g2d, const TokamakMagneticField& c):
+            m_f(dg::evaluate(dg::one, g2d)), m_g(m_f), m_heavi(m_f),
+            m_psi( dg::evaluate( c.psip(), g2d)),
+            m_w2d ( dg::create::weights( g2d))
+    {
+    }
+
+    /**
+     * @brief Set the left function to integrate
+     *
+     * @param f the container containing the discretized function
+     */
+    void set_left( const container& f){
+        dg::blas1::copy( f, m_f);
+    }
+    /**
+     * @brief Set the right function to integrate
+     *
+     * @param f the container containing the discretized function
+     */
+    void set_right( const container& g){
+        dg::blas1::copy( g, m_g);
+    }
+    /**
+     * @brief Calculate the Flux Volume Integral
+     *
+     * @param psip0 the actual psi value of the flux surface
+     * @return Int_0^psip0 (f,g)
+     */
+    double operator()(double psip0)
+    {
+        dg::Heaviside heavi( psip0, -1);
+        dg::blas1::evaluate( m_heavi, dg::equals(), heavi, m_psi);
+        dg::blas1::pointwiseDot( 1., m_heavi, m_f, m_g, 0., m_heavi);
+        return dg::blas1::dot( m_heavi, m_w2d);
+    }
+    private:
+    double m_eps;
+    container m_f, m_g, m_heavi, m_psi;
+    const container m_w2d;
+};
+
+
+/**
+ * @brief Flux surface average (differential volume average) over quantity
+ \f[ \langle f\rangle(\psi_0) = \frac{1}{A} \int dR dZ \delta(\psi_p(R,Z)-\psi_0) f(R,Z)H(R,Z) \f]
+
+ with \f$ A = \int dRdZ \delta(\psi_p(R,Z)-\psi_0)H(R,Z)\f$
  where \c H is a weight function that can be used to e.g. cut away parts of the domain below the X-point or contain a volume form
  * @ingroup misc_geo
  */
@@ -142,16 +203,18 @@ struct FluxSurfaceAverage
      * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point and/or contain a volume form without dg weights)
      * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
      */
-    FluxSurfaceAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, container weights, double width_factor = 1.) :
+    FluxSurfaceAverage( const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, container weights, double width_factor = 1.) :
     m_avg( g2d,c, width_factor), m_area( g2d, c, width_factor)
     {
         m_avg.set_left( f);
-        container gradpsi  = dg::evaluate( dg::geo::GradPsip( c), g2d);
-        dg::blas1::pointwiseDot( weights, gradpsi, weights);
+        //    container gradpsi  = dg::evaluate( dg::geo::GradPsip( c), g2d);
+        //    dg::blas1::pointwiseDot( weights, gradpsi, weights);
         m_avg.set_right( weights);
         m_area.set_right( weights);
     }
 
+    double get_deltapsi() const{return m_avg.get_deltapsi;}
+
     /**
      * @brief Reset the function to average
      *
@@ -174,6 +237,10 @@ struct FluxSurfaceAverage
     FluxSurfaceIntegral<container> m_avg, m_area;
 };
 
+
+
+
+
 /**
  * @brief Class for the evaluation of the safety factor q based on a flux-surface integral
  * \f[ q(\psi_0) = \frac{1}{2\pi} \int dRdZ \frac{I(\psi_p)}{R} \delta(\psi_p - \psi_0)H(R,Z) \f]
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 22c9f876b..60234cdff 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -145,7 +145,7 @@ int main( int argc, char* argv[])
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
     dg::geo::Iris iris( c.psip(), gp.psipmin, gp.psipmax);
-    dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,1, 2e3, 2e3, dg::PER, dg::PER);
+    dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,3, 2e2, 2e2, dg::PER, dg::PER);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::HVec R  = dg::evaluate( dg::cooX2d, g2dC);
     dg::HVec onesC = dg::evaluate( dg::one, g2dC);
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index c4e73f7ee..3e53998ba 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -17,7 +17,8 @@
 #include "magnetic_field.h"
 #include "average.h"
 #include "testfunctors.h"
-#include "flux.h" //to compute Safety factor
+#include "curvilinearX.h"
+#include "separatrix_orthogonal.h"
 
 struct Parameters
 {
@@ -153,6 +154,12 @@ int main( int argc, char* argv[])
     const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
     const double psip0 = c.psip()(gp.R_0, 0);
     std::cout << "psip( 1, 0) "<<psip0<<"\n";
+    double R_O = gp.R_0, Z_O = 0.;
+    dg::geo::findXpoint( c.get_psip(), R_O, Z_O);
+    const double psipmin = c.psip()(R_O, Z_O);
+
+    std::cout << "O-point "<<R_O<<" "<<Z_O<<" with Psip = "<<psipmin<<std::endl;
+
     std::cout << "TEST ACCURACY OF PSI (values must be close to 0!)\n";
     if( gp.hasXpoint())
         std::cout << "    Equilibrium with X-point!\n";
@@ -233,39 +240,48 @@ int main( int argc, char* argv[])
     dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
     if( gp.hasXpoint() )
         dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
-    double psipmin = psip0;//(float)thrust::reduce( psipog2d.begin(), psipog2d.end(), 0.0,thrust::minimum<double>()  );
     double psipmax = 0.;//dg::blas1::reduce( psipog2d, 0., thrust::maximum<double>()  );
     unsigned npsi = 3, Npsi = 150;//set number of psivalues
-    //psipmin += (gp.psipmax - psipmin)/(double)Npsi; //the inner value is not good
     dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
     dg::geo::SafetyFactor     qprof( c);
     dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, c, psipog2d, xpoint_weights);
     dg::HVec psi_fsa    = dg::evaluate( fsa,        grid1d);
     dg::HVec sf         = dg::evaluate( qprof,      grid1d);
     dg::HVec psi        = dg::evaluate( dg::cooX1d, grid1d), rho(psi);
-    dg::blas1::axpby( -1./psip0, rho, +1., 1., rho); //transform psi to rho
+    dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
 
     //other flux labels
     dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, c);
     fsi.set_right( xpoint_weights);
     dg::HVec vol_psip = dg::evaluate( fsi, grid1d);
     dg::HVec w1d = dg::create::weights( grid1d);
-    double volumeRZ = 2.*M_PI*dg::blas1::dot( vol_psip, w1d);
+    double volumeCoarea = 2.*M_PI*dg::blas1::dot( vol_psip, w1d);
 
     //area
     fsi.set_left( dg::evaluate( dg::geo::GradPsip(c), grid2d));
     dg::HVec area_psip = dg::evaluate( fsi, grid1d);
 
-    dg::geo::Pupil pupil( c.psip(), 0.);
-    dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,1, 2e3, 2e3, dg::PER, dg::PER);
-    xpoint_weights = dg::evaluate( dg::cooX2d, g2dC);
-    if( gp.hasXpoint() )
-        dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), g2dC), xpoint_weights);
-    dg::HVec vec  = dg::evaluate( pupil, g2dC);
-    dg::HVec g2d_weights = dg::create::weights( g2dC);
-    double volumeRZP = 2.*M_PI*dg::blas2::dot( vec, g2d_weights, xpoint_weights);
-    std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeRZ<<" "<<volumeRZP
-              <<" rel error = "<<fabs(volumeRZ-volumeRZP)/volumeRZP<<"\n";
+    //dg::geo::Pupil pupil( c.psip(), 0.);
+    dg::geo::FluxVolumeIntegral<dg::HVec> fvi( grid2d, c);
+    std::cout << "Delta Rho for Flux surface integrals = "<<-fsi.get_deltapsi()/psipmin<<"\n";
+
+    fvi.set_right( xpoint_weights);
+    dg::HVec psi_vol = dg::evaluate( fvi, grid1d);
+    dg::blas1::scal(psi_vol, 2.*M_PI);
+    double volumeFVI = 2.*M_PI*fvi(0);
+    std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeCoarea<<" "<<volumeFVI
+              <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
+    ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
+    std::cout << "Generate X-point flux-aligned grid!\n";
+    double RX = gp.R_0-1.1*gp.triangularity*gp.a;
+    double ZX = -1.1*gp.elongation*gp.a;
+    c = dg::geo::createSolovevField(gp);
+    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( c.get_psip(), RX, ZX) ;
+    dg::geo::SeparatrixOrthogonal generator(c.get_psip(), monitor_chi, psipmin, R_X, Z_X, c.R0(), 0, 0, true);
+    double fx_0 = 1./8.;
+    double psi_1 = -fx_0/(1.-fx_0)*psipmin;
+    std::cout << "psi 1 is          "<<psi_1<<"\n";
+    dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., 3, 64, 160, dg::DIR, dg::NEU);
 
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
@@ -280,18 +296,20 @@ int main( int argc, char* argv[])
     dim2d_ids[0] = dim3d_ids[1], dim2d_ids[1] = dim3d_ids[2];
 
     //write 1d vectors
-    int avgID[5];
+    int avgID[6];
     err = nc_def_var( ncid, "q-profile", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[0]);
     err = nc_def_var( ncid, "rho", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[1]);
     err = nc_def_var( ncid, "psip1d", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[2]);
     err = nc_def_var( ncid, "psi_fsa", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[3]);
-    err = nc_def_var( ncid, "area", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[4]);
+    err = nc_def_var( ncid, "psi_area", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[4]);
+    err = nc_def_var( ncid, "psi_vol", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[5]);
     err = nc_enddef( ncid);
     err = nc_put_var_double( ncid, avgID[0], sf.data());
     err = nc_put_var_double( ncid, avgID[1], rho.data());
     err = nc_put_var_double( ncid, avgID[2], psi.data());
     err = nc_put_var_double( ncid, avgID[3], psi_fsa.data());
     err = nc_put_var_double( ncid, avgID[4], area_psip.data());
+    err = nc_put_var_double( ncid, avgID[5], psi_vol.data());
     err = nc_redef(ncid);
 
     //write 2d vectors
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 0bc1210e9..5ac0007ab 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -97,9 +97,9 @@ int main( int argc, char* argv[])
     double psi_0 = -20;
     std::cin >> psi_0;
     std::cout << "Typed "<<psi_0<<"\n";
-    //std::cout << "Type fx and fy ( fx*Nx and fy*Ny must be integer) \n";
+    std::cout << "Type fx and fy ( fx*Nx and fy*Ny must be integer) 1/4, 1/22 \n";
     double fx_0=1./4., fy_0=1./22.;
-    //std::cin >> fx_0>> fy_0;
+    std::cin >> fx_0>> fy_0;
     std::cout << "Typed "<<fx_0<<" "<<fy_0<<"\n";
 
     std::cout << "Type add_x and add_y \n";
@@ -111,7 +111,10 @@ int main( int argc, char* argv[])
     t.tic();
     //dg::geo::TokamakMagneticField c = dg::geo::createTaylorField(gp);
     dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
-    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
+    double R_O = gp.R_0, Z_O = 0.;
+    dg::geo::findXpoint( c.get_psip(), R_O, Z_O);
+    const double psipmin = c.psip()(R_O, Z_O);
+    std::cout << "Psi min "<<psipmin<<"\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
     //dg::geo::CylindricalSymmTensorLvl1 monitor_chi;
@@ -189,7 +192,7 @@ int main( int argc, char* argv[])
     err = nc_close( ncid);
 
     std::cout << "TEST VOLUME IS:\n";
-    dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, Z_X, 1.2*gp.a*gp.elongation, 1, 5e3, 5e3, dg::PER, dg::PER);
+    dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, Z_X, 1.2*gp.a*gp.elongation, 1, 5e2, 5e2, dg::PER, dg::PER);
     gp.psipmax = 0., gp.psipmin = psi_0;
     dg::geo::Iris iris( c.psip(), gp.psipmin, gp.psipmax);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
diff --git a/inc/geometries/utilitiesX.h b/inc/geometries/utilitiesX.h
index 0134ff99d..e4e2c7a00 100644
--- a/inc/geometries/utilitiesX.h
+++ b/inc/geometries/utilitiesX.h
@@ -9,13 +9,14 @@ namespace dg
 namespace geo
 {
 /**
- * @brief This function finds the X-point via Newton iteration applied to the gradient of psi
+ * @brief This function finds the X-point (or O-point for that matter) via Newton iteration applied to the gradient of psi
  *
  * The inverse of the Hessian matrix is computed analytically
  * @param psi \f$ \psi(R,Z)\f$, where R, Z are cylindrical coordinates
  * @param R_X start value on input, X-point on output
  * @param Z_X start value on input, X-point on output
  * @ingroup misc_geo
+ * @note also finds the O-point or any other point with vanishing gradient
  */
 static inline void findXpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
 {
@@ -23,7 +24,7 @@ static inline void findXpoint( const CylindricalFunctorsLvl2& psi, double& R_X,
     std::array<double, 2> X{ {0,0} }, XN(X), X_OLD(X);
     X[0] = R_X, X[1] = Z_X;
     double eps = 1e10, eps_old= 2e10;
-    while( (eps < eps_old || eps > 1e-7) && eps > 1e-13)
+    while( (eps < eps_old || eps > 1e-7) && eps > 1e-10)
     {
         X_OLD = X; eps= eps_old;
         hessianRZtau.newton_iteration( X, XN);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 02b122837..0a328c697 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -581,7 +581,7 @@ We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
   S_{n_e}(R,Z,\varphi, t) &= \omega_s
     (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho_{s} -\rho(R,Z)) H(Z-Z_X)\\
-    \rho(R,Z) &:= \frac{\psi_p(R_0,0)- \psi_p(R,Z) }{\psi_p(R_0,0)},\\
+    \rho(R,Z) &:= \frac{\psi_{p\min}- \psi_p(R,Z) }{\psi_{p,\min}})},\\
     \psi_p(R,Z)&:= (1-\rho(R,Z))\psi_p(R_0,0)
 \end{align}
 with $0 < \rho_{s}<1$
@@ -1231,7 +1231,7 @@ correlationNPhi  & Dataset & 1 (time) & $\left( \int\dV e^\phi n_e \right)/ ||e^
 correlationNTildePhi  & Dataset & 1 (time) & $\left( \int\dV \phi \tilde n_e \right)/ ||\phi||||\tilde n_e||$ with $n_e \equiv \langle n_e \rangle_{\psi_p} ( 1 + \tilde n_e)$\\
 q                & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} \\
 psip1d           & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
-rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,0} - \psi_p)/\psi_{p,0}$ \\
+rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
 area             & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\nabla\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
 X                & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
 X\_avg           & Dataset & 3 (time,y,x) & Toroidal average $\langle X
-- 
GitLab


From 26a964c48afee0dea78559ac0fa241a1562da36d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Apr 2019 23:21:05 +0200
Subject: [PATCH 065/540] Debug geometry_diag X-point fsi and fvi

---
 inc/dg/functors.h               |   5 +-
 inc/dg/topology/average.h       |  11 ++-
 inc/geometries/average.h        |   9 +--
 inc/geometries/geometry_diag.cu | 116 +++++++++++++++++++-------------
 src/feltor/feltor.tex           |   6 +-
 5 files changed, 92 insertions(+), 55 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 9673bc450..f4e1f4fbc 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -766,6 +766,7 @@ struct PsiPupil
         0  \text{ if } x < x_b \\
         1  \text{ else}
      \end{cases}\f]
+  @note the 1 is inclusive i.e if x==x_b the functor always returns 1
  */
 struct Heaviside
 {
@@ -774,8 +775,8 @@ struct Heaviside
      * @brief Construct with xb and sign
      *
      * @param xb boundary value
-     * @param sign either +1 or -1, If -1, we swap 1 and 0 in the definition,
-     * thus mirror the Heaviside at the \c x=x_b axis
+     * @param sign either +1 or -1, If -1, we mirror the Heaviside at
+     *  the \c x=x_b axis, i.e. we swap the < sign in the definition to >
      */
     Heaviside( double xb, int sign = +1):
         m_xb(xb), m_s(sign){ }
diff --git a/inc/dg/topology/average.h b/inc/dg/topology/average.h
index f8b8adb4a..5e92276d3 100644
--- a/inc/dg/topology/average.h
+++ b/inc/dg/topology/average.h
@@ -11,12 +11,21 @@
 namespace dg{
 
 /**
- * @brief Class for average computations in a Cartesian topology
+ * @brief Topological average computations in a Cartesian topology
  *
+ * \f[
+ * \langle f \rangle_x := \frac{1}{L_x}\int_0^{L_x}dx f \quad
+ * \langle f \rangle_y := \frac{1}{L_y}\int_0^{L_y}dy f \quad
+ * \langle f \rangle_z := \frac{1}{L_z}\int_0^{L_z}dz f \\
+ * \langle f \rangle_{xy} := \frac{1}{L_xL_y}\int_0^{L_x}\int_0^{L_y}dxdy f \quad
+ * \langle f \rangle_{xz} := \frac{1}{L_xL_z}\int_0^{L_x}\int_0^{L_z}dxdz f \quad
+ * \langle f \rangle_{yz} := \frac{1}{L_yL_z}\int_0^{L_y}\int_0^{L_z}dydz f \quad
+ * \f]
  * Given a Cartesian topology it is possible to define a partial reduction of a given vector.
  * In two dimensions for example we can define a reduction over all points that are neighbors in the x (or y) direction.
  * We are then left with Ny (Nx) points. In three dimensions we can define the reduction along the x, y, z directions
  * but also over all points in the xy (xz or yz) planes. We are left with two- (respectively three-)dimensional vectors.
+ * @note The integrals include the dG weights but not the volume element (does not know about geometry)
  * @snippet topology/average_t.cu doxygen
  * @ingroup utilities
  */
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 6509fa6dc..ffa03dbe0 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -141,10 +141,11 @@ struct FluxVolumeIntegral
      * @param g2d grid
      * @param c contains psip
      */
-    FluxVolumeIntegral(const dg::Grid2d& g2d, const TokamakMagneticField& c):
-            m_f(dg::evaluate(dg::one, g2d)), m_g(m_f), m_heavi(m_f),
-            m_psi( dg::evaluate( c.psip(), g2d)),
-            m_w2d ( dg::create::weights( g2d))
+    template<class Geometry2d>
+    FluxVolumeIntegral(const Geometry2d& g2d, const TokamakMagneticField& c):
+        m_f(dg::evaluate(dg::one, g2d)), m_g(m_f), m_heavi(m_f),
+        m_psi( dg::pullback( c.psip(), g2d)),
+        m_w2d ( dg::create::volume( g2d))
     {
     }
 
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 3e53998ba..3cee6de9a 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -15,10 +15,10 @@
 //#include "taylor.h"
 #include "init.h"
 #include "magnetic_field.h"
-#include "average.h"
 #include "testfunctors.h"
 #include "curvilinearX.h"
 #include "separatrix_orthogonal.h"
+#include "average.h"
 
 struct Parameters
 {
@@ -154,6 +154,7 @@ int main( int argc, char* argv[])
     const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
     const double psip0 = c.psip()(gp.R_0, 0);
     std::cout << "psip( 1, 0) "<<psip0<<"\n";
+    //Find O-point
     double R_O = gp.R_0, Z_O = 0.;
     dg::geo::findXpoint( c.get_psip(), R_O, Z_O);
     const double psipmin = c.psip()(R_O, Z_O);
@@ -234,54 +235,85 @@ int main( int argc, char* argv[])
         {"Gaussian3d", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
             M_PI, p.sigma, p.sigma, p.sigma, p.amp)}
     };
-    //Compute flux average
     dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
     dg::DVec psipog2d   = dg::evaluate( c.psip(), grid2d);
+    std::map<std::string, dg::HVec> map1d;
+    ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
+    std::cout << "Generate X-point flux-aligned grid!\n";
+    double RX = gp.R_0-1.1*gp.triangularity*gp.a;
+    double ZX = -1.1*gp.elongation*gp.a;
+    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( c.get_psip(), RX, ZX) ;
+    dg::geo::SeparatrixOrthogonal generator(c.get_psip(), monitor_chi, psipmin, R_X, Z_X, c.R0(), 0, 0, true);
+    double fx_0 = 1./8.;
+    double psipmax = -fx_0/(1.-fx_0)*psipmin;
+    std::cout << "psi 1 is          "<<psipmax<<"\n";
+    unsigned npsi = 3, Npsi = 32;//set number of psivalues
+    dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., npsi, Npsi, 160, dg::DIR, dg::NEU);
+    std::cout << "DONE! Generate X-point flux-aligned grid!\n";
+    dg::Average<dg::HVec > avg_eta( gX2d.grid(), dg::coo2d::y);
+    std::vector<dg::HVec> coordsX = gX2d.map();
+    dg::SparseTensor<dg::HVec> metricX = gX2d.metric();
+    dg::HVec volX2d = dg::tensor::volume2d( metricX);
+    dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
+    dg::IHMatrix grid2gX2d = dg::create::interpolation( coordsX[0], coordsX[1], grid2d);
+    dg::HVec psip_X = dg::evaluate( dg::one, gX2d), area_psip_X;
+    dg::blas2::symv( grid2gX2d, (dg::HVec)psipog2d, psip_X);
+    dg::blas1::pointwiseDot( volX2d, psip_X, psip_X);
+    avg_eta( psip_X, map1d["X_psip1d"], false);
+    avg_eta( volX2d, area_psip_X, false);
+    dg::blas1::pointwiseDivide( map1d.at("X_psip1d"), area_psip_X, map1d.at("X_psip1d"));
+
+    //NOTE: VOLUME is WITHIN cells while AREA is ON gridpoints
+    dg::HVec gradPsipX = metricX.value(0,0);
+    dg::blas1::transform( gradPsipX, gradPsipX, dg::SQRT<double>());
+    dg::blas1::pointwiseDot( volX2d, gradPsipX, gradPsipX); //R\sqrt{g}|\nabla\zeta|
+    avg_eta( gradPsipX, map1d["X_psi_area"], false);
+    dg::blas1::scal( map1d.at("X_psi_area"), 4.*M_PI*M_PI);
+
+    dg::geo::FluxVolumeIntegral<dg::HVec> fvi_X( gX2d, c);
+    fvi_X.set_left( coordsX[0]);
+    dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
+    map1d["X_psi_vol"] = dg::evaluate( fvi_X, grid1d);
+    dg::blas1::scal( map1d.at("X_psi_vol"), 2.*M_PI);
+
+
+    ///////////////////Compute flux average////////////////////
     dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
     if( gp.hasXpoint() )
         dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
-    double psipmax = 0.;//dg::blas1::reduce( psipog2d, 0., thrust::maximum<double>()  );
-    unsigned npsi = 3, Npsi = 150;//set number of psivalues
-    dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
     dg::geo::SafetyFactor     qprof( c);
     dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, c, psipog2d, xpoint_weights);
-    dg::HVec psi_fsa    = dg::evaluate( fsa,        grid1d);
-    dg::HVec sf         = dg::evaluate( qprof,      grid1d);
-    dg::HVec psi        = dg::evaluate( dg::cooX1d, grid1d), rho(psi);
-    dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
+    map1d["psi_fsa"]    = dg::evaluate( fsa,        grid1d);
+    map1d["q-profile"]  = dg::evaluate( qprof,      grid1d);
+    map1d["psip1d"]     = dg::evaluate( dg::cooX1d, grid1d);
+    map1d["rho"] = map1d.at("psip1d");
+    dg::blas1::axpby( -1./psipmin, map1d.at("rho"), +1., 1., map1d.at("rho")); //transform psi to rho
 
     //other flux labels
     dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, c);
     fsi.set_right( xpoint_weights);
-    dg::HVec vol_psip = dg::evaluate( fsi, grid1d);
+
+    dg::HVec areaT_psip = dg::evaluate( fsi, grid1d);
     dg::HVec w1d = dg::create::weights( grid1d);
-    double volumeCoarea = 2.*M_PI*dg::blas1::dot( vol_psip, w1d);
+    double volumeCoarea = 2.*M_PI*dg::blas1::dot( areaT_psip, w1d);
 
     //area
     fsi.set_left( dg::evaluate( dg::geo::GradPsip(c), grid2d));
-    dg::HVec area_psip = dg::evaluate( fsi, grid1d);
+    map1d["psi_area"] = dg::evaluate( fsi, grid1d);
+    dg::blas1::scal( map1d.at("psi_area"), 2.*M_PI);
 
-    //dg::geo::Pupil pupil( c.psip(), 0.);
-    dg::geo::FluxVolumeIntegral<dg::HVec> fvi( grid2d, c);
+    dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, c);
     std::cout << "Delta Rho for Flux surface integrals = "<<-fsi.get_deltapsi()/psipmin<<"\n";
 
     fvi.set_right( xpoint_weights);
-    dg::HVec psi_vol = dg::evaluate( fvi, grid1d);
-    dg::blas1::scal(psi_vol, 2.*M_PI);
-    double volumeFVI = 2.*M_PI*fvi(0);
+    map1d["psi_vol"] = dg::evaluate( fvi, grid1d);
+    dg::blas1::scal(map1d["psi_vol"], 2.*M_PI);
+    double volumeFVI = 2.*M_PI*fvi(psipmax);
+    double volumeFVIX = 2.*M_PI*fvi_X(psipmax);
     std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeCoarea<<" "<<volumeFVI
               <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
-    ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
-    std::cout << "Generate X-point flux-aligned grid!\n";
-    double RX = gp.R_0-1.1*gp.triangularity*gp.a;
-    double ZX = -1.1*gp.elongation*gp.a;
-    c = dg::geo::createSolovevField(gp);
-    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( c.get_psip(), RX, ZX) ;
-    dg::geo::SeparatrixOrthogonal generator(c.get_psip(), monitor_chi, psipmin, R_X, Z_X, c.R0(), 0, 0, true);
-    double fx_0 = 1./8.;
-    double psi_1 = -fx_0/(1.-fx_0)*psipmin;
-    std::cout << "psi 1 is          "<<psi_1<<"\n";
-    dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., 3, 64, 160, dg::DIR, dg::NEU);
+    std::cout << "VOLUME TEST WITH X Grid FORMULA: "<<volumeFVIX<<" "<<volumeFVI
+              <<" rel error = "<<fabs(volumeFVIX-volumeFVI)/volumeFVI<<"\n";
 
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
@@ -296,27 +328,21 @@ int main( int argc, char* argv[])
     dim2d_ids[0] = dim3d_ids[1], dim2d_ids[1] = dim3d_ids[2];
 
     //write 1d vectors
-    int avgID[6];
-    err = nc_def_var( ncid, "q-profile", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[0]);
-    err = nc_def_var( ncid, "rho", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[1]);
-    err = nc_def_var( ncid, "psip1d", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[2]);
-    err = nc_def_var( ncid, "psi_fsa", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[3]);
-    err = nc_def_var( ncid, "psi_area", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[4]);
-    err = nc_def_var( ncid, "psi_vol", NC_DOUBLE, 1, &dim1d_ids[0], &avgID[5]);
-    err = nc_enddef( ncid);
-    err = nc_put_var_double( ncid, avgID[0], sf.data());
-    err = nc_put_var_double( ncid, avgID[1], rho.data());
-    err = nc_put_var_double( ncid, avgID[2], psi.data());
-    err = nc_put_var_double( ncid, avgID[3], psi_fsa.data());
-    err = nc_put_var_double( ncid, avgID[4], area_psip.data());
-    err = nc_put_var_double( ncid, avgID[5], psi_vol.data());
-    err = nc_redef(ncid);
-
+    for( auto pair : map1d)
+    {
+        int vid;
+        err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 1,
+            &dim1d_ids[0], &vid);
+        err = nc_enddef( ncid);
+        err = nc_put_var_double( ncid, vid, pair.second.data());
+        err = nc_redef(ncid);
+    }
     //write 2d vectors
     for(auto pair : map)
     {
         int vectorID;
-        err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 2, &dim2d_ids[0], &vectorID);
+        err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 2,
+            &dim2d_ids[0], &vectorID);
         err = nc_enddef( ncid);
         hvisual = dg::evaluate( pair.second, grid2d);
         err = nc_put_var_double( ncid, vectorID, hvisual.data());
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 0a328c697..30ea46ab8 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -581,8 +581,8 @@ We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
   S_{n_e}(R,Z,\varphi, t) &= \omega_s
     (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho_{s} -\rho(R,Z)) H(Z-Z_X)\\
-    \rho(R,Z) &:= \frac{\psi_{p\min}- \psi_p(R,Z) }{\psi_{p,\min}})},\\
-    \psi_p(R,Z)&:= (1-\rho(R,Z))\psi_p(R_0,0)
+    \rho(R,Z) &:= \frac{\psi_{p,\min}- \psi_p(R,Z) }{\psi_{p,\min}},\\
+    \psi_p(R,Z)&:= (1-\rho(R,Z))\psi_{p,\min}
 \end{align}
 with $0 < \rho_{s}<1$
 where $\omega_s$ is the source strength parameter.
@@ -1022,7 +1022,7 @@ of the flux 2-form and the metric
 to a parameterization of the flux-surface.
 In a flux-aligned coordinate system $\{\zeta, \eta, \varphi\}$ the pull-back is trivial ($\zeta=const$) and we have with $\hat\zeta = \hat \psi_p$
 \begin{align}
-\dA = \sqrt{g} |\nabla \zeta| \d\eta\d\varphi, \quad \vec\dA := \hat\zeta \dA
+\dA = \sqrt{g^{\zeta\zeta}} \sqrt{g} \d\eta\d\varphi, \quad \vec\dA := \hat\zeta \dA
 \label{}
 \end{align}
 where we used that $g^{\zeta\zeta} = (\nabla\zeta)^2$
-- 
GitLab


From 462b068a071b87aa0ee301b9d4e91541a2c8c67d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 11 Apr 2019 17:26:24 +0200
Subject: [PATCH 066/540] Derive, implement, test and document dg integral

the indefinite integral using dG polynomials
---
 .../dg_introduction/dg_introduction.tex       | 39 ++++++++++++++++-
 inc/dg/topology/evaluation_t.cu               | 12 +++++-
 inc/dg/topology/operator.h                    | 23 ++++++++++
 inc/dg/topology/weights.h                     | 42 +++++++++++++++++++
 4 files changed, 112 insertions(+), 4 deletions(-)

diff --git a/doc/related_pages/dg_introduction/dg_introduction.tex b/doc/related_pages/dg_introduction/dg_introduction.tex
index 73d08946a..06f661131 100644
--- a/doc/related_pages/dg_introduction/dg_introduction.tex
+++ b/doc/related_pages/dg_introduction/dg_introduction.tex
@@ -144,8 +144,8 @@ via Gauss--Legendre quadrature
 \end{align}
 \label{eq:gausslegendre}
 \end{subequations}
-With these formulas we have a simple, accurate, and fast 
-method to evaluate integrals. This is applied, for example, to compute
+With these formulas we have a simple, accurate, and fast
+method to evaluate integrals on the entire domain. This is applied, for example, to compute
 errors in the $L_2$-norm.
 
 We now define some useful quantities that simplify our notation (note that $i,j=0,\dots,P-1$) 
@@ -206,7 +206,42 @@ matrices is simplified to a large extent.
 %\end{align}
 %
 
+\subsection{Indefinite Integrals}\label{sec:integrals}
+The approximation of an indefinite integral $\int_a^x f_h(x')\dx'$ is a bit more involved since the integration
+boundary $x$ does not need to coincide with a cell boundary.
+First, integrating the Legendre differential equation
+$\frac{\d}{\dx}\left[ (1-x^2) \frac{\d}{\dx}p_j(x) \right] + j(j+1)p_j(x) = 0$
+and with the help of Eq.~\eqref{eq:recursion} and a second recursion for Legendre-Polynomials
+$(x^2-1) \d p_j(x)/\dx = jxp_j(x) - j p_{j-1}(x)$
+we compute
+\begin{align}
+\int_{-1}^{x} p_0(x')\dx' = p_1(x)+p_0(x) \quad \int_{-1}^{x} p_j(x') \dx'  = \frac{ p_{j+1}(x) - p_{j-1}(x)}{2j+1}\quad \forall\ j>0
+\end{align}
+Notice that the integral of a polynomial of order $P$ yields a polynomial of order $P+1$.
+The projection integral onto a base polynomial $p_i(x)$ yields
+\begin{align}
+ N_{00} &:= \int_{-1}^{1} p_0(x) \int_{-1}^{x} p_0(x') \dx' \dx = 2\quad
+ %N_{i0} := \int_{-1}^{1} p_i(x) \int_{-1}^{x} p_0(x') \dx' \dx = \frac{2}{2i+1}\delta_{i1}\quad
+ \nonumber\\
+ N_{ij} &:= \int_{-1}^{1} p_i(x) \int_{-1}^{x} p_j(x') \dx' \dx
+= \frac{2}{(2i+1)(2j+1)}\left[\delta_{i(j+1)}-\delta_{i(j-1)}\right] \quad i,j\neq 0,0
+\end{align}
 
+Our idea is to split the integral $F_h(x) = \int_a^x f_h(x')\dx'$ into two parts:
+\begin{align}
+F_h(x):=\int_a^x f_h(x') \dx' = \int_a^{x_{n-1/2}} f_h(x')\dx' + \int_{x_{n-1/2}}^x f_h(x') \dx'
+\end{align}
+where $n$ is the cell number such that $x_{n-1/2}\leq x < x_{n+1/2}$.
+Inserting $f_h(x) = \bar f_{nj} p_{nj}(x)$ and projecting onto a base polynomial $p_{ni}(x)$ yields
+\begin{align}
+\bar F_{nk} := T_{ki}\int_{x_{n-1/2}}^{x_{n+1/2}} p_{ni}(x) \int_a^x f_h(x') \dx'\dx
+= \delta_{k0} \sum_{m=1}^{n-1} h\bar f_{m0} + \frac{h^2}{4}T_{ki}N_{ij} \bar f_{nj}
+\end{align}
+where we choose the proper normalization $h^2/4$ due to the double integrals in $N_{ij}$.
+Notice that if $f_h(x)$ is discretized using $P$ polynomial coefficients $F_h(x)$ would in
+principle need $P+1$ coefficients to be the exact integral.
+However, we usually truncate the coefficients at $P$ to match the order of the discretization.
+Numerical tests show that the corresponding truncation error is extremely small.
 
 \subsection{ Discretization of first derivatives} \label{sec:firstderivatives}
 From here on we write
diff --git a/inc/dg/topology/evaluation_t.cu b/inc/dg/topology/evaluation_t.cu
index 7669b8b5b..94841848c 100644
--- a/inc/dg/topology/evaluation_t.cu
+++ b/inc/dg/topology/evaluation_t.cu
@@ -41,10 +41,10 @@ int main()
 {
     std::cout << "This program tests the exblas::dot function. The tests succeed only if the evaluation and grid functions but also the weights and especially the exblas::dot function are correctly implemented and compiled. Furthermore, the compiler implementation of the exp function in the math library must be consistent across platforms to get reproducible results\n";
     std::cout << "A TEST is PASSED if the number in the second column shows EXACTLY 0!\n";
-    unsigned n = 3, Nx = 12, Ny = 28, Nz = 100;
+    unsigned n = 1, Nx = 12, Ny = 28, Nz = 100;
     std::cout << "On Grid "<<n<<" x "<<Nx<<" x "<<Ny<<" x "<<Nz<<"\n";
 
-    dg::Grid1d g1d( 1, 2, n, Nx);
+    dg::Grid1d g1d( 0, M_PI/2., n, Nx);
     dg::Grid2d g2d( 1, 2, 3, 4, n, Nx, Ny);
     dg::Grid3d g3d( 1, 2, 3, 4, 5, 6, n, Nx, Ny, Nz,dg::PER,dg::PER,dg::PER);
 
@@ -104,6 +104,14 @@ int main()
     res.d = y[0];
     std::cout << "Result of exp:     "<<res.i<<"\n"
               << "          GCC:     4645210948416067678 (correct)"<<std::endl;
+
+    //TEST OF INTEGRAL
+    dg::HVec integral_num = dg::create::integral( dg::evaluate( cos, g1d), g1d);
+    dg::HVec integral_ana = dg::evaluate( sin, g1d);
+    dg::blas1::plus( integral_ana, -sin(g1d.x0()));
+    dg::blas1::axpby( 1., integral_ana, -1., integral_num);
+    norm = dg::blas2::dot( integral_num, dg::create::weights( g1d), integral_num);
+    std::cout << " Error norm of  1d integral function "<<norm<<"\n";
     std::cout << "\nFINISHED! Continue with topology/derivatives_t.cu !\n\n";
     return 0;
 }
diff --git a/inc/dg/topology/operator.h b/inc/dg/topology/operator.h
index c89ff566e..f30442e1e 100644
--- a/inc/dg/topology/operator.h
+++ b/inc/dg/topology/operator.h
@@ -615,6 +615,29 @@ Operator<real_type> lilj( unsigned n)
     return op;
 }
 
+/**
+ * @brief Create the N-matrix
+ *
+ * @param n # of polynomial coefficients
+ *
+ * @return Operator
+ */
+template<class real_type>
+Operator<real_type> ninj( unsigned n)
+{
+    Operator<real_type> op( n, 0.);
+    for( int i=0; i<(int)n; i++)
+        for( int j=0; j<(int)n; j++)
+        {
+            if( i == j+1)
+                op( i,j) = 2./(2*i+1)/(2*j+1);
+            if( i == j-1)
+                op( i,j) = -2./(2*i+1)/(2*j+1);
+        }
+    op(0,0) = 2;
+    return op;
+}
+
 
 /**
  * @brief Construct a diagonal operator with weights
diff --git a/inc/dg/topology/weights.h b/inc/dg/topology/weights.h
index ad78605b6..9cf509e59 100644
--- a/inc/dg/topology/weights.h
+++ b/inc/dg/topology/weights.h
@@ -2,6 +2,7 @@
 
 #include <thrust/host_vector.h>
 #include "grid.h"
+#include "operator.h"
 #include "../enums.h"
 
 /*! @file
@@ -56,6 +57,47 @@ thrust::host_vector<real_type> inv_weights( const RealGrid1d<real_type>& g)
     return v;
 }
 
+/*!@brief Create the integral of a function on a grid
+ * \f[ F_h(x) = \int_a^x f_h(x') dx' \f]
+ *
+ * This function computes the indefinite integral of a given input
+ * @param in Host vector discretized on g
+ * @param g The grid
+ * @return integral of in on the grid g
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ */
+template<class real_type>
+thrust::host_vector<real_type> integral( const thrust::host_vector<real_type>& in, const RealGrid1d<real_type>& g)
+{
+    double h = g.h();
+    unsigned n = g.n();
+    thrust::host_vector<real_type> out(in.size(), 0);
+    dg::Operator<real_type> forward = g.dlt().forward();
+    dg::Operator<real_type> backward = g.dlt().backward();
+    dg::Operator<real_type> ninj = create::ninj<real_type>( n );
+    Operator<real_type> t = create::pipj_inv<real_type>(n);
+    t *= h/2.;
+    ninj = backward*t*ninj*forward;
+    real_type constant = 0.;
+
+    for( unsigned i=0; i<g.N(); i++)
+    {
+        for( unsigned k=0; k<n; k++)
+        {
+            for( unsigned l=0; l<n; l++)
+            {
+                out[ i*n + k] += ninj(k,l)*in[ i*n + l];
+            }
+            out[ i*n + k] += constant;
+        }
+        for( unsigned l=0; l<n; l++)
+            constant += h*forward(0,l)*in[i*n+l];
+    }
+    return out;
+
+
+}
+
 ///@cond
 namespace detail{
 
-- 
GitLab


From f7304ad38300879ab3a944959a501b5e199c2817 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Apr 2019 00:07:18 +0200
Subject: [PATCH 067/540] geometry_diag with new integrate and interpolate

- Fix 1d size in topological average object
- overload integrate with Functor version and promote it to dg namespace
- fix evaluation_t again
- Redesign interpolate function and add a 1d version for it
- fix fieldaligned using the interpolate function
---
 inc/dg/enums.h                     |   6 ++
 inc/dg/topology/average.h          |  19 +++-
 inc/dg/topology/evaluation.h       |  66 ++++++++++++-
 inc/dg/topology/evaluation_t.cu    |   6 +-
 inc/dg/topology/interpolation.h    | 149 ++++++++++++++++++++++-------
 inc/dg/topology/interpolation_t.cu |  60 +++++-------
 inc/dg/topology/weights.h          |  42 --------
 inc/geometries/fieldaligned.h      |  18 ++--
 inc/geometries/geometry_diag.cu    |  19 ++--
 9 files changed, 243 insertions(+), 142 deletions(-)

diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index 65a873e88..8d0ed510d 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -91,6 +91,12 @@ enum direction{
     centered //!< centered derivative
 };
 
+///@brief Space of DG coefficients
+enum space{
+    lspace, //!< DG Polynomial space
+    xspace //!< Configuration space
+};
+
 ///@brief 2d coordinates
 enum class coo2d : char
 {
diff --git a/inc/dg/topology/average.h b/inc/dg/topology/average.h
index 5e92276d3..4f587552d 100644
--- a/inc/dg/topology/average.h
+++ b/inc/dg/topology/average.h
@@ -43,23 +43,30 @@ struct Average
     {
         m_nx = g.Nx()*g.n(), m_ny = g.Ny()*g.n();
         m_w=dg::construct<ContainerType>(dg::create::weights(g, direction));
-        m_temp1d = m_temp = m_w;
+        m_temp = m_w;
         m_transpose = false;
+        unsigned size1d = 0;
         if( direction == coo2d::x)
+        {
             dg::blas1::scal( m_w, 1./g.lx());
+            size1d = m_ny;
+        }
         else
         {
             m_transpose = true;
             dg::blas1::scal( m_temp, 1./g.ly());
             dg::transpose( m_nx, m_ny, m_temp, m_w);
+            size1d = m_nx;
         }
+        thrust::host_vector<double> t1d( size1d);
+        m_temp1d = dg::construct<ContainerType>( t1d);
     }
 
     ///@copydoc Average()
     Average( const aTopology3d& g, enum coo3d direction)
     {
         m_w = dg::construct<ContainerType>(dg::create::weights(g, direction));
-        m_temp1d = m_temp = m_w;
+        m_temp = m_w;
         m_transpose = false;
         unsigned nx = g.n()*g.Nx(), ny = g.n()*g.Ny(), nz = g.Nz();
         if( direction == coo3d::x) {
@@ -84,6 +91,12 @@ struct Average
         }
         else
             std::cerr << "Warning: this direction is not implemented\n";
+        if(!m_transpose)
+            m_temp1d = dg::construct<ContainerType>(
+                thrust::host_vector<double>( m_ny,0.));
+        else
+            m_temp1d = dg::construct<ContainerType>(
+                thrust::host_vector<double>( m_nx,0.));
     }
     /**
      * @brief Compute the average as configured in the constructor
@@ -100,6 +113,7 @@ struct Average
     {
         if( !m_transpose)
         {
+            //temp1d has size m_ny
             dg::average( m_nx, m_ny, src, m_w, m_temp1d);
             if( extend )
                 dg::extend_column( m_nx, m_ny, m_temp1d, res);
@@ -108,6 +122,7 @@ struct Average
         }
         else
         {
+            //temp1d has size m_nx
             dg::transpose( m_nx, m_ny, src, m_temp);
             dg::average( m_ny, m_nx, m_temp, m_w, m_temp1d);
             if( extend )
diff --git a/inc/dg/topology/evaluation.h b/inc/dg/topology/evaluation.h
index 6d603729c..6624a98cd 100644
--- a/inc/dg/topology/evaluation.h
+++ b/inc/dg/topology/evaluation.h
@@ -5,6 +5,7 @@
 #include <thrust/host_vector.h>
 #include "dg/backend/config.h"
 #include "grid.h"
+#include "operator.h"
 
 /*! @file
   @brief Function discretization routines
@@ -43,7 +44,6 @@ thrust::host_vector<real_type> abscissas( const RealGrid1d<real_type>& g)
 ///@{
 
 
-
 /**
  * @brief Evaluate a 1d function on grid coordinates
  *
@@ -171,6 +171,70 @@ thrust::host_vector<real_type> evaluate( real_type(f)(real_type, real_type, real
     return evaluate<real_type(real_type, real_type, real_type)>( *f, g);
 };
 ///@endcond
+/////////////////////////////////////INTEGRATE/////////////////
+
+/*!@brief Indefinite integral of a function on a grid
+ * \f[ F_h(x) = \int_a^x f_h(x') dx' \f]
+ *
+ * This function computes the indefinite integral of a given input
+ * @param in Host vector discretized on g
+ * @param g The grid
+ * @return integral of in on the grid g
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ */
+template<class real_type>
+thrust::host_vector<real_type> integrate( const thrust::host_vector<real_type>& in, const RealGrid1d<real_type>& g)
+{
+    double h = g.h();
+    unsigned n = g.n();
+    thrust::host_vector<real_type> out(g.size(), 0.);
+    dg::Operator<real_type> forward = g.dlt().forward();
+    dg::Operator<real_type> backward = g.dlt().backward();
+    dg::Operator<real_type> ninj = create::ninj<real_type>( n );
+    Operator<real_type> t = create::pipj_inv<real_type>(n);
+    t *= h/2.;
+    ninj = backward*t*ninj*forward;
+    real_type constant = 0.;
+
+    for( unsigned i=0; i<g.N(); i++)
+    {
+        for( unsigned k=0; k<n; k++)
+        {
+            for( unsigned l=0; l<n; l++)
+                out[ i*n + k] += ninj(k,l)*in[ i*n + l];
+            out[ i*n + k] += constant;
+        }
+        for( unsigned l=0; l<n; l++)
+            constant += h*forward(0,l)*in[i*n+l];
+    }
+    return out;
+}
+
+
+/*!@brief Indefinite integral of a function on a grid
+ * \f[ F_h(x) = \int_a^x f_h(x') dx' \f]
+ *
+ * This function first evaluates f on the given grid and then computes
+ *  and returns its indefinite integral
+ * @param f The function to evaluate and then integrate
+ * @param g The grid
+ * @return integral of f on the grid g
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ */
+template< class UnaryOp,class real_type>
+thrust::host_vector<real_type> integrate( UnaryOp f, const RealGrid1d<real_type>& g)
+{
+    thrust::host_vector<real_type> vector = evaluate( f, g);
+    return integrate<real_type>(vector, g);
+}
+///@cond
+template<class real_type>
+thrust::host_vector<real_type> integrate( real_type (f)(real_type), const RealGrid1d<real_type>& g)
+{
+    thrust::host_vector<real_type> vector = evaluate( f, g);
+    return integrate<real_type>(vector, g);
+};
+///@endcond
 
 ///@}
 }//namespace dg
diff --git a/inc/dg/topology/evaluation_t.cu b/inc/dg/topology/evaluation_t.cu
index 94841848c..d4417a4d2 100644
--- a/inc/dg/topology/evaluation_t.cu
+++ b/inc/dg/topology/evaluation_t.cu
@@ -41,10 +41,10 @@ int main()
 {
     std::cout << "This program tests the exblas::dot function. The tests succeed only if the evaluation and grid functions but also the weights and especially the exblas::dot function are correctly implemented and compiled. Furthermore, the compiler implementation of the exp function in the math library must be consistent across platforms to get reproducible results\n";
     std::cout << "A TEST is PASSED if the number in the second column shows EXACTLY 0!\n";
-    unsigned n = 1, Nx = 12, Ny = 28, Nz = 100;
+    unsigned n = 3, Nx = 12, Ny = 28, Nz = 100;
     std::cout << "On Grid "<<n<<" x "<<Nx<<" x "<<Ny<<" x "<<Nz<<"\n";
 
-    dg::Grid1d g1d( 0, M_PI/2., n, Nx);
+    dg::Grid1d g1d( 1, 2, n, Nx);
     dg::Grid2d g2d( 1, 2, 3, 4, n, Nx, Ny);
     dg::Grid3d g3d( 1, 2, 3, 4, 5, 6, n, Nx, Ny, Nz,dg::PER,dg::PER,dg::PER);
 
@@ -106,7 +106,7 @@ int main()
               << "          GCC:     4645210948416067678 (correct)"<<std::endl;
 
     //TEST OF INTEGRAL
-    dg::HVec integral_num = dg::create::integral( dg::evaluate( cos, g1d), g1d);
+    dg::HVec integral_num = dg::integrate( cos, g1d);
     dg::HVec integral_ana = dg::evaluate( sin, g1d);
     dg::blas1::plus( integral_ana, -sin(g1d.x0()));
     dg::blas1::axpby( 1., integral_ana, -1., integral_num);
diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index 21e138a5f..b87ece148 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -47,8 +47,8 @@ std::vector<real_type> coefficients( real_type xn, unsigned n)
     std::vector<real_type> px(n);
     if( xn == -1)
     {
-        for( unsigned i=0; i<n; i++)
-            px[i] = (real_type)pow( -1, i);
+        for( unsigned u=0; u<n; u++)
+            px[u] = (u%2 == 0) ? +1. : -1.;
     }
     else if( xn == 1)
     {
@@ -98,6 +98,13 @@ void assert_contains( real_type X, real_type x0, real_type x1, char const * poin
 ///@endcond
 ///@addtogroup interpolation
 ///@{
+/*!@class hide_bcx_doc
+ * @param bcx determines what to do when a point lies outside the boundary in x. If \c dg::PER, the point will be shifted topologically back onto the domain. Else the
+ * point will be mirrored at the boundary: \c dg::NEU will then simply interpolate at the resulting point, \c dg::DIR will take the negative of the interpolation.
+ (\c dg::DIR_NEU and \c dg::NEU_DIR apply \c dg::NEU / \c dg::DIR to the respective left or right boundary )
+ * This means the result of the interpolation is as if the interpolated function were Fourier transformed with the correct boundary condition and thus extended beyond the grid boundaries.
+*/
+
 /**
  * @brief Create interpolation matrix
  *
@@ -108,10 +115,7 @@ void assert_contains( real_type X, real_type x0, real_type x1, char const * poin
  * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * @param x X-coordinates of interpolation points
  * @param g The Grid on which to operate
- * @param bcx determines what to do when a point lies outside the boundary in x. If \c dg::PER, the point will be shifted topologically back onto the domain. Else the
- * point will be mirrored at the boundary: \c dg::NEU will then simply interpolate at the resulting point, \c dg::DIR will take the negative of the interpolation.
- (\c dg::DIR_NEU and \c dg::NEU_DIR apply \c dg::NEU / \c dg::DIR to the respective left or right boundary )
- * This means the result of the interpolation is as if the interpolated function were Fourier transformed with the correct boundary condition and thus extended beyond the grid boundaries.
+ * @copydoc hide_bcx_doc
  *
  * @return interpolation matrix
  */
@@ -168,10 +172,7 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
  * @param x X-coordinates of interpolation points
  * @param y Y-coordinates of interpolation points (\c y.size() must equal \c x.size())
  * @param g The Grid on which to operate
- * @param bcx determines what to do when a point lies outside the boundary in x. If \c dg::PER, the point will be shifted topologically back onto the domain. Else the
- * point will be mirrored at the boundary: \c dg::NEU will then simply interpolate at the resulting point, \c dg::DIR will take the negative of the interpolation.
- (\c dg::DIR_NEU and \c dg::NEU_DIR apply \c dg::NEU / \c dg::DIR to the respective left or right boundary )
- * This means the result of the interpolation is as if the interpolated function were Fourier transformed with the correct boundary condition and thus extended beyond the grid boundaries.
+ * @copydoc hide_bcx_doc
  * @param bcy analogous to \c bcx, applies to y direction
  *
  * @return interpolation matrix
@@ -319,10 +320,7 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
  * @param y Y-coordinates of interpolation points (\c y.size() must equal \c x.size())
  * @param z Z-coordinates of interpolation points (\c z.size() must equal \c x.size())
  * @param g The Grid on which to operate
- * @param bcx determines what to do when a point lies outside the boundary in x. If \c dg::PER, the point will be shifted topologically back onto the domain. Else the
- * point will be mirrored at the boundary: \c dg::NEU will then simply interpolate at the resulting point, \c dg::DIR will take the negative of the interpolation.
- (\c dg::DIR_NEU and \c dg::NEU_DIR apply \c dg::NEU / \c dg::DIR to the respective left or right boundary )
- * This means the result of the interpolation is as if the interpolated function were Fourier transformed with the correct boundary condition and thus extended beyond the grid boundaries.
+ * @copydoc hide_bcx_doc
  * @param bcy analogous to \c bcx, applies to y direction
  * @param bcz analogous to \c bcx, applies to z direction
  *
@@ -521,6 +519,8 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const aRealTo
 ///@}
 
 
+}//namespace create
+
 /**
  * @brief Transform a vector from XSPACE to LSPACE
  *
@@ -545,35 +545,101 @@ thrust::host_vector<real_type> forward_transform( const thrust::host_vector<real
     return out;
 }
 
-}//namespace create
 
 /**
- * @brief Interpolate a single point
+ * @brief Interpolate a vector on a single point on a 1d Grid
  *
+ * @param v The vector to interpolate in dg::xspace
  * @param x X-coordinate of interpolation point
- * @param y Y-coordinate of interpolation point
- * @param v The vector to interpolate in LSPACE, s.a. dg::forward_transform( )
  * @param g The Grid on which to operate
+ * @copydoc hide_bcx_doc
  *
  * @ingroup interpolation
  * @return interpolated point
  * @note \c g.contains(x,y) must return true
  */
 template<class real_type>
-real_type interpolate( real_type x, real_type y,  const thrust::host_vector<real_type>& v, const aRealTopology2d<real_type>& g )
+real_type interpolate(
+    const thrust::host_vector<real_type>& v,
+    real_type x,
+    const RealGrid1d<real_type>& g,
+    dg::bc bcx = dg::NEU)
 {
     assert( v.size() == g.size());
+    g.shift_topologic( x,x, bcx);
+    //mirror at boundary
+    bool negative = false;
+    create::detail::mirror( negative, x, g.x0(), g.x1(), bcx);
 
-    if (!(x >= g.x0() && x <= g.x1())) {
-        std::cerr << g.x0()<<"< xi = " << x <<" < "<<g.x1()<<std::endl;
-    }
+    //determine which cell (x) lies in
 
-    assert(x >= g.x0() && x <= g.x1());
+    real_type xnn = (x-g.x0())/g.h();
+    unsigned n = (unsigned)floor(xnn);
+    //determine normalized coordinates
 
-    if (!(y >= g.y0() && y <= g.y1())) {
-        std::cerr << g.y0()<<"< yi = " << y <<" < "<<g.y1()<<std::endl;
+    real_type xn =  2.*xnn - (real_type)(2*n+1);
+    //interval correction
+    if (n==g.N()) {
+        n-=1;
+        xn = 1.;
+    }
+    //evaluate 1d Legendre polynomials at (xn)...
+    std::vector<real_type> px = create::detail::coefficients( xn, g.n());
+    dg::Operator<real_type> forward( g.dlt().forward());
+    std::vector<real_type> pxF(g.n(),0);
+    for( unsigned l=0; l<g.n(); l++)
+        for( unsigned k=0; k<g.n(); k++)
+            pxF[l]+= px[k]*forward(k,l);
+    for( unsigned k=0; k<g.n(); k++)
+        px[k] = pxF[k];
+    //these are the matrix coefficients with which to multiply
+    unsigned col_begin = (n)*g.n();
+    //multiply x
+    real_type value = 0;
+    for( unsigned j=0; j<g.n(); j++)
+    {
+        if(negative)
+            value -= v[col_begin + j]*px[j];
+        else
+            value += v[col_begin + j]*px[j];
     }
-    assert( y >= g.y0() && y <= g.y1());
+    return value;
+}
+
+/**
+ * @brief Interpolate a vector on a single point on a 2d Grid
+ *
+ * @param sp Indicate whether the elements of the vector
+ * v are in xspace or lspace
+ *  (choose dg::xspace if you don't know what is going on here,
+ *      It is faster to interpolate in dg::lspace so consider
+ *      transforming v using dg::forward_transform( )
+ *      if you do it very many times)
+ * @param v The vector to interpolate in dg::xspace, or dg::lspace s.a. dg::forward_transform( )
+ * @param x X-coordinate of interpolation point
+ * @param y Y-coordinate of interpolation point
+ * @param g The Grid on which to operate
+ * @copydoc hide_bcx_doc
+ * @param bcy analogous to \c bcx, applies to y direction
+ *
+ * @ingroup interpolation
+ * @return interpolated point
+ * @note \c g.contains(x,y) must return true
+ */
+template<class real_type>
+real_type interpolate(
+    dg::space sp,
+    const thrust::host_vector<real_type>& v,
+    real_type x, real_type y,
+    const aRealTopology2d<real_type>& g,
+    dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU )
+{
+    assert( v.size() == g.size());
+    g.shift_topologic( x,y,x,y, bcx, bcy);
+    //mirror at boundary
+    bool negative = false;
+    create::detail::mirror( negative, x, g.x0(), g.x1(), bcx);
+    create::detail::mirror( negative, y, g.y0(), g.y1(), bcy);
 
     //determine which cell (x,y) lies in
 
@@ -596,23 +662,32 @@ real_type interpolate( real_type x, real_type y,  const thrust::host_vector<real
     }
     //evaluate 2d Legendre polynomials at (xn, yn)...
     std::vector<real_type> px = create::detail::coefficients( xn, g.n()),
-                        py = create::detail::coefficients( yn, g.n());
-    //dg::Operator<real_type> forward( g.dlt().forward());
-    //std::vector<real_type> pxF(g.n(),0), pyF(g.n(), 0);
-    //for( unsigned l=0; l<g.n(); l++)
-    //    for( unsigned k=0; k<g.n(); k++)
-    //    {
-    //        pxF[l]+= px[k]*forward(k,l);
-    //        pyF[l]+= py[k]*forward(k,l);
-    //    }
+                           py = create::detail::coefficients( yn, g.n());
+    if( sp == dg::xspace)
+    {
+        dg::Operator<real_type> forward( g.dlt().forward());
+        std::vector<real_type> pxF(g.n(),0), pyF(g.n(), 0);
+        for( unsigned l=0; l<g.n(); l++)
+            for( unsigned k=0; k<g.n(); k++)
+            {
+                pxF[l]+= px[k]*forward(k,l);
+                pyF[l]+= py[k]*forward(k,l);
+            }
+        for( unsigned k=0; k<g.n(); k++)
+            px[k] = pxF[k], py[k] = pyF[k];
+    }
     //these are the matrix coefficients with which to multiply
     unsigned col_begin = (m)*g.Nx()*g.n()*g.n() + (n)*g.n();
     //multiply x
     real_type value = 0;
     for( unsigned i=0; i<g.n(); i++)
         for( unsigned j=0; j<g.n(); j++)
-            value += v[col_begin + i*g.Nx()*g.n() + j]*px[j]*py[i];
-            //value += v[col_begin + i*g.Nx()*g.n() + j]*pxF[j]*pyF[i];
+        {
+            if(negative)
+                value -= v[col_begin + i*g.Nx()*g.n() + j]*px[j]*py[i];
+            else
+                value += v[col_begin + i*g.Nx()*g.n() + j]*px[j]*py[i];
+        }
     return value;
 }
 
diff --git a/inc/dg/topology/interpolation_t.cu b/inc/dg/topology/interpolation_t.cu
index 82346f507..25584b3b8 100644
--- a/inc/dg/topology/interpolation_t.cu
+++ b/inc/dg/topology/interpolation_t.cu
@@ -40,7 +40,7 @@ int main()
             y[i*g.Nx()*g.n() + j] =
                     g.y0() + 2*g.ly() + (i+0.5)*g.hy()/(double)(g.n());
         }
-    //typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
+    //use DIR because the coo.2d is zero on the right boundary
     Matrix B = dg::create::interpolation( x, y, g, dg::DIR, dg::DIR);
     //values outside the grid are mirrored back in
 
@@ -59,34 +59,17 @@ int main()
         std::cout<< "2D TEST FAILED!\n";
     else
         std::cout << "2D TEST PASSED!\n";
-    //cusp::print(A);
-    //cusp::print(B);
-    //ATTENTION: backscatter might delete zeroes in matrices
-    //for( unsigned i=0; i<A.values.size(); i++)
-    //{
-    //    if( (A.values[i] - B.values[i]) > 1e-14)
-    //    {
-    //        std::cerr << "NOT EQUAL "<<A.row_indices[i] <<" "<<A.column_indices[i]<<" "<<A.values[i] << "\t "<<B.row_indices[i]<<" "<<B.column_indices[i]<<" "<<B.values[i]<<"\n";
-    //        passed = false;
-    //    }
-    //}
-    //if( A.num_entries != B.num_entries)
-    //{
-    //    std::cerr << "Number of entries not equal!\n";
-    //    passed = false;
-    //}
 
 
     bool passed = true;
     thrust::host_vector<double> xs = dg::evaluate( dg::cooX2d, g);
     thrust::host_vector<double> ys = dg::evaluate( dg::cooY2d, g);
-    thrust::host_vector<double> xF = dg::create::forward_transform( xs, g);
-    thrust::host_vector<double> yF = dg::create::forward_transform( ys, g);
+    thrust::host_vector<double> xF = dg::forward_transform( xs, g);
     for( unsigned i=0; i<x.size(); i++)
     {
-        x[i] -= g.lx(), y[i] -= 2*g.ly();
-        double xi = dg::interpolate(x[i],y[i], xF, g);
-        double yi = dg::interpolate(x[i],y[i], yF, g);
+        //use DIR because the coo.2d is zero on the right boundary
+        double xi = dg::interpolate(dg::lspace, xF, x[i],y[i], g, dg::DIR, dg::DIR);
+        double yi = dg::interpolate(dg::xspace, ys, x[i],y[i], g, dg::DIR, dg::DIR);
         if( x[i] - xi > 1e-14)
         {
             std::cerr << "X NOT EQUAL "<<i<<"\t"<<x[i]<<"  \t"<<xi<<"\n";
@@ -101,6 +84,26 @@ int main()
     if( passed)
         std::cout << "2D INTERPOLATE TEST PASSED!\n";
     }
+    {
+    bool passed = true;
+    dg::Grid1d g1d( -M_PI, 0, n, Nx);
+    thrust::host_vector<double> xs = dg::evaluate( dg::cooX1d, g1d);
+    thrust::host_vector<double> x( g1d.size());
+    for( unsigned i=0; i<x.size(); i++)
+    {
+        //create equidistant values
+        x[i] = g1d.x0() + g1d.lx() + (i+0.5)*g1d.h()/(double)(g1d.n());
+        //use DIR because the cooX1d is zero on the right boundary
+        double xi = dg::interpolate( xs, x[i], g1d, dg::DIR);
+        if( x[i] - xi > 1e-14)
+        {
+            std::cerr << "X NOT EQUAL "<<i<<"\t"<<x[i]<<"  \t"<<xi<<"\n";
+            passed = false;
+        }
+    }
+    if( passed)
+        std::cout << "1D INTERPOLATE TEST PASSED!\n";
+    }
     ////////////////////////////////////////////////////////////////////////////
     {
     dg::Grid3d g( -10, 10, -5, 5, -7, -3, n, Nx, Ny, Nz);
@@ -123,7 +126,6 @@ int main()
                 z[(k*g.Ny()*g.n() + i)*g.Nx()*g.n() + j] =
                         g.z0() + (k+0.5)*g.hz();
             }
-    //typedef cusp::coo_matrix<int, double, cusp::host_memory> Matrix;
     Matrix B = dg::create::interpolation( x, y, z, g);
     const thrust::host_vector<double> vec = dg::evaluate( function, g);
     thrust::host_vector<double> inter(vec);
@@ -138,18 +140,6 @@ int main()
         std::cout<< "3D TEST FAILED!\n";
     else
         std::cout << "3D TEST PASSED!\n";
-
-    //bool passed = true;
-    //for( unsigned i=0; i<A.values.size(); i++)
-    //{
-    //    if( (A.values[i] - B.values[i]) > 1e-14)
-    //    {
-    //        std::cerr << "NOT EQUAL "<<A.row_indices[i] <<" "<<A.column_indices[i]<<" "<<A.values[i] << "\t "<<B.row_indices[i]<<" "<<B.column_indices[i]<<" "<<B.values[i]<<"\n";
-    //        passed = false;
-    //    }
-    //}
-    //if( passed)
-    //    std::cout << "3D TEST PASSED!\n";
     }
 
     return 0;
diff --git a/inc/dg/topology/weights.h b/inc/dg/topology/weights.h
index 9cf509e59..ad78605b6 100644
--- a/inc/dg/topology/weights.h
+++ b/inc/dg/topology/weights.h
@@ -2,7 +2,6 @@
 
 #include <thrust/host_vector.h>
 #include "grid.h"
-#include "operator.h"
 #include "../enums.h"
 
 /*! @file
@@ -57,47 +56,6 @@ thrust::host_vector<real_type> inv_weights( const RealGrid1d<real_type>& g)
     return v;
 }
 
-/*!@brief Create the integral of a function on a grid
- * \f[ F_h(x) = \int_a^x f_h(x') dx' \f]
- *
- * This function computes the indefinite integral of a given input
- * @param in Host vector discretized on g
- * @param g The grid
- * @return integral of in on the grid g
- * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
- */
-template<class real_type>
-thrust::host_vector<real_type> integral( const thrust::host_vector<real_type>& in, const RealGrid1d<real_type>& g)
-{
-    double h = g.h();
-    unsigned n = g.n();
-    thrust::host_vector<real_type> out(in.size(), 0);
-    dg::Operator<real_type> forward = g.dlt().forward();
-    dg::Operator<real_type> backward = g.dlt().backward();
-    dg::Operator<real_type> ninj = create::ninj<real_type>( n );
-    Operator<real_type> t = create::pipj_inv<real_type>(n);
-    t *= h/2.;
-    ninj = backward*t*ninj*forward;
-    real_type constant = 0.;
-
-    for( unsigned i=0; i<g.N(); i++)
-    {
-        for( unsigned k=0; k<n; k++)
-        {
-            for( unsigned l=0; l<n; l++)
-            {
-                out[ i*n + k] += ninj(k,l)*in[ i*n + l];
-            }
-            out[ i*n + k] += constant;
-        }
-        for( unsigned l=0; l<n; l++)
-            constant += h*forward(0,l)*in[i*n+l];
-    }
-    return out;
-
-
-}
-
 ///@cond
 namespace detail{
 
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index aa635b40e..bde927fe8 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -72,9 +72,9 @@ struct DSField
         dg::blas1::pointwiseDivide(v_zeta, v_phi, v_zeta);
         dg::blas1::pointwiseDivide(v_eta, v_phi, v_eta);
         dg::blas1::pointwiseDivide(1.,    v_phi, v_phi);
-        dzetadphi_  = dg::create::forward_transform( v_zeta, g );
-        detadphi_   = dg::create::forward_transform( v_eta, g );
-        dsdphi_     = dg::create::forward_transform( v_phi, g );
+        dzetadphi_  = dg::forward_transform( v_zeta, g );
+        detadphi_   = dg::forward_transform( v_eta, g );
+        dsdphi_     = dg::forward_transform( v_phi, g );
     }
     //interpolate the vectors given in the constructor on the given point
     //if point lies outside of grid boundaries zero is returned
@@ -82,16 +82,10 @@ struct DSField
     {
         double R = y[0], Z = y[1];
         g_->shift_topologic( y[0], y[1], R, Z); //shift R,Z onto domain
-        if( !g_->contains( R, Z))
         {
-            yp[0] = yp[1] = 0; //Let's hope this never happens?
-        }
-        else
-        {
-            //else interpolate
-            yp[0] = interpolate( R, Z, dzetadphi_, *g_);
-            yp[1] = interpolate( R, Z, detadphi_,  *g_);
-            yp[2] = interpolate( R, Z, dsdphi_,    *g_);
+            yp[0] = interpolate(dg::lspace, dzetadphi_, R, Z, *g_);
+            yp[1] = interpolate(dg::lspace, detadphi_,  R, Z, *g_);
+            yp[2] = interpolate(dg::lspace, dsdphi_,    R, Z, *g_);
         }
     }
     private:
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 3cee6de9a..62a3690dc 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -261,7 +261,11 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( volX2d, psip_X, psip_X);
     avg_eta( psip_X, map1d["X_psip1d"], false);
     avg_eta( volX2d, area_psip_X, false);
-    dg::blas1::pointwiseDivide( map1d.at("X_psip1d"), area_psip_X, map1d.at("X_psip1d"));
+    dg::Grid1d gX1d( gX2d.x0(), gX2d.x1(), npsi, Npsi, dg::NEU);
+    map1d["X_psi_vol"] = dg::integrate( area_psip_X, gX1d);
+    dg::blas1::scal( map1d.at("X_psi_vol"), 4.*M_PI*M_PI);
+    dg::blas1::pointwiseDivide( map1d.at("X_psip1d"), area_psip_X,
+        map1d.at("X_psip1d"));
 
     //NOTE: VOLUME is WITHIN cells while AREA is ON gridpoints
     dg::HVec gradPsipX = metricX.value(0,0);
@@ -270,24 +274,19 @@ int main( int argc, char* argv[])
     avg_eta( gradPsipX, map1d["X_psi_area"], false);
     dg::blas1::scal( map1d.at("X_psi_area"), 4.*M_PI*M_PI);
 
-    dg::geo::FluxVolumeIntegral<dg::HVec> fvi_X( gX2d, c);
-    fvi_X.set_left( coordsX[0]);
-    dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
-    map1d["X_psi_vol"] = dg::evaluate( fvi_X, grid1d);
-    dg::blas1::scal( map1d.at("X_psi_vol"), 2.*M_PI);
-
-
     ///////////////////Compute flux average////////////////////
     dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
     if( gp.hasXpoint() )
         dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
     dg::geo::SafetyFactor     qprof( c);
     dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, c, psipog2d, xpoint_weights);
+    dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
     map1d["psi_fsa"]    = dg::evaluate( fsa,        grid1d);
     map1d["q-profile"]  = dg::evaluate( qprof,      grid1d);
     map1d["psip1d"]     = dg::evaluate( dg::cooX1d, grid1d);
     map1d["rho"] = map1d.at("psip1d");
-    dg::blas1::axpby( -1./psipmin, map1d.at("rho"), +1., 1., map1d.at("rho")); //transform psi to rho
+    dg::blas1::axpby( -1./psipmin, map1d.at("rho"), +1., 1.,
+        map1d.at("rho")); //transform psi to rho
 
     //other flux labels
     dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, c);
@@ -309,7 +308,7 @@ int main( int argc, char* argv[])
     map1d["psi_vol"] = dg::evaluate( fvi, grid1d);
     dg::blas1::scal(map1d["psi_vol"], 2.*M_PI);
     double volumeFVI = 2.*M_PI*fvi(psipmax);
-    double volumeFVIX = 2.*M_PI*fvi_X(psipmax);
+    double volumeFVIX = dg::interpolate( map1d.at("X_psi_vol"), gX1d.x1(), gX1d);
     std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeCoarea<<" "<<volumeFVI
               <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
     std::cout << "VOLUME TEST WITH X Grid FORMULA: "<<volumeFVIX<<" "<<volumeFVI
-- 
GitLab


From ef3249f3fc005a37a58d871828b7315bb5887494 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Apr 2019 16:47:24 +0200
Subject: [PATCH 068/540] Work on vorticity equation

---
 inc/geometries/geometry_diag.cu | 79 ++++++++++++++++++---------------
 src/feltor/feltor.tex           | 31 +++++++++++--
 2 files changed, 70 insertions(+), 40 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 62a3690dc..6a86119b6 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -239,40 +239,46 @@ int main( int argc, char* argv[])
     dg::DVec psipog2d   = dg::evaluate( c.psip(), grid2d);
     std::map<std::string, dg::HVec> map1d;
     ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
-    std::cout << "Generate X-point flux-aligned grid!\n";
-    double RX = gp.R_0-1.1*gp.triangularity*gp.a;
-    double ZX = -1.1*gp.elongation*gp.a;
-    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( c.get_psip(), RX, ZX) ;
-    dg::geo::SeparatrixOrthogonal generator(c.get_psip(), monitor_chi, psipmin, R_X, Z_X, c.R0(), 0, 0, true);
-    double fx_0 = 1./8.;
-    double psipmax = -fx_0/(1.-fx_0)*psipmin;
-    std::cout << "psi 1 is          "<<psipmax<<"\n";
-    unsigned npsi = 3, Npsi = 32;//set number of psivalues
-    dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., npsi, Npsi, 160, dg::DIR, dg::NEU);
-    std::cout << "DONE! Generate X-point flux-aligned grid!\n";
-    dg::Average<dg::HVec > avg_eta( gX2d.grid(), dg::coo2d::y);
-    std::vector<dg::HVec> coordsX = gX2d.map();
-    dg::SparseTensor<dg::HVec> metricX = gX2d.metric();
-    dg::HVec volX2d = dg::tensor::volume2d( metricX);
-    dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
-    dg::IHMatrix grid2gX2d = dg::create::interpolation( coordsX[0], coordsX[1], grid2d);
-    dg::HVec psip_X = dg::evaluate( dg::one, gX2d), area_psip_X;
-    dg::blas2::symv( grid2gX2d, (dg::HVec)psipog2d, psip_X);
-    dg::blas1::pointwiseDot( volX2d, psip_X, psip_X);
-    avg_eta( psip_X, map1d["X_psip1d"], false);
-    avg_eta( volX2d, area_psip_X, false);
-    dg::Grid1d gX1d( gX2d.x0(), gX2d.x1(), npsi, Npsi, dg::NEU);
-    map1d["X_psi_vol"] = dg::integrate( area_psip_X, gX1d);
-    dg::blas1::scal( map1d.at("X_psi_vol"), 4.*M_PI*M_PI);
-    dg::blas1::pointwiseDivide( map1d.at("X_psip1d"), area_psip_X,
-        map1d.at("X_psip1d"));
+    unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
+    double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
+    double volumeXGrid;
+    if( gp.hasXpoint())
+    {
+        std::cout << "Generate X-point flux-aligned grid!\n";
+        double RX = gp.R_0-1.1*gp.triangularity*gp.a;
+        double ZX = -1.1*gp.elongation*gp.a;
+        dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( c.get_psip(), RX, ZX) ;
+        dg::geo::SeparatrixOrthogonal generator(c.get_psip(), monitor_chi, psipmin, R_X, Z_X, c.R0(), 0, 0, true);
+        double fx_0 = 1./8.;
+        psipmax = -fx_0/(1.-fx_0)*psipmin;
+        std::cout << "psi 1 is          "<<psipmax<<"\n";
+        dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., npsi, Npsi, 160, dg::DIR, dg::NEU);
+        std::cout << "DONE! Generate X-point flux-aligned grid!\n";
+        dg::Average<dg::HVec > avg_eta( gX2d.grid(), dg::coo2d::y);
+        std::vector<dg::HVec> coordsX = gX2d.map();
+        dg::SparseTensor<dg::HVec> metricX = gX2d.metric();
+        dg::HVec volX2d = dg::tensor::volume2d( metricX);
+        dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
+        dg::IHMatrix grid2gX2d = dg::create::interpolation( coordsX[0], coordsX[1], grid2d);
+        dg::HVec psip_X = dg::evaluate( dg::one, gX2d), area_psip_X;
+        dg::blas2::symv( grid2gX2d, (dg::HVec)psipog2d, psip_X);
+        dg::blas1::pointwiseDot( volX2d, psip_X, psip_X);
+        avg_eta( psip_X, map1d["X_psip1d"], false);
+        avg_eta( volX2d, area_psip_X, false);
+        dg::Grid1d gX1d( gX2d.x0(), gX2d.x1(), npsi, Npsi, dg::NEU);
+        map1d["X_psi_vol"] = dg::integrate( area_psip_X, gX1d);
+        dg::blas1::scal( map1d.at("X_psi_vol"), 4.*M_PI*M_PI);
+        dg::blas1::pointwiseDivide( map1d.at("X_psip1d"), area_psip_X,
+            map1d.at("X_psip1d"));
 
-    //NOTE: VOLUME is WITHIN cells while AREA is ON gridpoints
-    dg::HVec gradPsipX = metricX.value(0,0);
-    dg::blas1::transform( gradPsipX, gradPsipX, dg::SQRT<double>());
-    dg::blas1::pointwiseDot( volX2d, gradPsipX, gradPsipX); //R\sqrt{g}|\nabla\zeta|
-    avg_eta( gradPsipX, map1d["X_psi_area"], false);
-    dg::blas1::scal( map1d.at("X_psi_area"), 4.*M_PI*M_PI);
+        //NOTE: VOLUME is WITHIN cells while AREA is ON gridpoints
+        dg::HVec gradPsipX = metricX.value(0,0);
+        dg::blas1::transform( gradPsipX, gradPsipX, dg::SQRT<double>());
+        dg::blas1::pointwiseDot( volX2d, gradPsipX, gradPsipX); //R\sqrt{g}|\nabla\zeta|
+        avg_eta( gradPsipX, map1d["X_psi_area"], false);
+        dg::blas1::scal( map1d.at("X_psi_area"), 4.*M_PI*M_PI);
+        volumeXGrid = dg::interpolate( map1d.at("X_psi_vol"), gX1d.x1(), gX1d);
+    }
 
     ///////////////////Compute flux average////////////////////
     dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
@@ -308,11 +314,12 @@ int main( int argc, char* argv[])
     map1d["psi_vol"] = dg::evaluate( fvi, grid1d);
     dg::blas1::scal(map1d["psi_vol"], 2.*M_PI);
     double volumeFVI = 2.*M_PI*fvi(psipmax);
-    double volumeFVIX = dg::interpolate( map1d.at("X_psi_vol"), gX1d.x1(), gX1d);
     std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeCoarea<<" "<<volumeFVI
               <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
-    std::cout << "VOLUME TEST WITH X Grid FORMULA: "<<volumeFVIX<<" "<<volumeFVI
-              <<" rel error = "<<fabs(volumeFVIX-volumeFVI)/volumeFVI<<"\n";
+    if(gp.hasXpoint()){
+        std::cout << "VOLUME TEST WITH X Grid FORMULA: "<<volumeXGrid<<" "<<volumeFVI
+                  <<" rel error = "<<fabs(volumeXGrid-volumeFVI)/volumeFVI<<"\n";
+    };
 
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 30ea46ab8..b15902814 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -618,7 +618,24 @@ with the particle flux
 \begin{align} \label{eq:mass_flux}
 \vec{ j_{n_e}} := n_e\left(\vec v_E + \vec v_C + \vec v_{K} + U (\bhat + \vec{\tilde b}_\perp)\right)
 \end{align}
-\subsubsection{Charge/vorticity conservation}
+\subsubsection{Vorticity density}
+First of all let us formally re-express $N_i = \Gamma_{1,i}\Gamma^{-1}_{1,i}N_i = \Gamma_{1,i} N_i- \frac{T_i}{2 m_i \Omega_0^2} \Gamma_{1,i}\Delta_\perp  N_i$.
+With this identiy we can write the polarisation equation as
+\begin{align}\label{eq:general_polarisation}
+\nabla\cdot\left(\frac{N_i}{\Omega_i B}\nabla_\perp\phi\right) 
++ \frac{1}{2} \frac{T_i}{m_i\Omega_0^2} \Gamma_{1,i}\Delta_\perp N_i
+= n_e - N_i 
+\end{align}
+Let us now define
+\begin{align}
+\mathcal W_E &:= \Omega_{i0} \nabla\cdot\left(\frac{N_i}{\Omega_i B}\nabla_\perp\phi\right)
+=
+\frac{\Omega_{i0}}{\Omega_i}\bhat \cdot(\nabla\times N_i \vec u_E) - \Omega_{i0} N_i \vec{\mathcal K} \cdot \vec u_E
+\\
+\mathcal W_d &:= \frac{T_i}{m_i\Omega_{i0}} \Gamma_{1,i}\Delta_\perp N_i
+\end{align}
+which are the generalized \ExB and diamagnetic vorticity densities.
+
 Integrating the polarisation equation~\eqref{eq:polarisation_dimensional}
 and assuming $\Gamma_{1,i}S_{N_i} = S_{n_e}$ we find
 \begin{align} \label{eq:charge_conservation}
@@ -996,7 +1013,7 @@ A numerically tractable approximation to the delta-function reads
 where $\epsilon$ is a small, free parameter.
 In the DG framework the left-hand side
 of Eq.~\eqref{eq:dirac_delta} can thus readily be computed
-via Gauss-Legendre quadrature, which gives us an easy means to compute area
+via Gauss-Legendre quadrature, which we propse as a first method to compute area
 integrals even if our coordinate system is not aligned to the area.
 
 Furthermore, recall the {\bf co-area formula}
@@ -1022,10 +1039,14 @@ of the flux 2-form and the metric
 to a parameterization of the flux-surface.
 In a flux-aligned coordinate system $\{\zeta, \eta, \varphi\}$ the pull-back is trivial ($\zeta=const$) and we have with $\hat\zeta = \hat \psi_p$
 \begin{align}
-\dA = \sqrt{g^{\zeta\zeta}} \sqrt{g} \d\eta\d\varphi, \quad \vec\dA := \hat\zeta \dA
+\dA = \sqrt{g^{\zeta\zeta}} \sqrt{g} \d\eta\d\varphi, \quad \vec\dA := \hat\zeta \dA,\quad
+\hat\zeta=\frac{\nabla\psi_p}{|\nabla\psi_p|}
 \label{}
 \end{align}
-where we used that $g^{\zeta\zeta} = (\nabla\zeta)^2$
+where we used that $g^{\zeta\zeta} = (\nabla\zeta)^2 = f_0^2(\nabla\psi_p)^2$.
+Notice that numerically we can integrate in flux-aligned coordinates by generating a corresponding
+grid and pulling back (interpolating) the relevant fields to this grid. This is the second method
+to numerically compute area integrals.
 
 \subsection{Flux surface average}
 
@@ -1039,6 +1060,8 @@ of a function $f(R,Z,\varphi)$ is given by the formula
 \frac{ \oint_{\psi_p  } f(R,Z,\varphi)\dA}{\oint_{\psi_p } \dA} \nonumber \\
 =& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vec\nabla\psi_p| \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
 {\int_\Omega |\vec\nabla\psi_p|\delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z} \nonumber\\
+=& \frac{\oint \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g g^{\zeta\zeta}}\d\eta}
+         { \oint \sqrt{g g^{\zeta\zeta}}\d\eta}
 \end{align}
 %with $\dV := R\d R\d Z\d \varphi$ %(we define the average in computational space and omit one $R$)
 and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
-- 
GitLab


From 1b9f4fc092bf9dba381dac2853b3993ce1a60e8d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 14 Apr 2019 23:59:28 +0200
Subject: [PATCH 069/540] Think about diamagnetic vorticity

---
 doc/related_pages/newcommands.tex |   6 ++
 src/feltor/feltor.tex             | 101 +++++++++++++++++++++---------
 2 files changed, 79 insertions(+), 28 deletions(-)

diff --git a/doc/related_pages/newcommands.tex b/doc/related_pages/newcommands.tex
index 1a7a72786..594221907 100644
--- a/doc/related_pages/newcommands.tex
+++ b/doc/related_pages/newcommands.tex
@@ -50,3 +50,9 @@
 \newcommand{\Tp}{\mathcal T^+_{\Delta\varphi}}
 \newcommand{\Tm}{\mathcal T^-_{\Delta\varphi}}
 \newcommand{\Tpm}{\mathcal T^\pm_{\Delta\varphi}}
+%%%%%%%%Some useful abbreviations %%%%%%%%%%%%%%%%
+\def\feltor{{\textsc{Feltor }}}
+
+\def\fixme#1{\typeout{FIXME in page \thepage :{#1}}%
+ \textsc{\color{red}[{#1}]}}
+
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index b15902814..eec39d57e 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -618,40 +618,80 @@ with the particle flux
 \begin{align} \label{eq:mass_flux}
 \vec{ j_{n_e}} := n_e\left(\vec v_E + \vec v_C + \vec v_{K} + U (\bhat + \vec{\tilde b}_\perp)\right)
 \end{align}
-\subsubsection{Vorticity density}
-First of all let us formally re-express $N_i = \Gamma_{1,i}\Gamma^{-1}_{1,i}N_i = \Gamma_{1,i} N_i- \frac{T_i}{2 m_i \Omega_0^2} \Gamma_{1,i}\Delta_\perp  N_i$.
+\subsubsection{Polarisation charge density}
+First of all let us approximate FLR effects in the long wavelength limit (LWL)
+$\Gamma_{1,i}N_i \approx N_i + \frac{T_i}{2 m_i \Omega_0^2} \Delta_\perp  N_i$.
 With this identiy we can write the polarisation equation as
 \begin{align}\label{eq:general_polarisation}
 \nabla\cdot\left(\frac{N_i}{\Omega_i B}\nabla_\perp\phi\right) 
-+ \frac{1}{2} \frac{T_i}{m_i\Omega_0^2} \Gamma_{1,i}\Delta_\perp N_i
++ \frac{1}{2} \frac{T_i}{m_i\Omega_0^2} \Delta_\perp N_i
 = n_e - N_i 
 \end{align}
 Let us now define
 \begin{align}
-\mathcal W_E &:= \Omega_{i0} \nabla\cdot\left(\frac{N_i}{\Omega_i B}\nabla_\perp\phi\right)
+\rho_E &:= -\nabla\cdot\left(\frac{N_i}{\Omega_i B}\nabla_\perp\phi\right)
+\equiv \nabla\cdot\left(\frac{N_i}{\Omega_i} \bhat\times\vec u_E\right)
 =
-\frac{\Omega_{i0}}{\Omega_i}\bhat \cdot(\nabla\times N_i \vec u_E) - \Omega_{i0} N_i \vec{\mathcal K} \cdot \vec u_E
+-\frac{\Omega_{i0}}{\Omega_i}\bhat \cdot(\nabla\times N_i \vec u_E)
++ \Omega_{i0} N_i \vec{\mathcal K} \cdot \vec u_E
 \\
-\mathcal W_d &:= \frac{T_i}{m_i\Omega_{i0}} \Gamma_{1,i}\Delta_\perp N_i
+\mathcal \rho_D &:= -\frac{T_i}{m_i\Omega^2_{i0}} \Delta_\perp N_i
+\equiv \nabla\cdot\left( \frac{N_i}{\Omega_{i0}} \bhat\times \vec v_{d0}\right) \quad \vec v_{d0} := \frac{\bhat\times \nabla T_i N_i}{eB_0N_i}
 \end{align}
 which are the generalized \ExB and diamagnetic vorticity densities.
 
-Integrating the polarisation equation~\eqref{eq:polarisation_dimensional}
-and assuming $\Gamma_{1,i}S_{N_i} = S_{n_e}$ we find
-\begin{align} \label{eq:charge_conservation}
-  \frac{\partial}{\partial t} \int_\Omega \dV \mathcal W
-  + \int_{\partial\Omega} \vec\dA\cdot\vec{ j_{\mathcal W}}
-  =  \int_\Omega \dV \Lambda_{\mathcal W} \nonumber\\
-\end{align}
-with the definitions
+Now, we take the time derivative of Eq.~\eqref{eq:general_polarisation}
+to get (neglecting terms in the LWL, \fixme{Confirm, units!})
 \begin{align}
-\mathcal W &:= \vec{\nabla} \cdot\left(\frac{m_iN_i}{e B^2} \vec{\nabla}_\perp \phi\right) \nonumber\\
-\vec{j_{\mathcal W}} &:= \vec j_{n_e} - \vec j_{N_i} \nonumber\\
-\Lambda_{\mathcal W} &:= \Lambda_{n_e} - \Lambda_{N_i}
-\end{align}
-where the polarization charge density is approximately related to the \ExB vorticity through
-$\mathcal W = -\vec{\nabla} \cdot\left(\frac{m_iN_i}{e B} \bhat \times \vec u_E\right) \approx \bhat\cdot\left(\vec\nabla \times\frac{m_iN_i}{e B}  \vec u_E\right)$
-and we used that $\Gamma_{1,i}^\dagger 1 = 1$.
+ &\frac{\partial}{\partial t}\left[
+\rho_E + \rho_D
+\right]
++ \nabla\cdot\left( \vec u_E \left[
+\rho_E + \rho_D
+\right]
+\right)
+- \nabla\cdot(N_i \vec u_P)
+\nonumber\\
+&= \nabla\cdot\left( \frac{\bhat\times\vec\nabla (t_e n_e + T_i N_i)}{eN_iB}N_i\right)
+\nonumber\\
+&+ \nabla\cdot\left( \frac{\nabla\times\bhat}{B}\frac{1}{2}\left(m_e n_e u_e^2 + m_i N_i U_i^2\right) \right)
+\nonumber\\
+&+ \nabla\cdot\left(\bhat j_\parallel\right)
++ \nabla\cdot\left(\frac{\nabla\times A_\parallel \bhat}{B} \Delta_\perp A_\parallel\right)
++ \Lambda_{N_i} - \Lambda_{n_e}
+\\
+\vec u_P &:=\frac{\bhat\times \nabla u_E^2/2}{\Omega_i}\nonumber \\
+&\nabla\cdot \left(\vec j_{pol} + \vec j_D + \vec j_{\nabla\times\bhat} + \vec j_\parallel + \vec j_{MW} + \vec j_{Diff}\right) = 0
+\label{eq:current_divergence}
+\end{align}
+Notice that this equation is exact for $T_i=0$ and holds in the LWL
+otherwise. Furthermore, we assumed $\Gamma_{1,i}S_{N_i} = S_{n_e}$.
+In slab geometry $\bhat\equiv \hat z$ and $B=B(x)$ this equation reduces
+to the familiar drift-fluid quasineutrality equation \fixme{recompute, units!}
+\begin{align}
+ &\nabla\cdot\frac{\partial}{\partial t}\left[
+\frac{\zhat\times \nabla\cdot( N_i (\vec u_E ))}{\Omega_i}
++
+\frac{\zhat\times \nabla\cdot( N_i (\vec u_{d0}))}{\Omega_{i0}}
+\right]
++ \nabla\cdot\left(
+\frac{\zhat\times\vec \nabla\cdot(N_i\vec u_E \vec u_E)}{\Omega_i} 
++
+\frac{\zhat\times\vec \nabla\cdot(N_i\vec u_E \vec u_{d0})}{\Omega_{i0}} 
+\right)
+\nonumber\\
+&= \nabla\cdot\left( \frac{\zhat\times\vec\nabla (t_e n_e + T_i N_i)}{eN_iB}N_i\right)
+\nonumber\\
+%&+ \nabla\cdot\left( \frac{\nabla\times\bhat}{B}\frac{1}{2}\left(m_e n_e u_e^2 + m_i N_i U_i^2\right) \right)
+%\nonumber\\
+&+ \nabla\cdot\left(\zhat j_\parallel\right)
++ \nabla\cdot\left(\frac{\zhat\times \vec\nabla\cdot\left(
+\tilde{\vec B}_\perp\tilde{\vec B}_\perp\right)}{B_0} \right)
+\quad
+\tilde{\vec B}_\perp := \nabla \times A_\parallel \zhat
+%\label{eq:slab_divergence}
+\end{align}
+\fixme{use Mathematika for maybe slab/toroidal equation in 3d geometry}
 
 \subsubsection{Energy theorem}
 The terms of the energy theorem are
@@ -763,9 +803,9 @@ The terms of the particle conservation read
 \end{align}
 and the vorticity conservation
 \begin{align} \label{eq:vorticity_conservation}
-\mathcal W &= \vec{\nabla} \cdot\left(\frac{N_i}{B^2} \vec{\nabla}_\perp \phi\right) \nonumber\\
-\vec{j_{\mathcal W}} &= \vec j_{n_e} - \vec j_{N_i} \nonumber\\
-\Lambda_{\mathcal W} &= \Lambda_{n_e} - \Lambda_{N_i}
+\mathcal \rho_{pol} &= -\vec{\nabla} \cdot\left(\frac{N_i}{B^2} \vec{\nabla}_\perp \phi\right)  - \tau_i \Delta_\perp N_i\nonumber\\
+\vec{j_{pol}} &= \vec j_{N_i} - \vec j_{n_e} \nonumber\\
+\Lambda_{ pol} &= \Lambda_{N_i} - \Lambda_{n_e}
 \end{align}
 
 The terms of the energy theorem read (with $z_e=-1$ and $z_i=+1$)
@@ -1157,15 +1197,18 @@ Now, let us integrate the vorticity density over a flux aligned volume to get
 \int \dV \mathcal W =\int \dV \nabla\cdot\left(\frac{N_i\nabla_\perp\phi}{B^2}\right)  =  \int \dA u^{\hat \eta} \frac{N_i}{B}
 \end{align}
 from where we get the flux-surface (area) integrated vorticity conservation
+\fixme{This is the Eq for the radial current / or radial Force? / or poloidal/toroidal flux}
 \begin{align}
-\frac{\partial}{\partial t}\int\frac{N_iu^{\hat \eta}}{B}\dA
- = \int\frac{( \vec j \times\vec B)\cdot\hat\eta}{B} \dA
+\frac{\partial}{\partial t}\int\vec\dA\cdot\left(\frac{N_i\vec\nabla_\perp\phi}{B^2} +
+\vec \nabla_\perp N_i\right)
+ = \int \vec j  \cdot\vec \dA
  + \int\dV \Lambda_{n_e} - \Lambda_{N_i}
  \end{align}
  with $(\vec j\times \vec B)\cdot\hat \eta/B =(\vec j_{N_i} - \vec j_{n_e})\cdot \hat\zeta $ and
 \begin{align}
  (\vec j_{N_i} - \vec j_{n_e})\cdot\vec \nabla\psi_p  =
- \frac{N_i}{B}[\psi,\psi_p]_\perp - \frac{n_e}{B}[\phi,\psi_p]_\perp
+ \frac{\rho_E + \rho_d}{B}[\phi,\psi_p]_\perp 
+ - \frac{N_i}{B}[u_E^2/2,\psi_p]_\perp
  \nonumber\\
  + \left( \mu_iN_i U_i^2 - \mu_e n_e u_e^2\right) \mathcal K_{\nabla\times\bhat}(\psi_p)
  + \left(\tau_i N_i - \tau_e n_e  \right) \mathcal K(\psi_p)
@@ -1179,10 +1222,12 @@ that the prefactors are always positive since $\tau_e$ and $\mu_e$ are negative.
 The curvature vectors counter-align with $\hat \zeta$ ($\mathcal K(\psi_p) < 0$, decelerate) on the top 
 and align ($\mathcal K(\psi_p) > 0$, accelerate) on the bottom of the tokamak.
 (Is this related to the X-point orbit loss mechanism? Chang, Ku, et al. )
-
 A surplus of electrons in the domain will lead to counter-clockwise poloidal acceleration, while positive
 charge leads to clockwise poloidal acceleration.
 
+Notice that $\hat\zeta\cdot\nabla N_i < 0$.
+This means that a radial
+pressure gradient balances the radial electric field.
 
 
 
-- 
GitLab


From e332716c1f1772c4004fe8db43867141fb83129c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 2 May 2019 23:41:31 +0200
Subject: [PATCH 070/540] Add another curvature relation

---
 src/feltor/feltor.tex | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index eec39d57e..1a1884066 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -55,7 +55,8 @@ we can define various differential operations.
     $\mathcal K_{\nabla B}(f) := \vec{\mathcal K_{\nabla B}} \cdot \vec \nabla f = \frac{1}{B}(\bhat \times \vec \nabla \ln B)\cdot \vec \nabla f$ \\
     Curvature &
     $\mathcal K$ &
-    $\mathcal{K}(f):=\vec{\mathcal K} \cdot \vec \nabla f = \vec{\nabla} \cdot \left(\frac{ \bhat \times \vec{\nabla} f}{B} \right) = \mathcal K_{\nabla\times\bhat}(f) + \mathcal K_{\nabla B}(f)$,\\
+    $\mathcal{K}(f):=\vec{\mathcal K} \cdot \vec \nabla f = 
+     \nabla\times\frac{\bhat}{B}\cdot\nabla f$,\\
     Parallel derivative&
     $\nabla_\parallel $&
     $ \nabla_\parallel f := \bhat\cdot\vec{\nabla} f$ \\
@@ -76,6 +77,9 @@ Note that we have
     \vec \nabla \cdot \vec{\mathcal K_{\nabla\times\bhat}}
     &= -\vec\nabla \cdot \vec{\mathcal K_{\nabla B}} = -\vec{ \mathcal K_{\nabla\times\bhat}}\cdot\nabla\ln B, \\
     \vec\nabla\cdot\vec{ \mathcal K} &= 0, \\
+    \vec{\mathcal K} &=
+    \vec{\nabla} \cdot \left(\frac{ \bhat \times \vec{\nabla} f}{B} \right)
+    = \mathcal K_{\nabla\times\bhat}(f) + \mathcal K_{\nabla B}(f),\\
     \vec{ \mathcal K_{\nabla\times\bhat}} - \vec{ \mathcal K_{\nabla B}} &= \frac{1}{B^2} (\vec \nabla \times \vec B), \\
     \nabla_\parallel \ln B &= -\vec\nabla\cdot\bhat.
     \label{eq:curl_curvature}
-- 
GitLab


From 871b4e390121882c6f0e96a9180f9c303849084f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 5 May 2019 17:12:00 +0200
Subject: [PATCH 071/540] Reorganize writeup without dimensional quantities

- BUGFIX: missing 1/beta factor in magnetic energy
- FIX: balance initial condition should contain half diamagnetic
vorticity
- remove dimensional equations from feltor writeup - still need to
update the vorticity equation
---
 src/feltor/feltor.cu     |   9 +-
 src/feltor/feltor.cuh    |  28 +++-
 src/feltor/feltor.tex    | 335 +++++++++++++++++++--------------------
 src/feltor/feltor_hpc.cu |   7 +-
 4 files changed, 185 insertions(+), 194 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index e255908f0..cccdda925 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -131,14 +131,7 @@ int main( int argc, char* argv[])
     y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<dg::DVec>(profile);
     dg::blas1::axpby( 1., dg::construct<dg::DVec>(ntilde), 1., y0[0][0]);
     std::cout << "initialize ni" << std::endl;
-    if( p.initphi == "zero")
-    {
-        feltor.initializeni( y0[0][0], y0[0][1]);
-    }
-    else if( p.initphi == "balance")
-        dg::blas1::copy( y0[0][0], y0[0][1]); //set N_i = n_e
-    else
-        std::cerr <<"WARNING: Unknown initial condition for phi!\n";
+    feltor.initializeni( y0[0][0], y0[0][1]);
 
     dg::blas1::copy( 0., y0[1][0]); //set we = 0
     dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 417bd71c1..643d001d2 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -454,14 +454,30 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::initializene(
 }
 template<class Geometry, class IMatrix, class Matrix, class Container>
 void Explicit<Geometry, IMatrix, Matrix, Container>::initializeni(
-    const Container& src, Container& target)
+    const Container& src, Container& target, std::string initphi)
 {
     // Ni = ne
     dg::blas1::copy( src, target);
     if (m_p.tau[1] != 0.) {
-        //add FLR correction -0.5*tau*mu*Delta n_e
-        dg::blas2::symv( 0.5*m_p.tau[1]*m_p.mu[1],
-            m_lapperpN, src, 1.0, target);
+        if( p.initphi == "zero")
+        {
+            //add FLR correction -0.5*tau*mu*Delta n_e
+            dg::blas2::symv( 0.5*m_p.tau[1]*m_p.mu[1],
+                m_lapperpN, src, 1.0, target);
+        }
+        else if( p.initphi == "balance")
+            //add FLR correction +0.5*tau*mu*Delta n_e
+            dg::blas2::symv( -0.5*m_p.tau[1]*m_p.mu[1],
+                m_lapperpN, src, 1.0, target);
+        else
+        {
+            #ifdef MPI_VERSION
+                int rank;
+                MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+                if(rank==0)
+            #endif
+            std::cerr <<"WARNING: Unknown initial condition for phi!\n";
+        }
     }
 }
 
@@ -703,7 +719,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_energies(
         m_q.Tpar[i] = z[i]*0.5*m_p.mu[i]*dg::blas2::dot(
             fields[0][i], m_vol3d, m_temp0);
     }
-    //= 0.5 beta (grad_perp Apar)^2
+    //= 0.5 beta^{-1} (grad_perp Apar)^2
     if( m_p.beta != 0)
     {
         dg::tensor::multiply3d( m_hh, m_dxA, m_dyA, m_dzA,
@@ -711,7 +727,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_energies(
         dg::blas1::subroutine( routines::ComputePsi(),
             m_temp0, m_dxA, m_dyA, m_dzA,
             m_temp0, m_temp1, m_temp2);
-        m_q.Apar = 0.5*dg::blas1::dot( m_vol3d, m_temp0);
+        m_q.Apar = 0.5*dg::blas1::dot( m_vol3d, m_temp0)/m_beta;
     }
     //= 0.5 mu_i N_i u_E^2
     m_q.Tperp = 0.5*m_p.mu[1]*dg::blas2::dot( fields[0][1], m_vol3d, m_UE2);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 1a1884066..22e167dd8 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -6,13 +6,25 @@
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \begin{document}
 
-\title{The feltor project}
-\author{ M.~Held and M.~Wiesenberger}
+\title{
+The full-F electromagnetic model in toroidal geometry \textsc{Feltor}}
+\author{ M.~Wiesenberger and M.~Held}
 \maketitle
 
 \begin{abstract}
-The full-F electromagnetic model in toroidal geoemtry (\textsc{Feltor}).
-This is a program for global 3d isothermal electromagnetic full-F gyro-fluid simulations.
+The purpose of this document is to describe the programs
+\texttt{feltor\_hpc.cu, feltor.cu, feltor\_diag.cu} and to an extend
+\texttt{geometry\_diag.cu}. The goal is to provide
+enough information such that the user never needs to look
+into the actual codes on the one side and to be able to connect
+the presented formulas to relevant journal publications on the other.
+
+The program \texttt{feltor/inc/geometries/geometry\_diag.cu}
+analyses the magnetic field geometry.
+\texttt{feltor\_hpc.cu} and \texttt{feltor.cu} are programs for global 3d isothermal electromagnetic full-F gyro-fluid simulations.
+\texttt{feltor/diag/feltor\_diag.cu} is a program to analyse the output
+file(s) of \texttt{feltor\_hpc.cu}.
+
 \end{abstract}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -49,14 +61,14 @@ we can define various differential operations.
     = \nabla\cdot( h\cdot\nabla f)$  \\
     Curl-b Curvature &
     $\mathcal K_{\nabla\times\bhat}$ &
-    $\mathcal K_{\nabla\times\bhat}(f) := \vec{ \mathcal K_{\nabla\times\bhat} }\cdot \vec \nabla f = \frac{1}{B}(\nabla \times \bhat)\cdot \vec \nabla f$ \\
+    $\mathcal K_{\nabla\times\bhat}(f) := \vec{ \mathcal K_{\nabla\times\bhat} }\cdot \vec \nabla f = \frac{1}{B}(\nabla \times \bhat)\cdot \vec \nabla f$ \\[4pt]
     Grad-B Curvature &
     $\mathcal K_{\nabla B} $ &
-    $\mathcal K_{\nabla B}(f) := \vec{\mathcal K_{\nabla B}} \cdot \vec \nabla f = \frac{1}{B}(\bhat \times \vec \nabla \ln B)\cdot \vec \nabla f$ \\
+    $\mathcal K_{\nabla B}(f) := \vec{\mathcal K_{\nabla B}} \cdot \vec \nabla f = \frac{1}{B}(\bhat \times \vec \nabla \ln B)\cdot \vec \nabla f$ \\[4pt]
     Curvature &
     $\mathcal K$ &
-    $\mathcal{K}(f):=\vec{\mathcal K} \cdot \vec \nabla f = 
-     \nabla\times\frac{\bhat}{B}\cdot\nabla f$,\\
+    $\mathcal{K}(f):=\vec{\mathcal K} \cdot \vec \nabla f =
+     \vec{\nabla}\cdot\left(\frac{\bhat\times\vec{\nabla} f}{B}\right)$,\\[4pt]
     Parallel derivative&
     $\nabla_\parallel $&
     $ \nabla_\parallel f := \bhat\cdot\vec{\nabla} f$ \\
@@ -78,7 +90,7 @@ Note that we have
     &= -\vec\nabla \cdot \vec{\mathcal K_{\nabla B}} = -\vec{ \mathcal K_{\nabla\times\bhat}}\cdot\nabla\ln B, \\
     \vec\nabla\cdot\vec{ \mathcal K} &= 0, \\
     \vec{\mathcal K} &=
-    \vec{\nabla} \cdot \left(\frac{ \bhat \times \vec{\nabla} f}{B} \right)
+     \nabla\times\frac{\bhat}{B}\cdot\nabla f
     = \mathcal K_{\nabla\times\bhat}(f) + \mathcal K_{\nabla B}(f),\\
     \vec{ \mathcal K_{\nabla\times\bhat}} - \vec{ \mathcal K_{\nabla B}} &= \frac{1}{B^2} (\vec \nabla \times \vec B), \\
     \nabla_\parallel \ln B &= -\vec\nabla\cdot\bhat.
@@ -350,7 +362,7 @@ K_{\nabla\times\bhat}^\varphi &= \frac{R_0}{R^2B^2}\left(
 Our computational domain is a box and in particular not aligned with the
 magnetic flux surfaces. This means that particularly in the corners of
 the domain the field lines inside the domain are very short (in the
-sense that the distance between the enter point and leave point is short).
+sense that the distance between the entry point and leave point is short).
 It turns out that this behaviour is numerically disadvantageous (may
 blow up the simulation in the worst case) in the
 computation of parallel derivatives. In order to remedy this situation
@@ -390,41 +402,47 @@ Note that $\Theta_\alpha(0) = 0.5$ and $\theta_\alpha(0) = 35\alpha/256$.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
-\subsection{Dimensional Equations}
-% \(n_e\) is the electron density, \(N_i\) is the ion gyro-centre density, \(\phi\) is the electric potential,
-% \\
-% This model is an isothermal 3D gyro-fluid model, which exploits the toroidal field line approximation in the curvature operator terms (cf.~\ref{sec:torfieldlineapprox}).
-% It incorporates a Solov'ev equilibrium (cf.~\ref{sec:solovev}) for the magnetic field, which allows a
-% realistically shaped axisymmetric magnetic field. The coordinate system of choice is a non-aligned  cylindrical coordinate system (cf.~\ref{sec:cylmetric} and~\ref{sec:nonparallelalignedmetric}), which allows
-% the correct numerical treatment of singular points (e.g. X- and O- Points). These points occur naturally in realistic magnetic field configurations.
-% Since it is based on a global geometry, treating the complete poloidal flux surface, it is suited for coupled simulations of core, edge and SOL.
-% However, our isothermal model misses important core physics ingredients, such as the ion temperature gradient (ITG) and trapped electron mode (TEM)~\cite{Wesson07}. Thus the validity the global model is limited to certain
-% parameter regimes.\\
-We set up an isothermal 3d gyro-fluid model with up to 2nd order FLR effects
-on in the electric potential $\phi$ and 0th order FLR effects in the parallel magnetic
-potential $A_\parallel$.
-The continuity equation for the electron density \(n_e\) and the ion gyro-centre
-density \(N_i\) and the momentum conservation equation for
-the parallel electron velocity \(u_e\) and the parallel ion gyro-centre velocity \(U_i\) are
-(omitting species labels)~\cite{WiesenbergerPhD, HeldPhD}
+\subsection{Conservative form}
+We scale all spatial lengths by $\rho_s = \sqrt{T_e m_i}/(eB_0)$ and time by the ion gyro-frequency $\Omega_0 = eB_0/m_i$.
+The magnetic field is scaled with $B_0$, densities with $n_0$ and the parallel velocity is scaled with $c_s = \sqrt{T_e/m_i}$.
+The potential is scaled with $\hat \phi = e/T_e$ and the vector potential with
+$\hat A_\parallel = \rho_s B_0$.
+We introduce the dimensionless parameters
+\begin{align}
+  \tau_a = \frac{T_a}{z_aT_e}~,\quad \mu_a = \frac{m_a}{z_am_i}\text{ and } 
+  \beta:=\frac{\mu_0 n_0 T_e}{B_0^2}
+  \label{}
+\end{align}
+where $a\in\{e,i\}$ is the species label and $z$ is the charge number. We define with 
+$\eta_\parallel := \frac{0.51 m_e \nu_{ei}}{n_e e^2}$
+\begin{align}
+  \eta:=\frac{en_0\eta_\parallel}{B_0} = 8.45\cdot 10^{-5}\ln \lambda \left(\frac{n_0}{10^{19}\text{m}^3}\right) \left(\frac{T_e}{\text{eV}}\right)^{-3/2} \left(\frac{B_0}{\text{T}}\right)^{-1},
+    \label{eq:resistivity}
+\end{align}
+with $\ln \lambda \approx 10$.
+ The approximate Spitzer current \(J_{\parallel,s}:= n_e \left(U_i - u_e\right)\)
+ determines the parallel resistive terms to $R_\parallel:= n_e\eta J_{\parallel,s}$.
+Omitting the species label we arrive at (dividing the density equation by $\Omega_0n_0$ and the velocity equation by $\Omega_0 c_s$)
 \begin{align}
 \frac{\partial}{\partial t} N &+ \vec\nabla\cdot\left( N \left(
     \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + \tilde{\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
-mN \frac{\partial}{\partial t} U &+ mN \left(
+\mu N \frac{\partial}{\partial t} U &+ \mu N \left(
     \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + \tilde{\vec b}_\perp\right)
     \right)\cdot \vec\nabla U  \nonumber \\
-    &+ 2m\vec \nabla \cdot ( NU \vec v_{\nabla\times\bhat})
-    -mNU\vec \nabla\cdot \vec v_{\nabla\times\bhat}
-    + mNU\mathcal K_{\nabla\times\bhat}(\psi) \nonumber\\
-    &= -T \left(\bhat + \tilde{\vec b}_\perp\right)\cdot \nabla N -qN \left( \left(\bhat+\tilde{\vec b}_\perp\right)\cdot \nabla \psi + \frac{\partial A_\parallel}{\partial t}\right) + mN R_{\eta_\parallel} + mN(\Lambda_U + S_U)
+    &+ 2\mu \vec \nabla \cdot ( NU \vec v_{\nabla\times\bhat})
+    -\mu NU\vec \nabla\cdot \vec v_{\nabla\times\bhat}
+    + \mu NU\mathcal K_{\nabla\times\bhat}(\psi) \nonumber\\
+    &= -\tau \left(\bhat + \tilde{\vec b}_\perp\right)\cdot \nabla N 
+    -N \left( \left(\bhat+\tilde{\vec b}_\perp\right)\cdot \nabla \psi + \frac{\partial A_\parallel}{\partial t}\right) 
+    - \eta n_e^2(U_i-u_e) + \mu N(\Lambda_U + S_U)
 \label{}
 \end{align}
 with
 \begin{align}
 \vec v_E := \frac{\bhat\times\nabla\psi}{B},\quad
-\vec v_{K} := \frac{T}{q}\left(\vec{\mathcal K_{\nabla B}} + \vec{\mathcal K_{\nabla\times\bhat}}\right)=\frac{T}{q}\vec{\mathcal K}  ,\nonumber\\
-\vec v_C := \frac{mU^2}{q}\vec{\mathcal K_{\nabla\times\bhat}},\quad
-\vec v_{\nabla\times\bhat} := \frac{T}{q}\vec{\mathcal K_{\nabla\times\bhat}},\quad
+\vec v_{K} := \tau \left(\vec{\mathcal K_{\nabla B}} + \vec{\mathcal K_{\nabla\times\bhat}}\right)=\tau\vec{\mathcal K}  ,\nonumber\\
+\vec v_C := \mu U^2\vec{\mathcal K_{\nabla\times\bhat}},\quad
+\vec v_{\nabla\times\bhat} := \tau\vec{\mathcal K_{\nabla\times\bhat}},\quad
 \tilde{\vec b}_\perp = \frac{\nabla\times A_\parallel \bhat}{B}.
 \label{}
 \end{align}
@@ -432,33 +450,23 @@ with
 The electric potential \(\phi\) and parallel magnetic vector potential \(A_\parallel\) are
 computed by the polarisation and induction equations (with $q_e=-e$ and $q_i=+e$)
 \begin{align}
- -\vec{\nabla} \cdot\left(\frac{m_iN_i}{e B^2} \vec{\nabla}_\perp \phi\right) &=  \Gamma_{1,i} N_i -n_e, \quad \Gamma_{1,i}^{-1} := 1-\frac{m_i T_i}{2e^2 B_0^2} \Delta_\perp , \\
-  -\frac{1}{\mu_0} \Delta_\perp A_\parallel &= e\left(N_i U_i-n_e u_e \right)
+ -\vec{\nabla} \cdot\left(\frac{\mu_iN_i}{B^2} \vec{\nabla}_\perp \phi\right) &=  \Gamma_{1,i} N_i -n_e, \quad \Gamma_{1,i}^{-1} := 1-\frac{1}{2}\mu_i\tau_i\Delta_\perp , \\
+  -\frac{1}{\beta} \Delta_\perp A_\parallel &= \left(N_i U_i-n_e u_e \right)
   \label{eq:polarisation_dimensional}
 \end{align}
 Given $\phi$ we define the generalised electric potential
 \begin{align}
-    \psi_e := \phi,\quad \psi_i&:= \Gamma_{1,i} \phi - \frac{m_i }{2 e}\left(\frac{\vec \nabla_\perp\phi}{B}\right)^2
-\end{align}
-\subsection{Relation to diamagnetic drift}
-Notice that the term $\nabla (n_e \vec v_K ) \equiv -\nabla (T_e n_e \vec K /e) $
-forms the familiar form of the divergence of the diamagnetic flux.
-\begin{align}
-\nabla\cdot\left( n_e \frac{\bhat \times \nabla n_e T_e}{en_e B}\right) \equiv  \nabla\cdot \left(\frac{T_e n_e}{e} \vec{ \mathcal K}\right)
+    \psi_e := \phi,\quad \psi_i&:= \Gamma_{1,i} \phi - \frac{\mu_i }{2}\left(\frac{\vec \nabla_\perp\phi}{B}\right)^2
 \end{align}
-which is evident with the identities in Section~\ref{sec:magnetic}.
+In total 
+we have an isothermal 3d gyro-fluid model with up to 2nd order FLR effects
+on in the electric potential $\phi$ and 0th order FLR effects in the parallel magnetic
+potential $A_\parallel$.
+We have the continuity equation for the electron density \(n_e\) and the ion gyro-centre
+density \(N_i\) and the momentum conservation equation for
+the parallel electron velocity \(u_e\) and the parallel ion gyro-centre velocity \(U_i\)~\cite{WiesenbergerPhD, HeldPhD}.
 
-\subsection{Parallel Resistivity and diffusion}\label{sec:dissres}
-The terms $R_{e/i,\eta_\parallel}$ account for resistive friction.
-The parallel Spitzer resistivity
-\begin{align}
-\eta_\parallel := \frac{0.51 m_e \nu_{ei}}{n_e e^2}
-\end{align}
-and the approximate Spitzer current \(J_{\parallel,s}:= e n_e \left(U_i - u_e\right)\) determine the parallel resistive terms to:
-\begin{align}
-  R_{e,\eta_\parallel} &=  en_e\eta_\parallel J_{\parallel,s}/(m_en_e)  &
-  R_{i,\eta_\parallel} &=- en_e\eta_\parallel J_{\parallel,s}/(m_iN_i)
-\end{align}
+\subsection{Diffusive terms}\label{sec:dissres}
 The dissipative terms can be decomposed into perpendicular and parallel components
 \begin{align}
  \Lambda_{n_e} &= \Lambda_{n_e,\perp}+\Lambda_{n_e,\parallel}, &
@@ -542,10 +550,11 @@ zone that avoids numerical oscillations.
 
 We have two possibilities to initialize the ion density
 \begin{align} \label{eq:initphi}
-  N_i = \Gamma_{1,i}^{-1} n_e \text{ or } N_i = n_e
+  N_i = \Gamma_{1,i}^{-1} n_e \text{ or } N_i = \Gamma_{1,i}n_e\approx \left(1+\frac{1}{2}\tau_i\mu_i\Delta_\perp\right)n_e
 \end{align}
-In the first case the potential $\phi= 0$ while in the second case
-the $E\times B$ and ion diamagnetic vorticity coincide $\Delta N_i \propto \Delta \phi$.
+In the first case the potential $\phi= 0$ while in the second (prefered)
+case
+the $E\times B$ and ion diamagnetic vorticity coincide $\Delta_\perp N_i \propto \Delta_\perp \phi$ in the long-wavelength limit.
 We can choose between several initial conditions for $\tilde n$:
 
 \subsubsection{Blob and Straight blob}
@@ -594,7 +603,7 @@ This will result in exponential adaption of the core and wall
 density profile of the form $n_e \propto n_{prof}+(n_{prof}-n_{e,0})e^{-\omega_st}$.
 For ions we use
 \begin{align}
-    S_{N_i} = \Gamma_{1,i}^{-1} S_{n_e} = \left(1-\frac{m_i T_i}{2q^2 B_0^2} \Delta_\perp\right) S_{n_e}
+    S_{N_i} = \Gamma_{1,i}^{-1} S_{n_e} = \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S_{n_e}
   \label{eq:ion_source}
 \end{align}
 Note that Eq.~\eqref{eq:ion_source} is explicitly chosen as to avoid vorticity generation
@@ -610,6 +619,52 @@ so that $\nabla_\perp^2 S_{n_e}$ is well defined.
 %\end{align}
 %with $\rho_d > 1$.
 
+\subsection{Implemented form}
+The form that we implement avoids derivatives on the product of
+two functions for which we have no boundary conditions
+\begin{subequations}
+    \begin{align}
+    \frac{\partial}{\partial t} N =&
+        - \frac{1}{B}[\psi, N]_{\perp}%\nonumber\\
+        - \bar \nabla_\parallel \left( NU\right)
+        - NU\left(\vec \nabla\cdot\bhat+\vec \nabla\cdot\tilde{\vec b}_\perp\right)
+        - \tau \mathcal K(N) \nonumber \\&
+        - N \mathcal K(\psi)
+        -\mu \mathcal K_{\nabla\times\bhat}(NU^2)
+        -\mu NU^2\nabla\cdot \vec{ \mathcal K_{\nabla\times\bhat}}
+        + \nu_\perp\Delta_\perp N + \nu_\parallel \Delta_\parallel N + S_N, \\
+    \frac{\partial}{\partial t} W =&
+        - \frac{1}{B}\left[\psi, U\right]_{\perp}%& \nonumber\\
+        - \frac{1}{\mu} \bar \nabla_\parallel \psi% \nonumber\\
+        - \frac{1}{2}\bar \nabla_\parallel U^2
+        -\frac{\tau}{\mu} \bar \nabla_\parallel \ln N
+        - U\mathcal K_{\nabla\times\bhat}(\psi)
+        - \tau \mathcal K(U)
+        -\tau U\nabla\cdot\vec{ \mathcal K_{\nabla\times\bhat}}\nonumber\\&
+        - \left(2\tau + {\mu}U^2\right) \mathcal K_{\nabla\times\bhat} (U)
+        -2\tau U\mathcal K_{\nabla\times\bhat}(\ln N)
+        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
+        + \nu_\perp\Delta_\perp U
+        + \nu_\parallel \Delta_\parallel U + S_U,
+        \label{eq:EgyrofluidU} \\
+        W&:= \left( U + \frac{A_\parallel}{\mu}\right)
+    \end{align}
+    \label{eq:Egyrofluid}
+\end{subequations}
+together with
+$\bar\nabla_\parallel f = \nabla_\parallel f + A_\parallel \mathcal K_{\nabla\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$
+and $\vec \nabla \cdot \tilde{ \vec b}_\perp = A_\parallel \vec \nabla\cdot\vec{ \mathcal{ K}_{\nabla\times\bhat}} - \mathcal K_{\nabla B}(A_\parallel) $
+and
+\begin{subequations} \label{eq:elliptic}
+  \begin{align}
+    -\nabla\cdot\left( \frac{N_i}{B^2}\nabla_\perp \phi \right) &= \Gamma_{1,i} N_i - n_e, \quad\quad
+    \Gamma_{1,i}^{-1} = 1-\frac{1}{2}\tau_i\mu_i \Delta_\perp \\
+    \psi_e = \phi, \quad \psi_i &= \Gamma_{1,i}\phi -\frac{\mu_i}{2}\frac{(\nabla_\perp\phi)^2}{B^2} \\
+    \left(\frac{\beta}{\mu_i}N_i - \frac{\beta}{\mu_e}n_e-\Delta_\perp\right)
+    A_\parallel &= \beta\left(N_iW_i-n_e w_e\right)
+  \end{align}
+\end{subequations}
+Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite.
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
 Integrating the density equation we directly get
@@ -618,10 +673,27 @@ Integrating the density equation we directly get
   + \int_{\partial\Omega} \vec\dA\cdot\vec{ j_{n_e}}
   =  \int_\Omega \dV (\Lambda_{n_e}+S_{n_e})
 \end{align}
-with the particle flux
-\begin{align} \label{eq:mass_flux}
-\vec{ j_{n_e}} := n_e\left(\vec v_E + \vec v_C + \vec v_{K} + U (\bhat + \vec{\tilde b}_\perp)\right)
+The terms of the particle conservation thus read
+\begin{align} \label{eq:mass_conservation}
+  n_e= & n_e,\\
+  \vec j_{n_e} =& n_e\left(
+  \vec v_E + \vec v_C + \vec v_{K} +u_e\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \nonumber\\
+  =& n_e \left(\frac{\bhat\times \nabla\phi}{B} 
+  + \tau_e \frac{\bhat\times\nabla n_e}{n_eB} 
+  + \mu_e u_e^2\vec K_{\nabla\times\bhat} 
+  + u_e(\bhat + \tilde{\vec b}_\perp) \right), \\
+  \Lambda_{n_e} =&
+  \nu_\perp\Delta_\perp n_e + \nu_\parallel\Delta_\parallel n_e
+\\
+  S_{n_e} =&  S_{n_e}
 \end{align}
+Notice that we used
+\begin{align}
+n_e \vec K = n_e\nabla\times\frac{\bhat}{B} = \nabla\times n_e\frac{\bhat}{B} + \frac{\bhat\times\nabla n_e}{B}
+\label{}
+\end{align}
+such that we can define the diamagnetic flux in the particle flux since
+the rotation vanishes under the divergence.
 \subsubsection{Polarisation charge density}
 First of all let us approximate FLR effects in the long wavelength limit (LWL)
 $\Gamma_{1,i}N_i \approx N_i + \frac{T_i}{2 m_i \Omega_0^2} \Delta_\perp  N_i$.
@@ -697,6 +769,13 @@ to the familiar drift-fluid quasineutrality equation \fixme{recompute, units!}
 \end{align}
 \fixme{use Mathematika for maybe slab/toroidal equation in 3d geometry}
 
+and the vorticity conservation
+\begin{align} \label{eq:vorticity_conservation}
+\mathcal \rho_{pol} &= -\vec{\nabla} \cdot\left(\frac{N_i}{B^2} \vec{\nabla}_\perp \phi\right)  - \tau_i \Delta_\perp N_i\nonumber\\
+\vec{j_{pol}} &= \vec j_{N_i} - \vec j_{n_e} \nonumber\\
+\Lambda_{ pol} &= \Lambda_{N_i} - \Lambda_{n_e}
+\end{align}
+
 \subsubsection{Energy theorem}
 The terms of the energy theorem are
 \begin{align} \label{eq:energy_theorem}
@@ -706,122 +785,18 @@ The terms of the energy theorem are
 +  S_{\mathcal E}
 -  R_{\mathcal E} \right)
 \end{align}
-with
-\begin{align} \label{eq:energy_conservation_dimensional}
-  \mathcal E = &
-  t_e n_e \ln{(n_e)} +T_i N_i\ln{(N_i)}
-    +\frac{1}{2} m_i N_i \left(\frac{\vec\nabla_\perp\phi}{B}\right)^2 \nonumber\\
-    &+\frac{1}{2} m_e  n_e u_e^2 +\frac{1}{2} m_i  N_i U_i^2
-    + \frac{(\vec \nabla_\perp A_\parallel)^2}{2\mu_0}, \\
-  \vec j_{\mathcal E} =& \sum_s \left[ N\left(
-  \vec v_E + \vec v_C + \vec v_{K} +U\left(\bhat+\tilde{\vec b}_\perp\right)  \right)
-  \left(T \ln N + \frac{1}{2}m U^2 + q\psi  \right) \right. \nonumber\\
-  &\qquad \left . + mNU^2\vec v_{\nabla\times\bhat} + \left(\bhat + \tilde{\vec b}_\perp\right) NUT\right], \\
-  \Lambda_{\mathcal E} =&  \sum_s \left[\left(T\left( 1+\ln{N}\right) +q \psi + \frac{1}{2} m U^2 \right)\Lambda_{N}  + mNU \Lambda_U\right]
-\nonumber \\
-  S_{\mathcal E} =&  \sum_s  \left[\left(T\left( 1+\ln{N}\right) +q \psi + \frac{1}{2} m U^2 \right)S_{N}  + mNU S_U\right]
-\nonumber \\
-  R_{\mathcal E} =&  \eta_\parallel  \left[e n_e(U_i-u_e)\right]^2.
-\end{align}
-where in the energy flux $\vec j_{\mathcal E}$
-we neglect terms  containing time derivatives
-of the eletric and magnetic potentials and we sum over all species.
-The energy density $\mathcal E$ consists of the Helmholtz free energy density for electrons and ions,
-the \(\vec{E} \times \vec{B}\) energy density, the parallel energy densities for electrons and ions and the perturbed magnetic field energy density.
-In \(\Lambda\) we insert the dissipative terms of Section~\ref{sec:dissres}. \\
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\subsection{Dimensionless form}
-We scale all spatial lengths by $\rho_s = \sqrt{T_e m_i}/(eB_0)$ and time by the ion gyro-frequency $\Omega_0 = eB_0/m_i$.
-The magnetic field is scaled with $B_0$, densities with $n_0$ and the parallel velocity is scaled with $c_s = \sqrt{T_e/m_i}$.
-The potential is scaled with $\hat \phi = e/T_e$ and the vector potential with
-$\hat A_\parallel = \rho_s B_0$.
-We introduce the dimensionless parameters
-\begin{align}
-  \tau_a = \frac{T_a}{z_aT_e}~,\quad \mu_a = \frac{m_a}{z_am_i}\text{ and } 
-  \beta:=\frac{\mu_0 n_0 T_e}{B_0^2}
-  \label{}
-\end{align}
-where $a\in\{e,i\}$ is the species label and $z$ is the charge number. Finally, we define
-\begin{align}
-  \eta:=\frac{en_0\eta_\parallel}{B_0} = 8.45\cdot 10^{-5}\ln \lambda \left(\frac{n_0}{10^{19}\text{m}^3}\right) \left(\frac{T_e}{\text{eV}}\right)^{-3/2} \left(\frac{B_0}{\text{T}}\right)^{-1}.
-    \label{eq:resistivity}
-\end{align}
-with $\ln \lambda \approx 10$.
-Omitting the species label we arrive at (dividing the density equation by $\Omega_0n_0$ and the velocity equation by $\Omega_0 c_s$)
-\begin{subequations}
-    \begin{align}
-    \frac{\partial}{\partial t} N =&
-        - \frac{1}{B}[\psi, N]_{\perp}%\nonumber\\
-        - \bar \nabla_\parallel \left( NU\right)
-        - NU\left(\vec \nabla\cdot\bhat+\vec \nabla\cdot\tilde{\vec b}_\perp\right)
-        - \tau \mathcal K(N) \nonumber \\&
-        - N \mathcal K(\psi)
-        -\mu \mathcal K_{\nabla\times\bhat}(NU^2)
-        -\mu NU^2\nabla\cdot \vec{ \mathcal K_{\nabla\times\bhat}}
-        + \nu_\perp\Delta_\perp N + \nu_\parallel \Delta_\parallel N + S_N, \\
-    \frac{\partial}{\partial t} W =&
-        - \frac{1}{B}\left[\psi, U\right]_{\perp}%& \nonumber\\
-        - \frac{1}{\mu} \bar \nabla_\parallel \psi% \nonumber\\
-        - \frac{1}{2}\bar \nabla_\parallel U^2
-        -\frac{\tau}{\mu} \bar \nabla_\parallel \ln N
-        - U\mathcal K_{\nabla\times\bhat}(\psi)
-        - \tau \mathcal K(U)
-        -\tau U\nabla\cdot\vec{ \mathcal K_{\nabla\times\bhat}}\nonumber\\&
-        - \left(2\tau + {\mu}U^2\right) \mathcal K_{\nabla\times\bhat} (U)
-        -2\tau U\mathcal K_{\nabla\times\bhat}(\ln N)
-        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
-        + \nu_\perp\Delta_\perp U
-        + \nu_\parallel \Delta_\parallel U + S_U,
-        \label{eq:EgyrofluidU} \\
-        W&:= \left( U + \frac{A_\parallel}{\mu}\right)
-    \end{align}
-    \label{eq:Egyrofluid}
-\end{subequations}
-together with
-$\bar\nabla_\parallel f = \nabla_\parallel f + A_\parallel \mathcal K_{\nabla\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$
-and $\vec \nabla \cdot \tilde{ \vec b}_\perp = A_\parallel \vec \nabla\cdot\vec{ \mathcal{ K}_{\nabla\times\bhat}} - \mathcal K_{\nabla B}(A_\parallel) $
-and
-\begin{subequations} \label{eq:elliptic}
-  \begin{align}
-    -\nabla\cdot\left( \frac{N_i}{B^2}\nabla_\perp \phi \right) &= \Gamma_{1,i} N_i - n_e, \quad\quad
-    \Gamma_{1,i}^{-1} = 1-\frac{1}{2}\tau_i\mu_i \Delta_\perp \\
-    \psi_e = \phi, \quad \psi_i &= \Gamma_{1,i}\phi -\frac{\mu_i}{2}\frac{(\nabla_\perp\phi)^2}{B^2} \\
-    \left(\frac{\beta}{\mu_i}N_i - \frac{\beta}{\mu_e}n_e-\Delta_\perp\right)
-    A_\parallel &= \beta\left(N_iW_i-n_e w_e\right)
-  \end{align}
-\end{subequations}
-Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite.
-
-The terms of the particle conservation read
-\begin{align} \label{eq:mass_conservation}
-  n_e= & n_e,\\
-  \vec j_{n_e} =& n_e\left(
-  \vec v_E + \vec v_C + \vec v_{K} +u_e\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \nonumber\\
-  =& n_e \left(\frac{\bhat\times \nabla\phi}{B} 
-  + (\tau_e +\mu_e u_e^2)\vec K_{\nabla\times\bhat} 
-  + \tau_e \vec K_{\vec \nabla B} + u_e(\bhat + \tilde{\vec b}_\perp) \right), \\
-  \Lambda_{n_e} =&
-  \nu_\perp\Delta_\perp n_e + \nu_\parallel\Delta_\parallel n_e
-\\
-  S_{n_e} =&  S_{n_e}
-\end{align}
-and the vorticity conservation
-\begin{align} \label{eq:vorticity_conservation}
-\mathcal \rho_{pol} &= -\vec{\nabla} \cdot\left(\frac{N_i}{B^2} \vec{\nabla}_\perp \phi\right)  - \tau_i \Delta_\perp N_i\nonumber\\
-\vec{j_{pol}} &= \vec j_{N_i} - \vec j_{n_e} \nonumber\\
-\Lambda_{ pol} &= \Lambda_{N_i} - \Lambda_{n_e}
-\end{align}
-
-The terms of the energy theorem read (with $z_e=-1$ and $z_i=+1$)
+with ( $z_e=-1$ and $z_i=+1$)
 \begin{align} \label{eq:energy_conservation}
   \mathcal{E}= & z_e\tau_e n_e \ln{(n_e)} +z_i\tau_i N_i\ln{(N_i)}
-  +\frac{1}{2}\left(\vec \nabla_\perp A_\parallel\right)^2
-   +  \frac{1}{2} \mu_i N_i u_E^2  \nonumber\\
+  +\frac{1}{2\beta}\left(\vec \nabla_\perp A_\parallel\right)^2
+   +  \frac{1}{2} z_i \mu_i N_i u_E^2  \nonumber\\
    & +\frac{1}{2} z_e\mu_e  n_e u_e^2
   +\frac{1}{2} z_i\mu_i  N_i U_i^2,\\
   \vec j_{\mathcal E} =& \sum_s z\left[
-  \left(\tau \ln N + \frac{1}{2}\mu U^2 + \psi \right) \vec j_N 
-  + \mu \tau NU^2\vec K_{\nabla\times\bhat} + \tau NU \left(\bhat + \tilde{\vec b}_\perp\right)\right], \\
+  \left(\tau \ln N + \frac{1}{2}\mu U^2 + \psi \right)N\left(
+  \vec v_E + \vec v_C + \vec v_{K} +U\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \right]
+  \nonumber\\
+  &+ \sum_z z\left[\mu \tau NU^2\vec K_{\nabla\times\bhat} + \tau NU \left(\bhat + \tilde{\vec b}_\perp\right)\right], \\
   \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U^2 \right)
   \left(\nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N\right)  +  \mu NU\left(\nu_\perp\Delta_\perp U + \nu_\parallel\Delta_\parallel U\right) \right]
 \nonumber \\
@@ -829,9 +804,18 @@ The terms of the energy theorem read (with $z_e=-1$ and $z_i=+1$)
 \nonumber \\
   R_{\mathcal E} =&  \eta_\parallel  \left[ n_e(U_i-u_e)\right]^2.
 \end{align}
+where in the energy flux $\vec j_{\mathcal E}$
+we neglect terms  containing time derivatives
+of the eletric and magnetic potentials and we sum over all species.
+The energy density $\mathcal E$ consists of the Helmholtz free energy density for electrons and ions,
+the \(\vec{E} \times \vec{B}\) energy density, the parallel energy densities for electrons and ions and the perturbed magnetic field energy density.
+In \(\Lambda\) we insert the dissipative terms of Section~\ref{sec:dissres}. \\
 Replace $\Delta_\perp$ with $-\Delta_\perp^2$ when hyperviscous diffusion is chosen
 for the diffusion terms in the above equations.
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+
 \subsection{Manufactured Solution}
 In order to test the implementation we manufacture a solution to Eqs.~\eqref{eq:Egyrofluid} and \eqref{eq:elliptic} of the form
 \begin{align*}
@@ -1267,6 +1251,9 @@ We can then directly integrate the safety factor as
 \frac{\d \varphi}{\d\Theta} = \frac{B^\varphi}{B^\Theta}\\
 q\equiv\frac{1}{2\pi}\oint \frac{B^\varphi}{B^\Theta} \d\Theta
 \end{align}
+We integrate this equation with the help of one of our grid
+construction algorithms, i.e. we use a high-order Runge-Kutta method
+and refine the stepsize until machine-precision is reached.
 
 Notice that the safety factor diverges on the last closed flux
 surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa_area}
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index f0ffe628e..3a8934e31 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -239,12 +239,7 @@ int main( int argc, char* argv[])
         y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile);
         dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
         MPI_OUT std::cout << "initialize ni" << std::endl;
-        if( p.initphi == "zero")
-            feltor.initializeni( y0[0][0], y0[0][1]);
-        else if( p.initphi == "balance")
-            dg::blas1::copy( y0[0][0], y0[0][1]); //set N_i = n_e
-        else
-            MPI_OUT std::cerr <<"WARNING: Unknown initial condition for phi!\n";
+        feltor.initializeni( y0[0][0], y0[0][1]);
 
         dg::blas1::copy( 0., y0[1][0]); //set we = 0
         dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
-- 
GitLab


From 2502596ccc978d073d2a8c580a3ac3633b7c9440 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 6 May 2019 16:31:59 +0200
Subject: [PATCH 072/540] Further reorganizing feltor.tex

---
 src/feltor/feltor.tex | 551 ++++++++++++++++++------------------------
 1 file changed, 230 insertions(+), 321 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 22e167dd8..f751c6d14 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -399,6 +399,163 @@ We now use $\theta_\alpha(\psi-\psi_0)+\psi_0$ instead of $\psi$ for the computa
 magnetic field, which introduces a shear layer around $\psi_0$ where the
 fieldlines are straightened to match $\ehat_\varphi$.
 Note that $\Theta_\alpha(0) = 0.5$ and $\theta_\alpha(0) = 35\alpha/256$.
+\section{Flux surface averaging and safety factor}
+\subsection{Preliminary}
+Recall that the {\bf Dirac delta-function} has the property (in any dimension):
+\begin{align} \label{eq:dirac_delta}
+\int_V f(\vec x) \delta(h(\vec x) - h') \dV = \int_{h=h'} \frac{f(\vec x)}{|\nabla h|} \dA
+\end{align}
+which means that the delta-function can be used to express area integrals of the
+submanifold given as a contour of the function $h(\vec x)$.
+A numerically tractable approximation to the delta-function reads
+\begin{align}\label{eq:delta}
+\delta(h(\vec x)-h') = \frac{1}{2\pi \epsilon^2}
+\exp\left( - \frac{\left(h(\vec x)-h'\right)^2}{2\epsilon^2}\right)
+\end{align}
+where $\epsilon$ is a small, free parameter.
+In the DG framework the left-hand side
+of Eq.~\eqref{eq:dirac_delta} can thus readily be computed
+via Gauss-Legendre quadrature, which we propse as a first method to compute area
+integrals even if our coordinate system is not aligned to the area.
+
+Furthermore, recall the {\bf co-area formula}
+\begin{align} \label{eq:coarea}
+\int_{\Omega_0} f(\vec x) \dV =
+\int_0^{h_0} \left( \int_{h=h'} \frac{f(\vec x)}{|\nabla h|}  \dA  \right) \d h'
+\end{align}
+where $\Omega_0$ is the volume enclosed by the contour $h=h_0$.
+The co-area formula can be viewed as a change of variables in the
+volume integral.
+
+We define the {\bf toroidal average} of a function $f(R,Z,\varphi)$ as
+\begin{align} \label{eq:phi_average}
+\langle f\rangle_\varphi(R,Z) := \frac{1}{2\pi}\oint f(R,Z,\varphi)\d \varphi
+\end{align}
+
+In arbitrary coordinates the area integral is defined by the pull back
+of the flux 2-form and the metric
+\begin{align}
+\label{}
+\dA^2 = i_{\hat \psi_p} vol^3 \quad \hat \psi_p = \frac{\nabla \psi_p}{|\nabla \psi_p|}
+\end{align}
+to a parameterization of the flux-surface.
+In a flux-aligned coordinate system $\{\zeta, \eta, \varphi\}$ the pull-back is trivial ($\zeta=const$) and we have with $\hat\zeta = \hat \psi_p$
+\begin{align}
+\dA = \sqrt{g^{\zeta\zeta}} \sqrt{g} \d\eta\d\varphi, \quad \vec\dA := \hat\zeta \dA,\quad
+\hat\zeta=\frac{\nabla\psi_p}{|\nabla\psi_p|}
+\label{}
+\end{align}
+where we used that $g^{\zeta\zeta} = (\nabla\zeta)^2 = f_0^2(\nabla\psi_p)^2$.
+Notice that numerically we can integrate in flux-aligned coordinates by generating a corresponding
+grid and pulling back (interpolating) the relevant fields to this grid. This is the second method
+to numerically compute area integrals.
+
+\subsection{Flux surface average}
+
+There is two possible ways to introduce a flux-surface average.
+The first one is the straightforward average on the actual area of the
+flux-surface.
+The {\bf area average}
+of a function $f(R,Z,\varphi)$ is given by the formula
+\begin{align}\label{eq:fsa_area}
+\langle f \rangle^{area}_{\psi_{p}} :=&
+\frac{ \oint_{\psi_p  } f(R,Z,\varphi)\dA}{\oint_{\psi_p } \dA} \nonumber \\
+=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vec\nabla\psi_p| \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
+{\int_\Omega |\vec\nabla\psi_p|\delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z} \nonumber\\
+=& \frac{\oint \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g g^{\zeta\zeta}}\d\eta}
+         { \oint \sqrt{g g^{\zeta\zeta}}\d\eta}
+\end{align}
+%with $\dV := R\d R\d Z\d \varphi$ %(we define the average in computational space and omit one $R$)
+and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
+in our domain $\Omega$.
+
+The second one (the {\bf volume average} after \cite{haeseleer}) defines an average on a
+small volume - a shell centered around the flux-surface - defined by two neighboring flux-surfaces.
+If we define $v(\psi) := \int_0^\psi \dV$ as the volume
+flux label, we define
+\begin{align} \label{eq:fsa_vol}
+\langle f \rangle^{vol}_\psi :=& \frac{\partial}{\partial v} \int \dV f
+ = \frac{1}{\int \dA |\nabla\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\nabla\psi_p|} \dA \nonumber\\
+=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
+{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}\nonumber\\
+ =& \frac{1}{\int \sqrt{g}\d\eta } \int_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
+\end{align}
+where we used the co-area formula Eq.~\eqref{eq:coarea} for the second
+identity. We immediately see that this definition differs from the first
+Eq.~\eqref{eq:fsa_area} by the weight factor $|\nabla\psi_p|$ and that it is particularly easy to compute
+in a flux-aligned coordinate system. Notice however that the volume element does appear (unlike e.g. Tokam3X papers)
+
+Both averages fulfill the basic identities
+\begin{align}
+\label{eq:fsa_identities}
+\langle \mu f + \lambda g\rangle &= \mu\langle f\rangle + \lambda \langle g\rangle \\
+\langle f(\psi_p) \rangle &= f(\psi_p)
+\end{align}
+
+
+The volume average is better suited for density-like quantities
+than the area average as we can see with the following identity.
+Assume we have a quantity $X$ with $\partial_t X + \nabla \cdot \vec j_X = \Lambda_X$. Then we can use the volume average to write
+\begin{align}
+\frac{\partial}{\partial t} \langle X \rangle^{vol} + \frac{\partial}{
+  \partial v} \langle \vec j_X\cdot \nabla v\rangle^{vol}  = \langle \Lambda_X\rangle^{vol}
+\label{eq:fsa_balance}
+\end{align}
+where again $v=v(\psi_p)$ is the volume flux label.
+The {\bf total flux} of a given flux density $\vec j_X$ though the
+flux surface $\psi_p = \psi_{p0}$ is given by
+\begin{align}
+J_X:=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
+ \left\langle\vec j_X\cdot\nabla v\right\rangle^{vol} = \frac{\d v}{\d\psi_p} \langle \vec j_X\cdot\nabla\psi_p \rangle^{vol}
+%2\pi\int_\Omega \vec \langle \vec j\cdot \vec\nabla\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
+\label{eq:total_flux}
+\end{align}
+Once we have the flux-surface averaged equation we can easily get the volume integrated version
+\begin{align}
+\frac{\partial}{\partial t} \int_0^{v(\psi_p)}\langle X \rangle^{vol} \d v 
++ \langle \vec j_X\cdot \nabla v\rangle^{vol}(v(\psi_p))  = \int_0^{v(\psi_p)}\langle \Lambda_X\rangle^{vol}\d v
+\label{eq:integral_balance}
+\end{align}
+
+\subsection{The safety factor}
+Assume that we pick a random field line and follow it (integrate it) for exactly one
+poloidal turn. The {\bf safety factor} is defined as the ratio between
+the resulting toroidal angle ($\Delta\varphi$) to the poloidal angle ($2\pi$)
+\begin{align}
+q := \frac{\Delta\varphi}{2\pi}
+\label{}
+\end{align}
+Since our magnetic field is symmetric in $\varphi$ and we used one
+full poloidal turn this definition is independent of which
+fieldline we pick on a given flux surface.
+
+%Let us define the poloidal length $s$ as the fieldline following
+%parameter i.e. $\vec B\cdot \nabla s \equiv B_p = R_0|\nabla \psi_p|/R$
+%and $\d\varphi/\d s = B^\varphi(R(s), Z(s)) / B_p(R(s),Z(s))$.
+%We can then express the safety factor as the line integral
+%\begin{align}
+%q=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vec\nabla\psi_p|} \d s
+%= \frac{1}{2\pi}\int \frac{I(\psi_p)}{R}\delta(\psi_p-\psi_{p0}) H(Z-Z_X) \d R\d Z
+%\end{align}
+%where we made use of Eq.~\eqref{eq:dirac_delta} in two dimensions in the
+%last equality and thus arrive at a numerical tractable expression
+%to evaluate the safety factor.
+Let us define the geometric poloidal angle $\Theta$ as the fieldline following
+parameter i.e. $\vec B\cdot\nabla\Theta = R_0(\psi_R (R-R_0) + \psi_Z Z)/r^2R$.
+We can then directly integrate the safety factor as
+\begin{align}\label{eq:safety_factor}
+\frac{\d R}{\d\Theta} = \frac{B^R}{B^\Theta}\quad 
+\frac{\d Z}{\d\Theta} = \frac{B^Z}{B^\Theta}\quad 
+\frac{\d \varphi}{\d\Theta} = \frac{B^\varphi}{B^\Theta}\\
+q\equiv\frac{1}{2\pi}\oint \frac{B^\varphi}{B^\Theta} \d\Theta
+\end{align}
+We integrate this equation with the help of one of our grid
+construction algorithms, i.e. we use a high-order Runge-Kutta method
+and refine the stepsize until machine-precision is reached.
+
+Notice that the safety factor diverges on the last closed flux
+surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa_area}
+remain finite due to the $\nabla\psi$ factor.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
@@ -667,11 +824,11 @@ and
 Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite.
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
-Integrating the density equation we directly get
+The density equation directly yields the particle conservation
 \begin{align} \label{eq:mass_theorem}
-  \frac{\partial}{\partial t} \int_\Omega \dV n_e
-  + \int_{\partial\Omega} \vec\dA\cdot\vec{ j_{n_e}}
-  =  \int_\Omega \dV (\Lambda_{n_e}+S_{n_e})
+  \frac{\partial}{\partial t} n_e
+  + \nabla\cdot\vec{ j_{n_e}}
+  =  \Lambda_{n_e}+S_{n_e}
 \end{align}
 The terms of the particle conservation thus read
 \begin{align} \label{eq:mass_conservation}
@@ -694,96 +851,15 @@ n_e \vec K = n_e\nabla\times\frac{\bhat}{B} = \nabla\times n_e\frac{\bhat}{B} +
 \end{align}
 such that we can define the diamagnetic flux in the particle flux since
 the rotation vanishes under the divergence.
-\subsubsection{Polarisation charge density}
-First of all let us approximate FLR effects in the long wavelength limit (LWL)
-$\Gamma_{1,i}N_i \approx N_i + \frac{T_i}{2 m_i \Omega_0^2} \Delta_\perp  N_i$.
-With this identiy we can write the polarisation equation as
-\begin{align}\label{eq:general_polarisation}
-\nabla\cdot\left(\frac{N_i}{\Omega_i B}\nabla_\perp\phi\right) 
-+ \frac{1}{2} \frac{T_i}{m_i\Omega_0^2} \Delta_\perp N_i
-= n_e - N_i 
-\end{align}
-Let us now define
-\begin{align}
-\rho_E &:= -\nabla\cdot\left(\frac{N_i}{\Omega_i B}\nabla_\perp\phi\right)
-\equiv \nabla\cdot\left(\frac{N_i}{\Omega_i} \bhat\times\vec u_E\right)
-=
--\frac{\Omega_{i0}}{\Omega_i}\bhat \cdot(\nabla\times N_i \vec u_E)
-+ \Omega_{i0} N_i \vec{\mathcal K} \cdot \vec u_E
-\\
-\mathcal \rho_D &:= -\frac{T_i}{m_i\Omega^2_{i0}} \Delta_\perp N_i
-\equiv \nabla\cdot\left( \frac{N_i}{\Omega_{i0}} \bhat\times \vec v_{d0}\right) \quad \vec v_{d0} := \frac{\bhat\times \nabla T_i N_i}{eB_0N_i}
-\end{align}
-which are the generalized \ExB and diamagnetic vorticity densities.
-
-Now, we take the time derivative of Eq.~\eqref{eq:general_polarisation}
-to get (neglecting terms in the LWL, \fixme{Confirm, units!})
-\begin{align}
- &\frac{\partial}{\partial t}\left[
-\rho_E + \rho_D
-\right]
-+ \nabla\cdot\left( \vec u_E \left[
-\rho_E + \rho_D
-\right]
-\right)
-- \nabla\cdot(N_i \vec u_P)
-\nonumber\\
-&= \nabla\cdot\left( \frac{\bhat\times\vec\nabla (t_e n_e + T_i N_i)}{eN_iB}N_i\right)
-\nonumber\\
-&+ \nabla\cdot\left( \frac{\nabla\times\bhat}{B}\frac{1}{2}\left(m_e n_e u_e^2 + m_i N_i U_i^2\right) \right)
-\nonumber\\
-&+ \nabla\cdot\left(\bhat j_\parallel\right)
-+ \nabla\cdot\left(\frac{\nabla\times A_\parallel \bhat}{B} \Delta_\perp A_\parallel\right)
-+ \Lambda_{N_i} - \Lambda_{n_e}
-\\
-\vec u_P &:=\frac{\bhat\times \nabla u_E^2/2}{\Omega_i}\nonumber \\
-&\nabla\cdot \left(\vec j_{pol} + \vec j_D + \vec j_{\nabla\times\bhat} + \vec j_\parallel + \vec j_{MW} + \vec j_{Diff}\right) = 0
-\label{eq:current_divergence}
-\end{align}
-Notice that this equation is exact for $T_i=0$ and holds in the LWL
-otherwise. Furthermore, we assumed $\Gamma_{1,i}S_{N_i} = S_{n_e}$.
-In slab geometry $\bhat\equiv \hat z$ and $B=B(x)$ this equation reduces
-to the familiar drift-fluid quasineutrality equation \fixme{recompute, units!}
-\begin{align}
- &\nabla\cdot\frac{\partial}{\partial t}\left[
-\frac{\zhat\times \nabla\cdot( N_i (\vec u_E ))}{\Omega_i}
-+
-\frac{\zhat\times \nabla\cdot( N_i (\vec u_{d0}))}{\Omega_{i0}}
-\right]
-+ \nabla\cdot\left(
-\frac{\zhat\times\vec \nabla\cdot(N_i\vec u_E \vec u_E)}{\Omega_i} 
-+
-\frac{\zhat\times\vec \nabla\cdot(N_i\vec u_E \vec u_{d0})}{\Omega_{i0}} 
-\right)
-\nonumber\\
-&= \nabla\cdot\left( \frac{\zhat\times\vec\nabla (t_e n_e + T_i N_i)}{eN_iB}N_i\right)
-\nonumber\\
-%&+ \nabla\cdot\left( \frac{\nabla\times\bhat}{B}\frac{1}{2}\left(m_e n_e u_e^2 + m_i N_i U_i^2\right) \right)
-%\nonumber\\
-&+ \nabla\cdot\left(\zhat j_\parallel\right)
-+ \nabla\cdot\left(\frac{\zhat\times \vec\nabla\cdot\left(
-\tilde{\vec B}_\perp\tilde{\vec B}_\perp\right)}{B_0} \right)
-\quad
-\tilde{\vec B}_\perp := \nabla \times A_\parallel \zhat
-%\label{eq:slab_divergence}
-\end{align}
-\fixme{use Mathematika for maybe slab/toroidal equation in 3d geometry}
-
-and the vorticity conservation
-\begin{align} \label{eq:vorticity_conservation}
-\mathcal \rho_{pol} &= -\vec{\nabla} \cdot\left(\frac{N_i}{B^2} \vec{\nabla}_\perp \phi\right)  - \tau_i \Delta_\perp N_i\nonumber\\
-\vec{j_{pol}} &= \vec j_{N_i} - \vec j_{n_e} \nonumber\\
-\Lambda_{ pol} &= \Lambda_{N_i} - \Lambda_{n_e}
-\end{align}
 
 \subsubsection{Energy theorem}
 The terms of the energy theorem are
 \begin{align} \label{eq:energy_theorem}
-\partial_t \int_\Omega \dV \mathcal E +
-\oint_{\partial\Omega} \vec\dA \cdot \vec j_{\mathcal E}
-= \int_\Omega \dV \left( \Lambda_{\mathcal E}
+\partial_t \mathcal E +
+\nabla \cdot \vec j_{\mathcal E}
+= \Lambda_{\mathcal E}
 +  S_{\mathcal E}
--  R_{\mathcal E} \right)
+-  R_{\mathcal E}
 \end{align}
 with ( $z_e=-1$ and $z_i=+1$)
 \begin{align} \label{eq:energy_conservation}
@@ -813,6 +889,70 @@ In \(\Lambda\) we insert the dissipative terms of Section~\ref{sec:dissres}. \\
 Replace $\Delta_\perp$ with $-\Delta_\perp^2$ when hyperviscous diffusion is chosen
 for the diffusion terms in the above equations.
 
+\subsection{ Particle, energy and vorticity fluxes}
+Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservation} and \eqref{eq:energy_conservation} through a flux surface
+\begin{align} \label{eq:particle_flux}
+ \vec j_{N}\cdot \vec \nabla\psi_p %=& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ %\nabla\psi_p \nonumber\\
+ =&
+  N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
+   \mathcal K_{\nabla\times\bhat}(\psi_p) + \tau  \mathcal K_{\nabla B}(\psi_p) \right] \nonumber\\
+ &+ NU\left [\left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
+ \vec j_{\mathcal E}\cdot \vec \nabla\psi_p =&
+ %\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right)
+ %N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ %\vec j_N \cdot \vec\nabla\psi_p
+ %\nonumber\\
+ %+ z\left(\mu NU^2 \vec v_{\nabla\times \bhat} + \tau NU(\bhat+\tilde{\vec b}_\perp )\right) \cdot\vec \nabla\psi_p \nonumber\\
+ %=
+\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
++ z \mu\tau NU^2 \mathcal K_{\nabla\times\bhat}(\psi_p) \nonumber\\
+&+ z \tau NU
+ \left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
+\label{eq:energy_flux}
+\end{align}
+
+In order to discuss poloidal flows let us first define orthogonal vectors $\{\vec{ \hat\zeta}, \vec{\hat\eta},\bhat\}$
+\begin{align}
+\vec{\hat\zeta} := \nabla\psi_p,\quad
+\vec{\hat \eta} := \frac{\bhat \times\hat \zeta}{B}
+\end{align}
+where $\vec{\hat\eta}$
+points in the counter-clockwise poloidal direction with our choice of the magnetic field direction.
+Notice that $\vec{\hat \eta}$ in general has a (small) toroidal component in addition to the dominant poloidal component.
+
+%We then have $\nabla\phi = \hat \zeta\partial_{\hat\zeta}\phi + \hat\eta\partial_{\hat\eta} \phi$
+%with $\partial_{\hat\zeta}:= \hat\zeta\cdot\nabla$ and $\partial_{\hat\eta}:=\hat\eta\cdot\nabla$.
+Furthermore, from $\vec u_E = \bhat\times \vec\nabla\phi/B$ and $\vec u_d = \tau_i \bhat\times\nabla \ln N_i$
+we can derive
+\begin{align}
+u_E^{\hat\eta} &:= \vec u_E\cdot \vec{\hat\eta} = \frac{\partial_{\hat\zeta} \phi}{B^2} \\
+u_E^{\hat\zeta}&:= \vec u_E\cdot \vec{\hat\zeta}  = -\partial_{\hat\eta} \phi \\
+u_d^{\hat\eta} &:= \vec u_d\cdot \vec{\hat\eta} =  \frac{\tau_i \partial_{\hat\zeta} \ln N_i}{B} \\
+u_d^{\hat\zeta}&:= \vec u_d\cdot \vec{\hat\zeta} = -B \tau_i \partial_{\hat\eta} \ln N_i
+\end{align}
+With this we write the force balance equation
+\begin{align}
+\frac{\partial}{\partial t} \left\langle \mu_i N_i \left(
+\frac{\partial_{\hat\zeta}\phi\,}{B^2} + \tau_i \partial_{\hat\zeta} \ln N_i\right) \right\rangle
++ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle \mu_i N_i\partial_{\hat\eta} \phi \left(\frac{\partial_{\hat\zeta}\phi }{B^2}
++ \tau_i \partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel \right\rangle
+\nonumber\\
+= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\nabla\times\bhat}\cdot\nabla\psi_p \right\rangle
+\end{align}
+Notice that 
+\begin{align}
+\left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p\right\rangle
+= \left\langle \frac{ \bhat\times \nabla (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cdot\nabla\psi_p\right\rangle
+= -\left\langle\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)\right\rangle
+\end{align}
+
+
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 
@@ -856,6 +996,7 @@ time & Multistep "Karniadakis" & \\
 \qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion and Resistive terms. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
 \bottomrule
 \end{longtable}
+
 \section{Usage}
 Compilation:\\
 \texttt{make feltor device=\{gpu,omp\}} Compile \texttt{feltor.cu} (only shared memory)\\
@@ -1026,238 +1167,6 @@ the whole simulation is lost. It is safer to just merge files afterwards with fo
 \texttt{ncrcat output1.nc output2.nc output.nc}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Diagnostics}\label{sec:diagnostics}
-\subsection{Preliminary}
-Recall that the {\bf Dirac delta-function} has the property (in any dimension):
-\begin{align} \label{eq:dirac_delta}
-\int_V f(\vec x) \delta(h(\vec x) - h') \dV = \int_{h=h'} \frac{f(\vec x)}{|\nabla h|} \dA
-\end{align}
-which means that the delta-function can be used to express area integrals of the
-submanifold given as a contour of the function $h(\vec x)$.
-A numerically tractable approximation to the delta-function reads
-\begin{align}\label{eq:delta}
-\delta(h(\vec x)-h') = \frac{1}{2\pi \epsilon^2}
-\exp\left( - \frac{\left(h(\vec x)-h'\right)^2}{2\epsilon^2}\right)
-\end{align}
-where $\epsilon$ is a small, free parameter.
-In the DG framework the left-hand side
-of Eq.~\eqref{eq:dirac_delta} can thus readily be computed
-via Gauss-Legendre quadrature, which we propse as a first method to compute area
-integrals even if our coordinate system is not aligned to the area.
-
-Furthermore, recall the {\bf co-area formula}
-\begin{align} \label{eq:coarea}
-\int_{\Omega_0} f(\vec x) \dV =
-\int_0^{h_0} \left( \int_{h=h'} \frac{f(\vec x)}{|\nabla h|}  \dA  \right) \d h'
-\end{align}
-where $\Omega_0$ is the volume enclosed by the contour $h=h_0$.
-The co-area formula can be viewed as a change of variables in the
-volume integral.
-
-We define the {\bf toroidal average} of a function $f(R,Z,\varphi)$ as
-\begin{align} \label{eq:phi_average}
-\langle f\rangle_\varphi(R,Z) := \frac{1}{2\pi}\oint f(R,Z,\varphi)\d \varphi
-\end{align}
-
-In arbitrary coordinates the area integral is defined by the pull back
-of the flux 2-form and the metric
-\begin{align}
-\label{}
-\dA^2 = i_{\hat \psi_p} vol^3 \quad \hat \psi_p = \frac{\nabla \psi_p}{|\nabla \psi_p|}
-\end{align}
-to a parameterization of the flux-surface.
-In a flux-aligned coordinate system $\{\zeta, \eta, \varphi\}$ the pull-back is trivial ($\zeta=const$) and we have with $\hat\zeta = \hat \psi_p$
-\begin{align}
-\dA = \sqrt{g^{\zeta\zeta}} \sqrt{g} \d\eta\d\varphi, \quad \vec\dA := \hat\zeta \dA,\quad
-\hat\zeta=\frac{\nabla\psi_p}{|\nabla\psi_p|}
-\label{}
-\end{align}
-where we used that $g^{\zeta\zeta} = (\nabla\zeta)^2 = f_0^2(\nabla\psi_p)^2$.
-Notice that numerically we can integrate in flux-aligned coordinates by generating a corresponding
-grid and pulling back (interpolating) the relevant fields to this grid. This is the second method
-to numerically compute area integrals.
-
-\subsection{Flux surface average}
-
-There is two possible ways to introduce a flux-surface average.
-The first one is the straightforward average on the actual area of the
-flux-surface.
-The {\bf area average}
-of a function $f(R,Z,\varphi)$ is given by the formula
-\begin{align}\label{eq:fsa_area}
-\langle f \rangle^{area}_{\psi_{p}} :=&
-\frac{ \oint_{\psi_p  } f(R,Z,\varphi)\dA}{\oint_{\psi_p } \dA} \nonumber \\
-=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vec\nabla\psi_p| \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
-{\int_\Omega |\vec\nabla\psi_p|\delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z} \nonumber\\
-=& \frac{\oint \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g g^{\zeta\zeta}}\d\eta}
-         { \oint \sqrt{g g^{\zeta\zeta}}\d\eta}
-\end{align}
-%with $\dV := R\d R\d Z\d \varphi$ %(we define the average in computational space and omit one $R$)
-and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
-in our domain $\Omega$.
-
-The second one (the {\bf volume average} after \cite{haeseleer}) defines an average on a
-small volume - a shell centered around the flux-surface - defined by two neighboring flux-surfaces.
-If we define $v(\psi) := \int_0^\psi \dV$ as the volume
-flux label, we define
-\begin{align} \label{eq:fsa_vol}
-\langle f \rangle^{vol}_\psi :=& \frac{\partial}{\partial v} \int \dV f
- = \frac{1}{\int \dA |\nabla\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\nabla\psi_p|} \dA \nonumber\\
-=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
-{\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}\nonumber\\
- =& \frac{1}{\int \sqrt{g}\d\eta } \int_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
-\end{align}
-where we used the co-area formula Eq.~\eqref{eq:coarea} for the second
-identity. We immediately see that this definition differs from the first
-Eq.~\eqref{eq:fsa_area} by the weight factor $|\nabla\psi_p|$ and that it is particularly easy to compute
-in a flux-aligned coordinate system. Notice however that the volume element does appear (unlike e.g. Tokam3X papers)
-
-Both averages fulfill the basic identities
-\begin{align}
-\label{eq:fsa_identities}
-\langle \mu f + \lambda g\rangle &= \mu\langle f\rangle + \lambda \langle g\rangle \\
-\langle f(\psi_p) \rangle &= f(\psi_p)
-\end{align}
-
-
-The volume average is better suited for density-like quantities
-than the area average as we can see with the following identity.
-Assume we have a quantity $X$ with $\partial_t X + \nabla \cdot \vec j_X = \Lambda_X$. Then we can use the volume average to write
-\begin{align}
-\frac{\partial}{\partial t} \langle X \rangle^{vol} + \frac{\partial}{
-  \partial v} J_X  = \langle \Lambda_X\rangle^{vol}
-\label{eq:fsa_balance}
-\end{align}
-where again $v=v(\psi_p)$ is the volume flux label.
-The {\bf total flux} of a given flux density $\vec j_X$ though the
-flux surface $\psi_p = \psi_{p0}$ is given by
-\begin{align}
-J_X:=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
-2\pi\int_\Omega \vec \langle \vec j\cdot \vec\nabla\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
-\label{eq:total_flux}
-\end{align}
-\subsection{ Particle, energy and vorticity fluxes}
-Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservation} and \eqref{eq:energy_conservation} through a flux surface
-\begin{align} \label{eq:particle_flux}
- \vec j_{N}\cdot \vec \nabla\psi_p %=& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- %\nabla\psi_p \nonumber\\
- =&
-  N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
-   \mathcal K_{\nabla\times\bhat}(\psi_p) + \tau  \mathcal K_{\nabla B}(\psi_p) \right] \nonumber\\
- &+ NU\left [\left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
- \vec j_{\mathcal E}\cdot \vec \nabla\psi_p =&
- %\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right)
- %N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- %\vec j_N \cdot \vec\nabla\psi_p
- %\nonumber\\
- %+ z\left(\mu NU^2 \vec v_{\nabla\times \bhat} + \tau NU(\bhat+\tilde{\vec b}_\perp )\right) \cdot\vec \nabla\psi_p \nonumber\\
- %=
-\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
-+ z \mu\tau NU^2 \mathcal K_{\nabla\times\bhat}(\psi_p) \nonumber\\
-&+ z \tau NU
- \left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
-\label{eq:energy_flux}
-\end{align}
-
-In order to discuss poloidal flows let us first define orthogonal unit vectors $\{\hat\zeta, \hat\eta,\bhat\}$
-\begin{align}
-\hat\zeta := \frac{\nabla\psi_p}{|\nabla\psi_p|} \quad
-\hat \eta := \bhat \times\hat \zeta
-\end{align}
-where $\hat\eta$
-points in the counter-clockwise poloidal direction with our choice of the magnetic field direction.
-We then have $\nabla\phi = \hat \zeta\partial_{\hat\zeta}\phi + \hat\eta\partial_{\hat\eta} \phi$
-with $\partial_{\hat\zeta}:= \hat\zeta\cdot\nabla$ and $\partial_{\hat\eta}:=\hat\eta\cdot\nabla$.
-Notice that $\hat \eta$ in general has a (small) toroidal component in addition to the dominant poloidal component.
-Furthermore, from $\vec u_E = \bhat\times \vec\nabla\phi/B$ we can derive
-\begin{align}
-u^{\hat\eta} &= \frac{\partial_{\hat\zeta} \phi}{B} \\
-u^{\hat\zeta} &= -\frac{\partial_{\hat\eta} \phi}{B}
-\end{align}
-where $u^{\hat\zeta} := \hat\zeta\cdot\vec u_E$ and $u^{\hat\eta}:=\hat\eta\cdot\vec u_E$.
-Again, we emphasize that $u^{\hat\eta}$ has a toroidal component and is not a purely poloidal flow.
-
-Now, let us integrate the vorticity density over a flux aligned volume to get
-\begin{align}
-\int \dV \mathcal W =\int \dV \nabla\cdot\left(\frac{N_i\nabla_\perp\phi}{B^2}\right)  =  \int \dA u^{\hat \eta} \frac{N_i}{B}
-\end{align}
-from where we get the flux-surface (area) integrated vorticity conservation
-\fixme{This is the Eq for the radial current / or radial Force? / or poloidal/toroidal flux}
-\begin{align}
-\frac{\partial}{\partial t}\int\vec\dA\cdot\left(\frac{N_i\vec\nabla_\perp\phi}{B^2} +
-\vec \nabla_\perp N_i\right)
- = \int \vec j  \cdot\vec \dA
- + \int\dV \Lambda_{n_e} - \Lambda_{N_i}
- \end{align}
- with $(\vec j\times \vec B)\cdot\hat \eta/B =(\vec j_{N_i} - \vec j_{n_e})\cdot \hat\zeta $ and
-\begin{align}
- (\vec j_{N_i} - \vec j_{n_e})\cdot\vec \nabla\psi_p  =
- \frac{\rho_E + \rho_d}{B}[\phi,\psi_p]_\perp 
- - \frac{N_i}{B}[u_E^2/2,\psi_p]_\perp
- \nonumber\\
- + \left( \mu_iN_i U_i^2 - \mu_e n_e u_e^2\right) \mathcal K_{\nabla\times\bhat}(\psi_p)
- + \left(\tau_i N_i - \tau_e n_e  \right) \mathcal K(\psi_p)
- + \left(N_i U_i - n_eu_e \right)\vec{\tilde b_\perp}\cdot\vec\nabla\psi_p
-\end{align}
-This equation describes that a current through a flux surface feels the Lorentz force
-and thus drives a poloidal flux.
-In the first two terms of the Lorentz force density integral we find the terms related to the Reynolds/Favre stress.
-The remaining terms describe the currents related to the curvature drifts. Notice
-that the prefactors are always positive since $\tau_e$ and $\mu_e$ are negative.
-The curvature vectors counter-align with $\hat \zeta$ ($\mathcal K(\psi_p) < 0$, decelerate) on the top 
-and align ($\mathcal K(\psi_p) > 0$, accelerate) on the bottom of the tokamak.
-(Is this related to the X-point orbit loss mechanism? Chang, Ku, et al. )
-A surplus of electrons in the domain will lead to counter-clockwise poloidal acceleration, while positive
-charge leads to clockwise poloidal acceleration.
-
-Notice that $\hat\zeta\cdot\nabla N_i < 0$.
-This means that a radial
-pressure gradient balances the radial electric field.
-
-
-
-\subsection{The safety factor}
-Assume that we pick a random field line and follow it (integrate it) for exactly one
-poloidal turn. The {\bf safety factor} is defined as the ratio between
-the resulting toroidal angle ($\Delta\varphi$) to the poloidal angle ($2\pi$)
-\begin{align}
-q := \frac{\Delta\varphi}{2\pi}
-\label{}
-\end{align}
-Since our magnetic field is symmetric in $\varphi$ and we used one
-full poloidal turn this definition is independent of which
-fieldline we pick on a given flux surface.
-
-%Let us define the poloidal length $s$ as the fieldline following
-%parameter i.e. $\vec B\cdot \nabla s \equiv B_p = R_0|\nabla \psi_p|/R$
-%and $\d\varphi/\d s = B^\varphi(R(s), Z(s)) / B_p(R(s),Z(s))$.
-%We can then express the safety factor as the line integral
-%\begin{align}
-%q=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vec\nabla\psi_p|} \d s
-%= \frac{1}{2\pi}\int \frac{I(\psi_p)}{R}\delta(\psi_p-\psi_{p0}) H(Z-Z_X) \d R\d Z
-%\end{align}
-%where we made use of Eq.~\eqref{eq:dirac_delta} in two dimensions in the
-%last equality and thus arrive at a numerical tractable expression
-%to evaluate the safety factor.
-Let us define the geometric poloidal angle $\Theta$ as the fieldline following
-parameter i.e. $\vec B\cdot\nabla\Theta = R_0(\psi_R (R-R_0) + \psi_Z Z)/r^2R$.
-We can then directly integrate the safety factor as
-\begin{align}\label{eq:safety_factor}
-\frac{\d R}{\d\Theta} = \frac{B^R}{B^\Theta}\quad 
-\frac{\d Z}{\d\Theta} = \frac{B^Z}{B^\Theta}\quad 
-\frac{\d \varphi}{\d\Theta} = \frac{B^\varphi}{B^\Theta}\\
-q\equiv\frac{1}{2\pi}\oint \frac{B^\varphi}{B^\Theta} \d\Theta
-\end{align}
-We integrate this equation with the help of one of our grid
-construction algorithms, i.e. we use a high-order Runge-Kutta method
-and refine the stepsize until machine-precision is reached.
-
-Notice that the safety factor diverges on the last closed flux
-surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa_area}
-remain finite due to the $\nabla\psi$ factor.
 \subsection{Program and files}
 We have the program \texttt{feltor/diag/feltordiag.cu}.
 This program reads a previously generated simulation file \texttt{input.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows. \\
-- 
GitLab


From f507275fba9d21f75688df64909c7af7289f9a3c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 6 May 2019 23:46:08 +0200
Subject: [PATCH 073/540] Test CF-conventions in geometry-diag

- BUGFIX: correct Zonneveld typo
- introduce attributes in coordinates of nc_utilities
- introduce global and variable attributes in geometry_diag
- test xc, yc, zc coordinates, seem to work well with paraview
---
 inc/dg/tableau.h                |  4 +--
 inc/file/nc_utilities.h         | 52 +++++++++++++++++++++++++------
 inc/geometries/geometry_diag.cu | 55 +++++++++++++++++++++++++++++++--
 3 files changed, 97 insertions(+), 14 deletions(-)

diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index 060cf0d81..125e60288 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -842,7 +842,7 @@ enum tableau_identifier{
     HEUN_EULER_2_1_2,//!< <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Heun-Euler-2-1-2</a>
     BOGACKI_SHAMPINE_4_2_3,//!< <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Bogacki-Shampine-4-2-3</a>
     ARK324L2SA_ERK_4_2_3,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-4-2-3 (explicit)</a>
-    ZONNEVELD_5_3_4,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Zonnveld-5-3-4</a>
+    ZONNEVELD_5_3_4,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Zonneveld-5-3-4</a>
     ARK436L2SA_ERK_6_3_4,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-6-3-4 (explicit)</a>
     SAYFY_ABURUB_6_3_4,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Sayfy-Aburub-6-3-4</a>
     CASH_KARP_6_4_5,//!< <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Cash-Karp-6-4-5</a>
@@ -1005,7 +1005,7 @@ ButcherTableau<real_type> tableau( std::string name)
  *   Heun-Euler-2-1-2       | dg::HEUN_EULER_2_1_2       | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Heun-Euler-2-1-2</a>
  *   Bogacki-Shampine-4-2-3 | dg::BOGACKI_SHAMPINE_4_2_3 | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Bogacki-Shampine</a> (fsal)
  *   ARK-4-2-3 (explicit)   | dg::ARK324L2SA_ERK_4_2_3   | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-4-2-3 (explicit)</a>
- *   Zonneveld-5-3-4        | dg::ZONNEVELD_5_3_4        | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Zonnveld-5-3-4</a>
+ *   Zonneveld-5-3-4        | dg::ZONNEVELD_5_3_4        | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Zonneveld-5-3-4</a>
  *   ARK-6-3-4 (explicit)   | dg::ARK436L2SA_ERK_6_3_4   | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-6-3-4 (explicit)</a>
  *   Sayfy_Aburub-6-3-4     | dg::SAYFY_ABURUB_6_3_4     | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Sayfy_Aburub_6_3_4</a>
  *   Cash_Karp-6-4-5        | dg::CASH_KARP_6_4_5        | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Cash-Karp</a>
diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index 89e4998e7..4d25a85e1 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -165,6 +165,38 @@ static inline int define_dimension( int ncid, const char* name, int* dimID, cons
     return define_dimension( ncid, name, dimID, points.data(), points.size());
 }
 
+///@cond
+namespace detail{
+static inline int define_x_dimension( int ncid, int* dimID, const dg::Grid1d& g)
+{
+    int retval;
+    std::string long_name = "x-coordinate in Computational coordinate system";
+    if( (retval = define_dimension( ncid, "x", dimID, g))){ return retval;}
+    retval = nc_put_att_text( ncid, *dimID, "axis", 1, "X");
+    retval = nc_put_att_text( ncid, *dimID, "long_name", long_name.size(), long_name.data());
+    return retval;
+}
+static inline int define_y_dimension( int ncid, int* dimID, const dg::Grid1d& g)
+{
+    int retval;
+    std::string long_name = "y-coordinate in Computational coordinate system";
+    if( (retval = define_dimension( ncid, "y", dimID, g))){ return retval;}
+    retval = nc_put_att_text( ncid, *dimID, "axis", 1, "Y");
+    retval = nc_put_att_text( ncid, *dimID, "long_name", long_name.size(), long_name.data());
+    return retval;
+}
+static inline int define_z_dimension( int ncid, int* dimID, const dg::Grid1d& g)
+{
+    int retval;
+    std::string long_name = "z-coordinate in Computational coordinate system";
+    if( (retval = define_dimension( ncid, "z", dimID, g))){ return retval;}
+    retval = nc_put_att_text( ncid, *dimID, "axis", 1, "Z");
+    retval = nc_put_att_text( ncid, *dimID, "long_name", long_name.size(), long_name.data());
+    return retval;
+}
+}//namespace detail
+///@endcond
+
 /**
  * @brief Define a 1d time-dependent dimension variable together with its data points
  *
@@ -179,7 +211,7 @@ static inline int define_dimension( int ncid, const char* name, int* dimID, cons
 static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::Grid1d& g)
 {
     int retval;
-    if( (retval = define_dimension( ncid, "x", &dimsIDs[1], g))){ return retval;}
+    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[1], g))){ return retval;}
     if( (retval = define_time( ncid, "time", &dimsIDs[0], tvarID)) ){ return retval;}
 
     return retval;
@@ -200,8 +232,8 @@ static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology
     dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
     int retval;
-    if( (retval = define_dimension( ncid, "x", &dimsIDs[1], gx))){ return retval;}
-    if( (retval = define_dimension( ncid, "y", &dimsIDs[0], gy))){ return retval;}
+    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[1], gx))){ return retval;}
+    if( (retval = detail::define_y_dimension( ncid, &dimsIDs[0], gy))){ return retval;}
 
     return retval;
 }
@@ -222,8 +254,8 @@ static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const
     dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
     int retval;
-    if( (retval = define_dimension( ncid, "x", &dimsIDs[2], gx))){ return retval;}
-    if( (retval = define_dimension( ncid, "y", &dimsIDs[1], gy))){ return retval;}
+    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[2], gx))){ return retval;}
+    if( (retval = detail::define_y_dimension( ncid, &dimsIDs[1], gy))){ return retval;}
     if( (retval = define_time( ncid, "time", &dimsIDs[0], tvarID)) ){ return retval;}
 
     return retval;
@@ -247,8 +279,8 @@ static inline int define_limtime_xy( int ncid, int* dimsIDs, int size, int* tvar
     dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
     int retval;
-    if( (retval = define_dimension( ncid, "x", &dimsIDs[2], gx))){ return retval;}
-    if( (retval = define_dimension( ncid, "y", &dimsIDs[1], gy))){ return retval;}
+    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[2], gx)));
+    if( (retval = detail::define_y_dimension( ncid, &dimsIDs[1], gy)));
     if( (retval = define_limited_time( ncid, "time", size, &dimsIDs[0], tvarID)) ){ return retval;}
 
     return retval;
@@ -270,9 +302,9 @@ static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
     dg::Grid1d gz( g.z0(), g.z1(), 1, g.Nz());
     int retval;
-    if( (retval = define_dimension( ncid, "x", &dimsIDs[2], gx)));
-    if( (retval = define_dimension( ncid, "y", &dimsIDs[1], gy)));
-    if( (retval = define_dimension( ncid, "z", &dimsIDs[0], gz)));
+    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[2], gx)));
+    if( (retval = detail::define_y_dimension( ncid, &dimsIDs[1], gy)));
+    if( (retval = detail::define_z_dimension( ncid, &dimsIDs[0], gz)));
     return retval;
 }
 
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 6a86119b6..75b3a0cd3 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -4,6 +4,7 @@
 #include <fstream>
 #include <functional>
 #include <sstream>
+#include <ctime>
 #include <cmath>
 
 #include "json/json.h"
@@ -325,8 +326,28 @@ int main( int argc, char* argv[])
     file::NC_Error_Handle err;
     int ncid;
     err = nc_create( newfilename.data(), NC_NETCDF4|NC_CLOBBER, &ncid);
-    err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    err = nc_put_att_text( ncid, NC_GLOBAL, "geomfile", geom.size(), geom.data());
+    /// Set global attributes
+    std::map<std::string, std::string> att;
+    att["title"] = "Output file of feltor/inc/geometries/geometry_diag.cu";
+    att["Conventions"] = "CF-1.7";
+    ///Get local time and begin file history
+    auto t = std::time(nullptr);
+    auto tm = *std::localtime(&t);
+
+    std::ostringstream oss;
+    ///time string  + program-name + args
+    oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
+    for( int i=0; i<argc; i++) oss << " "<<argv[i];
+    att["history"] = oss.str();
+    att["comment"] = "Find more info in feltor/src/feltor.tex";
+    att["source"] = "FELTOR library";
+    att["references"] = "https://github.com/feltor-dev/feltor";
+    att["inputfile"] = input;
+    att["geomfile"] = geom;
+    for( auto pair : att)
+        err = nc_put_att_text( ncid, NC_GLOBAL,
+            pair.first.data(), pair.second.size(), pair.second.data());
+
     int dim1d_ids[1], dim2d_ids[2], dim3d_ids[3] ;
     err = file::define_dimension( ncid,"psi", &dim1d_ids[0], grid1d);
     dg::CylindricalGrid3d grid3d(Rmin,Rmax,Zmin,Zmax, 0, 2.*M_PI, n,Nx,Ny,Nz);
@@ -359,10 +380,40 @@ int main( int argc, char* argv[])
     dg::HVec vecR = dg::evaluate( dg::geo::BFieldR(c), grid3d);
     dg::HVec vecZ = dg::evaluate( dg::geo::BFieldZ(c), grid3d);
     dg::HVec vecP = dg::evaluate( dg::geo::BFieldP(c), grid3d);
+    std::string vec_long[3] = {"R-component of magnetic field", "Z-component of magnetic field", "Phi-component of magnetic field"};
     int vecID[3];
     err = nc_def_var( ncid, "B_R", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[0]);
     err = nc_def_var( ncid, "B_Z", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[1]);
     err = nc_def_var( ncid, "B_P", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[2]);
+    for(int i=0; i<3; i++)
+    {
+        err = nc_put_att_text( ncid, vecID[i], "long_name", vec_long[i].size(), vec_long[i].data());
+        std::string coordinates = "zc yc xc";
+        err = nc_put_att_text( ncid, vecID[i], "coordinates", coordinates.size(), coordinates.data());
+    }
+    err = nc_enddef( ncid);
+    err = nc_put_var_double( ncid, vecID[0], vecR.data());
+    err = nc_put_var_double( ncid, vecID[1], vecZ.data());
+    err = nc_put_var_double( ncid, vecID[2], vecP.data());
+    err = nc_redef(ncid);
+    vecR = dg::evaluate( dg::cooX3d, grid3d);
+    vecZ = dg::evaluate( dg::cooZ3d, grid3d);
+    vecP = dg::evaluate( dg::cooY3d, grid3d);
+    for( unsigned i=0; i<vecR.size(); i++)
+    {
+        double xc = vecR[i]*sin(vecZ[i]);
+        double yc = vecR[i]*cos(vecZ[i]);
+        vecR[i] = xc;
+        vecZ[i] = yc;
+    }
+    vec_long[0] = "x-coordinate in Cartesian coordinate system",
+    vec_long[1] = "y-coordinate in Cartesian coordinate system",
+    vec_long[2] = "z-coordinate in Cartesian coordinate system";
+    err = nc_def_var( ncid, "xc", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[0]);
+    err = nc_def_var( ncid, "yc", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[1]);
+    err = nc_def_var( ncid, "zc", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[2]);
+    for(int i=0; i<3; i++)
+        err = nc_put_att_text( ncid, vecID[i], "long_name", vec_long[i].size(), vec_long[i].data());
     err = nc_enddef( ncid);
     err = nc_put_var_double( ncid, vecID[0], vecR.data());
     err = nc_put_var_double( ncid, vecID[1], vecZ.data());
-- 
GitLab


From 3f7e6d933792fd06fdcbbe6bc89de6ebf4f511f0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 7 May 2019 19:09:56 +0200
Subject: [PATCH 074/540] Making feltor conform to CF-1.7

- standard name xc for Cartesian coordinates
- long names in geometry_diag
- attributes and long names in feltor_hpc
---
 inc/geometries/flux_t.cu                      |  4 +-
 inc/geometries/geometryX_elliptic_b.cu        |  4 +-
 .../geometryX_refined_elliptic_b.cu           |  4 +-
 inc/geometries/geometry_diag.cu               | 73 ++++++++++++-------
 inc/geometries/geometry_elliptic_b.cu         |  4 +-
 inc/geometries/geometry_elliptic_mpib.cu      |  4 +-
 inc/geometries/hector_t.cu                    |  4 +-
 inc/geometries/ribeiroX_t.cu                  |  4 +-
 inc/geometries/ribeiro_mpit.cu                |  4 +-
 inc/geometries/ribeiro_t.cu                   |  6 +-
 inc/geometries/separatrix_orthogonal_t.cu     |  4 +-
 inc/geometries/simple_orthogonal_t.cu         |  4 +-
 src/feltor/feltor.cu                          |  2 +-
 src/feltor/feltor.cuh                         |  8 +-
 src/feltor/feltor_hpc.cu                      | 69 +++++++++++++++---
 15 files changed, 132 insertions(+), 66 deletions(-)

diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 60234cdff..3fdaafdee 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -88,8 +88,8 @@ int main( int argc, char* argv[])
     int dim2d[2];
     err = file::define_dimensions(  ncid, dim2d, g2d_periodic);
     int coordsID[2];
-    err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
-    err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
+    err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
+    err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
     dg::HVec X=dg::pullback(dg::cooX2d, g2d), Y=dg::pullback(dg::cooY2d, g2d); //P = dg::pullback( dg::coo3, g);
     err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
     err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index 980e165c4..91064c33e 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -77,8 +77,8 @@ int main(int argc, char**argv)
     int dim2d[2];
     ncerr = file::define_dimensions(  ncid, dim2d, g2d.grid());
     int coordsID[2], psiID, functionID, function2ID, divBID;
-    ncerr = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
-    ncerr = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
+    ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
+    ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
     ncerr = nc_def_var( ncid, "error", NC_DOUBLE, 2, dim2d, &psiID);
     ncerr = nc_def_var( ncid, "num_solution", NC_DOUBLE, 2, dim2d, &functionID);
     ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
diff --git a/inc/geometries/geometryX_refined_elliptic_b.cu b/inc/geometries/geometryX_refined_elliptic_b.cu
index bcf714272..4c0121041 100644
--- a/inc/geometries/geometryX_refined_elliptic_b.cu
+++ b/inc/geometries/geometryX_refined_elliptic_b.cu
@@ -85,8 +85,8 @@ int main(int argc, char**argv)
     int dim2d[2];
     ncerr = file::define_dimensions(  ncid, dim2d, g2d_fine.grid());
     int coordsID[2], psiID, functionID, function2ID;
-    ncerr = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
-    ncerr = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
+    ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
+    ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
     ncerr = nc_def_var( ncid, "error", NC_DOUBLE, 2, dim2d, &psiID);
     ncerr = nc_def_var( ncid, "num_solution", NC_DOUBLE, 2, dim2d, &functionID);
     ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 75b3a0cd3..ac8ee0cdf 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -238,7 +238,7 @@ int main( int argc, char* argv[])
     };
     dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
     dg::DVec psipog2d   = dg::evaluate( c.psip(), grid2d);
-    std::map<std::string, dg::HVec> map1d;
+    std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
     ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
     unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
     double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
@@ -261,24 +261,29 @@ int main( int argc, char* argv[])
         dg::HVec volX2d = dg::tensor::volume2d( metricX);
         dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
         dg::IHMatrix grid2gX2d = dg::create::interpolation( coordsX[0], coordsX[1], grid2d);
-        dg::HVec psip_X = dg::evaluate( dg::one, gX2d), area_psip_X;
+        dg::HVec psip_X = dg::evaluate( dg::one, gX2d), area_psip_X, X_psip1d;
         dg::blas2::symv( grid2gX2d, (dg::HVec)psipog2d, psip_X);
         dg::blas1::pointwiseDot( volX2d, psip_X, psip_X);
-        avg_eta( psip_X, map1d["X_psip1d"], false);
+        avg_eta( psip_X, X_psip1d, false);
         avg_eta( volX2d, area_psip_X, false);
         dg::Grid1d gX1d( gX2d.x0(), gX2d.x1(), npsi, Npsi, dg::NEU);
-        map1d["X_psi_vol"] = dg::integrate( area_psip_X, gX1d);
-        dg::blas1::scal( map1d.at("X_psi_vol"), 4.*M_PI*M_PI);
-        dg::blas1::pointwiseDivide( map1d.at("X_psip1d"), area_psip_X,
-            map1d.at("X_psip1d"));
+        dg::HVec X_psi_vol = dg::integrate( area_psip_X, gX1d);
+        dg::blas1::scal( X_psi_vol, 4.*M_PI*M_PI);
+        map1d.emplace_back( "X_psi_vol", X_psi_vol,
+            "Flux volume on X-point grid");
+        dg::blas1::pointwiseDivide( X_psip1d, area_psip_X, X_psip1d);
+        map1d.emplace_back( "X_psip_fsa", X_psip1d,
+            "Flux surface average of psi on X-point grid");
 
         //NOTE: VOLUME is WITHIN cells while AREA is ON gridpoints
-        dg::HVec gradPsipX = metricX.value(0,0);
+        dg::HVec gradPsipX = metricX.value(0,0), X_psi_area;
         dg::blas1::transform( gradPsipX, gradPsipX, dg::SQRT<double>());
         dg::blas1::pointwiseDot( volX2d, gradPsipX, gradPsipX); //R\sqrt{g}|\nabla\zeta|
-        avg_eta( gradPsipX, map1d["X_psi_area"], false);
-        dg::blas1::scal( map1d.at("X_psi_area"), 4.*M_PI*M_PI);
-        volumeXGrid = dg::interpolate( map1d.at("X_psi_vol"), gX1d.x1(), gX1d);
+        avg_eta( gradPsipX, X_psi_area, false);
+        dg::blas1::scal( X_psi_area, 4.*M_PI*M_PI);
+        map1d.emplace_back( "X_psi_area", X_psi_area,
+            "Flux area on X-point grid");
+        volumeXGrid = dg::interpolate( X_psi_vol, gX1d.x1(), gX1d);
     }
 
     ///////////////////Compute flux average////////////////////
@@ -288,12 +293,16 @@ int main( int argc, char* argv[])
     dg::geo::SafetyFactor     qprof( c);
     dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, c, psipog2d, xpoint_weights);
     dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
-    map1d["psi_fsa"]    = dg::evaluate( fsa,        grid1d);
-    map1d["q-profile"]  = dg::evaluate( qprof,      grid1d);
-    map1d["psip1d"]     = dg::evaluate( dg::cooX1d, grid1d);
-    map1d["rho"] = map1d.at("psip1d");
-    dg::blas1::axpby( -1./psipmin, map1d.at("rho"), +1., 1.,
-        map1d.at("rho")); //transform psi to rho
+    map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
+        "Flux surface average of psi with delta function");
+    map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
+        "q-profile using direct integration");
+    map1d.emplace_back("psip1d",    dg::evaluate( dg::cooX1d, grid1d),
+        "Flux label psi evaluated on grid1d");
+    dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
+    dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
+    map1d.emplace_back("rho", rho,
+        "Alternative flux label rho = -psi/psimin + 1");
 
     //other flux labels
     dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, c);
@@ -305,15 +314,19 @@ int main( int argc, char* argv[])
 
     //area
     fsi.set_left( dg::evaluate( dg::geo::GradPsip(c), grid2d));
-    map1d["psi_area"] = dg::evaluate( fsi, grid1d);
-    dg::blas1::scal( map1d.at("psi_area"), 2.*M_PI);
+    dg::HVec psi_area = dg::evaluate( fsi, grid1d);
+    dg::blas1::scal(psi_area, 2.*M_PI);
+    map1d.emplace_back( "psi_area", psi_area,
+        "Flux area with delta function");
 
     dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, c);
     std::cout << "Delta Rho for Flux surface integrals = "<<-fsi.get_deltapsi()/psipmin<<"\n";
 
     fvi.set_right( xpoint_weights);
-    map1d["psi_vol"] = dg::evaluate( fvi, grid1d);
-    dg::blas1::scal(map1d["psi_vol"], 2.*M_PI);
+    dg::HVec psi_vol = dg::evaluate( fvi, grid1d);
+    dg::blas1::scal(psi_vol, 2.*M_PI);
+    map1d.emplace_back( "psi_vol", psi_vol,
+        "Flux volume with delta function");
     double volumeFVI = 2.*M_PI*fvi(psipmax);
     std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeCoarea<<" "<<volumeFVI
               <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
@@ -340,7 +353,7 @@ int main( int argc, char* argv[])
     for( int i=0; i<argc; i++) oss << " "<<argv[i];
     att["history"] = oss.str();
     att["comment"] = "Find more info in feltor/src/feltor.tex";
-    att["source"] = "FELTOR library";
+    att["source"] = "FELTOR";
     att["references"] = "https://github.com/feltor-dev/feltor";
     att["inputfile"] = input;
     att["geomfile"] = geom;
@@ -349,19 +362,24 @@ int main( int argc, char* argv[])
             pair.first.data(), pair.second.size(), pair.second.data());
 
     int dim1d_ids[1], dim2d_ids[2], dim3d_ids[3] ;
-    err = file::define_dimension( ncid,"psi", &dim1d_ids[0], grid1d);
+    err = file::define_dimension( ncid, "psi", &dim1d_ids[0], grid1d);
+    std::string psi_long_name = "Flux surface label";
+    err = nc_put_att_text( ncid, dim1d_ids[0], "long_name",
+        psi_long_name.size(), psi_long_name.data());
     dg::CylindricalGrid3d grid3d(Rmin,Rmax,Zmin,Zmax, 0, 2.*M_PI, n,Nx,Ny,Nz);
     err = file::define_dimensions( ncid, &dim3d_ids[0], grid3d);
     dim2d_ids[0] = dim3d_ids[1], dim2d_ids[1] = dim3d_ids[2];
 
     //write 1d vectors
-    for( auto pair : map1d)
+    for( auto tp : map1d)
     {
         int vid;
-        err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 1,
+        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 1,
             &dim1d_ids[0], &vid);
+        err = nc_put_att_text( ncid, vid, "long_name",
+            std::get<2>(tp).size(), std::get<2>(tp).data());
         err = nc_enddef( ncid);
-        err = nc_put_var_double( ncid, vid, pair.second.data());
+        err = nc_put_var_double( ncid, vid, std::get<1>(tp).data());
         err = nc_redef(ncid);
     }
     //write 2d vectors
@@ -380,7 +398,8 @@ int main( int argc, char* argv[])
     dg::HVec vecR = dg::evaluate( dg::geo::BFieldR(c), grid3d);
     dg::HVec vecZ = dg::evaluate( dg::geo::BFieldZ(c), grid3d);
     dg::HVec vecP = dg::evaluate( dg::geo::BFieldP(c), grid3d);
-    std::string vec_long[3] = {"R-component of magnetic field", "Z-component of magnetic field", "Phi-component of magnetic field"};
+    std::string vec_long[3] = {"R-component of magnetic field",
+        "Z-component of magnetic field", "Phi-component of magnetic field"};
     int vecID[3];
     err = nc_def_var( ncid, "B_R", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[0]);
     err = nc_def_var( ncid, "B_Z", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[1]);
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index 070225a75..17a0c6b4d 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -52,8 +52,8 @@ int main(int argc, char**argv)
     int dim2d[2];
     ncerr = file::define_dimensions(  ncid, dim2d, *g2d);
     int coordsID[2], psiID, functionID, function2ID;
-    ncerr = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
-    ncerr = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
+    ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
+    ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
     ncerr = nc_def_var( ncid, "error", NC_DOUBLE, 2, dim2d, &psiID);
     ncerr = nc_def_var( ncid, "num_solution", NC_DOUBLE, 2, dim2d, &functionID);
     ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 6cfb22500..93ffc775f 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -61,8 +61,8 @@ int main(int argc, char**argv)
     int dim2d[2];
     ncerr = file::define_dimensions(  ncid, dim2d, g2d->global());
     int coordsID[2], psiID, functionID, function2ID;
-    ncerr = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim2d, &coordsID[0]);
-    ncerr = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim2d, &coordsID[1]);
+    ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
+    ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
     ncerr = nc_def_var( ncid, "error", NC_DOUBLE, 2, dim2d, &psiID);
     ncerr = nc_def_var( ncid, "num_solution", NC_DOUBLE, 2, dim2d, &functionID);
     ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 212bb26e5..5cfac12f6 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -111,8 +111,8 @@ int main( int argc, char* argv[])
     int dim3d[2];
     err = file::define_dimensions(  ncid, dim3d, g2d_periodic);
     int coordsID[2], onesID, defID, confID,volID,divBID;
-    err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim3d, &coordsID[0]);
-    err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim3d, &coordsID[1]);
+    err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
+    err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
     //err = nc_def_var( ncid, "z_XYP", NC_DOUBLE, 3, dim3d, &coordsID[2]);
     err = nc_def_var( ncid, "psi", NC_DOUBLE, 2, dim3d, &onesID);
     err = nc_def_var( ncid, "deformation", NC_DOUBLE, 2, dim3d, &defID);
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index 1a3e43058..ebcbcfa9b 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -119,8 +119,8 @@ int main( int argc, char* argv[])
     err = file::define_dimension(  ncid, "i", dim1d, g1d);
     int coordsID[2], onesID, defID, volID, divBID;
     int coord1D[5];
-    err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 3, dim3d, &coordsID[0]);
-    err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 3, dim3d, &coordsID[1]);
+    err = nc_def_var( ncid, "xc", NC_DOUBLE, 3, dim3d, &coordsID[0]);
+    err = nc_def_var( ncid, "yc", NC_DOUBLE, 3, dim3d, &coordsID[1]);
     err = nc_def_var( ncid, "x_left", NC_DOUBLE, 1, dim1d, &coord1D[0]);
     err = nc_def_var( ncid, "y_left", NC_DOUBLE, 1, dim1d, &coord1D[1]);
     err = nc_def_var( ncid, "x_right", NC_DOUBLE, 1, dim1d, &coord1D[2]);
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 1db278b12..be3475518 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -70,8 +70,8 @@ int main( int argc, char* argv[])
     int dim3d[2];
     err = file::define_dimensions(  ncid, dim3d, g2d->global());
     int coordsID[2], onesID, defID,confID, volID, divBID;
-    err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim3d, &coordsID[0]);
-    err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim3d, &coordsID[1]);
+    err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
+    err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
     //err = nc_def_var( ncid, "z_XYP", NC_DOUBLE, 3, dim3d, &coordsID[2]);
     err = nc_def_var( ncid, "psi", NC_DOUBLE, 2, dim3d, &onesID);
     err = nc_def_var( ncid, "deformation", NC_DOUBLE, 2, dim3d, &defID);
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index 632b4751d..b72e7be04 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -80,9 +80,9 @@ int main( int argc, char* argv[])
     int dim3d[2];
     err = file::define_dimensions(  ncid, dim3d, g2d_periodic);
     int coordsID[2], onesID, defID, confID, volID,divBID;
-    err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim3d, &coordsID[0]);
-    err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim3d, &coordsID[1]);
-    //err = nc_def_var( ncid, "z_XYP", NC_DOUBLE, 3, dim3d, &coordsID[2]);
+    err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
+    err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
+    //err = nc_def_var( ncid, "zc", NC_DOUBLE, 3, dim3d, &coordsID[2]);
     err = nc_def_var( ncid, "psi", NC_DOUBLE, 2, dim3d, &onesID);
     err = nc_def_var( ncid, "deformation", NC_DOUBLE, 2, dim3d, &defID);
     err = nc_def_var( ncid, "conformal", NC_DOUBLE, 2, dim3d, &confID);
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 5ac0007ab..03ebd3cda 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -151,8 +151,8 @@ int main( int argc, char* argv[])
     err = nc_def_var( ncid, "divB", NC_DOUBLE, 3, dim3d, &divBID);
     err = nc_def_var( ncid, "num_solution", NC_DOUBLE, 3, dim3d, &gxxID);
     err = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 3, dim3d, &gyyID);
-    err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 3, dim3d, &coordsID[0]);
-    err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 3, dim3d, &coordsID[1]);
+    err = nc_def_var( ncid, "xc", NC_DOUBLE, 3, dim3d, &coordsID[0]);
+    err = nc_def_var( ncid, "yc", NC_DOUBLE, 3, dim3d, &coordsID[1]);
 
     thrust::host_vector<double> psi_p = dg::pullback( c.psip(), g2d);
     g2d.display();
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 7c721a1e9..d338ef07e 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -80,8 +80,8 @@ int main( int argc, char* argv[])
     int dim3d[2];
     err = file::define_dimensions(  ncid, dim3d, g2d_periodic);
     int coordsID[2], onesID, defID, confID,volID,divBID;
-    err = nc_def_var( ncid, "x_XYP", NC_DOUBLE, 2, dim3d, &coordsID[0]);
-    err = nc_def_var( ncid, "y_XYP", NC_DOUBLE, 2, dim3d, &coordsID[1]);
+    err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
+    err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
     //err = nc_def_var( ncid, "z_XYP", NC_DOUBLE, 3, dim3d, &coordsID[2]);
     err = nc_def_var( ncid, "psi", NC_DOUBLE, 2, dim3d, &onesID);
     err = nc_def_var( ncid, "conformal", NC_DOUBLE, 2, dim3d, &defID);
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index cccdda925..393d3ed55 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -131,7 +131,7 @@ int main( int argc, char* argv[])
     y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<dg::DVec>(profile);
     dg::blas1::axpby( 1., dg::construct<dg::DVec>(ntilde), 1., y0[0][0]);
     std::cout << "initialize ni" << std::endl;
-    feltor.initializeni( y0[0][0], y0[0][1]);
+    feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
 
     dg::blas1::copy( 0., y0[1][0]); //set we = 0
     dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 643d001d2..dbc16bbb7 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -200,7 +200,7 @@ struct Explicit
     //Given N_i-1 initialize n_e-1 such that phi=0
     void initializene( const Container& ni, Container& ne);
     //Given n_e-1 initialize N_i-1 such that phi=0
-    void initializeni( const Container& ne, Container& ni);
+    void initializeni( const Container& ne, Container& ni, std::string initphi);
 
     void operator()( double t,
         const std::array<std::array<Container,2>,2>& y,
@@ -459,13 +459,13 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::initializeni(
     // Ni = ne
     dg::blas1::copy( src, target);
     if (m_p.tau[1] != 0.) {
-        if( p.initphi == "zero")
+        if( m_p.initphi == "zero")
         {
             //add FLR correction -0.5*tau*mu*Delta n_e
             dg::blas2::symv( 0.5*m_p.tau[1]*m_p.mu[1],
                 m_lapperpN, src, 1.0, target);
         }
-        else if( p.initphi == "balance")
+        else if( m_p.initphi == "balance")
             //add FLR correction +0.5*tau*mu*Delta n_e
             dg::blas2::symv( -0.5*m_p.tau[1]*m_p.mu[1],
                 m_lapperpN, src, 1.0, target);
@@ -727,7 +727,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_energies(
         dg::blas1::subroutine( routines::ComputePsi(),
             m_temp0, m_dxA, m_dyA, m_dzA,
             m_temp0, m_temp1, m_temp2);
-        m_q.Apar = 0.5*dg::blas1::dot( m_vol3d, m_temp0)/m_beta;
+        m_q.Apar = 0.5*dg::blas1::dot( m_vol3d, m_temp0)/m_p.beta;
     }
     //= 0.5 mu_i N_i u_E^2
     m_q.Tperp = 0.5*m_p.mu[1]*dg::blas2::dot( fields[0][1], m_vol3d, m_UE2);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 3a8934e31..5521da10c 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -15,6 +15,16 @@
 #include "feltor.cuh"
 #include "implicit.h"
 
+struct Cylindrical2Cartesian{
+    void operator()( double R, double Z, double P, double& x, double& y, double& z)
+    {
+        //inplace possible
+        double xx = R*sin(P);
+        double yy = R*cos(P);
+        x = xx, y = yy, z = Z;
+    }
+};
+
 #ifdef FELTOR_MPI
 using HVec = dg::MHVec;
 using DVec = dg::MDVec;
@@ -239,7 +249,7 @@ int main( int argc, char* argv[])
         y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile);
         dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
         MPI_OUT std::cout << "initialize ni" << std::endl;
-        feltor.initializeni( y0[0][0], y0[0][1]);
+        feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
 
         dg::blas1::copy( 0., y0[1][0]); //set we = 0
         dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -352,8 +362,28 @@ int main( int argc, char* argv[])
 #else //FELTOR_MPI
     err = nc_create( argv[3],NC_NETCDF4|NC_CLOBBER, &ncid);
 #endif //FELTOR_MPI
-    err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    err = nc_put_att_text( ncid, NC_GLOBAL, "geomfile", geom.size(), geom.data());
+    /// Set global attributes
+    std::map<std::string, std::string> att;
+    att["title"] = "Output file of feltor/src/feltor_hpc.cu";
+    att["Conventions"] = "CF-1.7";
+    ///Get local time and begin file history
+    auto ttt = std::time(nullptr);
+    auto tm = *std::localtime(&ttt);
+
+    std::ostringstream oss;
+    ///time string  + program-name + args
+    oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
+    for( int i=0; i<argc; i++) oss << " "<<argv[i];
+    att["history"] = oss.str();
+    att["comment"] = "Find more info in feltor/src/feltor.tex";
+    att["source"] = "FELTOR";
+    att["references"] = "https://github.com/feltor-dev/feltor";
+    att["inputfile"] = input;
+    att["geomfile"] = geom;
+    for( auto pair : att)
+        err = nc_put_att_text( ncid, NC_GLOBAL,
+            pair.first.data(), pair.second.size(), pair.second.data());
+
     int dim_ids[4], tvarID;
 #ifdef FELTOR_MPI
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
@@ -381,23 +411,40 @@ int main( int argc, char* argv[])
         HVec vecZ = dg::pullback( fieldZ, grid);
         HVec vecP = dg::pullback( fieldP, grid);
         HVec psip = dg::pullback( mag.psip(), grid);
-        std::map<std::string, const HVec*> v3d{
-            {"BR", &vecR}, {"BZ", &vecZ}, {"BP", &vecP},
-            {"Psip", &psip}, {"Nprof", &profile },
-            {"Source", &source_damping }, {"Damping", &damping_damping}
-        };
+        HVec xc = dg::evaluate( dg::cooX3d, grid);
+        HVec yc = dg::evaluate( dg::cooY3d, grid);
+        HVec zc = dg::evaluate( dg::cooZ3d, grid);
+        dg::blas1::subroutine( Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
+
+        std::vector<std::tuple<std::string, const HVec*, std::string> > v3d;
+        v3d.emplace_back( "BR", &vecR,
+            "R-component of magnetic field in cylindrical coordinates");
+        v3d.emplace_back( "BZ", &vecZ,
+            "Z-component of magnetic field in cylindrical coordinates");
+        v3d.emplace_back( "BP", &vecP,
+            "P-component of magnetic field in cylindrical coordinates");
+        v3d.emplace_back( "Psip", &psip, "Flux-function psi");
+        v3d.emplace_back( "Nprof", &profile, "Density profile");
+        v3d.emplace_back( "Source", &source_damping, "Source region");
+        v3d.emplace_back( "Damping", &damping_damping, "Damping region");
+        v3d.emplace_back( "xc", &xc, "x-coordinate in Cartesian coordinate system");
+        v3d.emplace_back( "yc", &yc, "y-coordinate in Cartesian coordinate system");
+        v3d.emplace_back( "zc", &zc, "z-coordinate in Cartesian coordinate system");
+
         IHMatrix project = dg::create::projection( grid_out, grid);
         HVec transferH( dg::evaluate(dg::zero, grid_out));
-        for( auto pair : v3d)
+        for( auto tp : v3d)
         {
             int vecID;
-            err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 3,
+            err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 3,
                 &dim_ids[1], &vecID);
+            err = nc_put_att_text( ncid, vecID,
+                "long_name", std::get<2>(tp).size(), std::get<2>(tp).data());
             #ifdef FELTOR_MPI
             err = nc_var_par_access( ncid, vecID, NC_COLLECTIVE);
             #endif //FELTOR_MPI
             err = nc_enddef( ncid);
-            dg::blas2::symv( project, *pair.second, transferH);
+            dg::blas2::symv( project, *std::get<1>(tp), transferH);
             err = nc_put_vara_double( ncid, vecID, &start[1], &count[1],
                 #ifdef FELTOR_MPI
                 transferH.data().data()
-- 
GitLab


From 075a578280291c8b5d017ac46dd8ad040b6091f8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 8 May 2019 17:21:16 +0200
Subject: [PATCH 075/540] Start implementing feltordiag improved diagnostics

- BUGFIX: 1d interpolation boundary conditions
- use GridX in feltordiag
---
 diag/feltordiag.cu              | 172 ++++++++++++++++++++++++--------
 inc/dg/topology/interpolation.h |   2 +-
 inc/geometries/geometry_diag.cu |  13 +--
 src/feltor/feltor.tex           |  42 ++++----
 4 files changed, 162 insertions(+), 67 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index cf33a1819..f2ffdab1c 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -77,6 +77,10 @@ struct RadialEnergyFlux{
     double m_tau, m_mu;
 };
 
+struct Record{
+
+};
+
 int main( int argc, char* argv[])
 {
     if( argc != 3)
@@ -119,7 +123,6 @@ int main( int argc, char* argv[])
     const double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
     const double Rmax=gp.R_0+p.boxscaleRp*gp.a;
     const double Zmax=p.boxscaleZp*gp.a*gp.elongation;
-    const double Z_X = -1.1*gp.elongation*gp.a;
 
     dg::CylindricalGrid3d g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bcxN, p.bcyN, dg::PER);
@@ -130,13 +133,66 @@ int main( int argc, char* argv[])
     const double psip0 = mag.psip()(gp.R_0, 0);
     mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*psip0, p.alpha);
     dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
-    //double psipmin = (double)thrust::reduce(
-    //    psipog2d.begin(), psipog2d.end(), 0.0,thrust::minimum<double>() );
-    double psipmin = dg::blas1::reduce( psipog2d, 0,thrust::minimum<double>() );
-    double psipmax = dg::blas1::reduce( psipog2d, -1000,thrust::maximum<double>() );
 
-    unsigned Npsi = 50;//set number of psivalues
-    dg::Grid1d g1d_out(psipmin, psipmax, 3, Npsi, dg::NEU);
+    //Find O-point
+    double R_O = gp.R_0, Z_O = 0.;
+    dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
+    const double psipmin = mag.psip()(R_O, Z_O);
+
+
+    unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
+    std::cout << "Generate X-point flux-aligned grid!\n";
+    double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
+    double Z_X = -1.1*gp.elongation*gp.a;
+    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), R_X, Z_X) ;
+    dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, R_X, Z_X, mag.R0(), 0, 0, true);
+    double fx_0 = 1./8.;
+    double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
+    std::cout << "psi max is          "<<psipmax<<"\n";
+    psipmax = -fx_0/(1.-fx_0)*psipmin;
+    std::cout << "psi max is          "<<psipmax<<"\n";
+    dg::geo::CurvilinearGridX2d gridX2d( generator, fx_0, 0., npsi, Npsi, 160, dg::DIR_NEU, dg::NEU);
+    std::cout << "DONE!\n";
+    //Create 1d grids, one for psi and one for x
+    dg::Grid1d g1d_out(psipmin, psipmax, 3, Npsi, dg::DIR_NEU); //inner value is always 0
+    const double f0 = ( gridX2d.x1() - gridX2d.x0() ) / ( psipmax - psipmin );
+
+    //interpolation and metric
+    std::vector<dg::HVec > coordsX = gridX2d.map();
+    dg::SparseTensor<dg::HVec> metricX = gridX2d.metric();
+
+    std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
+    /// Compute flux volume label
+    dg::Average<dg::HVec > poloidal_average( gridX2d.grid(), dg::coo2d::y);
+    dg::HVec dvdpsip;
+    dg::HVec volX2d = dg::tensor::volume2d( metricX);
+    dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
+    poloidal_average( volX2d, dvdpsip, false);
+    dg::blas1::scal( dvdpsip, 4.*M_PI*M_PI*f0);
+    map1d.emplace_back( "dvdpsi", dvdpsip,
+        "Derivative of flux volume with respect to flux label psi");
+    dg::HVec X_psi_vol = dg::integrate( dvdpsip, g1d_out);
+    map1d.emplace_back( "psi_vol", X_psi_vol,
+        "Flux volume evaluated with X-point grid");
+
+    /// Compute flux area label
+    dg::HVec gradZetaX = metricX.value(0,0), X_psi_area;
+    dg::blas1::transform( gradZetaX, gradZetaX, dg::SQRT<double>());
+    dg::blas1::pointwiseDot( volX2d, gradZetaX, gradZetaX); //R\sqrt{g}|\nabla\zeta|
+    poloidal_average( gradZetaX, X_psi_area, false);
+    dg::blas1::scal( X_psi_area, 4.*M_PI*M_PI);
+    map1d.emplace_back( "psi_area", X_psi_area,
+        "Flux area evaluated with X-point grid");
+
+    dg::HVec rho = dg::evaluate( dg::cooX1d, g1d_out);
+    dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
+    map1d.emplace_back("rho", rho,
+        "Alternative flux label rho = -psi/psimin + 1");
+    dg::geo::SafetyFactor qprofile( mag);
+    map1d.emplace_back("q-profile", dg::evaluate( qprofile,   g1d_out),
+        "q-profile (Safety factor) using direct integration");
+    map1d.emplace_back("psi_psi",    dg::evaluate( dg::cooX1d, g1d_out),
+        "Flux label psi (same as coordinate)");
 
     // Construct weights and temporaries
 
@@ -147,15 +203,21 @@ int main( int argc, char* argv[])
     dg::HVec transfer2d = dg::evaluate(dg::zero,g2d_out);
     dg::HVec transfer1d = dg::evaluate(dg::zero,g1d_out);
 
+    dg::HVec transfer2dX = dg::evaluate( dg::zero, gridX2d);
+
     dg::DVec t1d = dg::evaluate( dg::zero, g1d_out);
     dg::DVec t2d = dg::evaluate( dg::zero, g2d_out);
     dg::DVec t3d = dg::evaluate( dg::zero, g3d_out);
 
 
-    // interpolate fsa back to 2d or 3d grid
 
-    dg::IDMatrix fsaonrzmatrix;
-    fsaonrzmatrix = dg::create::interpolation(psipog2d, g1d_out);
+
+    // interpolate from 2d grid to X-point points
+    dg::IHMatrix grid2gridX2d  = dg::create::interpolation(
+        coordsX[0], coordsX[1], g2d_out);
+    // interpolate fsa back to 2d or 3d grid
+    dg::IHMatrix fsaonrzmatrix = dg::create::interpolation(
+        psipog2d, g1d_out, dg::DIR_NEU);
 
     //perp laplacian for computation of vorticity
 
@@ -246,15 +308,35 @@ int main( int argc, char* argv[])
     // Create Netcdf output file and ids
     int ncid_out;
     err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
-    err = nc_put_att_text( ncid_out, NC_GLOBAL, "inputfile",
-        input.size(), input.data());
-    err = nc_put_att_text( ncid_out, NC_GLOBAL, "geomfile",
-        geom.size(), geom.data());
+
+    /// Set global attributes
+    std::map<std::string, std::string> att;
+    att["title"] = "Output file of feltor/diag/feltordiag.cu";
+    att["Conventions"] = "CF-1.7";
+    ///Get local time and begin file history
+    auto ttt = std::time(nullptr);
+    auto tm = *std::localtime(&ttt);
+    std::ostringstream oss;
+    ///time string  + program-name + args
+    oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
+    for( int i=0; i<argc; i++) oss << " "<<argv[i];
+    att["history"] = oss.str();
+    att["comment"] = "Find more info in feltor/src/feltor.tex";
+    att["source"] = "FELTOR";
+    att["references"] = "https://github.com/feltor-dev/feltor";
+    att["inputfile"] = input;
+    att["geomfile"] = geom;
+    for( auto pair : att)
+        err = nc_put_att_text( ncid, NC_GLOBAL,
+            pair.first.data(), pair.second.size(), pair.second.data());
     // define 2d and 1d and 0d dimensions and variables
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
     int dim_ids1d[2] = {dim_ids[0],dim_ids[1]}; //time , psi
     err = file::define_dimension( ncid_out, "psi", &dim_ids1d[1], g1d_out);
+    std::string psi_long_name = "Flux surface label";
+    err = nc_put_att_text( ncid, dim_ids1d[1], "long_name",
+        psi_long_name.size(), psi_long_name.data());
 
     std::map<std::string, int> id0d, id1d, id2d;
     for( auto pair : v3d)
@@ -262,7 +344,10 @@ int main( int argc, char* argv[])
         std::string name = pair.first + "_avg";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
-        name = pair.first + "_fsa_mp";
+        name = pair.first;
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        name = pair.first + "_fluc";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
         name = pair.first + "_fsa";
@@ -284,21 +369,18 @@ int main( int argc, char* argv[])
     size_t count3d[4] = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(), g3d_out.n()*g3d_out.Nx()};
     size_t start3d[4] = {0, 0, 0, 0};
 
-    //put safety factor and rho into file
-    std::cout << "Compute safety factor   "<< "\n";
-    dg::HVec xpoint_damping = dg::evaluate( dg::one, g2d_out);
-    if( gp.hasXpoint())
-        xpoint_damping = dg::evaluate( dg::geo::ZCutter(Z_X), g2d_out);
-    dg::geo::SafetyFactor qprofile(mag);
-    dg::HVec sf = dg::evaluate(qprofile, g1d_out);
-    int qID, rhoID;
-    err = nc_def_var( ncid_out, "q", NC_DOUBLE, 1, &dim_ids1d[1], &qID);
-    err = nc_def_var( ncid_out, "rho", NC_DOUBLE, 1, &dim_ids1d[1], &rhoID);
-    err = nc_enddef(ncid_out);
-    err = nc_put_vara_double( ncid_out, qID, &start1d[1], &count1d[1], sf.data());
-    dg::HVec rho = dg::evaluate( dg::cooX1d, g1d_out);//evaluate psi
-    dg::blas1::axpby( -1./psip0, rho, +1., 1., rho); //transform psi to rho
-    err = nc_put_vara_double( ncid_out, rhoID, &start1d[1], &count1d[1], rho.data());
+    //write 1d static vectors (psi, q-profile, ...) into file
+    for( auto tp : map1d)
+    {
+        int vid;
+        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 1,
+            &dim_ids1d[1], &vid);
+        err = nc_put_att_text( ncid, vid, "long_name",
+            std::get<2>(tp).size(), std::get<2>(tp).data());
+        err = nc_enddef( ncid);
+        err = nc_put_var_double( ncid, vid, std::get<1>(tp).data());
+        err = nc_redef(ncid);
+    }
     err = nc_close(ncid_out);
 
     /////////////////////////////////////////////////////////////////////////
@@ -312,8 +394,8 @@ int main( int argc, char* argv[])
     err = nc_close( ncid); //close 3d file
 
     dg::Average<dg::DVec> toroidal_average( g3d_out, dg::coo3d::z);
-    dg::geo::FluxSurfaceAverage<dg::DVec> fsa(g2d_out, mag, t2d, dg::DVec(xpoint_damping) );
 
+    steps = 3;
     for( unsigned i=0; i<steps; i++)//timestepping
     {
         start3d[0] = i;
@@ -393,28 +475,32 @@ int main( int argc, char* argv[])
         err = nc_open(argv[2], NC_WRITE, &ncid_out);
         for( auto pair : v3d)// {name, DVec}
         {
+            //toroidal average
             toroidal_average( pair.second, t2d, false);
             dg::blas1::transfer( t2d, transfer2d);
             err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_avg"),
                 start2d, count2d, transfer2d.data());
 
-            //computa fsa of quantities
-            fsa.set_container(t2d);
-            transfer1d = dg::evaluate(fsa, g1d_out);
+            //flux surface average
+            dg::blas2::symv( grid2gridX2d, transfer2d, transfer2dX); //interpolate onto X-point grid
+            dg::blas1::pointwiseDot( transfer2dX, volX2d, transfer2dX); //multiply by sqrt(g)
+            poloidal_average( transfer2dX, transfer1d, false); //average over eta
+            dg::blas1::scal( transfer1d, 4*M_PI*M_PI*f0); //
+            dg::blas1::pointwiseDivide( transfer1d, dvdpsip, transfer1d );
             err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_fsa"),
                 start1d, count1d, transfer1d.data());
 
-            //get 2d data of MidPlane
-            unsigned kmp = g3d_out.Nz()/2;
-            dg::DVec t2d_mp(pair.second.begin() + kmp*g2d_out.size(),
+            // 2d data of plane phi = 0
+            unsigned kmp = 0; //g3d_out.Nz()/2;
+            dg::HVec t2d_mp(pair.second.begin() + kmp*g2d_out.size(),
                 pair.second.begin() + (kmp+1)*g2d_out.size() );
+            err = nc_put_vara_double( ncid_out, id2d.at(pair.first),
+                start2d, count2d, t2d_mp.data() );
 
-            //compute delta f on midplane : df = f_mp - <f>
-            dg::assign( transfer1d, t1d);
-            dg::blas2::gemv(fsaonrzmatrix, t1d, t2d); //fsa on RZ grid
-            dg::blas1::axpby( 1.0, t2d_mp, -1.0, t2d);
-            dg::assign( t2d, transfer2d);
-            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_fsa_mp"),
+            // delta f on midplane : df = f_mp - <f>
+            dg::blas2::gemv(fsaonrzmatrix, transfer1d, transfer2d); //fsa on RZ grid
+            dg::blas1::axpby( 1.0, t2d_mp, -1.0, transfer2d);
+            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_fluc"),
                 start2d, count2d, transfer2d.data() );
 
         }
diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index b87ece148..4f2b2cac0 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -135,7 +135,7 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
         detail::mirror( negative, X, g.x0(), g.x1(), bcx);
 
         //determine which cell (x) lies in
-        real_type xnn = (x[i]-g.x0())/g.h();
+        real_type xnn = (X-g.x0())/g.h();
         unsigned n = (unsigned)floor(xnn);
         //determine normalized coordinates
         real_type xn = 2.*xnn - (real_type)(2*n+1);
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index ac8ee0cdf..eb8a332eb 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -261,17 +261,18 @@ int main( int argc, char* argv[])
         dg::HVec volX2d = dg::tensor::volume2d( metricX);
         dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
         dg::IHMatrix grid2gX2d = dg::create::interpolation( coordsX[0], coordsX[1], grid2d);
-        dg::HVec psip_X = dg::evaluate( dg::one, gX2d), area_psip_X, X_psip1d;
+        dg::HVec psip_X = dg::evaluate( dg::one, gX2d), dvdzeta, X_psip1d;
         dg::blas2::symv( grid2gX2d, (dg::HVec)psipog2d, psip_X);
         dg::blas1::pointwiseDot( volX2d, psip_X, psip_X);
         avg_eta( psip_X, X_psip1d, false);
-        avg_eta( volX2d, area_psip_X, false);
+        dg::blas1::scal( X_psip1d, 4.*M_PI*M_PI);
+        avg_eta( volX2d, dvdzeta, false);
+        dg::blas1::scal( dvdzeta, 4.*M_PI*M_PI);
         dg::Grid1d gX1d( gX2d.x0(), gX2d.x1(), npsi, Npsi, dg::NEU);
-        dg::HVec X_psi_vol = dg::integrate( area_psip_X, gX1d);
-        dg::blas1::scal( X_psi_vol, 4.*M_PI*M_PI);
+        dg::HVec X_psi_vol = dg::integrate( dvdzeta, gX1d);
         map1d.emplace_back( "X_psi_vol", X_psi_vol,
             "Flux volume on X-point grid");
-        dg::blas1::pointwiseDivide( X_psip1d, area_psip_X, X_psip1d);
+        dg::blas1::pointwiseDivide( X_psip1d, dvdzeta, X_psip1d);
         map1d.emplace_back( "X_psip_fsa", X_psip1d,
             "Flux surface average of psi on X-point grid");
 
@@ -296,7 +297,7 @@ int main( int argc, char* argv[])
     map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
         "Flux surface average of psi with delta function");
     map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
-        "q-profile using direct integration");
+        "q-profile (Safety factor) using direct integration");
     map1d.emplace_back("psip1d",    dg::evaluate( dg::cooX1d, grid1d),
         "Flux label psi evaluated on grid1d");
     dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index f751c6d14..bb85a70a4 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -439,10 +439,11 @@ of the flux 2-form and the metric
 \dA^2 = i_{\hat \psi_p} vol^3 \quad \hat \psi_p = \frac{\nabla \psi_p}{|\nabla \psi_p|}
 \end{align}
 to a parameterization of the flux-surface.
-In a flux-aligned coordinate system $\{\zeta, \eta, \varphi\}$ the pull-back is trivial ($\zeta=const$) and we have with $\hat\zeta = \hat \psi_p$
+In a flux-aligned coordinate system $\{\zeta, \eta, \varphi\}$ the pull-back is trivial ($\zeta=const$) and we have
 \begin{align}
-\dA = \sqrt{g^{\zeta\zeta}} \sqrt{g} \d\eta\d\varphi, \quad \vec\dA := \hat\zeta \dA,\quad
-\hat\zeta=\frac{\nabla\psi_p}{|\nabla\psi_p|}
+\dA &= \sqrt{g^{\zeta\zeta}} \sqrt{g} \d\eta\d\varphi = f_0|\nabla\psi_p|\sqrt{g}\d\eta\d\varphi,
+\\
+\vec\dA &:= \hat\psi_p \dA = f_0 (\nabla\psi_p) \sqrt{g}\d\eta\d\varphi,\quad
 \label{}
 \end{align}
 where we used that $g^{\zeta\zeta} = (\nabla\zeta)^2 = f_0^2(\nabla\psi_p)^2$.
@@ -471,14 +472,19 @@ in our domain $\Omega$.
 
 The second one (the {\bf volume average} after \cite{haeseleer}) defines an average on a
 small volume - a shell centered around the flux-surface - defined by two neighboring flux-surfaces.
-If we define $v(\psi) := \int_0^\psi \dV$ as the volume
-flux label, we define
+With the help of the volume
+flux label (notice that both the volume $v$ as well as the poloidal flux $\psi_p$ have physical 
+meaning while the coordinate $\zeta(\psi_p)$ is an arbitrary choice) we define
 \begin{align} \label{eq:fsa_vol}
+v(\psi_p) :=& \int_{\psi_{p,\min}}^\psi \dV = \int^{\zeta(\psi_p)} \sqrt{g}\d\zeta\d\eta\d\varphi,
+\\
+\frac{\d v}{\d\psi_p} =& \int\dA |\nabla\psi_p|^{-1} = 2\pi f_0\oint_{\zeta(\psi_p)} \sqrt{g}\d\eta \\
 \langle f \rangle^{vol}_\psi :=& \frac{\partial}{\partial v} \int \dV f
  = \frac{1}{\int \dA |\nabla\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\nabla\psi_p|} \dA \nonumber\\
 =& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
 {\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}\nonumber\\
- =& \frac{1}{\int \sqrt{g}\d\eta } \int_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
+ =& \left(\frac{\d v}{\d\psi_p }\right)^{-1} 2\pi f_0 \oint_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
+ = \frac{1}{\oint \sqrt{g}\d\eta } \oint_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
 \end{align}
 where we used the co-area formula Eq.~\eqref{eq:coarea} for the second
 identity. We immediately see that this definition differs from the first
@@ -505,12 +511,14 @@ where again $v=v(\psi_p)$ is the volume flux label.
 The {\bf total flux} of a given flux density $\vec j_X$ though the
 flux surface $\psi_p = \psi_{p0}$ is given by
 \begin{align}
-J_X:=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
- \left\langle\vec j_X\cdot\nabla v\right\rangle^{vol} = \frac{\d v}{\d\psi_p} \langle \vec j_X\cdot\nabla\psi_p \rangle^{vol}
+\left\langle\vec j_X\cdot\nabla v\right\rangle^{vol} &:= J_X=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
+ \frac{\d v}{\d\psi_p} \langle \vec j_X\cdot\nabla\psi_p \rangle^{vol}\\
+ &=
+   2\pi f_0 \oint_0^{2\pi} \langle \vec j_X\cdot\nabla\psi_p\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
 %2\pi\int_\Omega \vec \langle \vec j\cdot \vec\nabla\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
 \label{eq:total_flux}
 \end{align}
-Once we have the flux-surface averaged equation we can easily get the volume integrated version
+Once we have the flux-surface averaged equation we can easily get the volume integrated version (again with the help of the co-area formula)
 \begin{align}
 \frac{\partial}{\partial t} \int_0^{v(\psi_p)}\langle X \rangle^{vol} \d v 
 + \langle \vec j_X\cdot \nabla v\rangle^{vol}(v(\psi_p))  = \int_0^{v(\psi_p)}\langle \Lambda_X\rangle^{vol}\d v
@@ -752,10 +760,10 @@ We thus define a particle sink/source for electrons as
   S_{n_e}(R,Z,\varphi, t) &= \omega_s
     (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho_{s} -\rho(R,Z)) H(Z-Z_X)\\
     \rho(R,Z) &:= \frac{\psi_{p,\min}- \psi_p(R,Z) }{\psi_{p,\min}},\\
-    \psi_p(R,Z)&:= (1-\rho(R,Z))\psi_{p,\min}
+    \psi_p(R,Z)&:= (1-\rho(R,Z))\psi_{p,\min},\quad \text{In general we have }\psi_{p,\min} = \psi_p(R_O, Z_O) \neq\psi_{p}(R_0,0)
 \end{align}
 with $0 < \rho_{s}<1$
-where $\omega_s$ is the source strength parameter.
+where $\omega_s$ is the source strength parameter and $R_O$, $Z_O$ are the coordinates of the O-point.
 This will result in exponential adaption of the core and wall
 density profile of the form $n_e \propto n_{prof}+(n_{prof}-n_{e,0})e^{-\omega_st}$.
 For ions we use
@@ -1121,9 +1129,9 @@ y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, co
 z                & Coord. Var. & 1 (z) & $\varphi$-coordinate (computational space, size: $N_z$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
 energy\_time     & Coord. Var. & 1 (energy\_time) & timesteps at which 1d variables are written (variable size: itstp$\cdot$maxout$+1$, dimension size: unlimited ) \\
-x\_XYZ           & Dataset & 3 (z,y,x) & Cartesian x-coordinate $x=R\sin(\varphi)$ \\
-y\_XYZ           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
-z\_XYZ           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
+xc           & Dataset & 3 (z,y,x) & Cartesian x-coordinate $x=R\sin(\varphi)$ \\
+yc           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
+zc           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 Psip             & Dataset & 3 (z,y,x) & Flux function $\psi_p(R,Z)$ \\
 Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ \\
 Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta_\alpha(\rho_{s} - \rho(R,Z)) H(Z-Z_X)$\\
@@ -1197,14 +1205,14 @@ aligned          & Dataset & 1 (time) & $\int \dV (\nabla_\parallel n_e)^2 /n_e$
 perp\_aligned    & Dataset & 1 (time) & $\int \dV (\vec\nabla_\perp n_e)^2 /n_e$ \\
 correlationNPhi  & Dataset & 1 (time) & $\left( \int\dV e^\phi n_e \right)/ ||e^\phi||||n_e||$\\
 correlationNTildePhi  & Dataset & 1 (time) & $\left( \int\dV \phi \tilde n_e \right)/ ||\phi||||\tilde n_e||$ with $n_e \equiv \langle n_e \rangle_{\psi_p} ( 1 + \tilde n_e)$\\
-q                & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} \\
+q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} \\
 psip1d           & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
 rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
-area             & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\nabla\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
+psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\nabla\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
 X                & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
 X\_avg           & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
-X\_fluc       & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,\pi) - \langle X\rangle_{\psi_{p}}$ \\
+X\_fluc       & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,0) - \langle X\rangle_{\psi_{p}}$ \\
 X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
 particle\_flux   & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{n_e} \cdot \vec \dA$ Eq.~\eqref{eq:particle_flux} \\
 energy\_flux     & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{\mathcal E} \cdot \vec \dA$ Eq.~\eqref{eq:energy_flux} \\
-- 
GitLab


From 5559b1ea4cbd63a964461b84c7bdc96b232d251b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 8 May 2019 21:53:07 +0200
Subject: [PATCH 076/540] Add flux averages and integrals in feltordiag

---
 diag/feltordiag.cu    | 130 ++++++++++++++++++++++++++++--------------
 src/feltor/feltor.tex |  18 +++---
 2 files changed, 97 insertions(+), 51 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index f2ffdab1c..98d86b63c 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -148,9 +148,9 @@ int main( int argc, char* argv[])
     dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, R_X, Z_X, mag.R0(), 0, 0, true);
     double fx_0 = 1./8.;
     double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
-    std::cout << "psi max is          "<<psipmax<<"\n";
+    std::cout << "psi max is            "<<psipmax<<"\n";
     psipmax = -fx_0/(1.-fx_0)*psipmin;
-    std::cout << "psi max is          "<<psipmax<<"\n";
+    std::cout << "psi max in g1d_out is "<<psipmax<<"\n";
     dg::geo::CurvilinearGridX2d gridX2d( generator, fx_0, 0., npsi, Npsi, 160, dg::DIR_NEU, dg::NEU);
     std::cout << "DONE!\n";
     //Create 1d grids, one for psi and one for x
@@ -273,6 +273,7 @@ int main( int argc, char* argv[])
     dg::DVec vol = dg::tensor::volume( metric);
     const dg::DVec binv = dg::evaluate( dg::geo::InvB(mag) , g3d_out) ;
     dg::blas1::pointwiseDivide( binv, vol, vol); //1/vol/B
+    dg::DVec bphi = bhat[2];//save bphi for momentum conservation
     for( int i=0; i<3; i++)
         dg::blas1::pointwiseDot( vol, bhat[i], bhat[i]); //b_i/vol/B
     dg::DMatrix dxA = dg::create::dx( g3d_out, p.bcxU);
@@ -294,16 +295,13 @@ int main( int argc, char* argv[])
         "electrons","ions","Ue","Ui", "potential","induction"
     };
     std::vector<std::string> names_derived{
-        "vorticity","fluxe","Lperpinv","Lparallelinv"
+        "vorticity", "apar_vorticity", "neue", "NiUi",
+        "neuebphi", "NiUibphi", "fluxe",
+        "Lperpinv","Lparallelinv"
     };
     std::vector<std::string> names_0d{
         "aligned", "perp_aligned", "correlationNPhi", "total_flux"
     };
-    std::map<std::string, dg::DVec> v3d;
-    for( auto name : names_direct)
-        v3d[name] = t3d;
-    for( auto name : names_derived)
-        v3d[name] = t3d;
 
     // Create Netcdf output file and ids
     int ncid_out;
@@ -339,25 +337,43 @@ int main( int argc, char* argv[])
         psi_long_name.size(), psi_long_name.data());
 
     std::map<std::string, int> id0d, id1d, id2d;
+    // construct 0d variables
+    std::map<std::string, double> v0d;
+    for( auto name : names_0d)
+    {
+        v0d[name] = 0.;
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
+            &id0d[name]);
+    }
+
+    // construct 3d and 2d and add to 0d variables
+    std::map<std::string, dg::DVec> v3d;
+    for( auto name : names_direct)
+        v3d[name] = t3d;
+    for( auto name : names_derived)
+        v3d[name] = t3d;
     for( auto pair : v3d)
     {
-        std::string name = pair.first + "_avg";
+        std::string name = pair.first + "_ta";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
-        name = pair.first;
+        name = pair.first + "_plane";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
         name = pair.first + "_fluc";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        name = pair.first + "_fsa2";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
         name = pair.first + "_fsa";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
             &id1d[name]);
-    }
-    std::map<std::string, double> v0d;
-    for( auto name : names_0d)
-    {
-        v0d[name] = 0.;
+        name = pair.first + "_fsi";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        name = pair.first + "_fsi_lcfs";
+        v0d.at(name) = 0.; //this needs to go into v0d
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
             &id0d[name]);
     }
@@ -403,7 +419,7 @@ int main( int argc, char* argv[])
         start1d[0] = i;
         err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
         err = nc_get_vara_double( ncid, timeID, start3d, count3d, &time);
-        std::cout << "Timestep = " << i << "  time = " << time << "\n";
+        std::cout << "Timestep = " << i << "  time = " << time << "\t";
         //read in Ne,Ni,Ue,Ui,Phi,Apar
         for( auto name : names_direct)
         {
@@ -417,29 +433,44 @@ int main( int argc, char* argv[])
 
         //----------------vorticity computation
 
-        dg::blas2::gemv( laplacianM, v3d["potential"], v3d["vorticity"]);
+        dg::blas2::gemv( laplacianM, v3d.at("potential"), v3d.at("vorticity"));
+        dg::blas2::gemv( laplacianM, v3d.at("induction"), v3d.at("apar_vorticity"));
+        //----------------currents
+        //
+        dg::blas1::pointwiseDot( v3d.at("electrons"), v3d.at("Ue"), v3d.at("neue"));
+        dg::blas1::pointwiseDot( v3d.at("ions"), v3d.at("Ui"), v3d.at("NiUi"));
+        dg::blas1::pointwiseDot( v3d.at("neue"), bphi, v3d.at("neuebphi"));
+        dg::blas1::pointwiseDot( v3d.at("NiUi"), bphi, v3d.at("NiUibphi"));
+
+        //----------------Test if induction equation holds
+        dg::blas1::axpbypgz( p.beta, v3d.at("neue"), -p.beta, v3d.at("NiUi"),
+            0., t3d);
+        double norm  = dg::blas2::dot( t3d, w3d, t3d);
+        dg::blas1::axpby( -1., v3d.at("apar_vorticity"), 1., t3d);
+        double error = dg::blas2::dot( t3d, w3d, t3d);
+        std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
 
         //----------------radial flux computation
 
-        dg::blas2::symv( dxA, v3d["induction"], dx_A);
-        dg::blas2::symv( dyA, v3d["induction"], dy_A);
-        dg::blas2::symv( dz , v3d["induction"], dz_A);
-        dg::blas2::symv( dxP, v3d["potential"], dx_P);
-        dg::blas2::symv( dyP, v3d["potential"], dy_P);
-        dg::blas2::symv( dz , v3d["potential"], dz_P);
-        dg::blas1::evaluate( v3d["fluxe"], dg::equals(),
+        dg::blas2::symv( dxA, v3d.at("induction"), dx_A);
+        dg::blas2::symv( dyA, v3d.at("induction"), dy_A);
+        dg::blas2::symv( dz , v3d.at("induction"), dz_A);
+        dg::blas2::symv( dxP, v3d.at("potential"), dx_P);
+        dg::blas2::symv( dyP, v3d.at("potential"), dy_P);
+        dg::blas2::symv( dz , v3d.at("potential"), dz_P);
+        dg::blas1::evaluate( v3d.at("fluxe"), dg::equals(),
             RadialParticleFlux( p.tau[0], p.mu[0]),
-            v3d["electrons"], v3d["Ue"], v3d["induction"],
+            v3d.at("electrons"), v3d.at("Ue"), v3d.at("induction"),
             dx_A, dy_A, dz_A, dx_P, dy_P, dz_P, psipR, psipZ, psipP,
             bhat[0], bhat[1], bhat[2],
             curvNabla[0], curvNabla[1], curvNabla[2],
             curvKappa[0], curvKappa[1], curvKappa[2]
         );
-        v0d["total_flux"] = dg::blas1::dot( w3d, v3d["fluxe"]);
+        v0d.at("total_flux") = dg::blas1::dot( w3d, v3d.at("fluxe"));
 
         //----------------perp length scale computation
 
-        dg::blas1::axpby( 1., v3d["electrons"], -1., 1., t3d);
+        dg::blas1::axpby( 1., v3d.at("electrons"), -1., 1., t3d);
         dg::blas2::symv( dxN, t3d, dx_N);
         dg::blas2::symv( dyN, t3d, dy_N);
         dg::blas2::symv( dz , t3d, dz_N);
@@ -447,27 +478,27 @@ int main( int argc, char* argv[])
             dx_N, dy_N, dz_N, dx_A, dy_A, dz_A);
         dg::blas1::subroutine( feltor::routines::ComputePsi(),
             t3d, dx_N, dy_N, dz_N, dx_A, dy_A, dz_A);
-        dg::blas1::pointwiseDivide( t3d, v3d["electrons"], t3d);
-        v0d["perp_aligned"] = dg::blas1::dot( t3d, w3d);
-        dg::blas1::pointwiseDivide( t3d, v3d["electrons"], t3d);
-        dg::blas1::transform( t3d, v3d["Lperpinv"], dg::SQRT<double>());
+        dg::blas1::pointwiseDivide( t3d, v3d.at("electrons"), t3d);
+        v0d.at("perp_aligned") = dg::blas1::dot( t3d, w3d);
+        dg::blas1::pointwiseDivide( t3d, v3d.at("electrons"), t3d);
+        dg::blas1::transform( t3d, v3d.at("Lperpinv"), dg::SQRT<double>());
 
         //----------------parallel length scale computation
 
-        dg::blas1::axpby( 1., v3d["electrons"], -1., 1., t3d);
+        dg::blas1::axpby( 1., v3d.at("electrons"), -1., 1., t3d);
         dsN.centered( t3d, dx_N);
         dg::blas1::pointwiseDot ( dx_N, dx_N, t3d);
-        dg::blas1::pointwiseDivide( t3d, v3d["electrons"], t3d);
-        v0d["aligned"] = dg::blas1::dot( t3d, w3d);
-        dg::blas1::pointwiseDivide( t3d, v3d["electrons"], t3d);
-        dg::blas1::transform( t3d, v3d["Lparallelinv"], dg::SQRT<double>());
+        dg::blas1::pointwiseDivide( t3d, v3d.at("electrons"), t3d);
+        v0d.at("aligned") = dg::blas1::dot( t3d, w3d);
+        dg::blas1::pointwiseDivide( t3d, v3d.at("electrons"), t3d);
+        dg::blas1::transform( t3d, v3d.at("Lparallelinv"), dg::SQRT<double>());
 
         //------------------correlation------------//
 
-        dg::blas1::transform( v3d["potential"], t3d, dg::EXP<double>());
+        dg::blas1::transform( v3d.at("potential"), t3d, dg::EXP<double>());
         double norm1 = sqrt(dg::blas2::dot(t3d, w3d, t3d));
-        double norm2 = sqrt(dg::blas2::dot(v3d["electrons"], w3d, v3d["electrons"]));
-        v0d["correlationNPhi"] = dg::blas2::dot( t3d, w3d, v3d["electrons"])
+        double norm2 = sqrt(dg::blas2::dot(v3d.at("electrons"), w3d, v3d.at("electrons")));
+        v0d.at("correlationNPhi") = dg::blas2::dot( t3d, w3d, v3d.at("electrons"))
             /norm1/norm2;  //<e^phi, N>/||e^phi||/||N||
 
 
@@ -478,7 +509,7 @@ int main( int argc, char* argv[])
             //toroidal average
             toroidal_average( pair.second, t2d, false);
             dg::blas1::transfer( t2d, transfer2d);
-            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_avg"),
+            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_ta"),
                 start2d, count2d, transfer2d.data());
 
             //flux surface average
@@ -490,19 +521,32 @@ int main( int argc, char* argv[])
             err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_fsa"),
                 start1d, count1d, transfer1d.data());
 
-            // 2d data of plane phi = 0
+
+            // 2d data of plane varphi = 0
             unsigned kmp = 0; //g3d_out.Nz()/2;
             dg::HVec t2d_mp(pair.second.begin() + kmp*g2d_out.size(),
                 pair.second.begin() + (kmp+1)*g2d_out.size() );
-            err = nc_put_vara_double( ncid_out, id2d.at(pair.first),
+            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_plane"),
                 start2d, count2d, t2d_mp.data() );
 
-            // delta f on midplane : df = f_mp - <f>
+            // fsa on 2d plane : <f>
             dg::blas2::gemv(fsaonrzmatrix, transfer1d, transfer2d); //fsa on RZ grid
+            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_fsa2"),
+                start2d, count2d, transfer2d.data() );
+
+            // delta f on midplane : df = f_mp - <f>
             dg::blas1::axpby( 1.0, t2d_mp, -1.0, transfer2d);
             err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_fluc"),
                 start2d, count2d, transfer2d.data() );
 
+            //flux surface integral
+            transfer1d = dg::integrate( transfer1d, g1d_out);
+            dg::blas1::pointwiseDot( dvdpsip, transfer1d, transfer1d);
+            err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_fsi"),
+                start1d, count1d, transfer1d.data());
+            //flux surface integral on last closed flux surface
+            v0d.at(pair.first+"fsi_lcfs") = dg::interpolate( transfer1d, 0., g1d_out);
+
         }
         //and the 0d quantities
         for( auto pair : v0d) //{name, double}
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index bb85a70a4..5ad2cb7d5 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1209,23 +1209,25 @@ q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:s
 psip1d           & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
 rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
 psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\nabla\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
-X                & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
-X\_avg           & Dataset & 3 (time,y,x) & Toroidal average $\langle X
+X\_plane         & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
+X\_ta            & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
-X\_fluc       & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,0) - \langle X\rangle_{\psi_{p}}$ \\
+X\_fluc          & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,0) - \langle X\rangle_{\psi_{p}}$ \\
+X\_fsa2          & Dataset & 3 (time, y,x) & Flux surface average $\langle X\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
 X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
+X\_fsi           & Dataset & 2 (time, psi) & Flux surface integral $\int\d v\langle X\rangle_{\psi_p}$ \\
+X\_fsi\_lcfs     & Dataset & 1 (time) & Flux surface integral evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle X\rangle_{\psi_p}$ \\
 particle\_flux   & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{n_e} \cdot \vec \dA$ Eq.~\eqref{eq:particle_flux} \\
 energy\_flux     & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{\mathcal E} \cdot \vec \dA$ Eq.~\eqref{eq:energy_flux} \\
 \bottomrule
 \end{longtable}
-where X $\in$ \{ electrons, ions, Ue, Ui, potential,
-induction, vorticity, maxwell, Lperpinv, Lparallelinv,
-current, momentum
+where X $\in$ \{ electrons, ions, neue, NiUi, neuebphi, NiUibphi,
+potential,
+induction, vorticity, apar\_vorticity, Lperpinv, Lparallelinv
 \} corresponding to \{
-    $n_e$, $N_i$, $u_e$, $U_i$, $\phi$, $A_\parallel$,
+    $n_e$, $N_i$, $n_eu_e$, $N_iU_i$, $\phi$, $A_\parallel$,
     $-\Delta_\perp \phi$, $-\Delta_\perp A_\parallel$,
     $L_\perp^{-1}$, $L_\parallel^{-1}$,
-    $j_\parallel$, $j_\rho$
     \}
 and we define
 
-- 
GitLab


From 7795099d0cc89f4927000047c19040c36305cf32 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 10 May 2019 13:46:51 +0200
Subject: [PATCH 077/540] Update feltor writeup with vorticity equation

---
 src/feltor/feltor.tex | 66 +++++++++++++++++++++++++------------------
 1 file changed, 38 insertions(+), 28 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 5ad2cb7d5..b4474b275 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -860,6 +860,19 @@ n_e \vec K = n_e\nabla\times\frac{\bhat}{B} = \nabla\times n_e\frac{\bhat}{B} +
 such that we can define the diamagnetic flux in the particle flux since
 the rotation vanishes under the divergence.
 
+Let us here also derive the particle flux \eqref{eq:mass_conservation} through a flux surface
+\begin{align} \label{eq:particle_flux}
+ \vec j_{N}\cdot \vec \nabla v %=& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
+ %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
+ %\nabla\psi_p \nonumber\\
+ =
+  \frac{\d v}{\d \psi_p} &N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
+   \mathcal K_{\nabla\times\bhat}(\psi_p) + \tau  \mathcal K_{\nabla B}(\psi_p) \right] \nonumber\\
+ &+ NU\frac{\d v}{\d \psi_p}\left [\left( A_\parallel \mathcal
+ K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
+\end{align}
+
+
 \subsubsection{Energy theorem}
 The terms of the energy theorem are
 \begin{align} \label{eq:energy_theorem}
@@ -897,26 +910,10 @@ In \(\Lambda\) we insert the dissipative terms of Section~\ref{sec:dissres}. \\
 Replace $\Delta_\perp$ with $-\Delta_\perp^2$ when hyperviscous diffusion is chosen
 for the diffusion terms in the above equations.
 
-\subsection{ Particle, energy and vorticity fluxes}
-Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservation} and \eqref{eq:energy_conservation} through a flux surface
-\begin{align} \label{eq:particle_flux}
- \vec j_{N}\cdot \vec \nabla\psi_p %=& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- %\nabla\psi_p \nonumber\\
- =&
-  N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
-   \mathcal K_{\nabla\times\bhat}(\psi_p) + \tau  \mathcal K_{\nabla B}(\psi_p) \right] \nonumber\\
- &+ NU\left [\left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
- \vec j_{\mathcal E}\cdot \vec \nabla\psi_p =&
- %\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right)
- %N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- %\vec j_N \cdot \vec\nabla\psi_p
- %\nonumber\\
- %+ z\left(\mu NU^2 \vec v_{\nabla\times \bhat} + \tau NU(\bhat+\tilde{\vec b}_\perp )\right) \cdot\vec \nabla\psi_p \nonumber\\
- %=
-\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
+We have the energy flux through a flux surface
+\begin{align}
+ \vec j_{\mathcal E}\cdot \vec \nabla v =&%\frac{\d v}{\d \psi_p} \vec j_{\mathcal E}\cdot \vec \nabla \psi_p  =
+\frac{\d v}{\d \psi_p}\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
 + z \mu\tau NU^2 \mathcal K_{\nabla\times\bhat}(\psi_p) \nonumber\\
 &+ z \tau NU
  \left( A_\parallel \mathcal
@@ -924,6 +921,7 @@ Let us here also derive the particle and energy fluxes \eqref{eq:mass_conservati
 \label{eq:energy_flux}
 \end{align}
 
+\subsection{ Force balance equation}
 In order to discuss poloidal flows let us first define orthogonal vectors $\{\vec{ \hat\zeta}, \vec{\hat\eta},\bhat\}$
 \begin{align}
 \vec{\hat\zeta} := \nabla\psi_p,\quad
@@ -938,26 +936,38 @@ Notice that $\vec{\hat \eta}$ in general has a (small) toroidal component in add
 Furthermore, from $\vec u_E = \bhat\times \vec\nabla\phi/B$ and $\vec u_d = \tau_i \bhat\times\nabla \ln N_i$
 we can derive
 \begin{align}
-u_E^{\hat\eta} &:= \vec u_E\cdot \vec{\hat\eta} = \frac{\partial_{\hat\zeta} \phi}{B^2} \\
-u_E^{\hat\zeta}&:= \vec u_E\cdot \vec{\hat\zeta}  = -\partial_{\hat\eta} \phi \\
-u_d^{\hat\eta} &:= \vec u_d\cdot \vec{\hat\eta} =  \frac{\tau_i \partial_{\hat\zeta} \ln N_i}{B} \\
-u_d^{\hat\zeta}&:= \vec u_d\cdot \vec{\hat\zeta} = -B \tau_i \partial_{\hat\eta} \ln N_i
+u_E^{\hat\eta} &:= \vec u_E\cdot \vec{\hat\eta} = \frac{\partial_{\hat\zeta} \phi}{B^2}
+&&u_d^{\hat\eta} := \vec u_d\cdot \vec{\hat\eta} =  \frac{\tau_i \partial_{\hat\zeta} \ln N_i}{B} \\
+u_E^{\hat\zeta}&:= \vec u_E\cdot \vec{\hat\zeta}  = -\partial_{\hat\eta} \phi
+&&u_d^{\hat\zeta}:= \vec u_d\cdot \vec{\hat\zeta} = -B \tau_i \partial_{\hat\eta} \ln N_i
 \end{align}
-With this we write the force balance equation
+With this we write the force balance equation (in the LWL)
 \begin{align}
-\frac{\partial}{\partial t} \left\langle \mu_i N_i \left(
+&\frac{\partial}{\partial t} \left\langle \mu_i N_i \left(
 \frac{\partial_{\hat\zeta}\phi\,}{B^2} + \tau_i \partial_{\hat\zeta} \ln N_i\right) \right\rangle
-+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle \mu_i N_i\partial_{\hat\eta} \phi \left(\frac{\partial_{\hat\zeta}\phi }{B^2}
+\nonumber\\
+&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle \mu_i N_i\partial_{\hat\eta} \phi \left(\frac{\partial_{\hat\zeta}\phi }{B^2}
 + \tau_i \partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel \right\rangle
 \nonumber\\
-= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\nabla\times\bhat}\cdot\nabla\psi_p \right\rangle
+&= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\nabla\times\bhat}\cdot\nabla\psi_p \right\rangle
 \end{align}
+where the right hand side represents the Lorentz force $\vec j\times\vec B$.
 Notice that 
 \begin{align}
 \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p\right\rangle
 = \left\langle \frac{ \bhat\times \nabla (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cdot\nabla\psi_p\right\rangle
 = -\left\langle\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)\right\rangle
 \end{align}
+We can interpret the force balance as the flux surface average of a 
+continuity equation 
+with
+\begin{align}
+&\partial_t \Omega + \nabla\cdot \vec j_\Omega = S_\Omega \\
+\Omega &:= \mu_i N_i \left(\frac{\nabla\psi_p\cdot\nabla\phi}{B^2} + \tau_i \nabla\psi_p \cdot\nabla \ln N_i\right) \\
+\vec j_{\Omega} &:= -\mu_i N_i \Omega \vec u_E 
+    - \frac{1}{\beta} \nabla\psi_p\cdot\nabla A_\parallel \frac{\bhat\times\nabla A_\parallel}{B} \\
+    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\nabla\times\bhat}(\psi_p)
+\end{align}
 
 
 
-- 
GitLab


From b99878e3a2cbcfab12238a07f212eade2cdff186 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 9 May 2019 23:21:02 +0200
Subject: [PATCH 078/540] Add to feltordiag

- Add long names to variables in netcdf file
- Make sfi the variable to contain flux derivatives
---
 diag/feltordiag.cu    | 165 ++++++++++++++++++++++++++++++------------
 src/feltor/feltor.tex |  27 ++++---
 2 files changed, 132 insertions(+), 60 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index 98d86b63c..3f086a68a 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -15,9 +15,7 @@ struct RadialParticleFlux{
     RadialParticleFlux( double tau, double mu):
         m_tau(tau), m_mu(mu){
     }
-
-    DG_DEVICE double operator()( double ne, double ue, double A,
-        double d0A, double d1A, double d2A,
+    DG_DEVICE double operator()( double ne, double ue,
         double d0P, double d1P, double d2P, //Phi
         double d0S, double d1S, double d2S, //Psip
         double b_0,         double b_1,         double b_2,
@@ -26,19 +24,29 @@ struct RadialParticleFlux{
         ){
         double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
         double curvNablaS = curvNabla0*d0S+curvNabla1*d1S+curvNabla2*d2S;
-        double SA = b_0*( d1S*d2A-d2S*d1A)+
-                    b_1*( d2S*d0A-d0S*d2A)+
-                    b_2*( d0S*d1A-d1S*d0A);
         double PS = b_0*( d1P*d2S-d2P*d1S)+
                     b_1*( d2P*d0S-d0P*d2S)+
                     b_2*( d0P*d1S-d1P*d0S);
         double JPsi =
-            ne*ue* (A*curvKappaS + SA )
             + ne * PS
             + ne * (m_tau + m_mu*ue*ue)*curvKappaS
             + ne * m_tau*curvNablaS;
         return JPsi;
     }
+    DG_DEVICE double operator()( double ne, double ue, double A,
+        double d0A, double d1A, double d2A,
+        double d0S, double d1S, double d2S, //Psip
+        double b_0,         double b_1,         double b_2,
+        double curvKappa0,  double curvKappa1,  double curvKappa2
+        ){
+        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
+        double SA = b_0*( d1S*d2A-d2S*d1A)+
+                    b_1*( d2S*d0A-d0S*d2A)+
+                    b_2*( d0S*d1A-d1S*d0A);
+        double JPsi =
+            ne*ue* (A*curvKappaS + SA );
+        return JPsi;
+    }
     private:
     double m_tau, m_mu;
 };
@@ -78,7 +86,6 @@ struct RadialEnergyFlux{
 };
 
 struct Record{
-
 };
 
 int main( int argc, char* argv[])
@@ -133,6 +140,7 @@ int main( int argc, char* argv[])
     const double psip0 = mag.psip()(gp.R_0, 0);
     mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*psip0, p.alpha);
     dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
+    dg::HVec psipog3d = dg::evaluate( mag.psip(), g3d_out);
 
     //Find O-point
     double R_O = gp.R_0, Z_O = 0.;
@@ -205,7 +213,7 @@ int main( int argc, char* argv[])
 
     dg::HVec transfer2dX = dg::evaluate( dg::zero, gridX2d);
 
-    dg::DVec t1d = dg::evaluate( dg::zero, g1d_out);
+    dg::HVec t1d = dg::evaluate( dg::zero, g1d_out), fsa1d( t1d);
     dg::DVec t2d = dg::evaluate( dg::zero, g2d_out);
     dg::DVec t3d = dg::evaluate( dg::zero, g3d_out);
 
@@ -216,8 +224,15 @@ int main( int argc, char* argv[])
     dg::IHMatrix grid2gridX2d  = dg::create::interpolation(
         coordsX[0], coordsX[1], g2d_out);
     // interpolate fsa back to 2d or 3d grid
-    dg::IHMatrix fsaonrzmatrix = dg::create::interpolation(
+    dg::IHMatrix fsa2rzmatrix = dg::create::interpolation(
         psipog2d, g1d_out, dg::DIR_NEU);
+    dg::DVec dvdpsip3d;
+    {
+        dg::IHMatrix fsa2rzpmatrix = dg::create::interpolation(
+            psipog3d, g1d_out, dg::DIR_NEU);
+        dg::blas2::symv(fsa2rzpmatrix, dvdpsip, transfer3d);
+        dg::assign( transfer3d, dvdpsip3d);
+    };//save some storage by deleting matrix immediately
 
     //perp laplacian for computation of vorticity
 
@@ -283,6 +298,8 @@ int main( int argc, char* argv[])
     dg::DMatrix dxN = dg::create::dx( g3d_out, p.bcxN);
     dg::DMatrix dyN = dg::create::dy( g3d_out, p.bcyN);
     dg::DMatrix dz = dg::create::dz( g3d_out, dg::PER);
+    dg::HMatrix dpsi = dg::create::dx( g1d_out, dg::DIR_NEU);
+
     dg::DVec dx_A( t3d), dy_A(t3d), dz_A(t3d);
     dg::DVec dx_P( t3d), dy_P(t3d), dz_P(t3d);
     dg::DVec dx_N( t3d), dy_N(t3d), dz_N(t3d);
@@ -291,16 +308,29 @@ int main( int argc, char* argv[])
     const dg::DVec psipP =  t3d; //zero
 
     /////////////////// Construct names to output /////////////
-    std::vector<std::string> names_direct{
-        "electrons","ions","Ue","Ui", "potential","induction"
+    std::map<std::string, std::string> names_direct{
+        {"electrons",   "Electron density"},
+        {"ions",        "Ion gyrocentre density"},
+        {"Ue",          "Electron parallel velocity"},
+        {"Ui",          "Ion parallel velocity"},
+        {"potential",   "Electric potential"},
+        {"induction",   "Magnetic potential"}
     };
-    std::vector<std::string> names_derived{
-        "vorticity", "apar_vorticity", "neue", "NiUi",
-        "neuebphi", "NiUibphi", "fluxe",
-        "Lperpinv","Lparallelinv"
+    std::map<std::string, std::string> names_derived{
+        {"vorticity",       "Minus Lap_perp of electric potential"},
+        {"apar_vorticity",  "Minus Lap_perp of magnetic potential"},
+        {"neue",            "Product of electron density and velocity"},
+        {"NiUi",            "Product of ion gyrocentre density and velocity"},
+        {"neuebphi",        "Product of neue and covariant phi component of magnetic field unit vector"},
+        {"NiUibphi",        "Product of NiUi and covariant phi component of magnetic field unit vector"},
+        {"Lperpinv",        "Perpendicular density gradient length scale"},
+        {"Lparallelinv",    "Parallel density gradient length scale"},
+        {"jvne",            "Radial electron flux without induction contribution"},
+        {"jvneA",           "Radial electron flux: induction contribution"},
     };
+    names_derived.insert( names_direct.begin(), names_direct.end());
     std::vector<std::string> names_0d{
-        "aligned", "perp_aligned", "correlationNPhi", "total_flux"
+        "aligned", "perp_aligned", "correlationNPhi"
     };
 
     // Create Netcdf output file and ids
@@ -346,36 +376,62 @@ int main( int argc, char* argv[])
             &id0d[name]);
     }
 
-    // construct 3d and 2d and add to 0d variables
+    // construct a vector for each name in the list
     std::map<std::string, dg::DVec> v3d;
-    for( auto name : names_direct)
-        v3d[name] = t3d;
-    for( auto name : names_derived)
-        v3d[name] = t3d;
-    for( auto pair : v3d)
+    for( auto pair : names_derived)
+        v3d[pair.first] = t3d;
+    //now create the variables in the netcdf file
+    for( auto pair : names_derived)
     {
-        std::string name = pair.first + "_ta";
+        std::string name = pair.first + "_ta2d";
+        std::string long_name = pair.second + " (Toroidal average)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
-        name = pair.first + "_plane";
+        err = nc_put_att_text( ncid, id2d[name], "long_name", pair.second.size(),
+            long_name.data());
+
+        name = pair.first + "_2d";
+        long_name = pair.second + " (Evaluated on phi = 0 plane)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
-        name = pair.first + "_fluc";
+        err = nc_put_att_text( ncid, id2d[name], "long_name", pair.second.size(),
+            long_name.data());
+
+        name = pair.first + "_fluc2d";
+        long_name = pair.second + " (Fluctuations wrt fsa on phi = 0 plane)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
-        name = pair.first + "_fsa2";
+        err = nc_put_att_text( ncid, id2d[name], "long_name", pair.second.size(),
+            long_name.data());
+
+        name = pair.first + "_fsa2d";
+        long_name = pair.second + " (Flux surface average interpolated to 2d plane)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
+        err = nc_put_att_text( ncid, id2d[name], "long_name", pair.second.size(),
+            long_name.data());
+
         name = pair.first + "_fsa";
+        long_name = pair.second + " (Flux surface average)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
             &id1d[name]);
-        name = pair.first + "_fsi";
+        err = nc_put_att_text( ncid, id1d[name], "long_name", pair.second.size(),
+            long_name.data());
+
+        name = pair.first + "_ifs";
+        long_name = pair.second + " (Integrated Flux surface average unless it is a current then it is the derived flux surface average)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
             &id1d[name]);
-        name = pair.first + "_fsi_lcfs";
+        err = nc_put_att_text( ncid, id1d[name], "long_name", pair.second.size(),
+            long_name.data());
+
+        name = pair.first + "_ifs_lcfs";
+        long_name = pair.second + " (Integrated Flux surface average evaluated on last closed flux surface unless it is a current then it is the fsa evaluated)";
         v0d.at(name) = 0.; //this needs to go into v0d
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
             &id0d[name]);
+        err = nc_put_att_text( ncid, id0d[name], "long_name", pair.second.size(),
+            long_name.data());
     }
 
     size_t count1d[2] = {1, g1d_out.n()*g1d_out.N()};
@@ -424,10 +480,10 @@ int main( int argc, char* argv[])
         for( auto name : names_direct)
         {
             int dataID;
-            err = nc_inq_varid(ncid, name.data(), &dataID);
+            err = nc_inq_varid(ncid, name.first.data(), &dataID);
             err = nc_get_vara_double( ncid, dataID, start3d, count3d,
                 transfer3d.data());
-            dg::assign( transfer3d, v3d[name]);
+            dg::assign( transfer3d, v3d[name.first]);
         }
         err = nc_close(ncid);  //close 3d file
 
@@ -458,15 +514,23 @@ int main( int argc, char* argv[])
         dg::blas2::symv( dxP, v3d.at("potential"), dx_P);
         dg::blas2::symv( dyP, v3d.at("potential"), dy_P);
         dg::blas2::symv( dz , v3d.at("potential"), dz_P);
-        dg::blas1::evaluate( v3d.at("fluxe"), dg::equals(),
+        dg::blas1::evaluate( t3d, dg::equals(),
             RadialParticleFlux( p.tau[0], p.mu[0]),
-            v3d.at("electrons"), v3d.at("Ue"), v3d.at("induction"),
-            dx_A, dy_A, dz_A, dx_P, dy_P, dz_P, psipR, psipZ, psipP,
+            v3d.at("electrons"), v3d.at("Ue"),
+            dx_P, dy_P, dz_P, psipR, psipZ, psipP,
             bhat[0], bhat[1], bhat[2],
             curvNabla[0], curvNabla[1], curvNabla[2],
             curvKappa[0], curvKappa[1], curvKappa[2]
         );
-        v0d.at("total_flux") = dg::blas1::dot( w3d, v3d.at("fluxe"));
+        dg::blas1::pointwiseDot( t3d, dvdpsip3d, v3d.at("jvne"));
+        dg::blas1::evaluate( t3d, dg::equals(),
+            RadialParticleFlux( p.tau[0], p.mu[0]),
+            v3d.at("electrons"), v3d.at("Ue"), v3d.at("induction"),
+            dx_A, dy_A, dz_A, psipR, psipZ, psipP,
+            bhat[0], bhat[1], bhat[2],
+            curvKappa[0], curvKappa[1], curvKappa[2]
+        );
+        dg::blas1::pointwiseDot( t3d, dvdpsip3d, v3d.at("jvneA"));
 
         //----------------perp length scale computation
 
@@ -515,11 +579,11 @@ int main( int argc, char* argv[])
             //flux surface average
             dg::blas2::symv( grid2gridX2d, transfer2d, transfer2dX); //interpolate onto X-point grid
             dg::blas1::pointwiseDot( transfer2dX, volX2d, transfer2dX); //multiply by sqrt(g)
-            poloidal_average( transfer2dX, transfer1d, false); //average over eta
-            dg::blas1::scal( transfer1d, 4*M_PI*M_PI*f0); //
-            dg::blas1::pointwiseDivide( transfer1d, dvdpsip, transfer1d );
+            poloidal_average( transfer2dX, t1d, false); //average over eta
+            dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
+            dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
             err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_fsa"),
-                start1d, count1d, transfer1d.data());
+                start1d, count1d, fsa1d.data());
 
 
             // 2d data of plane varphi = 0
@@ -530,7 +594,7 @@ int main( int argc, char* argv[])
                 start2d, count2d, t2d_mp.data() );
 
             // fsa on 2d plane : <f>
-            dg::blas2::gemv(fsaonrzmatrix, transfer1d, transfer2d); //fsa on RZ grid
+            dg::blas2::gemv(fsa2rzmatrix, fsa1d, transfer2d); //fsa on RZ grid
             err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_fsa2"),
                 start2d, count2d, transfer2d.data() );
 
@@ -539,13 +603,22 @@ int main( int argc, char* argv[])
             err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_fluc"),
                 start2d, count2d, transfer2d.data() );
 
-            //flux surface integral
-            transfer1d = dg::integrate( transfer1d, g1d_out);
-            dg::blas1::pointwiseDot( dvdpsip, transfer1d, transfer1d);
-            err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_fsi"),
+            //flux surface integral/derivative
+            if( pair.first[0] == 'j') //j indicates a flux
+            {
+                v0d.at(pair.first+"ifs_lcfs") = dg::interpolate( fsa1d, 0., g1d_out);
+                dg::blas2::symv( dpsi, fsa1d, t1d);
+                dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
+            }
+            else
+            {
+                t1d = dg::integrate( fsa1d, g1d_out);
+                dg::blas1::pointwiseDot( t1d, dvdpsip, transfer1d);
+                v0d.at(pair.first+"ifs_lcfs") = dg::interpolate( transfer1d, 0., g1d_out);
+            }
+            err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_ifs"),
                 start1d, count1d, transfer1d.data());
-            //flux surface integral on last closed flux surface
-            v0d.at(pair.first+"fsi_lcfs") = dg::interpolate( transfer1d, 0., g1d_out);
+            //flux surface integral/derivative on last closed flux surface
 
         }
         //and the 0d quantities
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index b4474b275..c103e6f01 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1219,26 +1219,25 @@ q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:s
 psip1d           & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
 rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
 psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\nabla\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
-X\_plane         & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
-X\_ta            & Dataset & 3 (time,y,x) & Toroidal average $\langle X
+X\_2d            & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
+X\_ta2d          & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
-X\_fluc          & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,0) - \langle X\rangle_{\psi_{p}}$ \\
-X\_fsa2          & Dataset & 3 (time, y,x) & Flux surface average $\langle X\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
+X\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,0) - \langle X\rangle_{\psi_{p}}$ \\
+X\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\langle X\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
 X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
-X\_fsi           & Dataset & 2 (time, psi) & Flux surface integral $\int\d v\langle X\rangle_{\psi_p}$ \\
-X\_fsi\_lcfs     & Dataset & 1 (time) & Flux surface integral evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle X\rangle_{\psi_p}$ \\
+X\_ifs           & Dataset & 2 (time, psi) & Integrated flux surface average $\int\d v\langle X\rangle_{\psi_p}$ unless X is a current, then it is the derived flux-surface average $\partial_v \langle X\rangle_{\psi_p}$ \\
+X\_ifs\_lcfs     & Dataset & 1 (time) & Integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle X\rangle_{\psi_p}$ unless X is a current, then it is the fsa evaluated \\
 particle\_flux   & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{n_e} \cdot \vec \dA$ Eq.~\eqref{eq:particle_flux} \\
 energy\_flux     & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{\mathcal E} \cdot \vec \dA$ Eq.~\eqref{eq:energy_flux} \\
 \bottomrule
 \end{longtable}
-where X $\in$ \{ electrons, ions, neue, NiUi, neuebphi, NiUibphi,
-potential,
-induction, vorticity, apar\_vorticity, Lperpinv, Lparallelinv
-\} corresponding to \{
-    $n_e$, $N_i$, $n_eu_e$, $N_iU_i$, $\phi$, $A_\parallel$,
-    $-\Delta_\perp \phi$, $-\Delta_\perp A_\parallel$,
-    $L_\perp^{-1}$, $L_\parallel^{-1}$,
-    \}
+where X $\in$ \{ electrons $n_e$, ions $N_i$, Ue $u_e$, Ui $U_i$,
+neue $n_e u_e$, NiUi $N_i U_i$, neuebphi $n_eu_eb_\varphi$,
+NiUibphi $N_iU_ib_\varphi$, potential $\phi$,
+induction $A_\parallel$, vorticity $-\Delta_\perp\phi$,
+apar\_vorticity $-\Delta_\perp A_\parallel$, Lperpinv $L_\perp^{-1}$,
+Lparallelinv $L_\parallel^{-1}$
+\}
 and we define
 
 \begin{align}
-- 
GitLab


From 9977e2ac5ad177da75a3ab2646a3cb44fcba880f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 11 May 2019 13:12:05 +0200
Subject: [PATCH 079/540] Add diagnostics member functions to Explicit

- First, add a set_fields member
- Second, change the derivatives to std::array to be able to return
them in one go
- Third, add bphi and Laplacian for P
---
 src/feltor/feltor.cuh | 166 ++++++++++++++++++++++++++++++------------
 src/feltor/feltor.tex |   8 +-
 2 files changed, 123 insertions(+), 51 deletions(-)

diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index dbc16bbb7..1ab6cc0b2 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -227,6 +227,71 @@ struct Explicit
         return m_s;
     }
 
+    /// ///////////////////DIAGNOSTIC MEMBERS //////////////////////
+    // Set the internal fields and derivatives (for diagnostics)
+    void set_fields( double time, const Container& ne, const Container& Ni,
+        const Container& ue, const Container& Ui, const Container& potential,
+        const Container& induction){
+
+        dg::blas1::copy( potential, m_phi[0]);
+
+        // set m_phi[1], m_d*phi[0], m_d*phi[1] and m_UE2 --- needs m_phi[0]
+        compute_psi( time);
+
+        dg::blas1::copy( ne, m_fields[0][0]);
+        dg::blas1::copy( Ni, m_fields[0][1]);
+        dg::blas1::transform( m_fields[0], m_logn, dg::LN<double>());
+
+        dg::blas1::copy( ue, m_fields[1][0]);
+        dg::blas1::copy( Ui, m_fields[1][1]);
+        if( m_p.beta != 0)
+        {
+            dg::blas1::copy( induction, m_apar);
+            //----------Compute Derivatives----------------------------//
+            dg::blas2::symv( m_dx_U, m_apar, m_dA[0]);
+            dg::blas2::symv( m_dy_U, m_apar, m_dA[1]);
+            if(!m_p.symmetric) dg::blas2::symv( m_dz, m_apar, m_dA[2]);
+        }
+        dg::blas1::axpby( 1., ne, -1., 1., m_temp0);
+        dg::blas2::symv( m_dx_N, m_temp0, m_dN[0][0]);
+        dg::blas2::symv( m_dy_N, m_temp0, m_dN[0][1]);
+        if(!m_p.symmetric) dg::blas2::symv( m_dz, m_temp0, m_dN[0][2]);
+        dg::blas1::axpby( 1., Ni, -1., 1., m_temp0);
+        dg::blas2::symv( m_dx_N, m_temp0, m_dN[1][0]);
+        dg::blas2::symv( m_dy_N, m_temp0, m_dN[1][1]);
+        if(!m_p.symmetric) dg::blas2::symv( m_dz, m_temp0, m_dN[1][2]);
+
+        for( unsigned i=0; i<2; i++)
+        {
+            dg::blas2::symv( m_dx_U, fields[1][i], m_dU[i][0]);
+            dg::blas2::symv( m_dy_U, fields[1][i], m_dU[i][1]);
+            if(!m_p.symmetric) dg::blas2::symv( m_dz, fields[1][i], m_dU[i][2]);
+        }
+
+     }
+
+    const Container& uE2() const {return m_UE2;}
+    const std::array<Container, 3> & dN (int i) const {
+        return m_dN[i];
+    }
+    const std::array<Container, 3> & dP (int i) const {
+        return m_dP[i];
+    }
+    const std::array<Container, 3> & dA () const {
+        return m_dA;
+    }
+    const std::array<Container, 3> & curv () const {
+        return m_curv;
+    }
+    const std::array<Container, 3> & curvKappa () const {
+        return m_curvKappa;
+    }
+    const Container& bphi( ) const { return m_bphi; }
+    //bhat / sqrt{g} / B
+    const std::array<Container, 3> & bhatgB () const {
+        return m_b;
+    }
+
     //source strength, profile - 1
     void set_source( Container profile, double omega_source, Container source)
     {
@@ -237,7 +302,7 @@ struct Explicit
     void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
   private:
     void compute_phi( double t, const std::array<Container,2>& y);
-    void compute_psi( double t, const std::array<Container,2>& y);
+    void compute_psi( double t);
     void compute_energies(
         const std::array<std::array<Container,2>,2>& fields);
     void compute_dissipation(
@@ -263,17 +328,17 @@ struct Explicit
     Container m_R, m_Z, m_P; //coordinates
 #endif //DG_MANUFACTURED
 
-    //these should be considered const
+    //these should be considered const // m_curv is full curvature
     std::array<Container,3> m_curv, m_curvKappa, m_b;
     Container m_divCurvKappa;
-    Container m_binv, m_divb;
+    Container m_bphi, m_binv, m_divb;
     Container m_source, m_profne;
     Container m_vol3d;
 
-    Container m_apar, m_dxA, m_dyA, m_dzA;
-    std::array<Container,2> m_phi, m_dxPhi, m_dyPhi, m_dzPhi;
-    std::array<Container,2> m_logn, m_dxN, m_dyN, m_dzN, m_dsN;
-    std::array<Container,2> m_dxU, m_dyU, m_dzU, m_dsU;
+    Container m_apar;
+    std::array<Container,2> m_phi, m_logn, m_dsN, m_dsU;
+    std::array<Container,3> m_dA;
+    std::array<std::array<Container,3>,2> m_dP, m_dN, m_dU;
     std::array<std::array<Container,2>,2> m_fields, m_s;
 
     std::vector<Container> m_multi_chi;
@@ -281,7 +346,7 @@ struct Explicit
     //matrices and solvers
     Matrix m_dx_N, m_dx_U, m_dx_P, m_dy_N, m_dy_U, m_dy_P, m_dz;
     dg::geo::DS<Geometry, IMatrix, Matrix, Container> m_ds_P, m_ds_N, m_ds_U;
-    dg::Elliptic3d< Geometry, Matrix, Container> m_lapperpN, m_lapperpU;
+    dg::Elliptic3d< Geometry, Matrix, Container> m_lapperpN, m_lapperpU, m_lapperpP;
     std::vector<dg::Elliptic3d< Geometry, Matrix, Container> > m_multi_pol;
     std::vector<dg::Helmholtz3d<Geometry, Matrix, Container> > m_multi_invgammaP,
         m_multi_invgammaN, m_multi_induction;
@@ -362,13 +427,16 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
                                         m_b[0], m_b[1], m_b[2]);
     Container vol = dg::tensor::volume( metric);
     dg::blas1::pointwiseDivide( m_binv, vol, vol); //1/vol/B
+    dg::assign( m_b[2], m_bphi); //save bphi for momentum conservation
     for( int i=0; i<3; i++)
         dg::blas1::pointwiseDot( vol, m_b[i], m_b[i]); //b_i/vol/B
     m_hh = dg::geo::createProjectionTensor( bhat, g);
     m_lapperpN.construct ( g, p.bcxN, p.bcyN, dg::PER, dg::normed, dg::centered),
     m_lapperpU.construct ( g, p.bcxU, p.bcyU, dg::PER, dg::normed, dg::centered),
+    m_lapperpP.construct ( g, p.bcxP, p.bcyP, dg::PER, dg::normed, dg::centered),
     m_lapperpN.set_chi( m_hh);
     m_lapperpU.set_chi( m_hh);
+    m_lapperpP.set_chi( m_hh);
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
 void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
@@ -425,11 +493,16 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     //--------------------------init vectors to 0-----------------//
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
     m_UE2 = m_temp2 = m_temp1 = m_temp0;
+    m_apar = m_temp0;
+
     m_phi[0] = m_phi[1] = m_temp0;
-    m_dxPhi = m_dyPhi = m_dzPhi = m_fields[0] = m_fields[1] = m_logn = m_phi;
+    m_dsN = m_dsU =  m_logn = m_phi;
+    m_dA[0] = m_dA[1] = m_dA[2] = m_temp0;
+    m_dP[0] = m_dP[1] = m_dA;
+    m_dN = m_dU = m_dP;
+    m_fields[0] = m_fields[1] = m_phi;
     m_s = m_fields;
-    m_dxN = m_dyN = m_dzN = m_dsN = m_dxU = m_dyU = m_dzU = m_dsU = m_phi;
-    m_apar = m_dxA = m_dyA = m_dzA = m_phi[0];
+
     //--------------------------Construct-------------------------//
     construct_mag( g, p, mag);
     construct_bhat( g, p, mag);
@@ -531,6 +604,12 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_phi(
     m_old_phi.update( time, m_phi[0]);
     if(  number[0] == m_multigrid.max_iter())
         throw dg::Fail( m_p.eps_pol);
+}
+
+template<class Geometry, class IMatrix, class Matrix, class Container>
+void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
+    double time)
+{
     //-----------Solve for Gamma Phi---------------------------//
     if (m_p.tau[1] == 0.) {
         dg::blas1::copy( m_phi[0], m_phi[1]);
@@ -551,30 +630,23 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_phi(
         if(  number[0] == m_multigrid.max_iter())
             throw dg::Fail( m_p.eps_gamma);
     }
-}
-
-template<class Geometry, class IMatrix, class Matrix, class Container>
-void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
-    double time, const std::array<Container,2>& y)
-{
     //-------Compute Psi and derivatives
-    dg::blas2::symv( m_dx_P, m_phi[0], m_dxPhi[0]);
-    dg::blas2::symv( m_dy_P, m_phi[0], m_dyPhi[0]);
-    if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[0], m_dzPhi[0]);
+    dg::blas2::symv( m_dx_P, m_phi[0], m_dP[0][0]);
+    dg::blas2::symv( m_dy_P, m_phi[0], m_dP[0][1]);
+    if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[0], m_dP[0][2]);
     dg::tensor::multiply3d( m_hh, //grad_perp
-        m_dxPhi[0], m_dyPhi[0], m_dzPhi[0], m_UE2, m_temp0, m_temp1);
-    dg::blas1::subroutine( routines::ComputePsi(),
-        m_phi[1], m_dxPhi[0], m_dyPhi[0], m_dzPhi[0],
-        m_UE2, m_temp0, m_temp1, m_binv);
+        m_dP[0][0], m_dP[0][1], m_dP[0][2], m_UE2, m_temp0, m_temp1);
+    dg::blas1::subroutine( routines::ComputePsi(), m_phi[1],
+        m_dP[0][0], m_dP[0][1], m_dP[0][2], m_UE2, m_temp0, m_temp1, m_binv);
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( m_phi[1], dg::plus_equals(), manufactured::SPhii{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
 #endif //DG_MANUFACTURED
     //m_UE2 now contains u_E^2; also update derivatives
-    dg::blas2::symv( m_dx_P, m_phi[1], m_dxPhi[1]);
-    dg::blas2::symv( m_dy_P, m_phi[1], m_dyPhi[1]);
-    if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[1], m_dzPhi[1]);
+    dg::blas2::symv( m_dx_P, m_phi[1], m_dP[1][0]);
+    dg::blas2::symv( m_dy_P, m_phi[1], m_dP[1][1]);
+    if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[1], m_dP[1][2]);
 }
 template<class Geometry, class IMatrix, class Matrix, class Container>
 void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
@@ -607,9 +679,9 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
     if(  number[0] == m_multigrid.max_iter())
         throw dg::Fail( m_p.eps_pol);
     //----------Compute Derivatives----------------------------//
-    dg::blas2::symv( m_dx_U, m_apar, m_dxA);
-    dg::blas2::symv( m_dy_U, m_apar, m_dyA);
-    if(!m_p.symmetric) dg::blas2::symv( m_dz, m_apar, m_dzA);
+    dg::blas2::symv( m_dx_U, m_apar, m_dA[0]);
+    dg::blas2::symv( m_dy_U, m_apar, m_dA[1]);
+    if(!m_p.symmetric) dg::blas2::symv( m_dz, m_apar, m_dA[2]);
 
     //----------Compute Velocities-----------------------------//
     dg::blas1::axpby( 1., fields[1][0], -1./m_p.mu[0], m_apar, fields[1][0]);
@@ -627,19 +699,19 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
     for( unsigned i=0; i<2; i++)
     {
         ////////////////////perpendicular dynamics////////////////////////
-        dg::blas2::symv( m_dx_N, y[0][i], m_dxN[i]);
-        dg::blas2::symv( m_dy_N, y[0][i], m_dyN[i]);
-        if(!m_p.symmetric) dg::blas2::symv( m_dz, y[0][i], m_dzN[i]);
-        dg::blas2::symv( m_dx_U, fields[1][i], m_dxU[i]);
-        dg::blas2::symv( m_dy_U, fields[1][i], m_dyU[i]);
-        if(!m_p.symmetric) dg::blas2::symv( m_dz, fields[1][i], m_dzU[i]);
+        dg::blas2::symv( m_dx_N, y[0][i], m_dN[i][0]);
+        dg::blas2::symv( m_dy_N, y[0][i], m_dN[i][1]);
+        if(!m_p.symmetric) dg::blas2::symv( m_dz, y[0][i], m_dN[i][2]);
+        dg::blas2::symv( m_dx_U, fields[1][i], m_dU[i][0]);
+        dg::blas2::symv( m_dy_U, fields[1][i], m_dU[i][1]);
+        if(!m_p.symmetric) dg::blas2::symv( m_dz, fields[1][i], m_dU[i][2]);
         if( m_p.beta == 0){
             dg::blas1::subroutine( routines::ComputePerpDrifts(
                 m_p.mu[i], m_p.tau[i]),
                 //species depdendent
-                fields[0][i], m_dxN[i], m_dyN[i], m_dzN[i],
-                fields[1][i], m_dxU[i], m_dyU[i], m_dzU[i],
-                m_dxPhi[i], m_dyPhi[i], m_dzPhi[i],
+                fields[0][i], m_dN[i][0], m_dN[i][1], m_dN[i][2],
+                fields[1][i], m_dU[i][0], m_dU[i][1], m_dU[i][2],
+                m_dP[i][0], m_dP[i][1], m_dP[i][2],
                 //magnetic parameters
                 m_b[0], m_b[1], m_b[2],
                 m_curv[0], m_curv[1], m_curv[2],
@@ -651,11 +723,11 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
             dg::blas1::subroutine( routines::ComputePerpDrifts(
                 m_p.mu[i], m_p.tau[i]),
                 //species depdendent
-                fields[0][i], m_dxN[i], m_dyN[i], m_dzN[i],
-                fields[1][i], m_dxU[i], m_dyU[i], m_dzU[i],
-                m_dxPhi[i], m_dyPhi[i], m_dzPhi[i],
+                fields[0][i], m_dN[i][0], m_dN[i][1], m_dN[i][2],
+                fields[1][i], m_dU[i][0], m_dU[i][1], m_dU[i][2],
+                m_dP[i][0], m_dP[i][1], m_dP[i][2],
                 //induction
-                m_apar, m_dxA, m_dyA, m_dzA,
+                m_apar, m_dA[0], m_dA[1], m_dA[2],
                 //magnetic parameters
                 m_b[0], m_b[1], m_b[2],
                 m_curv[0], m_curv[1], m_curv[2],
@@ -722,10 +794,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_energies(
     //= 0.5 beta^{-1} (grad_perp Apar)^2
     if( m_p.beta != 0)
     {
-        dg::tensor::multiply3d( m_hh, m_dxA, m_dyA, m_dzA,
+        dg::tensor::multiply3d( m_hh, m_dA[0], m_dA[1], m_dA[2],
             m_temp0, m_temp1, m_temp2);
         dg::blas1::subroutine( routines::ComputePsi(),
-            m_temp0, m_dxA, m_dyA, m_dzA,
+            m_temp0, m_dA[0], m_dA[1], m_dA[2],
             m_temp0, m_temp1, m_temp2);
         m_q.Apar = 0.5*dg::blas1::dot( m_vol3d, m_temp0)/m_p.beta;
     }
@@ -821,8 +893,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     // set m_phi[0]
     compute_phi( t, y[0]);
 
-    // set m_psi[0], m_d*Phi[0], m_d*Phi[1] and m_UE2 --- needs m_phi[0]
-    compute_psi( t, y[0]);
+    // set m_phi[1], m_dP[0], m_dP[1] and m_UE2 --- needs m_phi[0]
+    compute_psi( t);
 
     // Transform n-1 to n and n to logn
     dg::blas1::subroutine( routines::ComputeLogN(), y[0], m_fields[0], m_logn);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index c103e6f01..47623ece0 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -15,8 +15,8 @@ The full-F electromagnetic model in toroidal geometry \textsc{Feltor}}
 The purpose of this document is to describe the programs
 \texttt{feltor\_hpc.cu, feltor.cu, feltor\_diag.cu} and to an extend
 \texttt{geometry\_diag.cu}. The goal is to provide
-enough information such that the user never needs to look
-into the actual codes on the one side and to be able to connect
+information such that a user can avoid to look
+into the actual codes on the one side and connect
 the presented formulas to relevant journal publications on the other.
 
 The program \texttt{feltor/inc/geometries/geometry\_diag.cu}
@@ -865,8 +865,8 @@ Let us here also derive the particle flux \eqref{eq:mass_conservation} through a
  \vec j_{N}\cdot \vec \nabla v %=& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
  %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
  %\nabla\psi_p \nonumber\\
- =
-  \frac{\d v}{\d \psi_p} &N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
+ =&
+  \frac{\d v}{\d \psi_p} N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
    \mathcal K_{\nabla\times\bhat}(\psi_p) + \tau  \mathcal K_{\nabla B}(\psi_p) \right] \nonumber\\
  &+ NU\frac{\d v}{\d \psi_p}\left [\left( A_\parallel \mathcal
  K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
-- 
GitLab


From db2aafa4281830dae69d91bc1a5e3a85ed4f5916 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 12 May 2019 12:01:26 +0200
Subject: [PATCH 080/540] Restructure feltordiag

- in new file we now have a list of functions that generate output
- it compiles, yet untested
---
 diag/feltordiag.cu    | 456 +++++++++---------------------------------
 diag/feltordiag.h     | 321 +++++++++++++++++++++++++++++
 src/feltor/feltor.cuh |  54 ++++-
 3 files changed, 464 insertions(+), 367 deletions(-)
 create mode 100644 diag/feltordiag.h

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index 3f086a68a..d6e3233e9 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -3,90 +3,12 @@
 #include <iomanip>
 #include <vector>
 #include <string>
+#include <functional>
 #include "json/json.h"
 
-#include "dg/algorithm.h"
 #include "dg/file/nc_utilities.h"
-#include "dg/geometries/geometries.h"
-#include "feltor/feltor.cuh"
-#include "feltor/parameters.h"
+#include "feltordiag.h"
 
-struct RadialParticleFlux{
-    RadialParticleFlux( double tau, double mu):
-        m_tau(tau), m_mu(mu){
-    }
-    DG_DEVICE double operator()( double ne, double ue,
-        double d0P, double d1P, double d2P, //Phi
-        double d0S, double d1S, double d2S, //Psip
-        double b_0,         double b_1,         double b_2,
-        double curvNabla0,  double curvNabla1,  double curvNabla2,
-        double curvKappa0,  double curvKappa1,  double curvKappa2
-        ){
-        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
-        double curvNablaS = curvNabla0*d0S+curvNabla1*d1S+curvNabla2*d2S;
-        double PS = b_0*( d1P*d2S-d2P*d1S)+
-                    b_1*( d2P*d0S-d0P*d2S)+
-                    b_2*( d0P*d1S-d1P*d0S);
-        double JPsi =
-            + ne * PS
-            + ne * (m_tau + m_mu*ue*ue)*curvKappaS
-            + ne * m_tau*curvNablaS;
-        return JPsi;
-    }
-    DG_DEVICE double operator()( double ne, double ue, double A,
-        double d0A, double d1A, double d2A,
-        double d0S, double d1S, double d2S, //Psip
-        double b_0,         double b_1,         double b_2,
-        double curvKappa0,  double curvKappa1,  double curvKappa2
-        ){
-        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
-        double SA = b_0*( d1S*d2A-d2S*d1A)+
-                    b_1*( d2S*d0A-d0S*d2A)+
-                    b_2*( d0S*d1A-d1S*d0A);
-        double JPsi =
-            ne*ue* (A*curvKappaS + SA );
-        return JPsi;
-    }
-    private:
-    double m_tau, m_mu;
-};
-struct RadialEnergyFlux{
-    RadialEnergyFlux( double tau, double mu):
-        m_tau(tau), m_mu(mu){
-    }
-
-    DG_DEVICE double operator()( double ne, double ue, double A, double P,
-        double d0A, double d1A, double d2A,
-        double d0P, double d1P, double d2P, //Phi
-        double d0S, double d1S, double d2S, //Psip
-        double b_0,         double b_1,         double b_2,
-        double curvNabla0,  double curvNabla1,  double curvNabla2,
-        double curvKappa0,  double curvKappa1,  double curvKappa2
-        ){
-        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
-        double curvNablaS = curvNabla0*d0S+curvNabla1*d1S+curvNabla2*d2S;
-        double SA = b_0*( d1S*d2A-d2S*d1A)+
-                    b_1*( d2S*d0A-d0S*d2A)+
-                    b_2*( d0S*d1A-d1S*d0A);
-        double PS = b_0*( d1P*d2S-d2P*d1S)+
-                    b_1*( d2P*d0S-d0P*d2S)+
-                    b_2*( d0P*d1S-d1P*d0S);
-        double JN =
-            ne*ue* (A*curvKappaS + SA )
-            + ne * PS
-            + ne * (m_tau + m_mu*ue*ue)*curvKappaS
-            + ne * m_tau*curvNablaS;
-        double Je = (m_tau * log(ne) + 0.5*m_mu*ue*ue + P)*JN
-            + m_mu*m_tau*ne*ue*ue*curvKappaS
-            + m_tau*ne*ue* (A*curvKappaS + SA );
-        return Je;
-    }
-    private:
-    double m_tau, m_mu;
-};
-
-struct Record{
-};
 
 int main( int argc, char* argv[])
 {
@@ -110,8 +32,8 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "geomfile", &geom[0]);
     err = nc_close(ncid);
 
-    std::cout << "input "<<input<<std::endl;
-    std::cout << "geome "<<geom <<std::endl;
+    //std::cout << "input "<<input<<std::endl;
+    //std::cout << "geome "<<geom <<std::endl;
     Json::Value js,gs;
     Json::CharReaderBuilder parser;
     parser["collectComments"] = false;
@@ -124,6 +46,35 @@ int main( int argc, char* argv[])
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
+    std::vector<std::string> names_input{
+        "electrons", "ions", "Ue", "Ui", "potential", "induction"
+    };
+
+    //-----------------Create Netcdf output file with attributes----------//
+    int ncid_out;
+    err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
+
+    /// Set global attributes
+    std::map<std::string, std::string> att;
+    att["title"] = "Output file of feltor/diag/feltordiag.cu";
+    att["Conventions"] = "CF-1.7";
+    ///Get local time and begin file history
+    auto ttt = std::time(nullptr);
+    auto tm = *std::localtime(&ttt);
+    std::ostringstream oss;
+    ///time string  + program-name + args
+    oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
+    for( int i=0; i<argc; i++) oss << " "<<argv[i];
+    att["history"] = oss.str();
+    att["comment"] = "Find more info in feltor/src/feltor.tex";
+    att["source"] = "FELTOR";
+    att["references"] = "https://github.com/feltor-dev/feltor";
+    att["inputfile"] = input;
+    att["geomfile"] = geom;
+    for( auto pair : att)
+        err = nc_put_att_text( ncid, NC_GLOBAL,
+            pair.first.data(), pair.second.size(), pair.second.data());
+
     //-------------------Construct grids-------------------------------------//
 
     const double Rmin=gp.R_0-p.boxscaleRm*gp.a;
@@ -141,7 +92,23 @@ int main( int argc, char* argv[])
     mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*psip0, p.alpha);
     dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
     dg::HVec psipog3d = dg::evaluate( mag.psip(), g3d_out);
+    // Construct weights and temporaries
+
+    const dg::DVec w3d = dg::create::volume( g3d_out);
 
+    dg::HVec transfer3d = dg::evaluate(dg::zero,g3d_out);
+    dg::HVec transfer2d = dg::evaluate(dg::zero,g2d_out);
+
+    dg::DVec t2d = dg::evaluate( dg::zero, g2d_out);
+    dg::DVec t3d = dg::evaluate( dg::zero, g3d_out);
+    dg::DVec result(t3d);
+
+    std::array<dg::DVec, 3> dpsip;
+    dpsip[0] =  dg::evaluate( mag.psipR(), g3d_out);
+    dpsip[1] =  dg::evaluate( mag.psipZ(), g3d_out);
+    dpsip[2] =  t3d; //zero
+
+    ///--------------- Construct X-point grid ---------------------//
     //Find O-point
     double R_O = gp.R_0, Z_O = 0.;
     dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
@@ -164,15 +131,22 @@ int main( int argc, char* argv[])
     //Create 1d grids, one for psi and one for x
     dg::Grid1d g1d_out(psipmin, psipmax, 3, Npsi, dg::DIR_NEU); //inner value is always 0
     const double f0 = ( gridX2d.x1() - gridX2d.x0() ) / ( psipmax - psipmin );
+    const dg::DVec w1d = dg::create::weights( g1d_out);
+    dg::HVec t1d = dg::evaluate( dg::zero, g1d_out), fsa1d( t1d);
+    dg::HVec transfer1d = dg::evaluate(dg::zero,g1d_out);
+    dg::HVec transfer2dX = dg::evaluate( dg::zero, gridX2d);
 
-    //interpolation and metric
-    std::vector<dg::HVec > coordsX = gridX2d.map();
-    dg::SparseTensor<dg::HVec> metricX = gridX2d.metric();
+
+
+    /// ------------------- Compute 1d flux labels ---------------------//
 
     std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
     /// Compute flux volume label
     dg::Average<dg::HVec > poloidal_average( gridX2d.grid(), dg::coo2d::y);
     dg::HVec dvdpsip;
+    //metric and map
+    dg::SparseTensor<dg::HVec> metricX = gridX2d.metric();
+    std::vector<dg::HVec > coordsX = gridX2d.map();
     dg::HVec volX2d = dg::tensor::volume2d( metricX);
     dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
     poloidal_average( volX2d, dvdpsip, false);
@@ -202,23 +176,6 @@ int main( int argc, char* argv[])
     map1d.emplace_back("psi_psi",    dg::evaluate( dg::cooX1d, g1d_out),
         "Flux label psi (same as coordinate)");
 
-    // Construct weights and temporaries
-
-    const dg::DVec w3d = dg::create::volume( g3d_out);
-    const dg::DVec w1d = dg::create::weights( g1d_out);
-
-    dg::HVec transfer3d = dg::evaluate(dg::zero,g3d_out);
-    dg::HVec transfer2d = dg::evaluate(dg::zero,g2d_out);
-    dg::HVec transfer1d = dg::evaluate(dg::zero,g1d_out);
-
-    dg::HVec transfer2dX = dg::evaluate( dg::zero, gridX2d);
-
-    dg::HVec t1d = dg::evaluate( dg::zero, g1d_out), fsa1d( t1d);
-    dg::DVec t2d = dg::evaluate( dg::zero, g2d_out);
-    dg::DVec t3d = dg::evaluate( dg::zero, g3d_out);
-
-
-
 
     // interpolate from 2d grid to X-point points
     dg::IHMatrix grid2gridX2d  = dg::create::interpolation(
@@ -234,129 +191,12 @@ int main( int argc, char* argv[])
         dg::assign( transfer3d, dvdpsip3d);
     };//save some storage by deleting matrix immediately
 
-    //perp laplacian for computation of vorticity
-
-    dg::DVec vor3d    = dg::evaluate( dg::zero, g3d_out);
-    dg::Elliptic3d<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec>
-        laplacianM(g3d_out, p.bcxP, p.bcyP, dg::PER, dg::normed, dg::centered);
-    auto bhatF = dg::geo::createEPhi();
-    if( p.curvmode == "true")
-        bhatF = dg::geo::createBHat( mag);
-    dg::SparseTensor<dg::DVec> hh = dg::geo::createProjectionTensor( bhatF, g3d_out);
-    laplacianM.set_chi( hh);
-
-    // The curvature vectors
-
-    dg::geo::CylindricalVectorLvl0 curvNablaF, curvKappaF;
-    if( p.curvmode == "true" )
-    {
-        curvNablaF = dg::geo::createTrueCurvatureNablaB(mag);
-        curvKappaF = dg::geo::createTrueCurvatureKappa(mag);
-    }
-    else if( p.curvmode == "low beta")
-    {
-        curvNablaF = curvKappaF = dg::geo::createCurvatureNablaB(mag);
-    }
-    else if( p.curvmode == "toroidal")
-    {
-        curvNablaF = dg::geo::createCurvatureNablaB(mag);
-        curvKappaF = dg::geo::createCurvatureKappa(mag);
-    }
-    std::array<dg::DVec, 3> curvNabla, curvKappa, bhat;
-    dg::pushForward(curvNablaF.x(), curvNablaF.y(), curvNablaF.z(),
-        curvNabla[0], curvNabla[1], curvNabla[2], g3d_out);
-    dg::pushForward(curvKappaF.x(), curvKappaF.y(), curvKappaF.z(),
-        curvKappa[0], curvKappa[1], curvKappa[2], g3d_out);
-
-    //in DS we take the true bhat
-
-    bhatF = dg::geo::createBHat( mag);
-    dg::geo::DS<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
-        dsN( bhatF, g3d_out, p.bcxN, p.bcyN, dg::geo::NoLimiter(),
-        dg::forward, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz );
-
-    // create ingredients for perp Poisson bracket
-
-    bhatF = dg::geo::createEPhi();
-    if( p.curvmode == "true")
-        bhatF = dg::geo::createBHat(mag);
-    dg::pushForward(bhatF.x(), bhatF.y(), bhatF.z(),
-        bhat[0], bhat[1], bhat[2], g3d_out);
-    dg::SparseTensor<dg::DVec> metric = g3d_out.metric();
-    dg::tensor::inv_multiply3d( metric, bhat[0], bhat[1], bhat[2],
-                                        bhat[0], bhat[1], bhat[2]);
-    dg::DVec vol = dg::tensor::volume( metric);
-    const dg::DVec binv = dg::evaluate( dg::geo::InvB(mag) , g3d_out) ;
-    dg::blas1::pointwiseDivide( binv, vol, vol); //1/vol/B
-    dg::DVec bphi = bhat[2];//save bphi for momentum conservation
-    for( int i=0; i<3; i++)
-        dg::blas1::pointwiseDot( vol, bhat[i], bhat[i]); //b_i/vol/B
-    dg::DMatrix dxA = dg::create::dx( g3d_out, p.bcxU);
-    dg::DMatrix dyA = dg::create::dy( g3d_out, p.bcyU);
-    dg::DMatrix dxP = dg::create::dx( g3d_out, p.bcxP);
-    dg::DMatrix dyP = dg::create::dy( g3d_out, p.bcyP);
-    dg::DMatrix dxN = dg::create::dx( g3d_out, p.bcxN);
-    dg::DMatrix dyN = dg::create::dy( g3d_out, p.bcyN);
-    dg::DMatrix dz = dg::create::dz( g3d_out, dg::PER);
     dg::HMatrix dpsi = dg::create::dx( g1d_out, dg::DIR_NEU);
-
-    dg::DVec dx_A( t3d), dy_A(t3d), dz_A(t3d);
-    dg::DVec dx_P( t3d), dy_P(t3d), dz_P(t3d);
-    dg::DVec dx_N( t3d), dy_N(t3d), dz_N(t3d);
-    const dg::DVec psipR =  dg::evaluate( mag.psipR(), g3d_out);
-    const dg::DVec psipZ =  dg::evaluate( mag.psipZ(), g3d_out);
-    const dg::DVec psipP =  t3d; //zero
-
-    /////////////////// Construct names to output /////////////
-    std::map<std::string, std::string> names_direct{
-        {"electrons",   "Electron density"},
-        {"ions",        "Ion gyrocentre density"},
-        {"Ue",          "Electron parallel velocity"},
-        {"Ui",          "Ion parallel velocity"},
-        {"potential",   "Electric potential"},
-        {"induction",   "Magnetic potential"}
-    };
-    std::map<std::string, std::string> names_derived{
-        {"vorticity",       "Minus Lap_perp of electric potential"},
-        {"apar_vorticity",  "Minus Lap_perp of magnetic potential"},
-        {"neue",            "Product of electron density and velocity"},
-        {"NiUi",            "Product of ion gyrocentre density and velocity"},
-        {"neuebphi",        "Product of neue and covariant phi component of magnetic field unit vector"},
-        {"NiUibphi",        "Product of NiUi and covariant phi component of magnetic field unit vector"},
-        {"Lperpinv",        "Perpendicular density gradient length scale"},
-        {"Lparallelinv",    "Parallel density gradient length scale"},
-        {"jvne",            "Radial electron flux without induction contribution"},
-        {"jvneA",           "Radial electron flux: induction contribution"},
-    };
-    names_derived.insert( names_direct.begin(), names_direct.end());
-    std::vector<std::string> names_0d{
-        "aligned", "perp_aligned", "correlationNPhi"
+    /// Construct Feltor Variables object
+    feltor::Variables var = {
+        feltor::Feltor( g3d_out, p, mag), p, dpsip, dpsip, dvdpsip3d
     };
 
-    // Create Netcdf output file and ids
-    int ncid_out;
-    err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
-
-    /// Set global attributes
-    std::map<std::string, std::string> att;
-    att["title"] = "Output file of feltor/diag/feltordiag.cu";
-    att["Conventions"] = "CF-1.7";
-    ///Get local time and begin file history
-    auto ttt = std::time(nullptr);
-    auto tm = *std::localtime(&ttt);
-    std::ostringstream oss;
-    ///time string  + program-name + args
-    oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
-    for( int i=0; i<argc; i++) oss << " "<<argv[i];
-    att["history"] = oss.str();
-    att["comment"] = "Find more info in feltor/src/feltor.tex";
-    att["source"] = "FELTOR";
-    att["references"] = "https://github.com/feltor-dev/feltor";
-    att["inputfile"] = input;
-    att["geomfile"] = geom;
-    for( auto pair : att)
-        err = nc_put_att_text( ncid, NC_GLOBAL,
-            pair.first.data(), pair.second.size(), pair.second.data());
     // define 2d and 1d and 0d dimensions and variables
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
@@ -367,7 +207,10 @@ int main( int argc, char* argv[])
         psi_long_name.size(), psi_long_name.data());
 
     std::map<std::string, int> id0d, id1d, id2d;
-    // construct 0d variables
+    /// Construct 0d names to output
+    std::vector<std::string> names_0d{
+        "correlationNPhi", "correlationNTildePhi"
+    };
     std::map<std::string, double> v0d;
     for( auto name : names_0d)
     {
@@ -376,63 +219,12 @@ int main( int argc, char* argv[])
             &id0d[name]);
     }
 
-    // construct a vector for each name in the list
+    // construct a vector for each name in the input
     std::map<std::string, dg::DVec> v3d;
-    for( auto pair : names_derived)
-        v3d[pair.first] = t3d;
+    for( auto name : names_input)
+        v3d[name] = t3d;
     //now create the variables in the netcdf file
-    for( auto pair : names_derived)
-    {
-        std::string name = pair.first + "_ta2d";
-        std::string long_name = pair.second + " (Toroidal average)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", pair.second.size(),
-            long_name.data());
-
-        name = pair.first + "_2d";
-        long_name = pair.second + " (Evaluated on phi = 0 plane)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", pair.second.size(),
-            long_name.data());
-
-        name = pair.first + "_fluc2d";
-        long_name = pair.second + " (Fluctuations wrt fsa on phi = 0 plane)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", pair.second.size(),
-            long_name.data());
-
-        name = pair.first + "_fsa2d";
-        long_name = pair.second + " (Flux surface average interpolated to 2d plane)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", pair.second.size(),
-            long_name.data());
-
-        name = pair.first + "_fsa";
-        long_name = pair.second + " (Flux surface average)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
-            &id1d[name]);
-        err = nc_put_att_text( ncid, id1d[name], "long_name", pair.second.size(),
-            long_name.data());
-
-        name = pair.first + "_ifs";
-        long_name = pair.second + " (Integrated Flux surface average unless it is a current then it is the derived flux surface average)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
-            &id1d[name]);
-        err = nc_put_att_text( ncid, id1d[name], "long_name", pair.second.size(),
-            long_name.data());
-
-        name = pair.first + "_ifs_lcfs";
-        long_name = pair.second + " (Integrated Flux surface average evaluated on last closed flux surface unless it is a current then it is the fsa evaluated)";
-        v0d.at(name) = 0.; //this needs to go into v0d
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
-            &id0d[name]);
-        err = nc_put_att_text( ncid, id0d[name], "long_name", pair.second.size(),
-            long_name.data());
-    }
+    feltor::create_records_in_file( ncid_out, dim_ids, dim_ids1d, id0d, id1d, id2d);
 
     size_t count1d[2] = {1, g1d_out.n()*g1d_out.N()};
     size_t start1d[2] = {0, 0};
@@ -477,86 +269,29 @@ int main( int argc, char* argv[])
         err = nc_get_vara_double( ncid, timeID, start3d, count3d, &time);
         std::cout << "Timestep = " << i << "  time = " << time << "\t";
         //read in Ne,Ni,Ue,Ui,Phi,Apar
-        for( auto name : names_direct)
+        for( auto name : names_input)
         {
             int dataID;
-            err = nc_inq_varid(ncid, name.first.data(), &dataID);
+            err = nc_inq_varid(ncid, name.data(), &dataID);
             err = nc_get_vara_double( ncid, dataID, start3d, count3d,
                 transfer3d.data());
-            dg::assign( transfer3d, v3d[name.first]);
+            dg::assign( transfer3d, v3d.at(name));
         }
         err = nc_close(ncid);  //close 3d file
-
-        //----------------vorticity computation
-
-        dg::blas2::gemv( laplacianM, v3d.at("potential"), v3d.at("vorticity"));
-        dg::blas2::gemv( laplacianM, v3d.at("induction"), v3d.at("apar_vorticity"));
-        //----------------currents
-        //
-        dg::blas1::pointwiseDot( v3d.at("electrons"), v3d.at("Ue"), v3d.at("neue"));
-        dg::blas1::pointwiseDot( v3d.at("ions"), v3d.at("Ui"), v3d.at("NiUi"));
-        dg::blas1::pointwiseDot( v3d.at("neue"), bphi, v3d.at("neuebphi"));
-        dg::blas1::pointwiseDot( v3d.at("NiUi"), bphi, v3d.at("NiUibphi"));
+        var.f.set_fields( time, v3d.at("electrons"), v3d.at("ions"), v3d.at("Ue"),
+            v3d.at("Ui"), v3d.at("potential"), v3d.at("induction") );
 
         //----------------Test if induction equation holds
-        dg::blas1::axpbypgz( p.beta, v3d.at("neue"), -p.beta, v3d.at("NiUi"),
-            0., t3d);
-        double norm  = dg::blas2::dot( t3d, w3d, t3d);
-        dg::blas1::axpby( -1., v3d.at("apar_vorticity"), 1., t3d);
+        dg::blas1::copy(var.f.lapMperpA(), result);
+        dg::blas1::pointwiseDot(
+            var.f.density(0), var.f.velocity(0), t3d);
+        dg::blas1::pointwiseDot( p.beta,
+            var.f.density(1), var.f.velocity(1), -p.beta, t3d);
+        double norm  = dg::blas2::dot( result, w3d, result);
+        dg::blas1::axpby( -1., result, 1., t3d);
         double error = dg::blas2::dot( t3d, w3d, t3d);
         std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
 
-        //----------------radial flux computation
-
-        dg::blas2::symv( dxA, v3d.at("induction"), dx_A);
-        dg::blas2::symv( dyA, v3d.at("induction"), dy_A);
-        dg::blas2::symv( dz , v3d.at("induction"), dz_A);
-        dg::blas2::symv( dxP, v3d.at("potential"), dx_P);
-        dg::blas2::symv( dyP, v3d.at("potential"), dy_P);
-        dg::blas2::symv( dz , v3d.at("potential"), dz_P);
-        dg::blas1::evaluate( t3d, dg::equals(),
-            RadialParticleFlux( p.tau[0], p.mu[0]),
-            v3d.at("electrons"), v3d.at("Ue"),
-            dx_P, dy_P, dz_P, psipR, psipZ, psipP,
-            bhat[0], bhat[1], bhat[2],
-            curvNabla[0], curvNabla[1], curvNabla[2],
-            curvKappa[0], curvKappa[1], curvKappa[2]
-        );
-        dg::blas1::pointwiseDot( t3d, dvdpsip3d, v3d.at("jvne"));
-        dg::blas1::evaluate( t3d, dg::equals(),
-            RadialParticleFlux( p.tau[0], p.mu[0]),
-            v3d.at("electrons"), v3d.at("Ue"), v3d.at("induction"),
-            dx_A, dy_A, dz_A, psipR, psipZ, psipP,
-            bhat[0], bhat[1], bhat[2],
-            curvKappa[0], curvKappa[1], curvKappa[2]
-        );
-        dg::blas1::pointwiseDot( t3d, dvdpsip3d, v3d.at("jvneA"));
-
-        //----------------perp length scale computation
-
-        dg::blas1::axpby( 1., v3d.at("electrons"), -1., 1., t3d);
-        dg::blas2::symv( dxN, t3d, dx_N);
-        dg::blas2::symv( dyN, t3d, dy_N);
-        dg::blas2::symv( dz , t3d, dz_N);
-        dg::tensor::multiply3d( hh, //grad_perp
-            dx_N, dy_N, dz_N, dx_A, dy_A, dz_A);
-        dg::blas1::subroutine( feltor::routines::ComputePsi(),
-            t3d, dx_N, dy_N, dz_N, dx_A, dy_A, dz_A);
-        dg::blas1::pointwiseDivide( t3d, v3d.at("electrons"), t3d);
-        v0d.at("perp_aligned") = dg::blas1::dot( t3d, w3d);
-        dg::blas1::pointwiseDivide( t3d, v3d.at("electrons"), t3d);
-        dg::blas1::transform( t3d, v3d.at("Lperpinv"), dg::SQRT<double>());
-
-        //----------------parallel length scale computation
-
-        dg::blas1::axpby( 1., v3d.at("electrons"), -1., 1., t3d);
-        dsN.centered( t3d, dx_N);
-        dg::blas1::pointwiseDot ( dx_N, dx_N, t3d);
-        dg::blas1::pointwiseDivide( t3d, v3d.at("electrons"), t3d);
-        v0d.at("aligned") = dg::blas1::dot( t3d, w3d);
-        dg::blas1::pointwiseDivide( t3d, v3d.at("electrons"), t3d);
-        dg::blas1::transform( t3d, v3d.at("Lparallelinv"), dg::SQRT<double>());
-
         //------------------correlation------------//
 
         dg::blas1::transform( v3d.at("potential"), t3d, dg::EXP<double>());
@@ -568,12 +303,13 @@ int main( int argc, char* argv[])
 
         //now write out 2d and 1d quantities
         err = nc_open(argv[2], NC_WRITE, &ncid_out);
-        for( auto pair : v3d)// {name, DVec}
+        for( auto record : feltor::records_list)// {name, DVec}
         {
+            record.function( t3d, var);
             //toroidal average
-            toroidal_average( pair.second, t2d, false);
+            toroidal_average( t3d, t2d, false);
             dg::blas1::transfer( t2d, transfer2d);
-            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_ta"),
+            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_ta"),
                 start2d, count2d, transfer2d.data());
 
             //flux surface average
@@ -582,31 +318,31 @@ int main( int argc, char* argv[])
             poloidal_average( transfer2dX, t1d, false); //average over eta
             dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
             dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
-            err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_fsa"),
+            err = nc_put_vara_double( ncid_out, id1d.at(record.name+"_fsa"),
                 start1d, count1d, fsa1d.data());
 
 
             // 2d data of plane varphi = 0
             unsigned kmp = 0; //g3d_out.Nz()/2;
-            dg::HVec t2d_mp(pair.second.begin() + kmp*g2d_out.size(),
-                pair.second.begin() + (kmp+1)*g2d_out.size() );
-            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_plane"),
+            dg::HVec t2d_mp(t3d.begin() + kmp*g2d_out.size(),
+                t3d.begin() + (kmp+1)*g2d_out.size() );
+            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_plane"),
                 start2d, count2d, t2d_mp.data() );
 
             // fsa on 2d plane : <f>
             dg::blas2::gemv(fsa2rzmatrix, fsa1d, transfer2d); //fsa on RZ grid
-            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_fsa2"),
+            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_fsa2"),
                 start2d, count2d, transfer2d.data() );
 
             // delta f on midplane : df = f_mp - <f>
             dg::blas1::axpby( 1.0, t2d_mp, -1.0, transfer2d);
-            err = nc_put_vara_double( ncid_out, id2d.at(pair.first+"_fluc"),
+            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_fluc"),
                 start2d, count2d, transfer2d.data() );
 
             //flux surface integral/derivative
-            if( pair.first[0] == 'j') //j indicates a flux
+            if( record.name[0] == 'j') //j indicates a flux
             {
-                v0d.at(pair.first+"ifs_lcfs") = dg::interpolate( fsa1d, 0., g1d_out);
+                v0d[record.name+"ifs_lcfs"] = dg::interpolate( fsa1d, 0., g1d_out); //create a new entry
                 dg::blas2::symv( dpsi, fsa1d, t1d);
                 dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
             }
@@ -614,9 +350,9 @@ int main( int argc, char* argv[])
             {
                 t1d = dg::integrate( fsa1d, g1d_out);
                 dg::blas1::pointwiseDot( t1d, dvdpsip, transfer1d);
-                v0d.at(pair.first+"ifs_lcfs") = dg::interpolate( transfer1d, 0., g1d_out);
+                v0d[record.name+"ifs_lcfs"] = dg::interpolate( transfer1d, 0., g1d_out); //create a new entry
             }
-            err = nc_put_vara_double( ncid_out, id1d.at(pair.first+"_ifs"),
+            err = nc_put_vara_double( ncid_out, id1d.at(record.name+"_ifs"),
                 start1d, count1d, transfer1d.data());
             //flux surface integral/derivative on last closed flux surface
 
diff --git a/diag/feltordiag.h b/diag/feltordiag.h
new file mode 100644
index 000000000..86f781553
--- /dev/null
+++ b/diag/feltordiag.h
@@ -0,0 +1,321 @@
+#include <string>
+#include <vector>
+#include <functional>
+
+#include "dg/algorithm.h"
+#include "dg/geometries/geometries.h"
+
+#include "feltor/feltor.cuh"
+#include "feltor/parameters.h"
+namespace feltor{
+struct Jacobian{
+    DG_DEVICE double operator()(
+        double d0P, double d1P, double d2P, //any three vectors
+        double d0S, double d1S, double d2S,
+        double b_0, double b_1, double b_2)
+    {
+        return      b_0*( d1P*d2S-d2P*d1S)+
+                    b_1*( d2P*d0S-d0P*d2S)+
+                    b_2*( d0P*d1S-d1P*d0S);
+    }
+};
+
+struct RadialParticleFlux{
+    RadialParticleFlux( double tau, double mu):
+        m_tau(tau), m_mu(mu){
+    }
+    DG_DEVICE double operator()( double ne, double ue,
+        double d0P, double d1P, double d2P, //Phi
+        double d0S, double d1S, double d2S, //Psip
+        double b_0,         double b_1,         double b_2,
+        double curv0,       double curv1,       double curv2,
+        double curvKappa0,  double curvKappa1,  double curvKappa2
+        ){
+        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
+        double curvS = curv0*d0S+curv1*d1S+curv2*d2S;
+        double PS = b_0*( d1P*d2S-d2P*d1S)+
+                    b_1*( d2P*d0S-d0P*d2S)+
+                    b_2*( d0P*d1S-d1P*d0S);
+        double JPsi =
+            + ne * PS
+            + ne * m_mu*ue*ue*curvKappaS
+            + ne * m_tau*curvS;
+        return JPsi;
+    }
+    DG_DEVICE double operator()( double ne, double ue, double A,
+        double d0A, double d1A, double d2A,
+        double d0S, double d1S, double d2S, //Psip
+        double b_0,         double b_1,         double b_2,
+        double curvKappa0,  double curvKappa1,  double curvKappa2
+        ){
+        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
+        double SA = b_0*( d1S*d2A-d2S*d1A)+
+                    b_1*( d2S*d0A-d0S*d2A)+
+                    b_2*( d0S*d1A-d1S*d0A);
+        double JPsi =
+            ne*ue* (A*curvKappaS + SA );
+        return JPsi;
+    }
+    private:
+    double m_tau, m_mu;
+};
+struct RadialEnergyFlux{
+    RadialEnergyFlux( double tau, double mu):
+        m_tau(tau), m_mu(mu){
+    }
+
+    DG_DEVICE double operator()( double ne, double ue, double A, double P,
+        double d0A, double d1A, double d2A,
+        double d0P, double d1P, double d2P, //Phi
+        double d0S, double d1S, double d2S, //Psip
+        double b_0,         double b_1,         double b_2,
+        double curv0,  double curv1,  double curv2,
+        double curvKappa0,  double curvKappa1,  double curvKappa2
+        ){
+        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
+        double curvS = curv0*d0S+curv1*d1S+curv2*d2S;
+        double SA = b_0*( d1S*d2A-d2S*d1A)+
+                    b_1*( d2S*d0A-d0S*d2A)+
+                    b_2*( d0S*d1A-d1S*d0A);
+        double PS = b_0*( d1P*d2S-d2P*d1S)+
+                    b_1*( d2P*d0S-d0P*d2S)+
+                    b_2*( d0P*d1S-d1P*d0S);
+        double JN =
+            ne*ue* (A*curvKappaS + SA )
+            + ne * PS
+            + ne * m_mu*ue*ue*curvKappaS
+            + ne * m_tau*curvS;
+        double Je = (m_tau * log(ne) + 0.5*m_mu*ue*ue + P)*JN
+            + m_mu*m_tau*ne*ue*ue*curvKappaS
+            + m_tau*ne*ue* (A*curvKappaS + SA );
+        return Je;
+    }
+    private:
+    double m_tau, m_mu;
+};
+
+template<class Container>
+void dot( const std::array<Container, 3>& v,
+          const std::array<Container, 3>& w,
+          Container& result)
+{
+    dg::blas1::evaluate( result, dg::equals(), dg::PairSum(),
+        v[0], w[0], v[1], w[1], v[2], w[2]);
+}
+template<class Container>
+void jacobian(
+          const std::array<Container, 3>& a,
+          const std::array<Container, 3>& b,
+          const std::array<Container, 3>& c,
+          Container& result)
+{
+    dg::blas1::evaluate( result, dg::equals(), Jacobian(),
+        a[0], a[1], a[2], b[0], b[1], b[2], c[0], c[1], c[2]);
+}
+
+
+using Feltor = feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>;
+
+struct Variables{
+    Feltor f;
+    feltor::Parameters p;
+    std::array<dg::DVec, 3> dpsip;
+    std::array<dg::DVec, 3> tmp;
+    dg::DVec dvdpsip3d;
+};
+
+struct Record{
+    std::string name;
+    std::string long_name;
+    std::function<void( dg::DVec&, Variables&)> function;
+};
+
+
+std::vector<Record> records_list = {
+    {"ne", "Electron density",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.density(0), result);
+        }
+    },
+    {"Ni", "Ion gyro-centre density",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.density(1), result);
+        }
+    },
+    {"ue", "Electron parallel velocity",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.velocity(0), result);
+        }
+    },
+    {"Ui", "Ion parallel velocity",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.velocity(1), result);
+        }
+    },
+    {"potential", "Electric potential",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.potential()[0], result);
+        }
+    },
+    {"apar", "Magnetic potential",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.induction(), result);
+        }
+    },
+    {"vorticity", "Minus Lap_perp of electric potential",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.lapMperpP(0), result);
+        }
+    },
+    {"apar_vorticity", "Minus Lap_perp of magnetic potential",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.lapMperpA(), result);
+        }
+    },
+    {"neue", "Product of electron density and velocity",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot(
+                v.f.density(0), v.f.velocity(0), result);
+        }
+    },
+    {"NiUi", "Product of ion gyrocentre density and velocity",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot(
+                v.f.density(1), v.f.velocity(1), result);
+        }
+    },
+    {"neuebphi", "Product of neue and covariant phi component of magnetic field unit vector",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 1.,
+                v.f.density(0), v.f.velocity(0), v.f.bphi(), 0., result);
+        }
+    },
+    {"NiUibphi", "Product of NiUi and covariant phi component of magnetic field unit vector",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 1.,
+                v.f.density(1), v.f.velocity(1), v.f.bphi(), 0., result);
+        }
+    },
+    {"Lperpinv", "Perpendicular density gradient length scale",
+        []( dg::DVec& result, Variables& v ) {
+            const std::array<dg::DVec, 3>& dN = v.f.gradN(0);
+            dg::tensor::multiply3d( v.f.projection(), //grad_perp
+                dN[0], dN[1], dN[2], v.tmp[0], v.tmp[1], v.tmp[2]);
+            dot(dN, v.tmp, result);
+            dg::blas1::pointwiseDivide( result, v.f.density(0), result);
+            dg::blas1::pointwiseDivide( result, v.f.density(0), result);
+            dg::blas1::transform( result, result, dg::SQRT<double>());
+        }
+    },
+    {"perp_aligned", "Perpendicular density alignement",
+        []( dg::DVec& result, Variables& v ) {
+            const std::array<dg::DVec, 3>& dN = v.f.gradN(0);
+            dg::tensor::multiply3d( v.f.projection(), //grad_perp
+                dN[0], dN[1], dN[2], v.tmp[0], v.tmp[1], v.tmp[2]);
+            dot(dN, v.tmp, result);
+            dg::blas1::pointwiseDivide( result, v.f.density(0), result);
+        }
+    },
+    {"Lparallelinv", "Parallel density gradient length scale",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot ( v.f.dsN(0), v.f.dsN(0), result);
+            dg::blas1::pointwiseDivide( result, v.f.density(0), result);
+            dg::blas1::pointwiseDivide( result, v.f.density(0), result);
+            dg::blas1::transform( result, result, dg::SQRT<double>());
+        }
+    },
+    {"aligned", "Parallel density alignement",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot ( v.f.dsN(0), v.f.dsN(0), result);
+            dg::blas1::pointwiseDivide( result, v.f.density(0), result);
+        }
+    },
+    {"jvne", "Radial electron particle flux without induction contribution",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
+                v.f.density(0), v.f.velocity(0),
+                v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
+                v.dpsip[0], v.dpsip[1], v.dpsip[2],
+                v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
+                v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
+                v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
+            );
+            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
+        }
+    },
+    {"jvneA", "Radial electron particle flux: induction contribution",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
+                v.f.density(0), v.f.velocity(0), v.f.induction(),
+                v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
+                v.dpsip[0], v.dpsip[1], v.dpsip[2],
+                v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
+                v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
+            );
+            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
+        }
+    }
+};
+
+void create_records_in_file( int ncid,
+    int dim_ids[3], int dim_ids1d[2],
+    std::map<std::string, int>& id0d,
+    std::map<std::string, int>& id1d,
+    std::map<std::string, int>& id2d)
+{
+    file::NC_Error_Handle err;
+    for( auto record : records_list)
+    {
+        std::string name = record.name + "_ta2d";
+        std::string long_name = record.long_name + " (Toroidal average)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);//creates a new id2d entry
+        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_2d";
+        long_name = record.long_name + " (Evaluated on phi = 0 plane)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_fluc2d";
+        long_name = record.long_name + " (Fluctuations wrt fsa on phi = 0 plane)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_fsa2d";
+        long_name = record.long_name + " (Flux surface average interpolated to 2d plane)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_fsa";
+        long_name = record.long_name + " (Flux surface average)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        err = nc_put_att_text( ncid, id1d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_ifs";
+        long_name = record.long_name + " (Integrated Flux surface average unless it is a current then it is the derived flux surface average)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        err = nc_put_att_text( ncid, id1d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_ifs_lcfs";
+        long_name = record.long_name + " (Integrated Flux surface average evaluated on last closed flux surface unless it is a current then it is the fsa evaluated)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 1, dim_ids,
+            &id0d[name]);
+        err = nc_put_att_text( ncid, id0d[name], "long_name", long_name.size(),
+            long_name.data());
+    }
+}
+}//namespace feltor
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 1ab6cc0b2..4a6198c9c 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -256,30 +256,47 @@ struct Explicit
         dg::blas2::symv( m_dx_N, m_temp0, m_dN[0][0]);
         dg::blas2::symv( m_dy_N, m_temp0, m_dN[0][1]);
         if(!m_p.symmetric) dg::blas2::symv( m_dz, m_temp0, m_dN[0][2]);
+        m_ds_N.centered( m_temp0, m_dsN[0]);
         dg::blas1::axpby( 1., Ni, -1., 1., m_temp0);
         dg::blas2::symv( m_dx_N, m_temp0, m_dN[1][0]);
         dg::blas2::symv( m_dy_N, m_temp0, m_dN[1][1]);
         if(!m_p.symmetric) dg::blas2::symv( m_dz, m_temp0, m_dN[1][2]);
+        m_ds_N.centered( m_temp0, m_dsN[1]);
 
         for( unsigned i=0; i<2; i++)
         {
-            dg::blas2::symv( m_dx_U, fields[1][i], m_dU[i][0]);
-            dg::blas2::symv( m_dy_U, fields[1][i], m_dU[i][1]);
-            if(!m_p.symmetric) dg::blas2::symv( m_dz, fields[1][i], m_dU[i][2]);
+            dg::blas2::symv( m_dx_U, m_fields[1][i], m_dU[i][0]);
+            dg::blas2::symv( m_dy_U, m_fields[1][i], m_dU[i][1]);
+            if(!m_p.symmetric) dg::blas2::symv( m_dz, m_fields[1][i], m_dU[i][2]);
         }
 
      }
 
     const Container& uE2() const {return m_UE2;}
-    const std::array<Container, 3> & dN (int i) const {
+    const Container& density(int i)const{
+        return m_fields[0][i];
+    }
+    const Container& velocity(int i)const{
+        return m_fields[1][i];
+    }
+    const std::array<Container, 3> & gradN (int i) const {
         return m_dN[i];
     }
-    const std::array<Container, 3> & dP (int i) const {
+    const std::array<Container, 3> & gradU (int i) const {
+        return m_dU[i];
+    }
+    const std::array<Container, 3> & gradP (int i) const {
         return m_dP[i];
     }
-    const std::array<Container, 3> & dA () const {
+    const std::array<Container, 3> & gradA () const {
         return m_dA;
     }
+    const Container & dsN (int i) const {
+        return m_dsN[i];
+    }
+    const dg::SparseTensor<Container>& projection() const{
+        return m_hh;
+    }
     const std::array<Container, 3> & curv () const {
         return m_curv;
     }
@@ -291,6 +308,29 @@ struct Explicit
     const std::array<Container, 3> & bhatgB () const {
         return m_b;
     }
+    const Container& lapMperpN (int i)
+    {
+        dg::blas1::axpby( 1., m_fields[0][i], -1., 1., m_temp0);
+        dg::blas2::gemv( m_lapperpN, m_temp0, m_temp1);
+        return m_temp1;
+    }
+    const Container& lapMperpU (int i)
+    {
+        dg::blas1::axpby( 1., m_fields[1][i], -1., 1., m_temp0);
+        dg::blas2::gemv( m_lapperpU, m_temp0, m_temp1);
+        return m_temp1;
+    }
+    const Container& lapMperpP (int i)
+    {
+        dg::blas2::gemv( m_lapperpP, m_phi[i], m_temp1);
+        return m_temp1;
+    }
+    const Container& lapMperpA ()
+    {
+        dg::blas2::gemv( m_lapperpU, m_apar, m_temp1);
+        return m_temp1;
+    }
+    /////////////////////////DIAGNOSTICS END////////////////////////////////
 
     //source strength, profile - 1
     void set_source( Container profile, double omega_source, Container source)
@@ -339,7 +379,7 @@ struct Explicit
     std::array<Container,2> m_phi, m_logn, m_dsN, m_dsU;
     std::array<Container,3> m_dA;
     std::array<std::array<Container,3>,2> m_dP, m_dN, m_dU;
-    std::array<std::array<Container,2>,2> m_fields, m_s;
+    std::array<std::array<Container,2>,2> m_fields, m_s; //fields, sources
 
     std::vector<Container> m_multi_chi;
 
-- 
GitLab


From 4630fbd1eaef28b06fb289229609a22659a421f9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 12 May 2019 22:52:48 +0200
Subject: [PATCH 081/540] Add some more terms in diagnostics

---
 diag/feltordiag.h     | 71 +++++++++++++++++++++++++++++++++++++++++++
 src/feltor/feltor.cuh | 15 +++++++++
 2 files changed, 86 insertions(+)

diff --git a/diag/feltordiag.h b/diag/feltordiag.h
index 86f781553..5c4423b2f 100644
--- a/diag/feltordiag.h
+++ b/diag/feltordiag.h
@@ -230,6 +230,7 @@ std::vector<Record> records_list = {
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
+    /// ------------------ Density terms ------------------------//
     {"jvne", "Radial electron particle flux without induction contribution",
         []( dg::DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
@@ -256,7 +257,77 @@ std::vector<Record> records_list = {
             );
             dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
         }
+    },
+    {"lneperp", "Perpendicular particle diffusion",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::copy( v.f.lapMperpN(0), result);
+            dg::blas1::scal( result, -v.p.nu_perp);
+        }
+    },
+    {"lneparallel", "Parallel particle diffusion",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::copy( v.f.lapParallelN(0), result);
+            dg::blas1::scal( result, v.p.nu_parallel);
+        }
+    },
+    /// ------------------- Energy terms ------------------------//
+    {"nelnne", "Entropy electrons",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::transform( v.f.density(0), result, dg::LN<double>());
+            dg::blas1::pointwiseDot( result, v.f.density(0), result);
+        }
+    },
+    {"NilnNi", "Entropy ions",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::transform( v.f.density(1), result, dg::LN<double>());
+            dg::blas1::pointwiseDot( v.p.tau[1], result, v.f.density(1), 0., result);
+        }
+    },
+    {"aperp2", "Magnetic energy",
+        []( dg::DVec& result, Variables& v ) {
+            if( v.p.beta == 0)
+            {
+                dg::blas1::scal( result, 0.);
+            }
+            else
+            {
+                dg::tensor::multiply3d( v.f.projection(), //grad_perp
+                    v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
+                    v.f.tmp[0], v.f.tmp[1], v.f.tmp[2])
+                dot( v.f.tmp, v.f.gradA(), result);
+                dg::blas1::scal( result, 1./2./v.p.beta);
+            }
+        }
+    },
+    {"UE2", "ExB energy",
+        []( dg::DVec& result, Variables& v ) {
+            dg::tensor::multiply3d( v.f.projection(), //grad_perp
+                v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
+                v.f.tmp[0], v.f.tmp[1], v.f.tmp[2])
+            dot( v.f.tmp, v.f.gradP(), result);
+            dg::blas1::pointwiseDot( 0.5, v.f.density(1), result, 0., result);
+        }
+    },
+    {"neue2", "Parallel electron energy",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( -0.5*v.p.mu[0], v.f.density(0),
+                v.f.velocity(0), v.f.velocity(0), 0., result);
+        }
+    },
+    {"NiUi2", "Parallel ion energy",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 0.5*v.p.mu[1], v.f.density(1),
+                v.f.velocity(1), v.f.velocity(1), 0., result);
+        }
+    },
+    {"resistivity", "Energy dissipation through resistivity",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::axpby( 1., v.f.velocity(1), -1., v.f.velocity(0), result);
+            dg::blas1::pointwiseDot( v.p.eta, v.f.density(0), result, result, 0., result)
+        }
     }
+
+
 };
 
 void create_records_in_file( int ncid,
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 4a6198c9c..42859859f 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -294,6 +294,21 @@ struct Explicit
     const Container & dsN (int i) const {
         return m_dsN[i];
     }
+    const Container& lapParallelN( int i){
+        dg::blas1::axpby( 1., m_fields[0][i], -1., 1., m_temp0);
+        m_ds_N.dss( m_temp0, m_temp1);
+        dg::blas1::pointwiseDot( 1., m_divb, m_dsN[i],
+                                 0., m_temp0);
+        dg::blas1::axpby( 1., m_temp1, 1., m_temp0);
+        return m_temp0;
+    }
+    const Container& lapParallelU( int i){
+        m_ds_N.dss( m_fields[1][i], m_temp1);
+        dg::blas1::pointwiseDot( 1., m_divb, m_dsN[i],
+                                 0., m_temp0);
+        dg::blas1::axpby( 1., m_temp1, 1., m_temp0);
+        return m_temp0;
+    }
     const dg::SparseTensor<Container>& projection() const{
         return m_hh;
     }
-- 
GitLab


From 0afb2e6ad023fda0834d16176cf37f22ca89673f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 13 May 2019 15:20:24 +0200
Subject: [PATCH 082/540] Implemented and documented remaining terms in diag

- should we avoid writing the 2d fields in some cases?
---
 diag/feltordiag.cu    |  71 ++++++++-
 diag/feltordiag.h     | 355 +++++++++++++++++++++++++++++++-----------
 src/feltor/feltor.cuh |  63 ++++----
 src/feltor/feltor.tex |  91 +++++++----
 4 files changed, 418 insertions(+), 162 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index d6e3233e9..b6d337604 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -9,6 +9,65 @@
 #include "dg/file/nc_utilities.h"
 #include "feltordiag.h"
 
+void create_records_in_file( int ncid,
+    int dim_ids[3], int dim_ids1d[2],
+    std::map<std::string, int>& id0d,
+    std::map<std::string, int>& id1d,
+    std::map<std::string, int>& id2d)
+{
+    file::NC_Error_Handle err;
+    for( auto record : feltor::records_list)
+    {
+        std::string name = record.name + "_ta2d";
+        std::string long_name = record.long_name + " (Toroidal average)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);//creates a new id2d entry
+        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_2d";
+        long_name = record.long_name + " (Evaluated on phi = 0 plane)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_fluc2d";
+        long_name = record.long_name + " (Fluctuations wrt fsa on phi = 0 plane)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_fsa2d";
+        long_name = record.long_name + " (Flux surface average interpolated to 2d plane)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_fsa";
+        long_name = record.long_name + " (Flux surface average)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        err = nc_put_att_text( ncid, id1d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_ifs";
+        long_name = record.long_name + " (Integrated Flux surface average unless it is a current then it is the derived flux surface average)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        err = nc_put_att_text( ncid, id1d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_ifs_lcfs";
+        long_name = record.long_name + " (Integrated Flux surface average evaluated on last closed flux surface unless it is a current then it is the fsa evaluated)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 1, dim_ids,
+            &id0d[name]);
+        err = nc_put_att_text( ncid, id0d[name], "long_name", long_name.size(),
+            long_name.data());
+    }
+}
 
 int main( int argc, char* argv[])
 {
@@ -103,10 +162,10 @@ int main( int argc, char* argv[])
     dg::DVec t3d = dg::evaluate( dg::zero, g3d_out);
     dg::DVec result(t3d);
 
-    std::array<dg::DVec, 3> dpsip;
-    dpsip[0] =  dg::evaluate( mag.psipR(), g3d_out);
-    dpsip[1] =  dg::evaluate( mag.psipZ(), g3d_out);
-    dpsip[2] =  t3d; //zero
+    std::array<dg::DVec, 3> gradPsip;
+    gradPsip[0] =  dg::evaluate( mag.psipR(), g3d_out);
+    gradPsip[1] =  dg::evaluate( mag.psipZ(), g3d_out);
+    gradPsip[2] =  t3d; //zero
 
     ///--------------- Construct X-point grid ---------------------//
     //Find O-point
@@ -194,7 +253,7 @@ int main( int argc, char* argv[])
     dg::HMatrix dpsi = dg::create::dx( g1d_out, dg::DIR_NEU);
     /// Construct Feltor Variables object
     feltor::Variables var = {
-        feltor::Feltor( g3d_out, p, mag), p, dpsip, dpsip, dvdpsip3d
+        feltor::Feltor( g3d_out, p, mag), p, gradPsip, gradPsip, dvdpsip3d
     };
 
     // define 2d and 1d and 0d dimensions and variables
@@ -224,7 +283,7 @@ int main( int argc, char* argv[])
     for( auto name : names_input)
         v3d[name] = t3d;
     //now create the variables in the netcdf file
-    feltor::create_records_in_file( ncid_out, dim_ids, dim_ids1d, id0d, id1d, id2d);
+    create_records_in_file( ncid_out, dim_ids, dim_ids1d, id0d, id1d, id2d);
 
     size_t count1d[2] = {1, g1d_out.n()*g1d_out.N()};
     size_t start1d[2] = {0, 0};
diff --git a/diag/feltordiag.h b/diag/feltordiag.h
index 5c4423b2f..cb17fd768 100644
--- a/diag/feltordiag.h
+++ b/diag/feltordiag.h
@@ -60,12 +60,11 @@ struct RadialParticleFlux{
     double m_tau, m_mu;
 };
 struct RadialEnergyFlux{
-    RadialEnergyFlux( double tau, double mu):
-        m_tau(tau), m_mu(mu){
+    RadialEnergyFlux( double tau, double mu, double z):
+        m_tau(tau), m_mu(mu), m_z(z){
     }
 
-    DG_DEVICE double operator()( double ne, double ue, double A, double P,
-        double d0A, double d1A, double d2A,
+    DG_DEVICE double operator()( double ne, double ue, double P,
         double d0P, double d1P, double d2P, //Phi
         double d0S, double d1S, double d2S, //Psip
         double b_0,         double b_1,         double b_2,
@@ -74,24 +73,39 @@ struct RadialEnergyFlux{
         ){
         double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
         double curvS = curv0*d0S+curv1*d1S+curv2*d2S;
-        double SA = b_0*( d1S*d2A-d2S*d1A)+
-                    b_1*( d2S*d0A-d0S*d2A)+
-                    b_2*( d0S*d1A-d1S*d0A);
-        double PS = b_0*( d1P*d2S-d2P*d1S)+
-                    b_1*( d2P*d0S-d0P*d2S)+
-                    b_2*( d0P*d1S-d1P*d0S);
+        double PS = b_0 * ( d1P * d2S - d2P * d1S )+
+                    b_1 * ( d2P * d0S - d0P * d2S )+
+                    b_2 * ( d0P * d1S - d1P * d0S );
         double JN =
-            ne*ue* (A*curvKappaS + SA )
             + ne * PS
             + ne * m_mu*ue*ue*curvKappaS
             + ne * m_tau*curvS;
-        double Je = (m_tau * log(ne) + 0.5*m_mu*ue*ue + P)*JN
-            + m_mu*m_tau*ne*ue*ue*curvKappaS
-            + m_tau*ne*ue* (A*curvKappaS + SA );
+        double Je = m_z*(m_tau * log(ne) + 0.5*m_mu*ue*ue + P)*JN
+            + m_z*m_mu*m_tau*ne*ue*ue*curvKappaS;
         return Je;
     }
+    DG_DEVICE double operator()( double ne, double ue, double P, double A,
+        double d0A, double d1A, double d2A,
+        double d0S, double d1S, double d2S, //Psip
+        double b_0,         double b_1,         double b_2,
+        double curvKappa0,  double curvKappa1,  double curvKappa2
+        ){
+        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
+        double SA = b_0 * ( d1S * d2A - d2S * d1A )+
+                    b_1 * ( d2S * d0A - d0S * d2A )+
+                    b_2 * ( d0S * d1A - d1S * d0A );
+        double JN = m_z*ne*ue* (A*curvKappaS + SA );
+        double Je = m_z*( m_tau * log(ne) + 0.5*m_mu*ue*ue + P )*JN
+                    + m_z*m_tau*ne*ue* (A*curvKappaS + SA );
+        return Je;
+    }
+    DG_DEVICE double operator()( double ne, double ue, double P,
+        double lapMperpN, double lapMperpU){
+        return m_z*(m_tau*(1+log(ne))+P+0.5*m_mu*ue*ue)*lapMperpN
+                + m_z*m_mu*ne*ue*lapMperpU;
+    }
     private:
-    double m_tau, m_mu;
+    double m_tau, m_mu, m_z;
 };
 
 template<class Container>
@@ -113,13 +127,12 @@ void jacobian(
         a[0], a[1], a[2], b[0], b[1], b[2], c[0], c[1], c[2]);
 }
 
-
 using Feltor = feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>;
 
 struct Variables{
     Feltor f;
     feltor::Parameters p;
-    std::array<dg::DVec, 3> dpsip;
+    std::array<dg::DVec, 3> gradPsip;
     std::array<dg::DVec, 3> tmp;
     dg::DVec dvdpsip3d;
 };
@@ -132,17 +145,17 @@ struct Record{
 
 
 std::vector<Record> records_list = {
-    {"ne", "Electron density",
+    {"electrons", "Electron density",
         []( dg::DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(0), result);
         }
     },
-    {"Ni", "Ion gyro-centre density",
+    {"ions", "Ion gyro-centre density",
         []( dg::DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(1), result);
         }
     },
-    {"ue", "Electron parallel velocity",
+    {"Ue", "Electron parallel velocity",
         []( dg::DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.velocity(0), result);
         }
@@ -157,7 +170,12 @@ std::vector<Record> records_list = {
              dg::blas1::copy(v.f.potential()[0], result);
         }
     },
-    {"apar", "Magnetic potential",
+    {"psi", "Ion potential psi",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.potential()[1], result);
+        }
+    },
+    {"induction", "Magnetic potential",
         []( dg::DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.induction(), result);
         }
@@ -178,7 +196,7 @@ std::vector<Record> records_list = {
                 v.f.density(0), v.f.velocity(0), result);
         }
     },
-    {"NiUi", "Product of ion gyrocentre density and velocity",
+    {"niui", "Product of ion gyrocentre density and velocity",
         []( dg::DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot(
                 v.f.density(1), v.f.velocity(1), result);
@@ -190,13 +208,13 @@ std::vector<Record> records_list = {
                 v.f.density(0), v.f.velocity(0), v.f.bphi(), 0., result);
         }
     },
-    {"NiUibphi", "Product of NiUi and covariant phi component of magnetic field unit vector",
+    {"niuibphi", "Product of NiUi and covariant phi component of magnetic field unit vector",
         []( dg::DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1.,
                 v.f.density(1), v.f.velocity(1), v.f.bphi(), 0., result);
         }
     },
-    {"Lperpinv", "Perpendicular density gradient length scale",
+    {"lperpinv", "Perpendicular density gradient length scale",
         []( dg::DVec& result, Variables& v ) {
             const std::array<dg::DVec, 3>& dN = v.f.gradN(0);
             dg::tensor::multiply3d( v.f.projection(), //grad_perp
@@ -207,7 +225,7 @@ std::vector<Record> records_list = {
             dg::blas1::transform( result, result, dg::SQRT<double>());
         }
     },
-    {"perp_aligned", "Perpendicular density alignement",
+    {"perpaligned", "Perpendicular density alignement",
         []( dg::DVec& result, Variables& v ) {
             const std::array<dg::DVec, 3>& dN = v.f.gradN(0);
             dg::tensor::multiply3d( v.f.projection(), //grad_perp
@@ -216,7 +234,7 @@ std::vector<Record> records_list = {
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
-    {"Lparallelinv", "Parallel density gradient length scale",
+    {"lparallelinv", "Parallel density gradient length scale",
         []( dg::DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot ( v.f.dsN(0), v.f.dsN(0), result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
@@ -237,7 +255,7 @@ std::vector<Record> records_list = {
                 RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
                 v.f.density(0), v.f.velocity(0),
                 v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
-                v.dpsip[0], v.dpsip[1], v.dpsip[2],
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
@@ -251,20 +269,20 @@ std::vector<Record> records_list = {
                 RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
                 v.f.density(0), v.f.velocity(0), v.f.induction(),
                 v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
-                v.dpsip[0], v.dpsip[1], v.dpsip[2],
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
             dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
         }
     },
-    {"lneperp", "Perpendicular particle diffusion",
+    {"lneperp", "Perpendicular electron diffusion",
         []( dg::DVec& result, Variables& v ) {
-            dg::blas1::copy( v.f.lapMperpN(0), result);
+            v.f.compute_diffusive_lapMperpN( v.f.density(0), v.tmp[0], result);
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
-    {"lneparallel", "Parallel particle diffusion",
+    {"lneparallel", "Parallel electron diffusion",
         []( dg::DVec& result, Variables& v ) {
             dg::blas1::copy( v.f.lapParallelN(0), result);
             dg::blas1::scal( result, v.p.nu_parallel);
@@ -277,7 +295,7 @@ std::vector<Record> records_list = {
             dg::blas1::pointwiseDot( result, v.f.density(0), result);
         }
     },
-    {"NilnNi", "Entropy ions",
+    {"nilnni", "Entropy ions",
         []( dg::DVec& result, Variables& v ) {
             dg::blas1::transform( v.f.density(1), result, dg::LN<double>());
             dg::blas1::pointwiseDot( v.p.tau[1], result, v.f.density(1), 0., result);
@@ -293,18 +311,19 @@ std::vector<Record> records_list = {
             {
                 dg::tensor::multiply3d( v.f.projection(), //grad_perp
                     v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
-                    v.f.tmp[0], v.f.tmp[1], v.f.tmp[2])
-                dot( v.f.tmp, v.f.gradA(), result);
+                    v.tmp[0], v.tmp[1], v.tmp[2]);
+                dot( v.tmp, v.f.gradA(), result);
                 dg::blas1::scal( result, 1./2./v.p.beta);
             }
         }
     },
-    {"UE2", "ExB energy",
+    {"ue2", "ExB energy",
         []( dg::DVec& result, Variables& v ) {
             dg::tensor::multiply3d( v.f.projection(), //grad_perp
                 v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
-                v.f.tmp[0], v.f.tmp[1], v.f.tmp[2])
-            dot( v.f.tmp, v.f.gradP(), result);
+                v.tmp[0], v.tmp[1], v.tmp[2]);
+            dot( v.tmp, v.f.gradP(0), result);
+            dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
             dg::blas1::pointwiseDot( 0.5, v.f.density(1), result, 0., result);
         }
     },
@@ -314,7 +333,7 @@ std::vector<Record> records_list = {
                 v.f.velocity(0), v.f.velocity(0), 0., result);
         }
     },
-    {"NiUi2", "Parallel ion energy",
+    {"niui2", "Parallel ion energy",
         []( dg::DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 0.5*v.p.mu[1], v.f.density(1),
                 v.f.velocity(1), v.f.velocity(1), 0., result);
@@ -323,70 +342,218 @@ std::vector<Record> records_list = {
     {"resistivity", "Energy dissipation through resistivity",
         []( dg::DVec& result, Variables& v ) {
             dg::blas1::axpby( 1., v.f.velocity(1), -1., v.f.velocity(0), result);
-            dg::blas1::pointwiseDot( v.p.eta, v.f.density(0), result, result, 0., result)
+            dg::blas1::pointwiseDot( result, v.f.density(0), result);
+            dg::blas1::pointwiseDot( v.p.eta, result, result, 0., result);
         }
-    }
+    },
+    /// ------------------ Energy flux terms ------------------------//
+    {"jvee", "Radial electron energy flux without induction contribution",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
+                v.f.density(0), v.f.velocity(0), v.f.potential()[0],
+                v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
+                v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
+                v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
+                v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
+            );
+            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
+        }
+    },
+    {"jveea", "Radial electron energy flux: induction contribution",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
+                v.f.density(0), v.f.velocity(0), v.f.potential()[0], v.f.induction(),
+                v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
+                v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
+                v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
+            );
+            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
+        }
+    },
+    {"jvei", "Radial ion energy flux without induction contribution",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
+                v.f.density(1), v.f.velocity(1), v.f.potential()[1],
+                v.f.gradP(1)[0], v.f.gradP(1)[1], v.f.gradP(1)[2],
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
+                v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
+                v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
+                v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
+            );
+            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
+        }
+    },
+    {"jveia", "Radial ion energy flux: induction contribution",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
+                v.f.density(1), v.f.velocity(1), v.f.potential()[1], v.f.induction(),
+                v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
+                v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
+                v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
+            );
+            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
+        }
+    },
+    /// ------------------------ Energy dissipation terms ------------------//
+    {"leeperp", "Perpendicular electron energy dissipation",
+        []( dg::DVec& result, Variables& v ) {
+            v.f.compute_diffusive_lapMperpN( v.f.density(0), result, v.tmp[0]);
+            v.f.compute_diffusive_lapMperpU( v.f.velocity(0), result, v.tmp[1]);
+            dg::blas1::evaluate( result, dg::times_equals(),
+                RadialEnergyFlux( v.p.tau[0], v.p.mu[0], 1.),
+                v.f.density(0), v.f.velocity(0), v.f.potential()[0],
+                v.tmp[0], v.tmp[1]
+            );
+            dg::blas1::scal( result, -v.p.nu_perp);
+        }
+    },
+    {"leiperp", "Perpendicular ion energy dissipation",
+        []( dg::DVec& result, Variables& v ) {
+            v.f.compute_diffusive_lapMperpN( v.f.density(1), result, v.tmp[0]);
+            v.f.compute_diffusive_lapMperpU( v.f.velocity(1), result, v.tmp[1]);
+            dg::blas1::evaluate( result, dg::times_equals(),
+                RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
+                v.f.density(1), v.f.velocity(1), v.f.potential()[1],
+                v.tmp[0], v.tmp[1]
+            );
+            dg::blas1::scal( result, -v.p.nu_perp);
+        }
+    },
+    {"leeparallel", "Parallel electron energy dissipation",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::copy(v.f.lapParallelN(0), v.tmp[0]);
+            dg::blas1::copy(v.f.lapParallelU(0), v.tmp[1]);
+            dg::blas1::evaluate( result, dg::times_equals(),
+                RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
+                v.f.density(0), v.f.velocity(0), v.f.potential()[0],
+                v.tmp[0], v.tmp[1]
+            );
+            dg::blas1::scal( result, v.p.nu_parallel);
+        }
+    },
+    {"leiparallel", "Parallel ion energy dissipation",
+        []( dg::DVec& result, Variables& v ) {
+            dg::blas1::copy(v.f.lapParallelN(0), v.tmp[0]);
+            dg::blas1::copy(v.f.lapParallelU(0), v.tmp[1]);
+            dg::blas1::evaluate( result, dg::times_equals(),
+                RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
+                v.f.density(1), v.f.velocity(1), v.f.potential()[1],
+                v.tmp[0], v.tmp[1]
+            );
+            dg::blas1::scal( result, v.p.nu_parallel);
+        }
+    },
+    /// ------------------------ Vorticity terms ---------------------------//
+    {"oexbi", "ExB vorticity term with ion density",
+        []( dg::DVec& result, Variables& v){
+            dot( v.f.gradP(0), v.gradPsip, result);
+            dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
+            dg::blas1::pointwiseDot( v.p.mu[1], result, v.f.density(1), 0., result);
+        }
+    },
+    {"oexbe", "ExB vorticity term with electron density",
+        []( dg::DVec& result, Variables& v){
+            dot( v.f.gradP(0), v.gradPsip, result);
+            dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
+            dg::blas1::pointwiseDot( v.p.mu[1], result, v.f.density(0), 0., result);
+        }
+    },
+    {"odiai", "Diamagnetic vorticity term with ion density",
+        []( dg::DVec& result, Variables& v){
+            dot( v.f.gradN(1), v.gradPsip, result);
+            dg::blas1::scal( result, v.p.mu[1]*v.p.tau[1]);
+        }
+    },
+    {"odiae", "Diamagnetic vorticity term with electron density",
+        []( dg::DVec& result, Variables& v){
+            dot( v.f.gradN(0), v.gradPsip, result);
+            dg::blas1::scal( result, v.p.mu[1]*v.p.tau[1]);
+        }
+    },
+    /// --------------------- Vorticity flux terms ---------------------------//
+    {"jvoexbi", "ExB vorticity flux term with ion density",
+        []( dg::DVec& result, Variables& v){
+            // - ExB Dot GradPsi
+            jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
 
+            // Omega
+            dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
+            dg::blas1::pointwiseDot( 1., v.tmp[0], v.f.binv(), v.f.binv(), 0., v.tmp[0]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], v.f.density(1), 0., v.tmp[0]);
 
-};
+            dot( v.f.gradN(1), v.gradPsip, v.tmp[1]);
+            dg::blas1::axpby( v.p.mu[1]*v.p.tau[1], v.tmp[1], 1., v.tmp[0] );
 
-void create_records_in_file( int ncid,
-    int dim_ids[3], int dim_ids1d[2],
-    std::map<std::string, int>& id0d,
-    std::map<std::string, int>& id1d,
-    std::map<std::string, int>& id2d)
-{
-    file::NC_Error_Handle err;
-    for( auto record : records_list)
-    {
-        std::string name = record.name + "_ta2d";
-        std::string long_name = record.long_name + " (Toroidal average)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);//creates a new id2d entry
-        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
-            long_name.data());
+            // Multiply everything
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0], v.dvdpsip3d, 0., result);
+        }
+    },
+    {"jvoexbe", "ExB vorticity flux term with electron density",
+        []( dg::DVec& result, Variables& v){
+            // - ExB Dot GradPsi
+            jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
 
-        name = record.name + "_2d";
-        long_name = record.long_name + " (Evaluated on phi = 0 plane)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
-            long_name.data());
+            // Omega
+            dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
+            dg::blas1::pointwiseDot( 1., v.tmp[0], v.f.binv(), v.f.binv(), 0., v.tmp[0]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], v.f.density(0), 0., v.tmp[0]);
 
-        name = record.name + "_fluc2d";
-        long_name = record.long_name + " (Fluctuations wrt fsa on phi = 0 plane)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
-            long_name.data());
+            dot( v.f.gradN(0), v.gradPsip, v.tmp[1]);
+            dg::blas1::axpby( v.p.mu[1]*v.p.tau[1], v.tmp[1], 1., v.tmp[0] );
 
-        name = record.name + "_fsa2d";
-        long_name = record.long_name + " (Flux surface average interpolated to 2d plane)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
-            long_name.data());
+            // Multiply everything
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0], v.dvdpsip3d, 0., result);
+        }
+    },
+    {"jvoapar", "A parallel vorticity flux term (Maxwell stress)",
+        []( dg::DVec& result, Variables& v){
+            if( v.p.beta == 0)
+                dg::blas1::scal( result, 0.);
+            else
+            {
+                // - AxB Dot GradPsi
+                jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradA(), result);
+                dot( v.f.gradA(), v.gradPsip, v.tmp[0]);
+                dg::blas1::pointwiseDot( 1./v.p.beta, result, v.tmp[0], v.dvdpsip3d, 0., result);
+            }
+        }
+    },
+    /// --------------------- Vorticity source terms ---------------------------//
+    {"socurve", "Vorticity source term electron curvature",
+        []( dg::DVec& result, Variables& v) {
+            dot( v.f.curv(), v.gradPsip, result);
+            dg::blas1::pointwiseDot( -v.p.tau[0], v.f.density(0), result, 0., result);
+        }
+    },
+    {"socurvi", "Vorticity source term ion curvature",
+        []( dg::DVec& result, Variables& v) {
+            dot( v.f.curv(), v.gradPsip, result);
+            dg::blas1::pointwiseDot( v.p.tau[1], v.f.density(1), result, 0., result);
+        }
+    },
+    {"socurvkappae", "Vorticity source term electron kappa curvature",
+        []( dg::DVec& result, Variables& v) {
+            dot( v.f.curvKappa(), v.gradPsip, result);
+            dg::blas1::pointwiseDot( 1., v.f.density(0), v.f.velocity(0), v.f.velocity(0), 0., v.tmp[0]);
+            dg::blas1::pointwiseDot( -v.p.mu[0], v.tmp[0], result, 0., result);
+        }
+    },
+    {"socurvkappai", "Vorticity source term ion kappa curvature",
+        []( dg::DVec& result, Variables& v) {
+            dot( v.f.curvKappa(), v.gradPsip, result);
+            dg::blas1::pointwiseDot( 1., v.f.density(1), v.f.velocity(1), v.f.velocity(1), 0., v.tmp[0]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], result, 0., result);
+        }
+    },
 
-        name = record.name + "_fsa";
-        long_name = record.long_name + " (Flux surface average)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 2, dim_ids1d,
-            &id1d[name]);
-        err = nc_put_att_text( ncid, id1d[name], "long_name", long_name.size(),
-            long_name.data());
 
-        name = record.name + "_ifs";
-        long_name = record.long_name + " (Integrated Flux surface average unless it is a current then it is the derived flux surface average)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 2, dim_ids1d,
-            &id1d[name]);
-        err = nc_put_att_text( ncid, id1d[name], "long_name", long_name.size(),
-            long_name.data());
+};
 
-        name = record.name + "_ifs_lcfs";
-        long_name = record.long_name + " (Integrated Flux surface average evaluated on last closed flux surface unless it is a current then it is the fsa evaluated)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 1, dim_ids,
-            &id0d[name]);
-        err = nc_put_att_text( ncid, id0d[name], "long_name", long_name.size(),
-            long_name.data());
-    }
-}
 }//namespace feltor
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 42859859f..98a2fbfc2 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -319,22 +319,11 @@ struct Explicit
         return m_curvKappa;
     }
     const Container& bphi( ) const { return m_bphi; }
+    const Container& binv( ) const { return m_binv; }
     //bhat / sqrt{g} / B
     const std::array<Container, 3> & bhatgB () const {
         return m_b;
     }
-    const Container& lapMperpN (int i)
-    {
-        dg::blas1::axpby( 1., m_fields[0][i], -1., 1., m_temp0);
-        dg::blas2::gemv( m_lapperpN, m_temp0, m_temp1);
-        return m_temp1;
-    }
-    const Container& lapMperpU (int i)
-    {
-        dg::blas1::axpby( 1., m_fields[1][i], -1., 1., m_temp0);
-        dg::blas2::gemv( m_lapperpU, m_temp0, m_temp1);
-        return m_temp1;
-    }
     const Container& lapMperpP (int i)
     {
         dg::blas2::gemv( m_lapperpP, m_phi[i], m_temp1);
@@ -355,6 +344,34 @@ struct Explicit
         m_source = source;
     }
     void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
+    void compute_diffusive_lapMperpN( const Container& density, Container& temp0, Container& result ){
+        // compute the negative diffusion contribution -Lambda N
+        // perp dissipation for N: nu_perp Delta_p N or -nu_perp Delta_p**2 N
+        if( m_p.perp_diff == "viscous")
+        {
+            dg::blas1::transform( density, temp0, dg::PLUS<double>(-1));
+            dg::blas2::gemv( m_lapperpN, temp0, result); //!minus
+        }
+        else
+        {
+            dg::blas1::transform( density, result, dg::PLUS<double>(-1));
+            dg::blas2::gemv( m_lapperpN, result, temp0);
+            dg::blas2::gemv( m_lapperpN, temp0, result); //!plus
+        }
+    }
+    void compute_diffusive_lapMperpU( const Container& velocity, Container& temp0, Container& result ){
+        // compute the negative diffusion contribution -Lambda U
+        // perp dissipation for U: nu_perp Delta_p U or -nu_perp Delta_p**2 U
+        if( m_p.perp_diff == "viscous")
+        {
+            dg::blas2::gemv( m_lapperpU, velocity, result); //!minus
+        }
+        else
+        {
+            dg::blas2::gemv( m_lapperpU, velocity, temp0);
+            dg::blas2::gemv( m_lapperpU, temp0, result); //!plus
+        }
+    }
   private:
     void compute_phi( double t, const std::array<Container,2>& y);
     void compute_psi( double t);
@@ -885,17 +902,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_dissipation(
         m_q.Dpar[i] = m_p.nu_parallel*dg::blas2::dot(
                         m_temp2, m_vol3d, m_dsN[i]);
         // perp dissipation for N: nu_perp Delta_p N or -nu_perp Delta_p**2 N
-        if( m_p.perp_diff == "viscous")
-        {
-            dg::blas1::transform( fields[0][i], m_temp1, dg::PLUS<double>(-1));
-            dg::blas2::gemv( m_lapperpN, m_temp1, m_temp0); //!minus
-        }
-        else
-        {
-            dg::blas1::transform( fields[0][i], m_temp0, dg::PLUS<double>(-1));
-            dg::blas2::gemv( m_lapperpN, m_temp0, m_temp1);
-            dg::blas2::gemv( m_lapperpN, m_temp1, m_temp0); //!plus
-        }
+        compute_diffusive_lapMperpN( fields[0][i], m_temp1, m_temp0);
         if( i==0)
             m_q.diff += -m_p.nu_perp*dg::blas1::dot( m_vol3d, m_temp0);
         m_q.Dperp[i] = -m_p.nu_perp*dg::blas2::dot(
@@ -910,15 +917,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_dissipation(
         m_q.Dpar[i+2] = m_p.nu_parallel*dg::blas2::dot(
             m_temp2, m_vol3d, m_dsU[i]);
         // perp dissipation for U: nu_perp Delta_p U or -nu_perp Delta_p**2 U
-        if( m_p.perp_diff == "viscous")
-        {
-            dg::blas2::gemv( m_lapperpU, fields[1][i], m_temp0); //!minus
-        }
-        else
-        {
-            dg::blas2::gemv( m_lapperpU, fields[1][i], m_temp1);
-            dg::blas2::gemv( m_lapperpU, m_temp1, m_temp0); //!plus
-        }
+        compute_diffusive_lapMperpU( fields[1][i], m_temp1, m_temp0);
         m_q.Dperp[i+2] = -m_p.nu_perp *dg::blas2::dot(
             m_temp2, m_vol3d, m_temp0);
     }
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 47623ece0..348849037 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -958,13 +958,12 @@ Notice that
 = \left\langle \frac{ \bhat\times \nabla (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cdot\nabla\psi_p\right\rangle
 = -\left\langle\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)\right\rangle
 \end{align}
-We can interpret the force balance as the flux surface average of a 
-continuity equation 
-with
+We can interpret the force balance as the flux surface average of a
+continuity equation
 \begin{align}
 &\partial_t \Omega + \nabla\cdot \vec j_\Omega = S_\Omega \\
 \Omega &:= \mu_i N_i \left(\frac{\nabla\psi_p\cdot\nabla\phi}{B^2} + \tau_i \nabla\psi_p \cdot\nabla \ln N_i\right) \\
-\vec j_{\Omega} &:= -\mu_i N_i \Omega \vec u_E 
+\vec j_{\Omega} &:= -\Omega \vec u_E 
     - \frac{1}{\beta} \nabla\psi_p\cdot\nabla A_\parallel \frac{\bhat\times\nabla A_\parallel}{B} \\
     S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\nabla\times\bhat}(\psi_p)
 \end{align}
@@ -1204,15 +1203,6 @@ x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, co
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
-mass             & Dataset & 1 (time) & Mass inside LCFS $\int_\Omega \dV n_e$ \\
-mass\_loss       & Dataset & 1 (time) & Mass flux through LCFS $\int_{\psi_p = 0} \vec \dA \cdot \vec j_{n_e}$\\
-mass\_diffusion  & Dataset & 1 (time) & Mass diffusion inside LCFS $\int_\Omega \dV \Lambda_{n_e}$ \\
-energy           & Dataset & 1 (time) & Energy inside LCFS $\int_\Omega \dV \mathcal E$ \\
-energy\_loss     & Dataset & 1 (time) & Energy flux through LCFS $\int_{\psi_p = 0} \vec \dA \cdot \vec j_{\mathcal E}$ \\
-energy\_diffusion& Dataset & 1 (time) & Energy diffusion inside LCFS $\int_\Omega \dV \Lambda_{\mathcal E}$ \\
-energy\_dissipation& Dataset & 1 (time) & Energy dissipation inside LCFS$\int_\Omega \dV R_{\mathcal E}$ \\
-aligned          & Dataset & 1 (time) & $\int \dV (\nabla_\parallel n_e)^2 /n_e$ \\
-perp\_aligned    & Dataset & 1 (time) & $\int \dV (\vec\nabla_\perp n_e)^2 /n_e$ \\
 correlationNPhi  & Dataset & 1 (time) & $\left( \int\dV e^\phi n_e \right)/ ||e^\phi||||n_e||$\\
 correlationNTildePhi  & Dataset & 1 (time) & $\left( \int\dV \phi \tilde n_e \right)/ ||\phi||||\tilde n_e||$ with $n_e \equiv \langle n_e \rangle_{\psi_p} ( 1 + \tilde n_e)$\\
 q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} \\
@@ -1227,25 +1217,66 @@ X\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\langle X\ran
 X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
 X\_ifs           & Dataset & 2 (time, psi) & Integrated flux surface average $\int\d v\langle X\rangle_{\psi_p}$ unless X is a current, then it is the derived flux-surface average $\partial_v \langle X\rangle_{\psi_p}$ \\
 X\_ifs\_lcfs     & Dataset & 1 (time) & Integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle X\rangle_{\psi_p}$ unless X is a current, then it is the fsa evaluated \\
-particle\_flux   & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{n_e} \cdot \vec \dA$ Eq.~\eqref{eq:particle_flux} \\
-energy\_flux     & Dataset & 2 (time, psi) & Integrated particle flux $\int_{\psi_p = \psi_{p0}}\vec j_{\mathcal E} \cdot \vec \dA$ Eq.~\eqref{eq:energy_flux} \\
 \bottomrule
 \end{longtable}
-where X $\in$ \{ electrons $n_e$, ions $N_i$, Ue $u_e$, Ui $U_i$,
-neue $n_e u_e$, NiUi $N_i U_i$, neuebphi $n_eu_eb_\varphi$,
-NiUibphi $N_iU_ib_\varphi$, potential $\phi$,
-induction $A_\parallel$, vorticity $-\Delta_\perp\phi$,
-apar\_vorticity $-\Delta_\perp A_\parallel$, Lperpinv $L_\perp^{-1}$,
-Lparallelinv $L_\parallel^{-1}$
-\}
-and we define
-
-\begin{align}
-   L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}\quad
-   L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e} \\
-   j_\parallel := N_iU_i-n_e u_e \quad
-   j_{\parallel,\rho} := \mu_i N_iU_i + \mu_e n_e u_e
-\end{align}
+where X $\in$
+\begin{longtable}{ll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}\\
+\midrule
+    electrons &$n_e$ \\
+    ions &$N_i$ \\
+    Ue &$u_e$ \\
+    Ui &$U_i$ \\
+    potential &$\phi$ \\
+    psi &$\psi$ \\
+    induction &$A_\parallel$ \\
+    vorticity &$-\Delta_\perp\phi$ \\
+    apar\_vorticity &$-\Delta_\perp A_\parallel$ \\
+    neue &$n_e u_e$ \\
+    niui &$N_i U_i$ \\
+    neuebphi &$n_eu_eb_\varphi$ \\
+    niuibphi &$N_iU_ib_\varphi$ \\
+    lperpinv &$L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}$ \\
+    perpaligned &$\frac{\vec\nabla_\perp n_e)^2}{n_e}$ \\
+    lparallelinv &$L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}$ \\
+    aligned &$\frac{\nabla_\parallel n_e)^2}{n_e}$ \\
+    jvne &$ n_e (\vec v_E + \vec v_K + \vec v_C )\cdot\nabla v$ \\
+    jvneA &$ n_e u_e \vec{\tilde b}_\perp  \cdot\nabla v$ \\
+    lneperp &$ \Lambda_{\perp,n_e}$ \\
+    lneparallel &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
+    nelnne &$ z_e\tau_e n_e \ln n_e$ \\
+    nilnni &$ z_i\tau_i N_i \ln N_i$ \\
+    aperp2 &$ (\nabla_\perp A_\parallel)^2/2/\beta$ \\
+    ue2   &$z_i\mu_i N_i u_E^2 /2$ \\
+    neue2 &$ z_e\mu_e n_e u_e^2/2$ \\
+    niui2 &$ z_i\mu_i N_i U_i^2/2$ \\
+    resistivity &$\eta_\parallel n_e^2 (U_i-u_e)^2$ \\
+    jvee &$z_e(\tau_e \ln n_e + \mu_e u_e^2/2 + \phi)n_e(\vec v_E + \vec v_C + \vec v_K)\cdot\nabla v
+        + z_e \tau_e n_e u_e^2 \vec K_{\nabla\times\bhat}\cdot\nabla v$ \\
+    jvei &$z_i(\tau_i \ln N_i + \mu_i U_i^2/2 + \psi_i)N_i(\vec v_E^i + \vec v_C + \vec v_K)\cdot\nabla v
+        + z_i \tau_i N_i U_i^2 \vec K_{\nabla\times\bhat}\cdot\nabla v$ \\
+    jveea &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec {\tilde b}_\perp\cdot\nabla v
+        + z_e \tau_e n_e u_e \vec{\tilde b}_\perp \cdot \nabla v $ \\
+    jveia &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec {\tilde b}_\perp\cdot\nabla v
+        + z_i \tau_i N_i U_i \vec{\tilde b}_\perp \cdot \nabla v $ \\
+    leeperp &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_e \nu_\perp \Delta_\perp u_e$ \\
+    leiperp &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_i \nu_\perp \Delta_\perp U_i$ \\
+    leeparallel &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_e \nu_\parallel \Delta_\parallel u_e$ \\
+    leiparallel &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_i \nu_\parallel \Delta_\parallel U_i$ \\
+    oexbi &$\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2$ \\
+    oexbe &$\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2$ \\
+    odiai &$\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i$ \\
+    odiae &$\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e$ \\
+    jvoexbi &$-(\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i)  \vec u_E\cdot\nabla\psi_p$ \\
+    jvoexbe &$-(\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e) \vec u_E\cdot\nabla\psi_p$ \\
+    jvoapar &$ \nabla\psi_p\cdot\nabla A_\parallel \bhat\times\nabla A_\parallel/B/\beta$ \\
+    socurve &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
+    socurvi &$z_i\tau_i N_i \mathcal K(\psi_p)$ \\
+    socurvkappae &$z_e\mu_e n_eu_e^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
+    socurvkappai &$z_i\mu_i N_iU_i^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
+\bottomrule
+\end{longtable}
 
 
 
-- 
GitLab


From f362df72f81656f99ec0f0226122245a8289534c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 13 May 2019 16:12:34 +0200
Subject: [PATCH 083/540] First test run successful

- the last remaining thing is the correlation function
---
 diag/feltordiag.cu    | 37 ++++++++++++++++++++++---------------
 src/feltor/feltor.tex |  4 ++--
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index b6d337604..a2d52a946 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -341,15 +341,18 @@ int main( int argc, char* argv[])
             v3d.at("Ui"), v3d.at("potential"), v3d.at("induction") );
 
         //----------------Test if induction equation holds
-        dg::blas1::copy(var.f.lapMperpA(), result);
-        dg::blas1::pointwiseDot(
-            var.f.density(0), var.f.velocity(0), t3d);
-        dg::blas1::pointwiseDot( p.beta,
-            var.f.density(1), var.f.velocity(1), -p.beta, t3d);
-        double norm  = dg::blas2::dot( result, w3d, result);
-        dg::blas1::axpby( -1., result, 1., t3d);
-        double error = dg::blas2::dot( t3d, w3d, t3d);
-        std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
+        if( p.beta != 0)
+        {
+            dg::blas1::copy(var.f.lapMperpA(), result);
+            dg::blas1::pointwiseDot(
+                var.f.density(0), var.f.velocity(0), t3d);
+            dg::blas1::pointwiseDot( p.beta,
+                var.f.density(1), var.f.velocity(1), -p.beta, t3d);
+            double norm  = dg::blas2::dot( result, w3d, result);
+            dg::blas1::axpby( -1., result, 1., t3d);
+            double error = dg::blas2::dot( t3d, w3d, t3d);
+            std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
+        }
 
         //------------------correlation------------//
 
@@ -358,6 +361,8 @@ int main( int argc, char* argv[])
         double norm2 = sqrt(dg::blas2::dot(v3d.at("electrons"), w3d, v3d.at("electrons")));
         v0d.at("correlationNPhi") = dg::blas2::dot( t3d, w3d, v3d.at("electrons"))
             /norm1/norm2;  //<e^phi, N>/||e^phi||/||N||
+        v0d.at("correlationNTildePhi") = dg::blas2::dot( t3d, w3d, v3d.at("electrons"))
+            /norm1/norm2;  //<phi, n-<n> >/||phi||/||n-<n>||
 
 
         //now write out 2d and 1d quantities
@@ -368,7 +373,7 @@ int main( int argc, char* argv[])
             //toroidal average
             toroidal_average( t3d, t2d, false);
             dg::blas1::transfer( t2d, transfer2d);
-            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_ta"),
+            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_ta2d"),
                 start2d, count2d, transfer2d.data());
 
             //flux surface average
@@ -385,23 +390,23 @@ int main( int argc, char* argv[])
             unsigned kmp = 0; //g3d_out.Nz()/2;
             dg::HVec t2d_mp(t3d.begin() + kmp*g2d_out.size(),
                 t3d.begin() + (kmp+1)*g2d_out.size() );
-            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_plane"),
+            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_2d"),
                 start2d, count2d, t2d_mp.data() );
 
             // fsa on 2d plane : <f>
             dg::blas2::gemv(fsa2rzmatrix, fsa1d, transfer2d); //fsa on RZ grid
-            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_fsa2"),
+            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_fsa2d"),
                 start2d, count2d, transfer2d.data() );
 
             // delta f on midplane : df = f_mp - <f>
             dg::blas1::axpby( 1.0, t2d_mp, -1.0, transfer2d);
-            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_fluc"),
+            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_fluc2d"),
                 start2d, count2d, transfer2d.data() );
 
             //flux surface integral/derivative
             if( record.name[0] == 'j') //j indicates a flux
             {
-                v0d[record.name+"ifs_lcfs"] = dg::interpolate( fsa1d, 0., g1d_out); //create a new entry
+                v0d[record.name+"_ifs_lcfs"] = dg::interpolate( fsa1d, 0., g1d_out); //create a new entry
                 dg::blas2::symv( dpsi, fsa1d, t1d);
                 dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
             }
@@ -409,7 +414,7 @@ int main( int argc, char* argv[])
             {
                 t1d = dg::integrate( fsa1d, g1d_out);
                 dg::blas1::pointwiseDot( t1d, dvdpsip, transfer1d);
-                v0d[record.name+"ifs_lcfs"] = dg::interpolate( transfer1d, 0., g1d_out); //create a new entry
+                v0d[record.name+"_ifs_lcfs"] = dg::interpolate( transfer1d, 0., g1d_out); //create a new entry
             }
             err = nc_put_vara_double( ncid_out, id1d.at(record.name+"_ifs"),
                 start1d, count1d, transfer1d.data());
@@ -418,8 +423,10 @@ int main( int argc, char* argv[])
         }
         //and the 0d quantities
         for( auto pair : v0d) //{name, double}
+        {
             err = nc_put_vara_double( ncid_out, id0d.at(pair.first),
                 start2d, count2d, &pair.second );
+        }
         //write time data
         err = nc_put_vara_double( ncid_out, tvarID, start2d, count2d, &time);
         //and close file
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 348849037..9b2029fa8 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1268,8 +1268,8 @@ where X $\in$
     oexbe &$\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2$ \\
     odiai &$\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i$ \\
     odiae &$\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e$ \\
-    jvoexbi &$-(\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i)  \vec u_E\cdot\nabla\psi_p$ \\
-    jvoexbe &$-(\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e) \vec u_E\cdot\nabla\psi_p$ \\
+    jvoexbi &$-(\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i)  \vec u_E\cdot\nabla v$ \\
+    jvoexbe &$-(\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e) \vec u_E\cdot\nabla v$ \\
     jvoapar &$ \nabla\psi_p\cdot\nabla A_\parallel \bhat\times\nabla A_\parallel/B/\beta$ \\
     socurve &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
     socurvi &$z_i\tau_i N_i \mathcal K(\psi_p)$ \\
-- 
GitLab


From 8e7f6825d82901d0b39c115684bf7099593e9294 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 15 May 2019 17:24:44 +0200
Subject: [PATCH 084/540] Insert curvature fsa test into geometry_diag

---
 inc/geometries/geometry_diag.cu | 230 ++++++++++++++++++++------------
 src/feltor/feltor.tex           |  20 ++-
 2 files changed, 154 insertions(+), 96 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index eb8a332eb..a8d6d78fe 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -143,8 +143,8 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
 
     //Test coefficients
-    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
-    c = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*c.psip()(c.R0(), 0.), p.alpha);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(), 0.), p.alpha);
     const double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     const double Z_X = -1.1*gp.elongation*gp.a;
     const double R_H = gp.R_0-gp.triangularity*gp.a;
@@ -153,12 +153,12 @@ int main( int argc, char* argv[])
     const double N1 = -(1.+alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.+alpha_);
     const double N2 =  (1.-alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.-alpha_);
     const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
-    const double psip0 = c.psip()(gp.R_0, 0);
+    const double psip0 = mag.psip()(gp.R_0, 0);
     std::cout << "psip( 1, 0) "<<psip0<<"\n";
     //Find O-point
     double R_O = gp.R_0, Z_O = 0.;
-    dg::geo::findXpoint( c.get_psip(), R_O, Z_O);
-    const double psipmin = c.psip()(R_O, Z_O);
+    dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
+    const double psipmin = mag.psip()(R_O, Z_O);
 
     std::cout << "O-point "<<R_O<<" "<<Z_O<<" with Psip = "<<psipmin<<std::endl;
 
@@ -167,93 +167,111 @@ int main( int argc, char* argv[])
         std::cout << "    Equilibrium with X-point!\n";
     else
         std::cout << "    NO X-point in flux function!\n";
-    std::cout << "psip( 1+e,0)           "<<c.psip()(gp.R_0 + gp.a, 0.)<<"\n";
-    std::cout << "psip( 1-e,0)           "<<c.psip()(gp.R_0 - gp.a, 0.)<<"\n";
-    std::cout << "psip( 1-de,ke)         "<<c.psip()(R_H, Z_H)<<"\n";
+    std::cout << "psip( 1+e,0)           "<<mag.psip()(gp.R_0 + gp.a, 0.)<<"\n";
+    std::cout << "psip( 1-e,0)           "<<mag.psip()(gp.R_0 - gp.a, 0.)<<"\n";
+    std::cout << "psip( 1-de,ke)         "<<mag.psip()(R_H, Z_H)<<"\n";
     if( !gp.hasXpoint())
-        std::cout << "psipR( 1-de,ke)        "<<c.psipR()(R_H, Z_H)<<"\n";
+        std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H, Z_H)<<"\n";
     else
     {
-        std::cout << "psip( 1-1.1de,-1.1ke)  "<<c.psip()(R_X, Z_X)<<"\n";
-        std::cout << "psipZ( 1+e,0)          "<<c.psipZ()(gp.R_0 + gp.a, 0.)<<"\n";
-        std::cout << "psipZ( 1-e,0)          "<<c.psipZ()(gp.R_0 - gp.a, 0.)<<"\n";
-        std::cout << "psipR( 1-de,ke)        "<<c.psipR()(R_H,Z_H)<<"\n";
-        std::cout << "psipR( 1-1.1de,-1.1ke) "<<c.psipR()(R_X,Z_X)<<"\n";
-        std::cout << "psipZ( 1-1.1de,-1.1ke) "<<c.psipZ()(R_X,Z_X)<<"\n";
+        std::cout << "psip( 1-1.1de,-1.1ke)  "<<mag.psip()(R_X, Z_X)<<"\n";
+        std::cout << "psipZ( 1+e,0)          "<<mag.psipZ()(gp.R_0 + gp.a, 0.)<<"\n";
+        std::cout << "psipZ( 1-e,0)          "<<mag.psipZ()(gp.R_0 - gp.a, 0.)<<"\n";
+        std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H,Z_H)<<"\n";
+        std::cout << "psipR( 1-1.1de,-1.1ke) "<<mag.psipR()(R_X,Z_X)<<"\n";
+        std::cout << "psipZ( 1-1.1de,-1.1ke) "<<mag.psipZ()(R_X,Z_X)<<"\n";
     }
-    std::cout << "psipZZ( 1+e,0)         "<<c.psipZZ()(gp.R_0+gp.a,0.)+N1*c.psipR()(gp.R_0+gp.a,0)<<"\n";
-    std::cout << "psipZZ( 1-e,0)         "<<c.psipZZ()(gp.R_0-gp.a,0.)+N2*c.psipR()(gp.R_0-gp.a,0)<<"\n";
-    std::cout << "psipRR( 1-de,ke)       "<<c.psipRR()(R_H,Z_H)+N3*c.psipZ()(R_H,Z_H)<<"\n";
+    std::cout << "psipZZ( 1+e,0)         "<<mag.psipZZ()(gp.R_0+gp.a,0.)+N1*mag.psipR()(gp.R_0+gp.a,0)<<"\n";
+    std::cout << "psipZZ( 1-e,0)         "<<mag.psipZZ()(gp.R_0-gp.a,0.)+N2*mag.psipR()(gp.R_0-gp.a,0)<<"\n";
+    std::cout << "psipRR( 1-de,ke)       "<<mag.psipRR()(R_H,Z_H)+N3*mag.psipZ()(R_H,Z_H)<<"\n";
 
 
     dg::HVec hvisual;
     //allocate mem for visual
     dg::HVec visual;
-    std::map< std::string, std::function<double(double,double)>> map{
-        {"Psip", c.psip()},
-        {"Ipol", c.ipol()},
-        {"Bmodule", dg::geo::Bmodule(c)},
-        {"InvB", dg::geo::InvB(c)},
-        {"LnB", dg::geo::LnB(c)},
-        {"GradLnB", dg::geo::GradLnB(c)},
-        {"Divb", dg::geo::Divb(c)},
-        {"BR", dg::geo::BR(c)},
-        {"BZ", dg::geo::BZ(c)},
-        {"CurvatureNablaBR", dg::geo::CurvatureNablaBR(c)},
-        {"CurvatureNablaBZ", dg::geo::CurvatureNablaBZ(c)},
-        {"CurvatureKappaR", dg::geo::CurvatureKappaR(c)},
-        {"CurvatureKappaZ", dg::geo::CurvatureKappaZ(c)},
-        {"DivCurvatureKappa", dg::geo::DivCurvatureKappa(c)},
-        {"DivCurvatureNablaB", dg::geo::DivCurvatureNablaB(c)},
-        {"TrueCurvatureNablaBR", dg::geo::TrueCurvatureNablaBR(c)},
-        {"TrueCurvatureNablaBZ", dg::geo::TrueCurvatureNablaBZ(c)},
-        {"TrueCurvatureNablaBP", dg::geo::TrueCurvatureNablaBP(c)},
-        {"TrueCurvatureKappaR", dg::geo::TrueCurvatureKappaR(c)},
-        {"TrueCurvatureKappaZ", dg::geo::TrueCurvatureKappaZ(c)},
-        {"TrueCurvatureKappaP", dg::geo::TrueCurvatureKappaP(c)},
-        {"TrueDivCurvatureKappa", dg::geo::TrueDivCurvatureKappa(c)},
-        {"TrueDivCurvatureNablaB", dg::geo::TrueDivCurvatureNablaB(c)},
-        {"BFieldR", dg::geo::BFieldR(c)},
-        {"BFieldZ", dg::geo::BFieldZ(c)},
-        {"BFieldP", dg::geo::BFieldP(c)},
-        {"BHatR", dg::geo::BHatR(c)},
-        {"BHatZ", dg::geo::BHatZ(c)},
-        {"BHatP", dg::geo::BHatP(c)},
-        {"GradBHatR", dg::geo::BHatR(c)},
-        {"GradBHatZ", dg::geo::BHatZ(c)},
-        {"GradBHatP", dg::geo::BHatP(c)},
+    std::vector< std::tuple<std::string, std::string, std::function<double(double,double)> > > map{
+        {"Psip", "Flux function", mag.psip()},
+        {"PsipR", "Flux function derivative in R", mag.psipR()},
+        {"PsipZ", "Flux function derivative in Z", mag.psipZ()},
+        {"Ipol", "Poloidal current", mag.ipol()},
+        {"Bmodule", "Magnetic field strength", dg::geo::Bmodule(mag)},
+        {"InvB", "Inverse of Bmodule", dg::geo::InvB(mag)},
+        {"LnB", "Natural logarithm of Bmodule", dg::geo::LnB(mag)},
+        {"GradLnB", "The parallel derivative of LnB", dg::geo::GradLnB(mag)},
+        {"Divb", "The divergence of the magnetic unit vector", dg::geo::Divb(mag)},
+        {"BR", "Derivative of Bmodule in R", dg::geo::BR(mag)},
+        {"BZ", "Derivative of Bmodule in Z", dg::geo::BZ(mag)},
+        {"CurvatureNablaBR",  "R-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBR(mag)},
+        {"CurvatureNablaBZ",  "Z-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBZ(mag)},
+        {"CurvatureKappaR",   "R-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaR(mag)},
+        {"CurvatureKappaZ",   "Z-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaZ(mag)},
+        {"DivCurvatureKappa", "Divergence of the (toroidal) Kappa B curvature vector", dg::geo::DivCurvatureKappa(mag)},
+        {"DivCurvatureNablaB","Divergence of the (toroidal) Nabla B curvature vector", dg::geo::DivCurvatureNablaB(mag)},
+        {"TrueCurvatureNablaBR", "R-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBR(mag)},
+        {"TrueCurvatureNablaBZ", "Z-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBZ(mag)},
+        {"TrueCurvatureNablaBP", "Contravariant Phi-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBP(mag)},
+        {"TrueCurvatureKappaR", "R-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaR(mag)},
+        {"TrueCurvatureKappaZ", "Z-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaZ(mag)},
+        {"TrueCurvatureKappaP", "Contravariant Phi-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaP(mag)},
+        {"TrueDivCurvatureKappa", "Divergence of the (true) Kappa B curvature vector", dg::geo::TrueDivCurvatureKappa(mag)},
+        {"TrueDivCurvatureNablaB","Divergence of the (true) Nabla B curvature vector",  dg::geo::TrueDivCurvatureNablaB(mag)},
+        {"BFieldR", "R-component of the magnetic field vector", dg::geo::BFieldR(mag)},
+        {"BFieldZ", "Z-component of the magnetic field vector", dg::geo::BFieldZ(mag)},
+        {"BFieldP", "Contravariant Phi-component of the magnetic field vector", dg::geo::BFieldP(mag)},
+        {"BHatR", "R-component of the magnetic field unit vector", dg::geo::BHatR(mag)},
+        {"BHatZ", "Z-component of the magnetic field unit vector", dg::geo::BHatZ(mag)},
+        {"BHatP", "Contravariant Phi-component of the magnetic field unit vector", dg::geo::BHatP(mag)},
+        {"GradBHatR", "Parallel derivative of BHatR", dg::geo::BHatR(mag)},
+        {"GradBHatZ", "Parallel derivative of BHatZ", dg::geo::BHatZ(mag)},
+        {"GradBHatP", "Parallel derivative of BHatP", dg::geo::BHatP(mag)},
         //////////////////////////////////
-        {"Iris", dg::geo::Iris(c.psip(), gp.psipmin, gp.psipmax)},
-        {"Pupil", dg::geo::Pupil(c.psip(), gp.psipmaxcut)},
-        {"GaussianDamping", dg::geo::GaussianDamping(c.psip(), gp.psipmaxcut, gp.alpha)},
-        {"ZonalFlow", dg::geo::ZonalFlow(c.psip(), p.amp, 0., 2.*M_PI*p.k_psi )},
-        {"PsiLimiter", dg::geo::PsiLimiter(c.psip(), gp.psipmaxlim)},
-        {"Nprofile", dg::geo::Nprofile(c.psip(), p.nprofileamp/c.psip()(c.R0(),0.), p.bgprofamp )},
-        {"Delta", dg::geo::DeltaFunction( c, gp.alpha*gp.alpha, psip0*0.2)},
-        {"TanhDamping", dg::geo::TanhDamping(c.psip(), -3*gp.alpha, gp.alpha, -1)},
+        {"Iris", "A flux aligned Heaviside", dg::geo::Iris(mag.psip(), gp.psipmin, gp.psipmax)},
+        {"Pupil", "A flux aligned Heaviside", dg::geo::Pupil(mag.psip(), gp.psipmaxcut)},
+        {"GaussianDamping", "A flux aligned Heaviside with Gaussian damping", dg::geo::GaussianDamping(mag.psip(), gp.psipmaxcut, gp.alpha)},
+        {"ZonalFlow",  "Flux aligned zonal flows", dg::geo::ZonalFlow(mag.psip(), p.amp, 0., 2.*M_PI*p.k_psi )},
+        {"PsiLimiter", "A flux aligned Heaviside", dg::geo::PsiLimiter(mag.psip(), gp.psipmaxlim)},
+        {"Nprofile", "A flux aligned profile", dg::geo::Nprofile(mag.psip(), p.nprofileamp/mag.psip()(mag.R0(),0.), p.bgprofamp )},
+        {"Delta", "A flux aligned delta function", dg::geo::DeltaFunction( mag, gp.alpha*gp.alpha, psip0*0.2)},
+        {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::geo::TanhDamping(mag.psip(), -3*gp.alpha, gp.alpha, -1)},
         ////
-        {"BathRZ", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,5., p.amp)},
-        {"Gaussian3d", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
+        {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,5., p.amp)},
+        {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
             M_PI, p.sigma, p.sigma, p.sigma, p.amp)}
     };
     dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
-    dg::DVec psipog2d   = dg::evaluate( c.psip(), grid2d);
+    dg::DVec psipog2d   = dg::evaluate( mag.psip(), grid2d);
     std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
     ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
-    unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
+    unsigned npsi = 3, Npsi = 128;//set number of psivalues (NPsi % 8 == 0)
     double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     double volumeXGrid;
+    /// -------  Elements for fsa of curvature operators ----------------
+    // This is a numerical test because the fsa of the curvature operators exactly vanishes (at least for I = cte)
+    dg::geo::CylindricalVectorLvl0 bhat_ = dg::geo::createBHat( mag);
+    dg::geo::CylindricalVectorLvl0 curvB_ = dg::geo::createTrueCurvatureNablaB( mag);
+    dg::geo::CylindricalVectorLvl0 curvK_ = dg::geo::createTrueCurvatureKappa( mag);
+    dg::HVec curvNablaBR = dg::evaluate( curvB_.x(), grid2d);
+    dg::HVec curvNablaBZ = dg::evaluate( curvB_.y(), grid2d);
+    dg::HVec curvKappaKR = dg::evaluate( curvK_.x(), grid2d);
+    dg::HVec curvKappaKZ = dg::evaluate( curvK_.y(), grid2d);
+    dg::HVec gradPsipR = dg::evaluate( mag.psipR(), grid2d);
+    dg::HVec gradPsipZ = dg::evaluate( mag.psipZ(), grid2d);
+    dg::HVec curvNablaBGradPsip(curvNablaBR), curvKappaKGradPsip( curvNablaBR), gradPsip(curvNablaBR);
+    dg::blas1::pointwiseDot( 1., curvNablaBR, gradPsipR, 1., curvNablaBZ, gradPsipZ, 0., curvNablaBGradPsip);
+    dg::blas1::pointwiseDot( 1., curvKappaKR, gradPsipR, 1., curvKappaKZ, gradPsipZ, 0., curvKappaKGradPsip);
+    dg::blas1::pointwiseDot( 1., gradPsipR, gradPsipR, 1., gradPsipZ, gradPsipZ, 0., gradPsip);
+    dg::blas1::transform( gradPsip, gradPsip, dg::SQRT<double>());
     if( gp.hasXpoint())
     {
         std::cout << "Generate X-point flux-aligned grid!\n";
         double RX = gp.R_0-1.1*gp.triangularity*gp.a;
         double ZX = -1.1*gp.elongation*gp.a;
-        dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( c.get_psip(), RX, ZX) ;
-        dg::geo::SeparatrixOrthogonal generator(c.get_psip(), monitor_chi, psipmin, R_X, Z_X, c.R0(), 0, 0, true);
+        dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), RX, ZX) ;
+        dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, R_X, Z_X, mag.R0(), 0, 0, true);
         double fx_0 = 1./8.;
         psipmax = -fx_0/(1.-fx_0)*psipmin;
         std::cout << "psi 1 is          "<<psipmax<<"\n";
-        dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., npsi, Npsi, 160, dg::DIR, dg::NEU);
+        dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., npsi, Npsi, 360, dg::DIR, dg::NEU);
         std::cout << "DONE! Generate X-point flux-aligned grid!\n";
         dg::Average<dg::HVec > avg_eta( gX2d.grid(), dg::coo2d::y);
         std::vector<dg::HVec> coordsX = gX2d.map();
@@ -277,22 +295,47 @@ int main( int argc, char* argv[])
             "Flux surface average of psi on X-point grid");
 
         //NOTE: VOLUME is WITHIN cells while AREA is ON gridpoints
-        dg::HVec gradPsipX = metricX.value(0,0), X_psi_area;
-        dg::blas1::transform( gradPsipX, gradPsipX, dg::SQRT<double>());
-        dg::blas1::pointwiseDot( volX2d, gradPsipX, gradPsipX); //R\sqrt{g}|\nabla\zeta|
-        avg_eta( gradPsipX, X_psi_area, false);
+        dg::HVec gradZetaX = metricX.value(0,0), X_psi_area;
+        dg::blas1::transform( gradZetaX, gradZetaX, dg::SQRT<double>());
+        dg::blas1::pointwiseDot( volX2d, gradZetaX, gradZetaX); //R\sqrt{g}|\nabla\zeta|
+        avg_eta( gradZetaX, X_psi_area, false);
         dg::blas1::scal( X_psi_area, 4.*M_PI*M_PI);
         map1d.emplace_back( "X_psi_area", X_psi_area,
             "Flux area on X-point grid");
         volumeXGrid = dg::interpolate( X_psi_vol, gX1d.x1(), gX1d);
+
+        //Compute FSA of curvature operators
+        dg::HVec X_curvNablaBGradPsip( psip_X), X_curvKappaKGradPsip( psip_X), X_gradPsip(psip_X);
+        dg::HVec X_curvNablaB_fsa, X_curvKappaK_fsa, X_gradPsip_fsa;
+        dg::blas2::symv( grid2gX2d, curvNablaBGradPsip, X_curvNablaBGradPsip);
+        dg::blas2::symv( grid2gX2d, curvKappaKGradPsip, X_curvKappaKGradPsip);
+        dg::blas2::symv( grid2gX2d, gradPsip, X_gradPsip);
+        dg::blas1::pointwiseDot( volX2d, X_curvNablaBGradPsip, X_curvNablaBGradPsip);
+        dg::blas1::pointwiseDot( volX2d, X_curvKappaKGradPsip, X_curvKappaKGradPsip);
+        dg::blas1::pointwiseDot( volX2d, X_gradPsip, X_gradPsip);
+        avg_eta( X_curvNablaBGradPsip, X_curvNablaB_fsa, false);
+        avg_eta( X_curvKappaKGradPsip, X_curvKappaK_fsa, false);
+        avg_eta( X_gradPsip, X_gradPsip_fsa, false);
+        dg::blas1::scal( X_curvNablaB_fsa, 4*M_PI*M_PI); //
+        dg::blas1::scal( X_curvKappaK_fsa, 4*M_PI*M_PI); //
+        dg::blas1::scal( X_gradPsip_fsa, 4*M_PI*M_PI); //
+        dg::blas1::pointwiseDivide( X_curvNablaB_fsa, dvdzeta, X_curvNablaB_fsa );
+        dg::blas1::pointwiseDivide( X_curvKappaK_fsa, dvdzeta, X_curvKappaK_fsa );
+        dg::blas1::pointwiseDivide( X_gradPsip_fsa, dvdzeta, X_gradPsip_fsa );
+        map1d.emplace_back( "X_curvNablaB_fsa", X_curvNablaB_fsa,
+            "Flux surface average of true Nabla B curvature Dot Grad Psip");
+        map1d.emplace_back( "X_curvKappaK_fsa", X_curvKappaK_fsa,
+            "Flux surface average of true Kappa curvature Dot Grad Psip");
+        map1d.emplace_back( "X_gradPsip_fsa", X_gradPsip_fsa,
+            "Flux surface average of |Grad Psip|");
     }
 
     ///////////////////Compute flux average////////////////////
     dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
     if( gp.hasXpoint() )
         dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
-    dg::geo::SafetyFactor     qprof( c);
-    dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, c, psipog2d, xpoint_weights);
+    dg::geo::SafetyFactor     qprof( mag);
+    dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, mag, psipog2d, xpoint_weights);
     dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
     map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
         "Flux surface average of psi with delta function");
@@ -304,9 +347,18 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
     map1d.emplace_back("rho", rho,
         "Alternative flux label rho = -psi/psimin + 1");
+    fsa.set_container( (dg::DVec)curvNablaBGradPsip);
+    map1d.emplace_back("curvNablaB_fsa",   dg::evaluate( fsa,      grid1d),
+        "Flux surface average of true Nabla B curvature Dot Grad Psip with delta function");
+    fsa.set_container( (dg::DVec)curvKappaKGradPsip);
+    map1d.emplace_back("curvKappaK_fsa",   dg::evaluate( fsa,      grid1d),
+        "Flux surface average of true Kappa curvature Dot Grad Psip with delta function");
+    fsa.set_container( (dg::DVec)gradPsip);
+    map1d.emplace_back("gradPsip_fsa",   dg::evaluate( fsa,      grid1d),
+        "Flux surface average of |Grad Psip| with delta function");
 
     //other flux labels
-    dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, c);
+    dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, mag);
     fsi.set_right( xpoint_weights);
 
     dg::HVec areaT_psip = dg::evaluate( fsi, grid1d);
@@ -314,13 +366,13 @@ int main( int argc, char* argv[])
     double volumeCoarea = 2.*M_PI*dg::blas1::dot( areaT_psip, w1d);
 
     //area
-    fsi.set_left( dg::evaluate( dg::geo::GradPsip(c), grid2d));
+    fsi.set_left( dg::evaluate( dg::geo::GradPsip(mag), grid2d));
     dg::HVec psi_area = dg::evaluate( fsi, grid1d);
     dg::blas1::scal(psi_area, 2.*M_PI);
     map1d.emplace_back( "psi_area", psi_area,
         "Flux area with delta function");
 
-    dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, c);
+    dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, mag);
     std::cout << "Delta Rho for Flux surface integrals = "<<-fsi.get_deltapsi()/psipmin<<"\n";
 
     fvi.set_right( xpoint_weights);
@@ -384,23 +436,26 @@ int main( int argc, char* argv[])
         err = nc_redef(ncid);
     }
     //write 2d vectors
-    for(auto pair : map)
+    for(auto tp : map)
     {
         int vectorID;
-        err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 2,
+        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 2,
             &dim2d_ids[0], &vectorID);
+        err = nc_put_att_text( ncid, vectorID, "long_name",
+            std::get<1>(tp).size(), std::get<1>(tp).data());
         err = nc_enddef( ncid);
-        hvisual = dg::evaluate( pair.second, grid2d);
+        hvisual = dg::evaluate( std::get<2>(tp), grid2d);
         err = nc_put_var_double( ncid, vectorID, hvisual.data());
         err = nc_redef(ncid);
 
     }
     //compute & write 3d vectors
-    dg::HVec vecR = dg::evaluate( dg::geo::BFieldR(c), grid3d);
-    dg::HVec vecZ = dg::evaluate( dg::geo::BFieldZ(c), grid3d);
-    dg::HVec vecP = dg::evaluate( dg::geo::BFieldP(c), grid3d);
-    std::string vec_long[3] = {"R-component of magnetic field",
-        "Z-component of magnetic field", "Phi-component of magnetic field"};
+    dg::HVec vecR = dg::evaluate( dg::geo::BFieldR(mag), grid3d);
+    dg::HVec vecZ = dg::evaluate( dg::geo::BFieldZ(mag), grid3d);
+    dg::HVec vecP = dg::evaluate( dg::geo::BFieldP(mag), grid3d);
+    std::string vec_long[3] = {"R-component of the magnetic field vector (3d version of BFieldR)",
+        "Z-component of the magnetic field vector (3d version of BFieldZ)",
+        "Contravariant Phi-component of the magnetic field vector (3d version of BFieldP)"};
     int vecID[3];
     err = nc_def_var( ncid, "B_R", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[0]);
     err = nc_def_var( ncid, "B_Z", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[1]);
@@ -442,9 +497,6 @@ int main( int argc, char* argv[])
     //////////////////////////////Finalize////////////////////////////////////
     err = nc_close(ncid);
     std::cout << "TEST ACCURACY OF CURVATURES (values must be close to 0!)\n";
-    dg::geo::CylindricalVectorLvl0 bhat_ = dg::geo::createBHat( c);
-    dg::geo::CylindricalVectorLvl0 curvB_ = dg::geo::createTrueCurvatureNablaB( c);
-    dg::geo::CylindricalVectorLvl0 curvK_ = dg::geo::createTrueCurvatureKappa( c);
     std::array<dg::HVec, 3> bhat, curvB, curvK;
     dg::pushForward( bhat_.x(), bhat_.y(), bhat_.z(),
             bhat[0], bhat[1], bhat[2], grid3d);
@@ -466,14 +518,14 @@ int main( int argc, char* argv[])
     dg::pushForward( curvK_.x(), curvK_.y(), curvK_.z(),
             curvK[0], curvK[1], curvK[2], grid3d);
     dg::blas1::axpby( 1., curvK, -1., curvB);
-    dg::HVec Bmodule = dg::pullback( dg::geo::Bmodule(c), grid3d);
+    dg::HVec Bmodule = dg::pullback( dg::geo::Bmodule(mag), grid3d);
     dg::blas1::pointwiseDot( Bmodule, Bmodule, Bmodule);
     for( int i=0; i<3; i++)
         dg::blas1::pointwiseDot( Bmodule, curvB[i], curvB[i]);
     dg::HVec R = dg::pullback( dg::cooX3d, grid3d);
-    dg::HVec IR =  dg::pullback( c.ipolR(), grid3d);
+    dg::HVec IR =  dg::pullback( mag.ipolR(), grid3d);
     dg::blas1::pointwiseDivide( gp.R_0, IR, R, 0., IR);
-    dg::HVec IZ =  dg::pullback( c.ipolZ(), grid3d);
+    dg::HVec IZ =  dg::pullback( mag.ipolZ(), grid3d);
     dg::blas1::pointwiseDivide( gp.R_0, IZ, R, 0., IZ);
     dg::HVec IP =  dg::pullback( IPhi( gp), grid3d);
     dg::blas1::pointwiseDivide( gp.R_0, IP, R, 0., IP);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 9b2029fa8..aefd57d47 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1203,8 +1203,14 @@ x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, co
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
-correlationNPhi  & Dataset & 1 (time) & $\left( \int\dV e^\phi n_e \right)/ ||e^\phi||||n_e||$\\
-correlationNTildePhi  & Dataset & 1 (time) & $\left( \int\dV \phi \tilde n_e \right)/ ||\phi||||\tilde n_e||$ with $n_e \equiv \langle n_e \rangle_{\psi_p} ( 1 + \tilde n_e)$\\
+correlationNTildePhi  & Dataset & 1 (time) & $ \int\d v \langle \tilde \phi \tilde n_e\rangle / ||\tilde \phi||||\tilde n_e||$
+with $\tilde n_e \equiv (n_e - \langle n_e \rangle)/\langle n_e\rangle$, 
+$\tilde\phi = \phi - \langle\phi\rangle$
+and $||\phi|| = \left( \int \d v \langle \phi^2 \rangle\right)^{1/2}$
+\\
+covarianceNTildePhi  & Dataset & 1 (time) & $ \int\d v \langle \tilde \phi \tilde n_e\rangle $
+\\
+correlationNPhi  & Dataset & 1 (time) & $\int\d v \langle e^\phi n_e\rangle/ ||e^\phi||||n_e||$\\
 q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} \\
 psip1d           & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
 rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
@@ -1237,13 +1243,13 @@ where X $\in$
     niui &$N_i U_i$ \\
     neuebphi &$n_eu_eb_\varphi$ \\
     niuibphi &$N_iU_ib_\varphi$ \\
-    lperpinv &$L_\perp^{-1} := \frac{|\vec\nabla_\perp n_e|}{n_e}$ \\
-    perpaligned &$\frac{\vec\nabla_\perp n_e)^2}{n_e}$ \\
-    lparallelinv &$L_\parallel^{-1} := \frac{|\nabla_\parallel n_e|}{n_e}$ \\
-    aligned &$\frac{\nabla_\parallel n_e)^2}{n_e}$ \\
+    lperpinv &$L_\perp^{-1} := |\vec\nabla_\perp n_e|/n_e$ \\
+    perpaligned &$(\vec\nabla_\perp n_e)^2/n_e$ \\
+    lparallelinv &$L_\parallel^{-1} := |\nabla_\parallel n_e|/n_e$ \\
+    aligned &$ (\nabla_\parallel n_e)^2/n_e$ \\
     jvne &$ n_e (\vec v_E + \vec v_K + \vec v_C )\cdot\nabla v$ \\
     jvneA &$ n_e u_e \vec{\tilde b}_\perp  \cdot\nabla v$ \\
-    lneperp &$ \Lambda_{\perp,n_e}$ \\
+    lneperp &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ \\
     lneparallel &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
     nelnne &$ z_e\tau_e n_e \ln n_e$ \\
     nilnni &$ z_i\tau_i N_i \ln N_i$ \\
-- 
GitLab


From 630eb9b0f72fc4ea9b270ee5bd36d9bb83ab3691 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 20 May 2019 14:21:52 +0200
Subject: [PATCH 085/540] BUGFIX: forgotten fix in mpi fieldaligned evaluate

- this is probably the reason why the mpi feltor runs all failed
---
 inc/geometries/ds_mpit.cu         | 3 +--
 inc/geometries/mpi_fieldaligned.h | 6 ++++--
 src/feltor/feltor.cu              | 4 ++--
 src/feltor/feltor.cuh             | 2 +-
 src/feltor/feltor_hpc.cu          | 6 +++---
 5 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index f8a409d20..35f298a86 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -110,11 +110,10 @@ int main(int argc, char* argv[])
     }
 
     ///##########################################################///
-
     if(rank==0)std::cout << "TEST FIELDALIGNED EVALUATION of a Gaussian\n";
     dg::Gaussian init0(R_0+0.5, 0, 0.2, 0.2, 1);
     dg::GaussianZ modulate(0., M_PI/3., 1);
-    dg::MDVec aligned = ds.fieldaligned().evaluate( init0, modulate, Nz/2, 2);
+    dg::MDVec aligned = dsFA.evaluate( init0, modulate, Nz/2, 2);
     ds( aligned, derivative);
     double norm = dg::blas2::dot(vol3d, derivative);
     if(rank==0)std::cout << "Norm Centered Derivative "<<sqrt( norm)<<" (compare with that of ds_t)\n";
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index d9b7cbc69..6108519b8 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -293,9 +293,11 @@ MPI_Vector<container> Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::e
             unsigned rep = r*globalNz + i0;
             for(unsigned k=0; k<rep; k++)
             {
-                dg::blas2::symv( m_plus, tempP, temp);
+                //!!! The value of f at the plus plane is I^- of the current plane
+                dg::blas2::symv( m_minus, tempP, temp);
                 temp.swap( tempP);
-                dg::blas2::symv( m_minus, tempM, temp);
+                //!!! The value of f at the minus plane is I^+ of the current plane
+                dg::blas2::symv( m_plus, tempM, temp);
                 temp.swap( tempM);
             }
             dg::blas1::scal( tempP, unary(  (double)rep*m_g->hz() ) );
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 393d3ed55..ccc3c24e7 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -220,7 +220,7 @@ int main( int argc, char* argv[])
     std::cout << std::scientific << std::setprecision( 2);
     dg::Average<dg::HVec> toroidal_average( grid, dg::coo3d::z);
     title << std::setprecision(2) << std::scientific;
-    unsigned failed_counter = 0;
+    //unsigned failed_counter = 0;
     while ( !glfwWindowShouldClose( w ))
     {
         title << std::fixed;
@@ -325,7 +325,7 @@ int main( int argc, char* argv[])
         t.toc();
         std::cout << "\n\t Step "<<step << " at time  "<<time;
         std::cout << "\n\t Average time for one step: "<<t.diff()/(double)p.itstp/(double)p.inner_loop;
-        std::cout << "\n\t Total # of failed steps:   "<<failed_counter<<"\n\n";
+        //std::cout << "\n\t Total # of failed steps:   "<<failed_counter<<"\n\n";
     }
     glfwTerminate();
     ////////////////////////////////////////////////////////////////////
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 98a2fbfc2..04c849ba0 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -9,7 +9,7 @@
 #endif
 
 
-//Latest measurement: m = 60.000 per step
+//Latest measurement: m = 10.000 per step
 
 namespace feltor
 {
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 5521da10c..11e414eda 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -539,7 +539,7 @@ int main( int argc, char* argv[])
     //        "Trapezoidal-2-2", grid, p, mag);
     dg::Timer t;
     t.tic();
-    unsigned step = 0, failed_counter = 0;
+    unsigned step = 0;//, failed_counter = 0;
     MPI_OUT q.display(std::cout);
     for( unsigned i=1; i<=p.maxout; i++)
     {
@@ -616,8 +616,8 @@ int main( int argc, char* argv[])
                     << p.inner_loop*p.itstp*p.maxout << " at time "<<time;
         MPI_OUT std::cout << "\n\t Average time for one step: "
                     << ti.diff()/(double)p.itstp/(double)p.inner_loop<<"s";
-        MPI_OUT std::cout << "\n\t Total number of failed steps: "
-                    << failed_counter;
+        //MPI_OUT std::cout << "\n\t Total number of failed steps: "
+        //            << failed_counter;
         ti.tic();
         //////////////////////////write fields////////////////////////
         start[0] = i;
-- 
GitLab


From d93ceff0c26495ca06e6d9a28ba6936ecb86ada4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 20 May 2019 15:30:49 +0200
Subject: [PATCH 086/540] Make all mpi comms cuda-unaware

---
 inc/dg/backend/config.h           |  2 +-
 inc/dg/backend/mpi_collective.h   | 37 +++++++++++++++++++++++++++++--
 inc/dg/backend/mpi_vector.h       | 29 ++++++++++++++----------
 inc/dg/cluster_mpib.cu            | 12 +++++-----
 inc/geometries/mpi_fieldaligned.h | 19 ++++++++++++++++
 5 files changed, 78 insertions(+), 21 deletions(-)

diff --git a/inc/dg/backend/config.h b/inc/dg/backend/config.h
index 607beb9b1..6afedb0fb 100644
--- a/inc/dg/backend/config.h
+++ b/inc/dg/backend/config.h
@@ -69,5 +69,5 @@
 #define _DG_CUDA_UNAWARE_MPI
 #endif //MPIX_CUDA
 
-#endif //THRUST = CUDA
+#endif //THRUST == CUDA
 #endif //MPI_VERSION
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 42e344453..98e4d3670 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -59,6 +59,7 @@ struct Collective
         thrust::exclusive_scan( sendTo.begin(),   sendTo.end(),   accS.begin());
         thrust::exclusive_scan( recvFrom.begin(), recvFrom.end(), accR.begin());
         m_sendTo=sendTo, m_recvFrom=recvFrom, m_accS=accS, m_accR=accR;
+    
     }
     /**
      * @brief Number of processes in the communicator
@@ -89,18 +90,36 @@ struct Collective
     private:
     unsigned sendTo( unsigned pid) const {return m_sendTo[pid];}
     unsigned recvFrom( unsigned pid) const {return m_recvFrom[pid];}
+#ifdef _DG_CUDA_UNAWARE_MPI
+    thrust::host_vector<int> m_sendTo,   m_accS;
+    thrust::host_vector<int> m_recvFrom, m_accR;
+    dg::Buffer<thrust::host_vector<get_value_type<Vector> >> m_values, m_store;
+#else
     Index m_sendTo,   m_accS; //accumulated send
     Index m_recvFrom, m_accR; //accumulated recv
+#endif // _DG_CUDA_UNAWARE_MPI
     MPI_Comm m_comm;
 };
 
 template< class Index, class Device>
 void Collective<Index, Device>::scatter( const Device& values, Device& store) const
 {
-    assert( store.size() == store_size() );
+    //assert( store.size() == store_size() );
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
     cudaDeviceSynchronize(); //needs to be called
 #endif //THRUST_DEVICE_SYSTEM
+#ifdef _DG_CUDA_UNAWARE_MPI
+    m_values.data() = values;
+    m_store.data().resize( store.size());
+    MPI_Alltoallv(
+            thrust::raw_pointer_cast( m_values.data().data()),
+            thrust::raw_pointer_cast( m_sendTo.data()),
+            thrust::raw_pointer_cast( m_accS.data()), getMPIDataType<get_value_type<Device> >(),
+            thrust::raw_pointer_cast( m_store.data().data()),
+            thrust::raw_pointer_cast( m_recvFrom.data()),
+            thrust::raw_pointer_cast( m_accR.data()), getMPIDataType<get_value_type<Device> >(), m_comm);
+    store = m_store.data();
+#else
     MPI_Alltoallv(
             thrust::raw_pointer_cast( values.data()),
             thrust::raw_pointer_cast( m_sendTo.data()),
@@ -108,16 +127,29 @@ void Collective<Index, Device>::scatter( const Device& values, Device& store) co
             thrust::raw_pointer_cast( store.data()),
             thrust::raw_pointer_cast( m_recvFrom.data()),
             thrust::raw_pointer_cast( m_accR.data()), getMPIDataType<get_value_type<Device> >(), m_comm);
+#endif //_DG_CUDA_UNAWARE_MPI
 }
 
 template< class Index, class Device>
 void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values) const
 {
-    assert( gatherFrom.size() == store_size() );
+    //assert( gatherFrom.size() == store_size() );
     values.resize( values_size() );
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
     cudaDeviceSynchronize(); //needs to be called
 #endif //THRUST_DEVICE_SYSTEM
+#ifdef _DG_CUDA_UNAWARE_MPI
+    m_store.data() = gatherFrom;
+    m_values.data().resize( values.size());
+    MPI_Alltoallv(
+            thrust::raw_pointer_cast( m_store.data().data()),
+            thrust::raw_pointer_cast( m_recvFrom.data()),
+            thrust::raw_pointer_cast( m_accR.data()), getMPIDataType<get_value_type<Device> >(),
+            thrust::raw_pointer_cast( m_values.data().data()),
+            thrust::raw_pointer_cast( m_sendTo.data()),
+            thrust::raw_pointer_cast( m_accS.data()), getMPIDataType<get_value_type<Device> >(), m_comm);
+    values = m_values.data();
+#else
     MPI_Alltoallv(
             thrust::raw_pointer_cast( gatherFrom.data()),
             thrust::raw_pointer_cast( m_recvFrom.data()),
@@ -125,6 +157,7 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
             thrust::raw_pointer_cast( values.data()),
             thrust::raw_pointer_cast( m_sendTo.data()),
             thrust::raw_pointer_cast( m_accS.data()), getMPIDataType<get_value_type<Device> >(), m_comm);
+#endif //_DG_CUDA_UNAWARE_MPI
 }
 //BijectiveComm ist der Spezialfall, dass jedes Element nur ein einziges Mal gebraucht wird.
 ///@endcond
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 396e7d685..680f39497 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -305,12 +305,13 @@ struct NearestNeighborComm
         MPI_Waitall( 4, rqst, MPI_STATUSES_IGNORE );
 #ifdef _DG_CUDA_UNAWARE_MPI
         unsigned size = buffer_size();
-        cudaMemcpy( thrust::raw_pointer_cast(&internal_host_buffer_.data()[0*size],
-                    thrust::raw_pointer_cast(&internal_buffer_.data()[0*size],
-                    buffer_size*sizeof(get_value_type<V>), cudaMemcpyHostToDevice);
-        cudaMemcpy( thrust::raw_pointer_cast(&internal_host_buffer_.data()[5*size],
-                    thrust::raw_pointer_cast(&internal_buffer_.data()[5*size],
-                    buffer_size*sizeof(get_value_type<V>), cudaMemcpyHostToDevice);
+        cudaMemcpy( thrust::raw_pointer_cast(&m_internal_buffer.data()[0*size]), //dst
+                    thrust::raw_pointer_cast(&m_internal_host_buffer.data()[0*size]), //src
+                    size*sizeof(get_value_type<Vector>), cudaMemcpyHostToDevice);
+
+        cudaMemcpy( thrust::raw_pointer_cast(&m_internal_buffer.data()[5*size]), //dst
+                    thrust::raw_pointer_cast(&m_internal_host_buffer.data()[5*size]), //src
+                    size*sizeof(get_value_type<Vector>), cudaMemcpyHostToDevice);
 #endif
     }
     private:
@@ -327,7 +328,8 @@ struct NearestNeighborComm
     Index m_gather_map_middle;
     dg::Buffer<Vector> m_internal_buffer;
 #ifdef _DG_CUDA_UNAWARE_MPI
-    dg::Buffer<thrust::host_vector<get_value_type<Vector>> m_internal_host_buffer; //a copy of the data on the host
+    //a copy of the data on the host (we need to send data manually through the host)
+    dg::Buffer<thrust::host_vector<get_value_type<Vector>>> m_internal_host_buffer;
 #endif
 
     void sendrecv(const_pointer_type, const_pointer_type, pointer_type, pointer_type, MPI_Request rqst[4])const;
@@ -402,7 +404,7 @@ void NearestNeighborComm<I,B,V>::construct( unsigned n, const unsigned dimension
     m_gather_map_middle = mid_gather; //transfer to device
     m_internal_buffer.data().resize( 6*buffer_size() );
 #ifdef _DG_CUDA_UNAWARE_MPI
-    internal_host_buffer.data().resize( 6*buffer_size() );
+    m_internal_host_buffer.data().resize( 6*buffer_size() );
 #endif
     }
 }
@@ -466,23 +468,26 @@ void NearestNeighborComm<I,B,V>::sendrecv( const_pointer_type sb1_ptr, const_poi
 {
     unsigned size = buffer_size();
 #ifdef _DG_CUDA_UNAWARE_MPI
-    cudaMemcpy( thrust::raw_pointer_cast(&m_internal_host_buffer.data()[1*size]), sb1_ptr, buffer_size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost);
-    cudaMemcpy( thrust::raw_pointer_cast(&m_internal_host_buffer.data()[4*size]), sb2_ptr, buffer_size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost);
+    cudaMemcpy( thrust::raw_pointer_cast(&m_internal_host_buffer.data()[1*size]),//dst
+        sb1_ptr, size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost); //src
+    cudaMemcpy( thrust::raw_pointer_cast(&m_internal_host_buffer.data()[4*size]),  //dst
+        sb2_ptr, size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost); //src
     sb1_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[1*size]);
     sb2_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[4*size]);
     rb1_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[0*size]);
     rb2_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[5*size]);
+//This is a mistake if called with a host_vector
 #endif
     MPI_Isend( sb1_ptr, size,
                getMPIDataType<get_value_type<V>>(),  //sender
-               m_dest[0], 3, comm_, &rqst[0]); //destination
+               m_dest[0], 3, m_comm, &rqst[0]); //destination
     MPI_Irecv( rb2_ptr, size,
                getMPIDataType<get_value_type<V>>(), //receiver
                m_source[0], 3, m_comm, &rqst[1]); //source
 
     MPI_Isend( sb2_ptr, size,
                getMPIDataType<get_value_type<V>>(),  //sender
-               m_dest[1], 9, comm_, &rqst[2]);  //destination
+               m_dest[1], 9, m_comm, &rqst[2]);  //destination
     MPI_Irecv( rb1_ptr, size,
                getMPIDataType<get_value_type<V>>(), //receiver
                m_source[1], 9, m_comm, &rqst[3]); //source
diff --git a/inc/dg/cluster_mpib.cu b/inc/dg/cluster_mpib.cu
index eea8d586c..6f4765f57 100644
--- a/inc/dg/cluster_mpib.cu
+++ b/inc/dg/cluster_mpib.cu
@@ -12,7 +12,7 @@
 
 const double lx = 2*M_PI;
 const double ly = 2*M_PI;
-const double lz = 1.;
+double lz = 1.;
 
 dg::bc bcx = dg::PER;
 dg::bc bcy = dg::PER;
@@ -78,6 +78,11 @@ int main(int argc, char* argv[])
 
 
     dg::CartesianMPIGrid3d grid( 0, lx, 0, ly, 0,lz, n, Nx, Ny, Nz, bcx, bcy, dg::PER, comm);
+    periods[0] = false, periods[1] = false;
+    MPI_Comm commEll;
+    MPI_Cart_create( MPI_COMM_WORLD, 3, dims, periods, true, &commEll);
+    if( Nz > 2) lz = 2.*M_PI;
+    dg::CylindricalMPIGrid3d gridEll( R_0, R_0+lx, 0., ly, 0.,lz, n, Nx, Ny,Nz, dg::DIR, dg::DIR, dg::PER, commEll);
     dg::Timer t;
     Vector w3d, lhs, rhs, jac, x, y, z;
     try{
@@ -169,13 +174,9 @@ int main(int argc, char* argv[])
     t.toc();
     if(rank==0)std::cout<<" "<<t.diff()/(double)multi<<std::flush;
     //The Elliptic scheme
-    periods[0] = false, periods[1] = false;
-    MPI_Comm commEll;
-    MPI_Cart_create( MPI_COMM_WORLD, 3, dims, periods, true, &commEll);
     exblas::udouble res;
     if( !(Nz > 2))
     {
-        dg::CylindricalMPIGrid3d gridEll( R_0, R_0+lx, 0., ly, 0.,lz, n, Nx, Ny,Nz, dg::DIR, dg::DIR, dg::PER, commEll);
         const Vector ellw3d = dg::create::volume(gridEll);
         const Vector ellv3d = dg::create::inv_volume(gridEll);
         dg::Elliptic<dg::CylindricalMPIGrid3d, Matrix, Vector> laplace(gridEll, dg::not_normed, dg::centered);
@@ -196,7 +197,6 @@ int main(int argc, char* argv[])
     else
     {
         //Elliptic3d
-        dg::CylindricalMPIGrid3d gridEll( R_0, R_0+lx, 0., ly, 0., 2.*M_PI, n, Nx, Ny,Nz, dg::DIR, dg::DIR, dg::PER, commEll);
         const Vector ellw3d = dg::create::volume(gridEll);
         const Vector ellv3d = dg::create::inv_volume(gridEll);
         dg::Elliptic3d<dg::CylindricalMPIGrid3d, Matrix, Vector> laplace(gridEll, dg::not_normed, dg::centered);
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index d9b7cbc69..71852fa8b 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -156,6 +156,10 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
     std::vector<MPI_Vector<dg::View<LocalContainer>> > m_temp;
     dg::ClonePtr<ProductMPIGeometry> m_g;
     unsigned m_coords2, m_sizeZ; //number of processes in z
+#ifdef _DG_CUDA_UNAWARE_MPI
+    //we need to manually send data through the host
+    thrust::host_vector<double> m_send_buffer, m_recv_buffer; //2d size
+#endif
 };
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
@@ -184,6 +188,9 @@ Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     dg::assign( dg::pullback(limit, *grid_coarse), m_limiter);
     dg::assign( dg::evaluate(zero, *grid_coarse), m_left);
     m_ghostM = m_ghostP = m_right = m_left;
+#ifdef _DG_CUDA_UNAWARE_MPI
+    m_recv_buffer = m_send_buffer = m_ghostP.data();
+#endif
     ///%%%%%%%%%%Set starting points and integrate field lines%%%%%%%%%%%//
 #ifdef DG_BENCHMARK
     dg::Timer t;
@@ -358,8 +365,14 @@ void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichM
     if( m_sizeZ != 1)
     {
         unsigned i0 = m_Nz-1;
+#ifdef _DG_CUDA_UNAWARE_MPI
+        thrust::copy( m_temp[i0].data().cbegin(), m_temp[i0].data().cend(), m_send_buffer.begin());
+        detail::sendBackward( m_send_buffer, m_recv_buffer, m_g->communicator());
+        thrust::copy( m_recv_buffer.cbegin(), m_recv_buffer.cend(), m_temp[i0].data().begin());
+#else
         detail::sendBackward( m_temp[i0].data(), m_ghostM.data(), m_g->communicator());
         dg::blas1::copy( m_ghostM, m_temp[i0]);
+#endif //_DG_CUDA_UNAWARE_MPI
     }
 
     //3. apply right boundary conditions in last plane
@@ -398,8 +411,14 @@ void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::eMinus( enum which
     if( m_sizeZ != 1)
     {
         unsigned i0 = 0;
+#ifdef _DG_CUDA_UNAWARE_MPI
+        thrust::copy( m_temp[i0].data().cbegin(), m_temp[i0].data().cend(), m_send_buffer.begin());
+        detail::sendForward( m_send_buffer, m_recv_buffer, m_g->communicator());
+        thrust::copy( m_recv_buffer.cbegin(), m_recv_buffer.cend(), m_temp[i0].data().begin());
+#else
         detail::sendForward( m_temp[i0].data(), m_ghostP.data(), m_g->communicator());
         dg::blas1::copy( m_ghostP, m_temp[i0]);
+#endif //_DG_CUDA_UNAWARE_MPI
     }
 
     //3. apply left boundary conditions in first plane
-- 
GitLab


From 6634f46bc4eb93cae1c9e05e08605035fdf017bc Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 20 May 2019 17:57:52 +0200
Subject: [PATCH 087/540] Improvements to feltor and feltor_hpc

- introduce alpha_mag to separately change profiles and magnetic field
- introduce induction test in feltor and feltor_hpc
- introduce sanity check for Ni in feltor and feltor_hpc
---
 inc/geometries/geometry_diag.cu |  8 ++---
 src/feltor/feltor.cu            | 35 +++++++++++++-------
 src/feltor/feltor.cuh           |  2 ++
 src/feltor/feltor.tex           | 13 +++++---
 src/feltor/feltor_hpc.cu        | 32 ++++++++++++++----
 src/feltor/input/default.json   | 57 +++++++++++++++++----------------
 src/feltor/parameters.h         |  6 ++--
 7 files changed, 97 insertions(+), 56 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index a8d6d78fe..3786699ab 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -28,7 +28,7 @@ struct Parameters
     double boxscaleZm, boxscaleZp;
     double amp, k_psi, bgprofamp, nprofileamp;
     double sigma, posX, posY;
-    double rho_damping, alpha;
+    double rho_damping, alpha_mag;
     Parameters( const Json::Value& js){
         n = js.get("n",3).asUInt();
         Nx = js.get("Nx",100).asUInt();
@@ -46,7 +46,7 @@ struct Parameters
         posX = js.get("posX", 0.5).asDouble();
         posY = js.get("posY", 0.5).asDouble();
         rho_damping = js.get("rho_damping", 1.2).asDouble();
-        alpha = js.get("alpha", 0.05).asDouble();
+        alpha_mag = js.get("alpha_mag", 0.05).asDouble();
     }
     void display( std::ostream& os = std::cout ) const
     {
@@ -144,7 +144,7 @@ int main( int argc, char* argv[])
 
     //Test coefficients
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(), 0.), p.alpha);
+    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(), 0.), p.alpha_mag);
     const double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     const double Z_X = -1.1*gp.elongation*gp.a;
     const double R_H = gp.R_0-gp.triangularity*gp.a;
@@ -242,7 +242,7 @@ int main( int argc, char* argv[])
     dg::DVec psipog2d   = dg::evaluate( mag.psip(), grid2d);
     std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
     ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
-    unsigned npsi = 3, Npsi = 128;//set number of psivalues (NPsi % 8 == 0)
+    unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
     double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     double volumeXGrid;
     /// -------  Elements for fsa of curvature operators ----------------
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index ccc3c24e7..11e5d814c 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -45,8 +45,9 @@ int main( int argc, char* argv[])
     //Make grid
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
+    dg::DVec vol3d = dg::create::volume( grid);
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha);
+    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
     //create RHS
     std::cout << "Constructing Explicit...\n";
@@ -132,6 +133,13 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( 1., dg::construct<dg::DVec>(ntilde), 1., y0[0][0]);
     std::cout << "initialize ni" << std::endl;
     feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
+    double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
+    if( minimalni <= -1)
+    {
+        std::cerr << "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n";
+        std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
+        return -1;
+    }
 
     dg::blas1::copy( 0., y0[1][0]); //set we = 0
     dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -193,15 +201,6 @@ int main( int argc, char* argv[])
     dg::HVec hvisual( grid.size(), 0.), visual(hvisual), avisual(hvisual);
     dg::IHMatrix equi = dg::create::backscatter( grid);
     draw::ColorMapRedBlueExtMinMax colors(-1.0, 1.0);
-    //perp laplacian for computation of vorticity
-
-    dg::Elliptic3d<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec>
-        laplacianM(grid, p.bcxP, p.bcyP, dg::PER, dg::normed, dg::centered);
-    auto bhatF = dg::geo::createEPhi();
-    if( p.curvmode == "true")
-        bhatF = dg::geo::createBHat( mag);
-    dg::SparseTensor<dg::DVec> hh = dg::geo::createProjectionTensor( bhatF, grid);
-    laplacianM.set_chi( hh);
 
     /////////glfw initialisation ////////////////////////////////////////////
     //
@@ -229,8 +228,8 @@ int main( int argc, char* argv[])
         {
             if(pair.first == "Ome / ")
             {
-                dg::blas2::gemv( laplacianM, *pair.second, dvisual);
-                dg::assign( dvisual, hvisual);
+                //dg::blas2::gemv( laplacianM, *pair.second, dvisual);
+                dg::assign( feltor.lapMperpP(0), hvisual);
                 dg::assign( *pair.second, hvisual);
             }
             else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
@@ -320,6 +319,18 @@ int main( int argc, char* argv[])
             std::cout <<" d M/dt = " << dMdt
                       <<" Lambda = " << *v0d["diff"]
                       <<" -> Accuracy: " << accuracyM << "\n";
+            //----------------Test if induction equation holds
+            if( p.beta != 0)
+            {
+                dg::blas1::pointwiseDot(
+                    feltor.density(0), feltor.velocity(0), dvisual);
+                dg::blas1::pointwiseDot( p.beta,
+                    feltor.density(1), feltor.velocity(1), -p.beta, dvisual);
+                double norm  = dg::blas2::dot( dvisual, vol3d, dvisual);
+                dg::blas1::axpby( -1., feltor.lapMperpA(), 1., dvisual);
+                double error = dg::blas2::dot( dvisual, vol3d, dvisual);
+                std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
+            }
 
         }
         t.toc();
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 04c849ba0..dcd7a882c 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -609,11 +609,13 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::initializeni(
             //add FLR correction -0.5*tau*mu*Delta n_e
             dg::blas2::symv( 0.5*m_p.tau[1]*m_p.mu[1],
                 m_lapperpN, src, 1.0, target);
+            //wird stark negativ falls alpha klein!!
         }
         else if( m_p.initphi == "balance")
             //add FLR correction +0.5*tau*mu*Delta n_e
             dg::blas2::symv( -0.5*m_p.tau[1]*m_p.mu[1],
                 m_lapperpN, src, 1.0, target);
+            //wird stark negativ falls alpha klein!!
         else
         {
             #ifdef MPI_VERSION
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index aefd57d47..5119db07f 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -392,10 +392,14 @@ An integral of this function is
     0 &\text{ for } \psi > \alpha
 \end{cases}
     \approx \psi H(-\psi)
-\label{eq:modified_psi}
 \end{align}
 
-We now use $\theta_\alpha(\psi-\psi_0)+\psi_0$ instead of $\psi$ for the computation of the
+We now use 
+\begin{align}
+\theta_\alpha(\psi-\psi_0)+\psi_0
+\label{eq:modified_psip}
+\end{align}
+instead of $\psi$ for the computation of the
 magnetic field, which introduces a shear layer around $\psi_0$ where the
 fieldlines are straightened to match $\ehat_\varphi$.
 Note that $\Theta_\alpha(0) = 0.5$ and $\theta_\alpha(0) = 35\alpha/256$.
@@ -1094,11 +1098,12 @@ posY       & float &0.0    & - & blob Z-position in units of $a$ \\
 sigma\_z    & float &0.25   & - & variance in units of $R_0$  \\
 k\_psi     & float &0    & - & zonal mode wave number  \\
 nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} \\
-alpha       & float & 0.02 & - & Width $\alpha$ of Heaviside profile in Eq.~\eqref{eq:approx_heaviside} and \eqref{eq:modified_psi} \\
+alpha\_mag   & float & 0.05 & - & Width $\alpha$ of the Heaviside in the modified $\psi_p$ function \eqref{eq:modified_psip}\\
+alpha       & float & 0.2 & - & Width $\alpha$ of the Heaviside Eq.~\eqref{eq:approx_heaviside} in the density and source profiles (should be small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes) \\
 source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
 rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source}  \\
 %damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
-rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0$ in Eq.~\eqref{eq:modified_psi}  \\
+rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0$ in Eq.~\eqref{eq:modified_psip}  \\
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 11e414eda..61665d92a 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -155,8 +155,10 @@ int main( int argc, char* argv[])
         , comm
         #endif //FELTOR_MPI
         );
+    DVec vol3d = dg::create::volume( grid);
+    DVec temp(vol3d); //needed for Apar test
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha);
+    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
     //create RHS
     MPI_OUT std::cout << "Constructing Explicit...\n";
@@ -250,6 +252,13 @@ int main( int argc, char* argv[])
         dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
         MPI_OUT std::cout << "initialize ni" << std::endl;
         feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
+        double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
+        if( minimalni <= -1)
+        {
+            MPI_OUT std::cerr << "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n";
+            MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
+            return -1;
+        }
 
         dg::blas1::copy( 0., y0[1][0]); //set we = 0
         dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -426,7 +435,7 @@ int main( int argc, char* argv[])
         v3d.emplace_back( "Psip", &psip, "Flux-function psi");
         v3d.emplace_back( "Nprof", &profile, "Density profile");
         v3d.emplace_back( "Source", &source_damping, "Source region");
-        v3d.emplace_back( "Damping", &damping_damping, "Damping region");
+        v3d.emplace_back( "Damping", &profile_damping, "Damping region for initial profile");
         v3d.emplace_back( "xc", &xc, "x-coordinate in Cartesian coordinate system");
         v3d.emplace_back( "yc", &yc, "y-coordinate in Cartesian coordinate system");
         v3d.emplace_back( "zc", &zc, "z-coordinate in Cartesian coordinate system");
@@ -596,6 +605,9 @@ int main( int argc, char* argv[])
             for( auto pair : v0d)
                 err = nc_put_vara_double( ncid, id0d.at(pair.first),
                     Estart, Ecount, pair.second);
+            #ifndef FELTOR_MPI
+            err = nc_close(ncid);
+            #endif //FELTOR_MPI
 
             MPI_OUT q.display(std::cout);
             MPI_OUT std::cout << "(m_tot-m_0)/m_0: "<< (*v0d["mass"]-mass0)/mass0<<"\t";
@@ -606,10 +618,18 @@ int main( int argc, char* argv[])
             MPI_OUT std::cout <<" d M/dt = " << dMdt
                       <<" Lambda = " << *v0d["diff"]
                       <<" -> Accuracy: " << accuracyM << "\n";
-            #ifndef FELTOR_MPI
-            err = nc_close(ncid);
-            #endif //FELTOR_MPI
-
+            //----------------Test if induction equation holds
+            if( p.beta != 0)
+            {
+                dg::blas1::pointwiseDot(
+                    feltor.density(0), feltor.velocity(0), temp);
+                dg::blas1::pointwiseDot( p.beta,
+                    feltor.density(1), feltor.velocity(1), -p.beta, temp);
+                double norm  = dg::blas2::dot( temp, vol3d, temp);
+                dg::blas1::axpby( -1., feltor.lapMperpA(), 1., temp);
+                double error = dg::blas2::dot( temp, vol3d, temp);
+                MPI_OUT std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
+            }
         }
         ti.toc();
         MPI_OUT std::cout << "\n\t Step "<<step <<" of "
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index 00c820046..32f51abe0 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -1,31 +1,31 @@
 {
     "n"  : 3,
-    "Nx" : 64,
-    "Ny" : 64,
-    "Nz" : 20,
-    "dt" : 1e-6,
+    "Nx" : 48,
+    "Ny" : 96,
+    "Nz" : 32,
+    "dt" : 1e-2,
     "compression" : [2,2],
-    "refineDS" : [5,5],
-    "rk4eps" : 1e-5,
-    "inner_loop" : 1,
-    "itstp"      : 5,
-    "maxout"     : 100,
-    "eps_pol"    : 1e-5,
+    "refineDS" : [1,1],
+    "rk4eps" : 1e-6,
+    "inner_loop" : 2,
+    "itstp"  : 2,
+    "maxout" : 5,
+    "eps_pol"    : 1e-6,
     "jumpfactor" : 1,
     "eps_gamma"  : 1e-6,
     "stages"     : 3,
-    "eps_time"   : 1e-9,
-    "rtol"       : 1e-5,
+    "eps_time"   : 1e-7,
+    "rtol"       : 1e-4,
     "mu"          : -0.000272121,
-    "tau"         : 0,
+    "tau"         : 0.5,
     "beta"        : 0,
     "nu_perp"     : 1e-3,
-    "perp_diff"   : "viscous",
-    "nu_parallel" : 1,
+    "perp_diff"   : "hyperviscous",
+    "nu_parallel" : 10,
     "resistivity" : 1e-4,
-    "amplitude" : 0.01,
+    "amplitude" : 0.2,
     "sigma"     : 2.0,
-    "posX"      : 0.3,
+    "posX"      : 0.6,
     "posY"      : 0,
     "sigma_z"   : 0.5,
     "k_psi"     : 0,
@@ -33,18 +33,19 @@
     "bgprofamp"   : 1,
     "bc" :
     {
-        "density" : ["DIR", "DIR"],
-        "velocity": ["DIR", "DIR"],
+        "density" : ["NEU", "NEU"],
+        "velocity": ["NEU", "NEU"],
         "potential":["DIR", "DIR"]
     },
-    "boxscaleR" :  [1.1,1.1],
-    "boxscaleZ" :  [1.2,1.1],
-    "initne"    : "turbulence",
-    "initphi"   : "zero",
-    "curvmode"  : "true",
-    "symmetric" : false,
-    "alpha"       : 0.05,
-    "source"      : 0,
-    "rho_source"  : 0.2,
+    "boxscaleR" :  [1.15,1.2],
+    "boxscaleZ" :  [1.2,1.15],
+    "initne"     : "turbulence",
+    "initphi"    : "balance",
+    "curvmode"   : "true",
+    "symmetric"  : false,
+    "alpha_mag"  : 0.05,
+    "alpha"      : 0.2,
+    "source"     : 0,
+    "rho_source" : 0.2,
     "rho_damping" : 1.2
 }
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index f168478cf..64c1bcc92 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -27,7 +27,7 @@ struct Parameters
 
     std::array<double,2> mu; // mu[0] = mu_e, m[1] = mu_i
     std::array<double,2> tau; // tau[0] = -1, tau[1] = tau_i
-    double alpha, beta;
+    double alpha_mag, alpha, beta;
     double rho_source, rho_damping;
 
     double nu_perp, nu_parallel;
@@ -92,7 +92,8 @@ struct Parameters
         nprofamp  = js["nprofileamp"].asDouble();
         omega_source  = js.get("source", 0.).asDouble();
         omega_damping = js.get("damping", 0.).asDouble();
-        alpha        = js.get("alpha", 0.02).asDouble();
+        alpha_mag    = js.get("alpha_mag", 0.05).asDouble();
+        alpha        = js.get("alpha", 0.2).asDouble();
         rho_source   = js.get("rho_source", 0.2).asDouble();
         rho_damping  = js.get("rho_damping", 1.2).asDouble();
 
@@ -141,6 +142,7 @@ struct Parameters
             <<"     rho_source:                   "<<rho_source<<"\n"
             <<"     omega_damping:                "<<omega_damping<<"\n"
             <<"     rho_damping:                  "<<rho_damping<<"\n"
+            <<"     alpha_mag:                    "<<alpha_mag<<"\n"
             <<"     alpha:                        "<<alpha<<"\n"
             <<"     density profile amplitude:    "<<nprofamp<<"\n"
             <<"     boxscale R+:                  "<<boxscaleRp<<"\n"
-- 
GitLab


From 3054a2cec69fdaf5429062e6bcc72aacfad5241d Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 20 May 2019 21:53:09 +0200
Subject: [PATCH 088/540] Add parallel Timer in feltor.cuh

---
 src/feltor/feltor.cuh | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index dcd7a882c..8b1ec6ed8 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -943,8 +943,6 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     const std::array<std::array<Container,2>,2>& y,
     std::array<std::array<Container,2>,2>& yp)
 {
-    //dg::Timer timer;
-    //timer.tic();
 
     // set m_phi[0]
     compute_phi( t, y[0]);
@@ -972,6 +970,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
 #endif
 
     // Add parallel dynamics --- needs m_logn
+    dg::Timer timer;
+    timer.tic();
 #if FELTORPARALLEL == 1
     compute_parallel( t, y, m_fields, yp);
 #endif
@@ -1002,12 +1002,12 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
 #endif //DG_MANUFACTURED
-    //timer.toc();
-    //#ifdef MPI_VERSION
-    //    int rank;
-    //    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-    //    if(rank==0)
-    //#endif
-    //std::cout << "#One rhs took "<<timer.diff()<<"s\n";
+    timer.toc();
+    #ifdef MPI_VERSION
+        int rank;
+        MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+        if(rank==0)
+    #endif
+    std::cout << "#Add parallel dynamics took "<<timer.diff()<<"s\n";
 }
 } //namespace feltor
-- 
GitLab


From 3edf2db28d4fcac11d0ee9cba336894824fe576f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 21 May 2019 18:28:55 +0200
Subject: [PATCH 089/540] Output minimal Ni value

---
 src/feltor/feltor.cu     | 3 ++-
 src/feltor/feltor_hpc.cu | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 11e5d814c..5c7975873 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -134,6 +134,7 @@ int main( int argc, char* argv[])
     std::cout << "initialize ni" << std::endl;
     feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
     double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
+    std::cout << "Minimum Ni value "<<minimalni+1<<std::endl;
     if( minimalni <= -1)
     {
         std::cerr << "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n";
@@ -335,7 +336,7 @@ int main( int argc, char* argv[])
         }
         t.toc();
         std::cout << "\n\t Step "<<step << " at time  "<<time;
-        std::cout << "\n\t Average time for one step: "<<t.diff()/(double)p.itstp/(double)p.inner_loop;
+        std::cout << "\n\t Average time for one step: "<<t.diff()/(double)p.itstp/(double)p.inner_loop<<"\n\n";
         //std::cout << "\n\t Total # of failed steps:   "<<failed_counter<<"\n\n";
     }
     glfwTerminate();
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 61665d92a..a4a61e06b 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -253,6 +253,7 @@ int main( int argc, char* argv[])
         MPI_OUT std::cout << "initialize ni" << std::endl;
         feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
         double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
+        MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
         if( minimalni <= -1)
         {
             MPI_OUT std::cerr << "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n";
-- 
GitLab


From dc48c7ebf6c251bb6224a672ece5ab7d9c9aa7c1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 22 May 2019 16:58:01 +0200
Subject: [PATCH 090/540] Update feltordiag with 3d output

---
 diag/feltordiag.cu       | 86 +++++++++++++++++++++++++++++++++-------
 diag/feltordiag.h        | 32 +++++++++++++++
 src/feltor/feltor.cuh    | 38 ++++++++++++++++++
 src/feltor/feltor.tex    | 11 +++++
 src/feltor/feltor_hpc.cu | 12 +-----
 5 files changed, 153 insertions(+), 26 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index a2d52a946..fcc164f46 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -10,12 +10,22 @@
 #include "feltordiag.h"
 
 void create_records_in_file( int ncid,
-    int dim_ids[3], int dim_ids1d[2],
+    int dim_ids4d[4], int dim_ids[3], int dim_ids1d[2],
     std::map<std::string, int>& id0d,
     std::map<std::string, int>& id1d,
-    std::map<std::string, int>& id2d)
+    std::map<std::string, int>& id2d,
+    std::map<std::string, int>& id3d)
 {
     file::NC_Error_Handle err;
+    for( auto record : feltor::records3d_list)
+    {
+        std::string name = record.name;
+        std::string long_name = record.long_name;
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 4, dim_ids4d,
+            &id3d[name]);//creates a new id3d entry
+        err = nc_put_att_text( ncid, id3d[name], "long_name", long_name.size(),
+            long_name.data());
+    }
     for( auto record : feltor::records_list)
     {
         std::string name = record.name + "_ta2d";
@@ -148,7 +158,7 @@ int main( int argc, char* argv[])
 
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     const double psip0 = mag.psip()(gp.R_0, 0);
-    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*psip0, p.alpha);
+    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*psip0, p.alpha_mag);
     dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
     dg::HVec psipog3d = dg::evaluate( mag.psip(), g3d_out);
     // Construct weights and temporaries
@@ -257,15 +267,24 @@ int main( int argc, char* argv[])
     };
 
     // define 2d and 1d and 0d dimensions and variables
-    int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
-    int dim_ids1d[2] = {dim_ids[0],dim_ids[1]}; //time , psi
+    int dim_ids4d[4], tvar4dID, tvarID;
+    err = file::define_dimensions( ncid_out, dim_ids4d, &tvarID, g3d_out);
+    int dim_ids[3] = {dim_ids4d[0], dim_ids4d[2], dim_ids4d[3]};
+    int dim_ids1d[2] = {dim_ids[0], 0}; //time , psi
     err = file::define_dimension( ncid_out, "psi", &dim_ids1d[1], g1d_out);
-    std::string psi_long_name = "Flux surface label";
+    err = file::define_time( ncid_out, "time3d", &dim_ids4d[0], &tvar4dID);
+    //Write long description
+    std::string long_name = "Time at which 2d fields are written";
+    err = nc_put_att_text( ncid, tvarID, "long_name", long_name.size(),
+            long_name.data());
+    long_name = "Time at which 3d fields are written";
+    err = nc_put_att_text( ncid, tvar4dID, "long_name", long_name.size(),
+            long_name.data());
+    long_name = "Flux surface label";
     err = nc_put_att_text( ncid, dim_ids1d[1], "long_name",
-        psi_long_name.size(), psi_long_name.data());
+        long_name.size(), long_name.data());
 
-    std::map<std::string, int> id0d, id1d, id2d;
+    std::map<std::string, int> id0d, id1d, id2d, id3d;
     /// Construct 0d names to output
     std::vector<std::string> names_0d{
         "correlationNPhi", "correlationNTildePhi"
@@ -283,7 +302,7 @@ int main( int argc, char* argv[])
     for( auto name : names_input)
         v3d[name] = t3d;
     //now create the variables in the netcdf file
-    create_records_in_file( ncid_out, dim_ids, dim_ids1d, id0d, id1d, id2d);
+    create_records_in_file( ncid_out, dim_ids4d, dim_ids, dim_ids1d, id0d, id1d, id2d, id3d);
 
     size_t count1d[2] = {1, g1d_out.n()*g1d_out.N()};
     size_t start1d[2] = {0, 0};
@@ -291,6 +310,7 @@ int main( int argc, char* argv[])
     size_t start2d[3] = {0, 0, 0};
     size_t count3d[4] = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(), g3d_out.n()*g3d_out.Nx()};
     size_t start3d[4] = {0, 0, 0, 0};
+    size_t start3d_out[4] = {0,0,0,0};
 
     //write 1d static vectors (psi, q-profile, ...) into file
     for( auto tp : map1d)
@@ -304,6 +324,28 @@ int main( int argc, char* argv[])
         err = nc_put_var_double( ncid, vid, std::get<1>(tp).data());
         err = nc_redef(ncid);
     }
+    //and the Cartesian 3d coordinates
+    {
+        std::vector<std::tuple<std::string, const dg::HVec*, std::string> > c3d;
+        dg::HVec xc = dg::evaluate( dg::cooX3d, g3d_out);
+        dg::HVec yc = dg::evaluate( dg::cooY3d, g3d_out);
+        dg::HVec zc = dg::evaluate( dg::cooZ3d, g3d_out);
+        dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
+        c3d.emplace_back( "xc", &xc, "x-coordinate in Cartesian coordinate system");
+        c3d.emplace_back( "yc", &yc, "y-coordinate in Cartesian coordinate system");
+        c3d.emplace_back( "zc", &zc, "z-coordinate in Cartesian coordinate system");
+        for( auto tp : c3d)
+        {
+            int vecID;
+            err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 3,
+                &dim_ids4d[1], &vecID);
+            err = nc_put_att_text( ncid, vecID,
+                "long_name", std::get<2>(tp).size(), std::get<2>(tp).data());
+            err = nc_enddef( ncid);
+            err = nc_put_var_double( ncid, vecID, std::get<1>(tp)->data());
+            err = nc_redef(ncid);
+        }
+    }
     err = nc_close(ncid_out);
 
     /////////////////////////////////////////////////////////////////////////
@@ -318,7 +360,7 @@ int main( int argc, char* argv[])
 
     dg::Average<dg::DVec> toroidal_average( g3d_out, dg::coo3d::z);
 
-    steps = 3;
+    //steps = 3;
     for( unsigned i=0; i<steps; i++)//timestepping
     {
         start3d[0] = i;
@@ -326,7 +368,7 @@ int main( int argc, char* argv[])
         start1d[0] = i;
         err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
         err = nc_get_vara_double( ncid, timeID, start3d, count3d, &time);
-        std::cout << "Timestep = " << i << "  time = " << time << "\t";
+        std::cout << "Timestep = " << i << "  time = " << time << std::endl;
         //read in Ne,Ni,Ue,Ui,Phi,Apar
         for( auto name : names_input)
         {
@@ -365,9 +407,23 @@ int main( int argc, char* argv[])
             /norm1/norm2;  //<phi, n-<n> >/||phi||/||n-<n>||
 
 
-        //now write out 2d and 1d quantities
+        //now write out quantities
         err = nc_open(argv[2], NC_WRITE, &ncid_out);
-        for( auto record : feltor::records_list)// {name, DVec}
+        if( i%10 == 0) //write 3d fields only every 10th timestep
+        {
+            start3d_out[0] = i/10;
+            for( auto record : feltor::records3d_list)
+            {
+                record.function( t3d, var);
+                //toroidal average
+                dg::blas1::transfer( t3d, transfer3d);
+                err = nc_put_vara_double( ncid_out, id3d.at(record.name),
+                    start3d_out, count3d, transfer3d.data());
+            }
+            //write time for 3d fields
+            err = nc_put_vara_double( ncid_out, tvar4dID, start3d_out, count3d, &time);
+        }
+        for( auto record : feltor::records_list)
         {
             record.function( t3d, var);
             //toroidal average
@@ -427,7 +483,7 @@ int main( int argc, char* argv[])
             err = nc_put_vara_double( ncid_out, id0d.at(pair.first),
                 start2d, count2d, &pair.second );
         }
-        //write time data
+        //write time
         err = nc_put_vara_double( ncid_out, tvarID, start2d, count2d, &time);
         //and close file
         err = nc_close(ncid_out);
diff --git a/diag/feltordiag.h b/diag/feltordiag.h
index cb17fd768..3063f9a64 100644
--- a/diag/feltordiag.h
+++ b/diag/feltordiag.h
@@ -143,6 +143,38 @@ struct Record{
     std::function<void( dg::DVec&, Variables&)> function;
 };
 
+std::vector<Record> records3d_list = {
+    {"dppne", "2nd varphi derivative of electron density",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.dppN(0), result);
+        }
+    },
+    {"dppphi", "2nd varphi derivative of electric potential",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.dppP(0), result);
+        }
+    },
+    {"dppue", "2nd varphi derivative of electron velocity",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.dppU(0), result);
+        }
+    },
+    {"dssne", "2nd fieldaligned derivative of electron density",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.dssN(0), result);
+        }
+    },
+    {"dssphi", "2nd fieldaligned derivative of electric potential",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.dssP(0), result);
+        }
+    },
+    {"dssue", "2nd fieldaligned derivative of electron velocity",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.dssU(0), result);
+        }
+    }
+};
 
 std::vector<Record> records_list = {
     {"electrons", "Electron density",
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 8b1ec6ed8..d4515a6b2 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -15,6 +15,16 @@ namespace feltor
 {
 
 namespace routines{
+struct Cylindrical2Cartesian{
+    void operator()( double R, double Z, double P, double& x, double& y, double& z)
+    {
+        //inplace possible
+        double xx = R*sin(P);
+        double yy = R*cos(P);
+        x = xx, y = yy, z = Z;
+    }
+};
+
 struct ComputePerpDrifts{
     ComputePerpDrifts( double mu, double tau):
         m_mu(mu), m_tau(tau){}
@@ -294,6 +304,34 @@ struct Explicit
     const Container & dsN (int i) const {
         return m_dsN[i];
     }
+    const Container & dssN(int i) { //2nd fieldaligned derivative
+        dg::blas1::axpby( 1., m_fields[0][i], -1., 1., m_temp0);
+        m_ds_N.dss( m_temp0, m_temp1);
+        return m_temp1;
+    }
+    const Container & dssP(int i) {
+        m_ds_P.dss( m_phi[i], m_temp1);
+        return m_temp1;
+    }
+    const Container & dssU(int i) {
+        m_ds_U.dss( m_fields[1][i], m_temp1);
+        return m_temp1;
+    }
+    const Container & dppN(int i) { //2nd varphi derivative
+        dg::blas2::symv( m_dz, m_fields[0][i], m_temp0);
+        dg::blas2::symv( m_dz, m_temp0, m_temp1);
+        return m_temp1;
+    }
+    const Container & dppP(int i) {
+        dg::blas2::symv( m_dz, m_phi[i], m_temp0);
+        dg::blas2::symv( m_dz, m_temp0, m_temp1);
+        return m_temp1;
+    }
+    const Container & dppU(int i) {
+        dg::blas2::symv( m_dz, m_fields[1][i], m_temp0);
+        dg::blas2::symv( m_dz, m_temp0, m_temp1);
+        return m_temp1;
+    }
     const Container& lapParallelN( int i){
         dg::blas1::axpby( 1., m_fields[0][i], -1., 1., m_temp0);
         m_ds_N.dss( m_temp0, m_temp1);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 5119db07f..119e9f53b 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1206,8 +1206,19 @@ inputfile  &     text attribute & - & verbose input file as a string (valid JSON
 geomfile   &     text attribute & - & verbose geometry input file as a string (valid JSON, no comments) \\
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
+z                & Coord. Var. & 1 (z) & $\varphi$-coordinate (computational space, compressed size: $N_z$)\\
 psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
+time3d           & Coord. Var. & 1 (time3d)& time at which 3d fields are written (variable size: maxout$/10+1$, dimension size: unlimited) \\
+xc           & Dataset & 3 (z,y,x) & Cartesian x-coordinate $x=R\sin(\varphi)$ \\
+yc           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
+zc           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
+dssne        & Dataset & 4 (time3d,z,y,x) & $\nabla_\parallel^2n_e$ \\
+dssue        & Dataset & 4 (time3d,z,y,x) & $\nabla_\parallel^2u_e$ \\
+dssphi       & Dataset & 4 (time3d,z,y,x) & $\nabla_\parallel^2\phi$ \\
+dppne        & Dataset & 4 (time3d,z,y,x) & $\partial_\varphi^2n_e$ \\
+dppue        & Dataset & 4 (time3d,z,y,x) & $\partial_\varphi^2u_e$ \\
+dppphi       & Dataset & 4 (time3d,z,y,x) & $\partial_\varphi^2\phi$ \\
 correlationNTildePhi  & Dataset & 1 (time) & $ \int\d v \langle \tilde \phi \tilde n_e\rangle / ||\tilde \phi||||\tilde n_e||$
 with $\tilde n_e \equiv (n_e - \langle n_e \rangle)/\langle n_e\rangle$, 
 $\tilde\phi = \phi - \langle\phi\rangle$
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index a4a61e06b..f452d747b 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -15,16 +15,6 @@
 #include "feltor.cuh"
 #include "implicit.h"
 
-struct Cylindrical2Cartesian{
-    void operator()( double R, double Z, double P, double& x, double& y, double& z)
-    {
-        //inplace possible
-        double xx = R*sin(P);
-        double yy = R*cos(P);
-        x = xx, y = yy, z = Z;
-    }
-};
-
 #ifdef FELTOR_MPI
 using HVec = dg::MHVec;
 using DVec = dg::MDVec;
@@ -424,7 +414,7 @@ int main( int argc, char* argv[])
         HVec xc = dg::evaluate( dg::cooX3d, grid);
         HVec yc = dg::evaluate( dg::cooY3d, grid);
         HVec zc = dg::evaluate( dg::cooZ3d, grid);
-        dg::blas1::subroutine( Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
+        dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
 
         std::vector<std::tuple<std::string, const HVec*, std::string> > v3d;
         v3d.emplace_back( "BR", &vecR,
-- 
GitLab


From 86777ae62ccfbef4c93f9ff86c3aee0a4225dca9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 23 May 2019 17:15:46 +0200
Subject: [PATCH 091/540] Rewrite feltor3d output with serial netcdf

This should
- improve the usability of files after a program crash
- improve the time spent on output
- avoid the need to install parallel netcdf
but might
- not scale to very many (10000) MPI processes
---
 inc/file/Makefile        |   1 +
 src/feltor/feltor_hpc.cu | 217 ++++++++++++++++++---------------------
 2 files changed, 102 insertions(+), 116 deletions(-)

diff --git a/inc/file/Makefile b/inc/file/Makefile
index 606bfc899..4cade7eca 100644
--- a/inc/file/Makefile
+++ b/inc/file/Makefile
@@ -4,6 +4,7 @@ include ../../config/default.mk
 include ../../config/*.mk
 include ../../config/devices/devices.mk
 
+INCLUDE+= -I../../ # other project libraries
 INCLUDE+= -I../    # other project libraries
 
 all: netcdf_t netcdf_mpit
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index f452d747b..a693396fd 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -8,7 +8,6 @@
 
 #ifdef FELTOR_MPI
 #include <mpi.h>
-#include "netcdf_par.h"
 #endif //FELTOR_MPI
 
 #include "dg/file/nc_utilities.h"
@@ -33,8 +32,6 @@ using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #endif //FELTOR_MPI
 
-int ncid = -1; //netcdf id (signal handler can close the file)
-
 #ifdef FELTOR_MPI
 //ATTENTION: in slurm should be used with --signal=SIGINT@30 (<signal>@<time in seconds>)
 void sigterm_handler(int signal)
@@ -42,14 +39,6 @@ void sigterm_handler(int signal)
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     std::cout << " pid "<<rank<<" sigterm_handler, got signal " << signal << std::endl;
-    std::cout << " pid "<<rank<<" ncid = " << ncid << std::endl;
-    if( ncid != -1)
-    {
-        file :: NC_Error_Handle err;
-        err = nc_close(ncid);
-        std::cout << " pid "<<rank<<" Closing NetCDF file with id " << ncid << std::endl;
-        ncid = -1;
-    }
     MPI_Finalize();
     exit(signal);
 }
@@ -68,6 +57,7 @@ int main( int argc, char* argv[])
 #endif
     int periods[3] = {false, false, true}; //non-, non-, periodic
     int rank, size;
+    MPI_Status status;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     MPI_Comm_size( MPI_COMM_WORLD, &size);
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
@@ -78,6 +68,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     int device = rank % num_devices; //assume # of gpus/node is fixed
+    std::cout << "# Rank "<<rank<<" computes with device "<<device<<" !"<<std::endl;
     cudaSetDevice( device);
 #endif//THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
     int np[3];
@@ -356,12 +347,9 @@ int main( int argc, char* argv[])
     };
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
-#ifdef FELTOR_MPI
-    MPI_Info info = MPI_INFO_NULL;
-    err = nc_create_par( argv[3], NC_NETCDF4|NC_MPIIO|NC_CLOBBER, comm, info, &ncid);
-#else //FELTOR_MPI
-    err = nc_create( argv[3],NC_NETCDF4|NC_CLOBBER, &ncid);
-#endif //FELTOR_MPI
+    std::string file_name = argv[3];
+    int ncid=-1;
+    MPI_OUT err = nc_create( file_name.data(), NC_NETCDF4|NC_CLOBBER, &ncid);
     /// Set global attributes
     std::map<std::string, std::string> att;
     att["title"] = "Output file of feltor/src/feltor_hpc.cu";
@@ -381,21 +369,17 @@ int main( int argc, char* argv[])
     att["inputfile"] = input;
     att["geomfile"] = geom;
     for( auto pair : att)
-        err = nc_put_att_text( ncid, NC_GLOBAL,
+        MPI_OUT err = nc_put_att_text( ncid, NC_GLOBAL,
             pair.first.data(), pair.second.size(), pair.second.data());
 
     int dim_ids[4], tvarID;
 #ifdef FELTOR_MPI
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
-    err = nc_var_par_access( ncid, tvarID, NC_COLLECTIVE);
-    int dims[3],  coords[3];
-    MPI_Cart_get( comm, 3, dims, periods, coords);
+    int coords[3];
+    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
+    size_t start[4] = {0, 0, 0, 0};
     size_t count[4] = {1, grid_out.local().Nz(),
         grid_out.n()*(grid_out.local().Ny()),
         grid_out.n()*(grid_out.local().Nx())};
-    size_t start[4] = {0, coords[2]*count[1],
-                          coords[1]*count[2],
-                          coords[0]*count[3]};
 #else //FELTOR_MPI
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     size_t start[4] = {0, 0, 0, 0};
@@ -436,50 +420,55 @@ int main( int argc, char* argv[])
         for( auto tp : v3d)
         {
             int vecID;
-            err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 3,
+            MPI_OUT err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 3,
                 &dim_ids[1], &vecID);
-            err = nc_put_att_text( ncid, vecID,
+            MPI_OUT err = nc_put_att_text( ncid, vecID,
                 "long_name", std::get<2>(tp).size(), std::get<2>(tp).data());
-            #ifdef FELTOR_MPI
-            err = nc_var_par_access( ncid, vecID, NC_COLLECTIVE);
-            #endif //FELTOR_MPI
-            err = nc_enddef( ncid);
+            MPI_OUT err = nc_enddef( ncid);
             dg::blas2::symv( project, *std::get<1>(tp), transferH);
+#ifdef FELTOR_MPI
+            if(rank==0)
+            {
+                for( int rrank=0; rrank<size; rrank++)
+                {
+                    if(rrank!=0)
+                        MPI_Recv( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
+                              rrank, rrank, grid_out.communicator(), &status);
+                    MPI_Cart_coords( grid_out.communicator(), rrank, 3, coords);
+                    start[1] = coords[2]*count[1],
+                    start[2] = coords[1]*count[2],
+                    start[3] = coords[0]*count[3];
+                    err = nc_put_vara_double( ncid, vecID, &start[1], &count[1],
+                        transferH.data().data());
+                }
+            }
+            else
+                MPI_Send( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
+                          0, rank, grid_out.communicator());
+            MPI_Barrier( grid_out.communicator());
+#else
             err = nc_put_vara_double( ncid, vecID, &start[1], &count[1],
-                #ifdef FELTOR_MPI
-                transferH.data().data()
-                #else //FELTOR_MPI
-                transferH.data()
-                #endif //FELTOR_MPI
-            );
-            err = nc_redef(ncid);
+                transferH.data());
+#endif // FELTOR_MPI
+            MPI_OUT err = nc_redef(ncid);
         }
     }
 
     //field IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
-#ifdef FELTOR_MPI
-    err = nc_var_par_access( ncid, EtimevarID, NC_COLLECTIVE);
-#endif //FELTOR_MPI
+    MPI_OUT err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     std::map<std::string, int> id0d, id4d;
     for( auto pair : v0d)
     {
-        err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 1,
+        MPI_OUT err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 1,
             &EtimeID, &id0d[pair.first]);
-#ifdef FELTOR_MPI
-        err = nc_var_par_access( ncid, id0d[pair.first], NC_COLLECTIVE);
-#endif //FELTOR_MPI
     }
     for( auto pair : v4d)
     {
-        err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 4,
+        MPI_OUT err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 4,
             dim_ids, &id4d[pair.first]);
-#ifdef FELTOR_MPI
-        err = nc_var_par_access( ncid, id4d[pair.first], NC_COLLECTIVE);
-#endif //FELTOR_MPI
     }
-    err = nc_enddef(ncid);
+    MPI_OUT err = nc_enddef(ncid);
     ///////////////////////////////////first output/////////////////////////
     double dt_new = p.dt;//, dt =0;
     MPI_OUT std::cout << "First output ... \n";
@@ -491,7 +480,7 @@ int main( int argc, char* argv[])
         } catch( dg::Fail& fail) {
             MPI_OUT std::cerr << "CG failed to converge in first step to "
                               <<fail.epsilon()<<"\n";
-            err = nc_close(ncid);
+            MPI_OUT err = nc_close(ncid);
             return -1;
         }
         feltor.update_quantities();
@@ -505,25 +494,40 @@ int main( int argc, char* argv[])
     {
         dg::blas2::symv( project, *pair.second, transferD);
         dg::assign( transferD, transferH);
-        err = nc_put_vara_double( ncid, id4d.at(pair.first), start, count,
-            #ifdef FELTOR_MPI
-            transferH.data().data()
-            #else //FELTOR_MPI
-            transferH.data()
-            #endif //FELTOR_MPI
-        );
+#ifdef FELTOR_MPI
+            if(rank==0)
+            {
+                for( int rrank=0; rrank<size; rrank++)
+                {
+                    if(rrank!=0)
+                        MPI_Recv( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
+                              rrank, rrank, grid_out.communicator(), &status);
+                    MPI_Cart_coords( grid_out.communicator(), rrank, 3, coords);
+                    start[1] = coords[2]*count[1],
+                    start[2] = coords[1]*count[2],
+                    start[3] = coords[0]*count[3];
+                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start, count,
+                        transferH.data().data());
+                }
+            }
+            else
+                MPI_Send( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
+                          0, rank, grid_out.communicator());
+            MPI_Barrier( grid_out.communicator());
+#else
+            err = nc_put_vara_double( ncid, id4d.at(pair.first), start, count,
+                transferH.data());
+#endif // FELTOR_MPI
     }
-    err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-    err = nc_put_vara_double( ncid, EtimevarID, start, count, &time);
+    MPI_OUT err = nc_put_vara_double( ncid, tvarID, start, count, &time);
+    MPI_OUT err = nc_put_vara_double( ncid, EtimevarID, start, count, &time);
 
     size_t Estart[] = {0};
     size_t Ecount[] = {1};
     for( auto pair : v0d)
-        err = nc_put_vara_double( ncid, id0d.at(pair.first),
+        MPI_OUT err = nc_put_vara_double( ncid, id0d.at(pair.first),
             Estart, Ecount, pair.second);
-#ifndef FELTOR_MPI
-    err = nc_close(ncid);
-#endif //FELTOR_MPI
+    MPI_OUT err = nc_close(ncid);
     MPI_OUT std::cout << "First write successful!\n";
     ///////////////////////////////////////Timeloop/////////////////////////////////
     dg::Karniadakis< std::array<std::array<DVec,2>,2 >,
@@ -531,15 +535,9 @@ int main( int argc, char* argv[])
             Geometry, IDMatrix, DMatrix, DVec>
         > karniadakis( grid, p, mag);
     karniadakis.init( feltor, im, time, y0, p.dt);
-    //dg::Adaptive< dg::ERKStep<std::array<std::array<DVec,2>,2>> > adaptive(
-    //    "Bogacki-Shampine-4-2-3", y0);
-    //adaptive.stepper().ignore_fsal();//necessary for splitting
-    //dg::ImplicitRungeKutta<std::array<std::array<DVec,2>,2>,
-    //    feltor::FeltorSpecialSolver< Geometry, IDMatrix, DMatrix, DVec>> dirk(
-    //        "Trapezoidal-2-2", grid, p, mag);
     dg::Timer t;
     t.tic();
-    unsigned step = 0;//, failed_counter = 0;
+    unsigned step = 0;
     MPI_OUT q.display(std::cout);
     for( unsigned i=1; i<=p.maxout; i++)
     {
@@ -553,29 +551,10 @@ int main( int argc, char* argv[])
             {
                 try{
                     karniadakis.step( feltor, im, time, y0);
-                    //do
-                    //{
-                    //    //Strang splitting
-                    //    dt = dt_new;
-                    //    dirk.step( im, time, y0, time, y0, dt/2.);
-                    //    adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
-                    //        dg::pid_control, dg::l2norm, p.rtol, 1e-10);
-                    //    if( adaptive.failed())
-                    //    {
-                    //        failed_counter++;
-                    //        MPI_OUT std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
-                    //        time -= dt; // time has to be reset here
-                    //        // in case of failure diffusion is applied twice?
-                    //    }
-                    //}while ( adaptive.failed());
-                    //dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
                 }
                 catch( dg::Fail& fail) {
                     MPI_OUT std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
                     MPI_OUT std::cerr << "Does Simulation respect CFL condition?\n";
-                    #ifdef FELTOR_MPI
-                    err = nc_close(ncid);
-                    #endif //FELTOR_MPI
                     return -1;
                 }
                 step++;
@@ -587,18 +566,14 @@ int main( int argc, char* argv[])
             E0 = *v0d["energy"], M0 = *v0d["mass"];
             accuracy  = 2.*fabs( (dEdt - *v0d["ediff"])/( dEdt + *v0d["ediff"]));
             accuracyM = 2.*fabs( (dMdt - *v0d["diff"])/( dMdt + *v0d["diff"]));
-            #ifndef FELTOR_MPI
-            err = nc_open(argv[3], NC_WRITE, &ncid);
-            //maybe MPI should use nc_open_par ? Or is problem if the ids are still the same?
-            #endif //FELTOR_MPI
+            MPI_OUT err = nc_open(file_name.data(), NC_WRITE, &ncid);
             Estart[0]++;
-            err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
+            MPI_OUT err = nc_put_vara_double( ncid, EtimevarID,
+                Estart, Ecount, &time);
             for( auto pair : v0d)
-                err = nc_put_vara_double( ncid, id0d.at(pair.first),
+                MPI_OUT err = nc_put_vara_double( ncid, id0d.at(pair.first),
                     Estart, Ecount, pair.second);
-            #ifndef FELTOR_MPI
-            err = nc_close(ncid);
-            #endif //FELTOR_MPI
+            MPI_OUT err = nc_close(ncid);
 
             MPI_OUT q.display(std::cout);
             MPI_OUT std::cout << "(m_tot-m_0)/m_0: "<< (*v0d["mass"]-mass0)/mass0<<"\t";
@@ -627,30 +602,41 @@ int main( int argc, char* argv[])
                     << p.inner_loop*p.itstp*p.maxout << " at time "<<time;
         MPI_OUT std::cout << "\n\t Average time for one step: "
                     << ti.diff()/(double)p.itstp/(double)p.inner_loop<<"s";
-        //MPI_OUT std::cout << "\n\t Total number of failed steps: "
-        //            << failed_counter;
         ti.tic();
         //////////////////////////write fields////////////////////////
         start[0] = i;
-        #ifndef FELTOR_MPI
-        err = nc_open(argv[3], NC_WRITE, &ncid);
-        #endif //FELTOR_MPI
-        err = nc_put_vara_double( ncid, tvarID, start, count, &time);
+        MPI_OUT err = nc_open(file_name.data(), NC_WRITE, &ncid);
+        MPI_OUT err = nc_put_vara_double( ncid, tvarID, start, count, &time);
         for( auto pair : v4d)
         {
             dg::blas2::symv( project, *pair.second, transferD);
             dg::assign( transferD, transferH);
+#ifdef FELTOR_MPI
+            if(rank==0)
+            {
+                for( int rrank=0; rrank<size; rrank++)
+                {
+                    if(rrank!=0)
+                        MPI_Recv( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
+                              rrank, rrank, grid_out.communicator(), &status);
+                    MPI_Cart_coords( grid_out.communicator(), rrank, 3, coords);
+                    start[1] = coords[2]*count[1],
+                    start[2] = coords[1]*count[2],
+                    start[3] = coords[0]*count[3];
+                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start, count,
+                        transferH.data().data());
+                }
+            }
+            else
+                MPI_Send( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
+                          0, rank, grid_out.communicator());
+            MPI_Barrier( grid_out.communicator());
+#else
             err = nc_put_vara_double( ncid, id4d.at(pair.first), start, count,
-                #ifdef FELTOR_MPI
-                transferH.data().data()
-                #else //FELTOR_MPI
-                transferH.data()
-                #endif //FELTOR_MPI
-            );
+                transferH.data());
+#endif // FELTOR_MPI
         }
-        #ifndef FELTOR_MPI
-        err = nc_close(ncid);
-        #endif //FELTOR_MPI
+        MPI_OUT err = nc_close(ncid);
         ti.toc();
         MPI_OUT std::cout << "\n\t Time for output: "<<ti.diff()<<"s\n\n"<<std::flush;
     }
@@ -662,7 +648,6 @@ int main( int argc, char* argv[])
     MPI_OUT std::cout <<"Computation Time \t"<<hour<<":"<<std::setw(2)<<minute<<":"<<second<<"\n";
     MPI_OUT std::cout <<"which is         \t"<<t.diff()/p.itstp/p.maxout/p.inner_loop<<"s/step\n";
 #ifdef FELTOR_MPI
-    err = nc_close(ncid);
     MPI_Finalize();
 #endif //FELTOR_MPI
 
-- 
GitLab


From 4e46462c323542ce6edc706f8dd757c0b2858620 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 19 Jun 2019 13:40:36 +0200
Subject: [PATCH 092/540] Implement 2d performance handle in Elliptic3d

This increases effective bandwidth by approx 30% and reduces
latency by a factor 5 for GPUs since we can avoid all communication
in x-y direction by only parallelizing z
---
 inc/dg/elliptic.h      | 33 ++++++++++++++++++++++++++-------
 inc/dg/elliptic2d_b.cu |  5 ++++-
 src/feltor/feltor.cuh  | 23 ++++++++++++++++++-----
 src/feltor/implicit.h  |  6 ++++++
 4 files changed, 54 insertions(+), 13 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 86760e35a..7c6c7b26e 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -429,6 +429,17 @@ class Elliptic3d
     ///@copydoc Elliptic::get_jfactor()
     value_type get_jfactor() const {return m_jfactor;}
 
+    /**
+     * @brief Restrict the problem to the first 2 dimensions
+     *
+     * This effectively makes the behaviour of dg::Elliptic3d
+     * identical to the dg::Elliptic class.
+     * @param compute_in_2d if true, the symv function avoids the derivative in z, false reverts to the original behaviour.
+     */
+    void set_compute_in_2d( bool compute_in_2d ) {
+        m_multiplyZ = !compute_in_2d;
+    }
+
     ///@copydoc Elliptic::symv(const ContainerType0&,ContainerType1&)
     template<class ContainerType0, class ContainerType1>
     void symv( const ContainerType0& x, ContainerType1& y){
@@ -441,14 +452,21 @@ class Elliptic3d
         //compute gradient
         dg::blas2::gemv( m_rightx, x, m_tempx); //R_x*f
         dg::blas2::gemv( m_righty, x, m_tempy); //R_y*f
-        dg::blas2::gemv( m_rightz, x, m_tempz); //R_z*f
-
-        //multiply with tensor (note the alias)
-        dg::tensor::multiply3d(m_chi, m_tempx, m_tempy, m_tempz, m_tempx, m_tempy, m_tempz);
+        if( m_multiplyZ )
+        {
+            dg::blas2::gemv( m_rightz, x, m_tempz); //R_z*f
 
-        //now take divergence
-        dg::blas2::symv( -1., m_leftz, m_tempz, 0., m_temp);
-        dg::blas2::symv( -1., m_lefty, m_tempy, 1., m_temp);
+            //multiply with tensor (note the alias)
+            dg::tensor::multiply3d(m_chi, m_tempx, m_tempy, m_tempz, m_tempx, m_tempy, m_tempz);
+            //now take divergence
+            dg::blas2::symv( -1., m_leftz, m_tempz, 0., m_temp);
+            dg::blas2::symv( -1., m_lefty, m_tempy, 1., m_temp);
+        }
+        else
+        {
+            dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
+            dg::blas2::symv( -1.,m_lefty, m_tempy, 0., m_temp);
+        }
         dg::blas2::symv( -1., m_leftx, m_tempx, 1., m_temp);
 
         //add jump terms
@@ -482,6 +500,7 @@ class Elliptic3d
     SparseTensor<Container> m_chi;
     Container m_sigma, m_vol;
     value_type m_jfactor;
+    bool m_multiplyZ = true;
 };
 ///@cond
 template< class G, class M, class V>
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 2f7762f2d..30cb3d183 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -136,7 +136,10 @@ int main()
     }
 
     {
-		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
+        //try the compute_in_2d handle of Elliptic3d
+	    dg::CartesianGrid3d grid( 0, lx, 0, ly, 0,1,n, Nx, Ny, 1, bcx, bcy, dg::PER);
+		dg::Elliptic3d<dg::CartesianGrid3d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
+        pol_backward.set_compute_in_2d(true);
 		pol_backward.set_chi( chi);
 		x = temp;
 		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index d4515a6b2..e0d595b83 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -547,6 +547,12 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
     m_lapperpN.set_chi( m_hh);
     m_lapperpU.set_chi( m_hh);
     m_lapperpP.set_chi( m_hh);
+    if( p.curvmode != "true")
+    {
+        m_lapperpN.set_compute_in_2d(true);
+        m_lapperpU.set_compute_in_2d(true);
+        m_lapperpP.set_compute_in_2d(true);
+    }
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
 void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
@@ -563,21 +569,28 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
     m_multi_induction.resize(p.stages);
     for( unsigned u=0; u<p.stages; u++)
     {
-        dg::SparseTensor<Container> hh = dg::geo::createProjectionTensor(
-            bhat, m_multigrid.grid(u));
         m_multi_pol[u].construct( m_multigrid.grid(u),
             p.bcxP, p.bcyP, dg::PER, dg::not_normed,
             dg::centered, p.jfactor);
-        m_multi_pol[u].set_chi( hh);
         m_multi_invgammaP[u].construct(  m_multigrid.grid(u),
             p.bcxP, p.bcyP, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered);
-        m_multi_invgammaP[u].elliptic().set_chi( hh);
         m_multi_invgammaN[u].construct(  m_multigrid.grid(u),
             p.bcxN, p.bcyN, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered);
-        m_multi_invgammaN[u].elliptic().set_chi( hh);
         m_multi_induction[u].construct(  m_multigrid.grid(u),
             p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
+
+        dg::SparseTensor<Container> hh = dg::geo::createProjectionTensor(
+            bhat, m_multigrid.grid(u));
+        m_multi_pol[u].set_chi( hh);
+        m_multi_invgammaP[u].elliptic().set_chi( hh);
+        m_multi_invgammaN[u].elliptic().set_chi( hh);
         m_multi_induction[u].elliptic().set_chi( hh);
+        if(p.curvmode != "true"){
+            m_multi_pol[u].set_compute_in_2d( true);
+            m_multi_invgammaP[u].elliptic().set_compute_in_2d( true);
+            m_multi_invgammaN[u].elliptic().set_compute_in_2d( true);
+            m_multi_induction[u].elliptic().set_compute_in_2d( true);
+        }
     }
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index d3149f8e1..754511e6a 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -53,6 +53,8 @@ struct ImplicitDensity
             = dg::geo::createProjectionTensor( bhat, g);
         //set perpendicular projection tensor h
         m_lapM_perpN.set_chi( hh);
+        if( p.curvmode != "true")
+            m_lapM_perpN.set_compute_in_2d( true);
     }
 
     void operator()( double t, const std::array<Container,2>& y,
@@ -119,6 +121,8 @@ struct ImplicitVelocity
             = dg::geo::createProjectionTensor( bhat, g);
         //set perpendicular projection tensor h
         m_lapM_perpU.set_chi( hh);
+        if( p.curvmode != "true")
+            m_lapM_perpU.set_compute_in_2d(true);
         //m_induction.construct(  g,
         //    p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
         //m_induction.elliptic().set_chi( hh);
@@ -133,6 +137,8 @@ struct ImplicitVelocity
             m_multi_induction[u].construct(  m_multigrid.grid(u),
                 p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
             m_multi_induction[u].elliptic().set_chi( hh);
+            if( p.curvmode != "true")
+                m_multi_induction[u].elliptic().set_compute_in_2d(true);
         }
         m_multi_chi = m_multigrid.project( m_temp);
         m_old_apar = dg::Extrapolation<Container>( 1, dg::evaluate( dg::zero, g));
-- 
GitLab


From 36d62bfa738ee1636580a2b8b8a28b1c913eefd0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 24 Jun 2019 13:32:53 +0200
Subject: [PATCH 093/540] Change default Bath parameters in feltor

To avoid structures that are too small
---
 diag/feltordiag.cu              |  2 +-
 inc/dg/multistep.h              | 92 ++++++++++++++++-----------------
 inc/geometries/geometry_diag.cu |  2 +-
 src/feltor/feltor.cu            |  2 +-
 src/feltor/feltor_hpc.cu        |  2 +-
 5 files changed, 50 insertions(+), 50 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index fcc164f46..168371f01 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -443,7 +443,7 @@ int main( int argc, char* argv[])
 
 
             // 2d data of plane varphi = 0
-            unsigned kmp = 0; //g3d_out.Nz()/2;
+            unsigned kmp = g3d_out.Nz()/2;
             dg::HVec t2d_mp(t3d.begin() + kmp*g2d_out.size(),
                 t3d.begin() + (kmp+1)*g2d_out.size() );
             err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_2d"),
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 394441352..077ecb4dc 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -68,8 +68,8 @@ struct AdamsBashforth
     */
     void construct(unsigned order, const ContainerType& copyable){
         m_k = order;
-        f_.assign( order, copyable);
-        u_ = copyable;
+        m_f.assign( order, copyable);
+        m_u = copyable;
         m_ab.resize( order);
         switch (order){
             case 1: m_ab = {1}; break;
@@ -82,7 +82,7 @@ struct AdamsBashforth
     }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
-    const ContainerType& copyable()const{ return u_;}
+    const ContainerType& copyable()const{ return m_u;}
 
     /**
      * @brief Initialize first step. Call before using the step function.
@@ -111,9 +111,9 @@ struct AdamsBashforth
     template< class RHS>
     void step( RHS& f, value_type& t, ContainerType& u);
   private:
-    value_type tu_, dt_;
-    std::vector<ContainerType> f_;
-    ContainerType u_;
+    value_type m_tu, m_dt;
+    std::vector<ContainerType> m_f;
+    ContainerType m_u;
     std::vector<value_type> m_ab;
     unsigned m_k;
 };
@@ -122,20 +122,20 @@ template< class ContainerType>
 template< class RHS>
 void AdamsBashforth<ContainerType>::init( RHS& f, value_type t0, const ContainerType& u0, value_type dt)
 {
-    tu_ = t0, dt_ = dt;
-    f( t0, u0, f_[0]);
+    m_tu = t0, m_dt = dt;
+    f( t0, u0, m_f[0]);
     //now do k Euler steps
     ContainerType u1(u0);
     for( unsigned i=1; i<m_k; i++)
     {
-        blas1::axpby( 1., u1, -dt, f_[i-1], u1);
-        tu_ -= dt;
-        f( tu_, u1, f_[i]);
+        blas1::axpby( 1., u1, -dt, m_f[i-1], u1);
+        m_tu -= dt;
+        f( m_tu, u1, m_f[i]);
     }
-    tu_ = t0;
-    blas1::copy(  u0, u_);
+    m_tu = t0;
+    blas1::copy(  u0, m_u);
     //finally evaluate f at u0 once more to set state in f
-    f( tu_, u_, f_[0]);
+    f( m_tu, m_u, m_f[0]);
 }
 
 template<class ContainerType>
@@ -143,13 +143,13 @@ template< class RHS>
 void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType& u)
 {
     for( unsigned i=0; i<m_k; i++)
-        blas1::axpby( dt_*m_ab[i], f_[i], 1., u_);
-    //permute f_[k-1]  to be the new f_[0]
+        blas1::axpby( m_dt*m_ab[i], m_f[i], 1., m_u);
+    //permute m_f[k-1]  to be the new m_f[0]
     for( unsigned i=m_k-1; i>0; i--)
-        f_[i-1].swap( f_[i]);
-    blas1::copy( u_, u);
-    t = tu_ = tu_ + dt_;
-    f( tu_, u_, f_[0]); //evaluate f at new point
+        m_f[i-1].swap( m_f[i]);
+    blas1::copy( m_u, u);
+    t = m_tu = m_tu + m_dt;
+    f( m_tu, m_u, m_f[0]); //evaluate f at new point
 }
 
 /**
@@ -199,7 +199,7 @@ struct Karniadakis
     ///@copydoc construct()
     template<class ...SolverParams>
     Karniadakis( SolverParams&& ...ps):m_solver( std::forward<SolverParams>(ps)...){
-        f_.fill(m_solver.copyable()), u_.fill(m_solver.copyable());
+        m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
         init_coeffs();
     }
     /**
@@ -211,12 +211,12 @@ struct Karniadakis
     template<class ...SolverParams>
     void construct( SolverParams&& ...ps){
         m_solver = Solver( std::forward<SolverParams>(ps)...);
-        f_.fill(m_solver.copyable()), u_.fill(m_solver.copyable());
+        m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
         init_coeffs();
     }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
-    const ContainerType& copyable()const{ return u_[0];}
+    const ContainerType& copyable()const{ return m_u[0];}
 
     ///Write access to the internal solver for the implicit part
     SolverType& solver() { return m_solver;}
@@ -257,8 +257,8 @@ struct Karniadakis
         a[2] = 2./11.;      b[2] = 6./11.;   //Karniadakis !!!
     }
     SolverType m_solver;
-    std::array<ContainerType,3> u_, f_;
-    value_type t_, dt_;
+    std::array<ContainerType,3> m_u, m_f;
+    value_type t_, m_dt;
     value_type a[3];
     value_type b[3], g0 = 6./11.;
 };
@@ -269,41 +269,41 @@ template< class RHS, class Diffusion>
 void Karniadakis<ContainerType, SolverType>::init( RHS& f, Diffusion& diff, value_type t0, const ContainerType& u0, value_type dt)
 {
     //operator splitting using explicit Euler for both explicit and implicit part
-    t_ = t0, dt_ = dt;
-    blas1::copy(  u0, u_[0]);
-    f( t0, u0, f_[0]); //f may not destroy u0
-    blas1::axpby( 1., u_[0], -dt, f_[0], f_[1]); //Euler step
+    t_ = t0, m_dt = dt;
+    blas1::copy(  u0, m_u[0]);
+    f( t0, u0, m_f[0]); //f may not destroy u0
+    blas1::axpby( 1., m_u[0], -dt, m_f[0], m_f[1]); //Euler step
     detail::Implicit<Diffusion, ContainerType> implicit( -dt, t0, diff);
-    implicit( f_[1], u_[1]); //explicit Euler step backwards
-    f( t0-dt, u_[1], f_[1]);
-    blas1::axpby( 1.,u_[1], -dt, f_[1], f_[2]);
+    implicit( m_f[1], m_u[1]); //explicit Euler step backwards
+    f( t0-dt, m_u[1], m_f[1]);
+    blas1::axpby( 1.,m_u[1], -dt, m_f[1], m_f[2]);
     implicit.time() = t0 - dt;
-    implicit( f_[2], u_[2]);
-    f( t0-2*dt, u_[2], f_[2]); //evaluate f at the latest step
-    f( t0, u0, f_[0]); // and set state in f to (t0,u0)
+    implicit( m_f[2], m_u[2]);
+    f( t0-2*dt, m_u[2], m_f[2]); //evaluate f at the latest step
+    f( t0, u0, m_f[0]); // and set state in f to (t0,u0)
 }
 
 template<class ContainerType, class SolverType>
 template< class RHS, class Diffusion>
 void Karniadakis<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, value_type& t, ContainerType& u)
 {
-    blas1::axpbypgz( dt_*b[0], f_[0], dt_*b[1], f_[1], dt_*b[2], f_[2]);
-    blas1::axpbypgz( a[0], u_[0], a[1], u_[1], a[2], u_[2]);
-    //permute f_[2], u_[2]  to be the new f_[0], u_[0]
+    blas1::axpbypgz( m_dt*b[0], m_f[0], m_dt*b[1], m_f[1], m_dt*b[2], m_f[2]);
+    blas1::axpbypgz( a[0], m_u[0], a[1], m_u[1], a[2], m_u[2]);
+    //permute m_f[2], m_u[2]  to be the new m_f[0], m_u[0]
     for( unsigned i=2; i>0; i--)
     {
-        f_[i-1].swap( f_[i]);
-        u_[i-1].swap( u_[i]);
+        m_f[i-1].swap( m_f[i]);
+        m_u[i-1].swap( m_u[i]);
     }
-    blas1::axpby( 1., f_[0], 1., u_[0]);
+    blas1::axpby( 1., m_f[0], 1., m_u[0]);
     //compute implicit part
     value_type alpha[2] = {2., -1.};
     //value_type alpha[2] = {1., 0.};
-    blas1::axpby( alpha[0], u_[1], alpha[1],  u_[2], u); //extrapolate previous solutions
-    t = t_ = t_+ dt_;
-    m_solver.solve( -dt_*g0, diff, t, u, u_[0]);
-    blas1::copy( u, u_[0]); //store result
-    f(t_, u_[0], f_[0]); //call f on new point
+    blas1::axpby( alpha[0], m_u[1], alpha[1],  m_u[2], u); //extrapolate previous solutions
+    t = t_ = t_+ m_dt;
+    m_solver.solve( -m_dt*g0, diff, t, u, m_u[0]);
+    blas1::copy( u, m_u[0]); //store result
+    f(t_, m_u[0], m_f[0]); //call f on new point
 }
 ///@endcond
 
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 3786699ab..08a3d7e81 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -234,7 +234,7 @@ int main( int argc, char* argv[])
         {"Delta", "A flux aligned delta function", dg::geo::DeltaFunction( mag, gp.alpha*gp.alpha, psip0*0.2)},
         {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::geo::TanhDamping(mag.psip(), -3*gp.alpha, gp.alpha, -1)},
         ////
-        {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,5., p.amp)},
+        {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
         {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
             M_PI, p.sigma, p.sigma, p.sigma, p.amp)}
     };
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 5c7975873..62ff80690 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -107,7 +107,7 @@ int main( int argc, char* argv[])
     else if( p.initne == "turbulence")
     {
         dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-        dg::BathRZ init0(16,16,Rmin,Zmin, 30.,5.,p.amp);
+        dg::BathRZ init0(16,16,Rmin,Zmin, 30.,2.,p.amp);
         if( p.symmetric)
             ntilde = dg::pullback( init0, grid);
         else
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index a693396fd..d048cdcf8 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -208,7 +208,7 @@ int main( int argc, char* argv[])
         else if( p.initne == "turbulence")
         {
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-            dg::BathRZ init0(16,16,Rmin,Zmin, 30.,5.,p.amp);
+            dg::BathRZ init0(16,16,Rmin,Zmin, 30.,2.,p.amp);
             if( p.symmetric)
                 ntilde = dg::pullback( init0, grid);
             else
-- 
GitLab


From e111c2d4b990dee91ed990e8ab7deb9e386bbb9b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 24 Jun 2019 23:04:52 +0200
Subject: [PATCH 094/540] First try at Quadrature class SimpsonsRule

---
 inc/dg/simpsons.h | 74 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 inc/dg/simpsons.h

diff --git a/inc/dg/simpsons.h b/inc/dg/simpsons.h
new file mode 100644
index 000000000..a2d64df79
--- /dev/null
+++ b/inc/dg/simpsons.h
@@ -0,0 +1,74 @@
+#pragma once
+
+#include "blas1.h"
+
+/*! @file
+ * @brief Equidistant time-integrators
+ */
+
+namespace dg{
+
+template<class ContainerType>
+struct SimpsonsRule
+{
+    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
+    using container_type = ContainerType; //!< the type of the vector class in use
+    SimpsonsRule() = default;
+    SimpsonsRule( const ContainerType& copyable): m_integral( m_integral), m_head( m_head)
+    {
+    }
+    const ContainerType& copyable() const{ return m_integral;}
+    void init(  value_type t0, const ContainerType& u0) {
+        dg::blas1::copy( u0, m_u0);
+        m_t[0] = t0;
+        flush();
+    }
+    const ContainerType& flush() {
+        m_counter = 0;
+        dg::blas1::scal( m_integral, 0.);
+    }
+    void add( value_type t_new, const ContainerType& u_new){
+        if( t_new < m_t0)
+            throw dg::Error(dg::Message()<<"New time must be strictly larger than old time!");
+        if( m_counter % 2 == 0)
+        {
+            //Trapezoidal rule
+            dg::blas1::axpbypgz( 0.5/(t_new - m_t[0]), u_new,
+                0.5/(t_new - m_t[0]), m_u[0] , 1., m_integral);
+            m_t[1] = m_t[0];
+            m_u[1].swap( m_u[0]);
+            m_t[0] = t_new;
+            dg::blas1::copy( u_new, m_u[0]);
+        }
+        else
+        {
+            //Simpson's rule
+            value_type t0 = m_t[1], t1 = m_t[0], t2 = t_new;
+            value_type pre = (t2-t0)/6./(t0-t1)/(t1-t2);
+            value_type pre2 = t0*t0+3.*t1*t1+2.*t0*t2-4.*t0*t1-2.*t1*t2;
+            value_type pre1 = -(t0-t2)*(t0-t2);
+            value_type pre0 = t2*t2+3.*t1*t1+2.*t0*t2-4.*t1*t2-2.*t0*t1;
+
+            dg::blas1::axpby( pre2/pre, u_new, 1., m_integral);
+            //Also subtract old Trapezoidal rule
+            dg::blas1::axpbypgz(
+                pre1/pre-0.5/(m_t[0] - m_t[1]), m_u[0],
+                pre0/pre-0.5/(m_t[0] - m_t[1]), m_u[1],
+                1., m_integral);
+            m_t[1] = m_t[0];
+            m_u[1].swap( m_u[0]);
+            m_t[0] = t_new;
+            dg::blas1::copy( u_new, m_u[0]);
+        }
+        m_counter++;
+    }
+    const ContainerType& get_sum() const{
+        return m_integral;
+    }
+    private:
+    unsigned m_counter;
+    ContainerType m_integral;
+    std::vector<value_type> m_t;
+    std::vector<ContainerType> m_u;
+};
+}//namespace dg
-- 
GitLab


From 67c70525b4ae1f1f71675941082f6323a5b388da Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 25 Jun 2019 16:02:53 +0200
Subject: [PATCH 095/540] Add test file for Simpsons algorithm

---
 inc/dg/simpsons.h    | 54 +++++++++++++++++++++++++++-----------------
 inc/dg/simpsons_t.cu | 35 ++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 21 deletions(-)
 create mode 100644 inc/dg/simpsons_t.cu

diff --git a/inc/dg/simpsons.h b/inc/dg/simpsons.h
index a2d64df79..86ad4aca6 100644
--- a/inc/dg/simpsons.h
+++ b/inc/dg/simpsons.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <list>
+#include "backend/exceptions.h"
 #include "blas1.h"
 
 /*! @file
@@ -14,51 +16,60 @@ struct SimpsonsRule
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
     SimpsonsRule() = default;
-    SimpsonsRule( const ContainerType& copyable): m_integral( m_integral), m_head( m_head)
+    SimpsonsRule( const ContainerType& copyable):
+        m_counter(0), m_integral( copyable),
+        m_t(2,0.), m_u( 2, copyable)
     {
     }
     const ContainerType& copyable() const{ return m_integral;}
     void init(  value_type t0, const ContainerType& u0) {
-        dg::blas1::copy( u0, m_u0);
-        m_t[0] = t0;
+        dg::blas1::copy( u0, m_u.front());
+        m_t.front() = t0;
         flush();
     }
-    const ContainerType& flush() {
+    void flush() {
         m_counter = 0;
         dg::blas1::scal( m_integral, 0.);
     }
     void add( value_type t_new, const ContainerType& u_new){
-        if( t_new < m_t0)
+        std::cout << "Time "<<t_new<<" Value "<<u_new<<"\n";
+        if( t_new < m_t.front())
             throw dg::Error(dg::Message()<<"New time must be strictly larger than old time!");
+        auto pt0 = m_t.begin();
+        auto pt1 = std::next( pt0);
+        auto pu0 = m_u.begin();
+        auto pu1 = std::next( pu0);
+        //if(true)
         if( m_counter % 2 == 0)
         {
             //Trapezoidal rule
-            dg::blas1::axpbypgz( 0.5/(t_new - m_t[0]), u_new,
-                0.5/(t_new - m_t[0]), m_u[0] , 1., m_integral);
-            m_t[1] = m_t[0];
-            m_u[1].swap( m_u[0]);
-            m_t[0] = t_new;
-            dg::blas1::copy( u_new, m_u[0]);
+            dg::blas1::axpbypgz( 0.5*(t_new - *pt0), u_new,
+                0.5*(t_new - *pt0), *pu0 , 1., m_integral);
+            m_t.splice( pt0, m_t, pt1, m_t.end());
+            m_u.splice( pu0, m_u, pu1, m_u.end());
+            m_t.front() = t_new; //and now remove zeroth element (it is moved now!)
+            dg::blas1::copy( u_new, m_u.front());
         }
         else
         {
             //Simpson's rule
-            value_type t0 = m_t[1], t1 = m_t[0], t2 = t_new;
+            value_type t0 = *pt1, t1 = *pt0, t2 = t_new;
             value_type pre = (t2-t0)/6./(t0-t1)/(t1-t2);
             value_type pre2 = t0*t0+3.*t1*t1+2.*t0*t2-4.*t0*t1-2.*t1*t2;
             value_type pre1 = -(t0-t2)*(t0-t2);
             value_type pre0 = t2*t2+3.*t1*t1+2.*t0*t2-4.*t1*t2-2.*t0*t1;
+            std::cout << "pre "<<pre*pre0<<" "<<pre*pre1<<" "<<pre*pre2<<std::endl;
 
-            dg::blas1::axpby( pre2/pre, u_new, 1., m_integral);
+            dg::blas1::axpby( pre2*pre, u_new, 1., m_integral);
             //Also subtract old Trapezoidal rule
             dg::blas1::axpbypgz(
-                pre1/pre-0.5/(m_t[0] - m_t[1]), m_u[0],
-                pre0/pre-0.5/(m_t[0] - m_t[1]), m_u[1],
+                pre1*pre-0.5*(t1-t0), *pu0,
+                pre0*pre-0.5*(t1-t0), *pu1,
                 1., m_integral);
-            m_t[1] = m_t[0];
-            m_u[1].swap( m_u[0]);
-            m_t[0] = t_new;
-            dg::blas1::copy( u_new, m_u[0]);
+            m_t.splice( pt0, m_t, pt1, m_t.end());
+            m_u.splice( pu0, m_u, pu1, m_u.end());
+            m_t.front() = t_new; //and now remove zeroth element (it is moved now!)
+            dg::blas1::copy( u_new, m_u.front());
         }
         m_counter++;
     }
@@ -68,7 +79,8 @@ struct SimpsonsRule
     private:
     unsigned m_counter;
     ContainerType m_integral;
-    std::vector<value_type> m_t;
-    std::vector<ContainerType> m_u;
+    std::list<value_type> m_t;
+    std::list<ContainerType> m_u;
 };
+
 }//namespace dg
diff --git a/inc/dg/simpsons_t.cu b/inc/dg/simpsons_t.cu
new file mode 100644
index 000000000..e46f0edf7
--- /dev/null
+++ b/inc/dg/simpsons_t.cu
@@ -0,0 +1,35 @@
+#include <iostream>
+#include <iomanip>
+#include <cmath>
+
+#include "simpsons.h"
+#include "topology/evaluation.h"
+#include "topology/geometry.h"
+
+double sine( double t ) { return sin(t);}
+double cosine( double t ) { return cos(t);}
+
+
+int main()
+{
+    dg::Grid1d g1d( 0, 2.*M_PI, 1, 20 );
+    dg::HVec times = dg::evaluate( dg::cooX1d, g1d);
+    dg::HVec values = dg::evaluate( cosine, g1d);
+
+    dg::SimpsonsRule<double> simpsons( values[0]);
+    simpsons.init( 0., 1.);
+    double integral = simpsons.get_sum();
+    std::cout << "Integral is "<<integral<<std::endl;
+    for ( unsigned i=0; i<10; i++)
+    {
+        simpsons.add( times[i], values[i]);
+        integral = simpsons.get_sum();
+        std::cout << "Integral is "<<integral<<std::endl;
+    }
+    simpsons.add( M_PI, -1.);
+    integral = simpsons.get_sum();
+    std::cout << "Integral is "<<integral<<std::endl;
+
+
+    return 0;
+}
-- 
GitLab


From bfac232a3f77088fbac4c280299e6cf22502b93e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 26 Jun 2019 16:40:25 +0200
Subject: [PATCH 096/540] Debug Simpsons Algorithm

Now the results converge with the expected order
---
 inc/dg/simpsons.h    | 34 +++++++++++++---------------------
 inc/dg/simpsons_t.cu | 22 ++++++++++++++--------
 2 files changed, 27 insertions(+), 29 deletions(-)

diff --git a/inc/dg/simpsons.h b/inc/dg/simpsons.h
index 86ad4aca6..e9d9e87b3 100644
--- a/inc/dg/simpsons.h
+++ b/inc/dg/simpsons.h
@@ -32,45 +32,37 @@ struct SimpsonsRule
         dg::blas1::scal( m_integral, 0.);
     }
     void add( value_type t_new, const ContainerType& u_new){
-        std::cout << "Time "<<t_new<<" Value "<<u_new<<"\n";
         if( t_new < m_t.front())
             throw dg::Error(dg::Message()<<"New time must be strictly larger than old time!");
         auto pt0 = m_t.begin();
         auto pt1 = std::next( pt0);
         auto pu0 = m_u.begin();
         auto pu1 = std::next( pu0);
-        //if(true)
+        value_type t0 = *pt1, t1 = *pt0, t2 = t_new;
         if( m_counter % 2 == 0)
         {
             //Trapezoidal rule
-            dg::blas1::axpbypgz( 0.5*(t_new - *pt0), u_new,
-                0.5*(t_new - *pt0), *pu0 , 1., m_integral);
-            m_t.splice( pt0, m_t, pt1, m_t.end());
-            m_u.splice( pu0, m_u, pu1, m_u.end());
-            m_t.front() = t_new; //and now remove zeroth element (it is moved now!)
-            dg::blas1::copy( u_new, m_u.front());
+            dg::blas1::axpbypgz( 0.5*(t2 - t1), u_new,
+                0.5*(t2 - t1), *pu0 , 1., m_integral);
         }
         else
         {
             //Simpson's rule
-            value_type t0 = *pt1, t1 = *pt0, t2 = t_new;
-            value_type pre = (t2-t0)/6./(t0-t1)/(t1-t2);
-            value_type pre2 = t0*t0+3.*t1*t1+2.*t0*t2-4.*t0*t1-2.*t1*t2;
-            value_type pre1 = -(t0-t2)*(t0-t2);
-            value_type pre0 = t2*t2+3.*t1*t1+2.*t0*t2-4.*t1*t2-2.*t0*t1;
-            std::cout << "pre "<<pre*pre0<<" "<<pre*pre1<<" "<<pre*pre2<<std::endl;
+            value_type pre0 = (2.*t0-3.*t1+t2)*(t2-t0)/(6.*(t0-t1));
+            value_type pre1 = (t2-t0)*(t2-t0)*(t2-t0)/(6.*(t0-t1)*(t1-t2));
+            value_type pre2 = (t0-3.*t1+2.*t2)*(t0-t2)/(6.*(t1-t2));
 
-            dg::blas1::axpby( pre2*pre, u_new, 1., m_integral);
+            dg::blas1::axpby( pre2, u_new, 1., m_integral);
             //Also subtract old Trapezoidal rule
             dg::blas1::axpbypgz(
-                pre1*pre-0.5*(t1-t0), *pu0,
-                pre0*pre-0.5*(t1-t0), *pu1,
+                pre1-0.5*(t1-t0), *pu0,
+                pre0-0.5*(t1-t0), *pu1,
                 1., m_integral);
-            m_t.splice( pt0, m_t, pt1, m_t.end());
-            m_u.splice( pu0, m_u, pu1, m_u.end());
-            m_t.front() = t_new; //and now remove zeroth element (it is moved now!)
-            dg::blas1::copy( u_new, m_u.front());
         }
+        m_t.splice( pt0, m_t, pt1, m_t.end());
+        m_u.splice( pu0, m_u, pu1, m_u.end());
+        m_t.front() = t_new; //and now remove zeroth element (it is moved now!)
+        dg::blas1::copy( u_new, m_u.front());
         m_counter++;
     }
     const ContainerType& get_sum() const{
diff --git a/inc/dg/simpsons_t.cu b/inc/dg/simpsons_t.cu
index e46f0edf7..5f1ecc493 100644
--- a/inc/dg/simpsons_t.cu
+++ b/inc/dg/simpsons_t.cu
@@ -12,23 +12,29 @@ double cosine( double t ) { return cos(t);}
 
 int main()
 {
-    dg::Grid1d g1d( 0, 2.*M_PI, 1, 20 );
+    std::cout << "Program to test the Simpson's rule quadrature algorithm\n";
+    unsigned N=20;
+    dg::Grid1d g1d( 0, M_PI/2., 3, N );
     dg::HVec times = dg::evaluate( dg::cooX1d, g1d);
     dg::HVec values = dg::evaluate( cosine, g1d);
 
     dg::SimpsonsRule<double> simpsons( values[0]);
     simpsons.init( 0., 1.);
+    for ( unsigned i=0; i<g1d.size(); i++)
+        simpsons.add( times[i], values[i]);
+    simpsons.add( M_PI/2., 0.);
     double integral = simpsons.get_sum();
-    std::cout << "Integral is "<<integral<<std::endl;
-    for ( unsigned i=0; i<10; i++)
-    {
+    std::cout << "Integral Error is "<<fabs(integral-1.)<<std::endl;
+
+    g1d = dg::Grid1d( M_PI/2., M_PI, 3, N );
+    times = dg::evaluate( dg::cooX1d, g1d);
+    values = dg::evaluate( cosine, g1d);
+    simpsons.flush();
+    for ( unsigned i=0; i<g1d.size(); i++)
         simpsons.add( times[i], values[i]);
-        integral = simpsons.get_sum();
-        std::cout << "Integral is "<<integral<<std::endl;
-    }
     simpsons.add( M_PI, -1.);
     integral = simpsons.get_sum();
-    std::cout << "Integral is "<<integral<<std::endl;
+    std::cout << "Integral Error is "<<fabs(integral+1.)<<std::endl;
 
 
     return 0;
-- 
GitLab


From f3652be58d6787d8950f7aaf33653abfe524495c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 26 Jun 2019 22:07:08 +0200
Subject: [PATCH 097/540] Document and clean-up SimpsonRule

---
 inc/dg/Doxyfile      |  1 +
 inc/dg/runge_kutta.h |  1 +
 inc/dg/simpsons.h    | 79 +++++++++++++++++++++++++++++++++++---------
 inc/dg/simpsons_t.cu | 29 ++++++++++------
 4 files changed, 85 insertions(+), 25 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index a17476b39..3bcb9934f 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -899,6 +899,7 @@ EXAMPLE_PATH           = cg2d_t.cu \
                          elliptic_b.cu \
                          elliptic2d_b.cu \
                          runge_kutta_t.cu \
+                         simpsons_t.cu \
                          adaptive_t.cu \
                          multistep_t.cu \
                          helmholtzg2_b.cu \
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 4386d1da1..b9acd2ca1 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -755,6 +755,7 @@ You can provide your own coefficients or use one of our predefined methods:
 *
 * @note Uses only \c dg::blas1 routines to integrate one step.
 * @copydoc hide_ContainerType
+* @copydoc hide_SolverType
 */
 template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
 struct ImplicitRungeKutta
diff --git a/inc/dg/simpsons.h b/inc/dg/simpsons.h
index e9d9e87b3..9440c677a 100644
--- a/inc/dg/simpsons.h
+++ b/inc/dg/simpsons.h
@@ -5,32 +5,76 @@
 #include "blas1.h"
 
 /*! @file
- * @brief Equidistant time-integrators
+ * @brief Equidistant time-integrator
  */
-
 namespace dg{
 
+/**
+* @brief Time integration based on Simpson's rule
+*
+* The intention of this class is to provide a means to continuously
+* integrate a sample of \f$( t_i, u_i)\f$ pairs that become available
+* one after the
+* other (e.g. from the integration of an ODE) and approximate
+*
+ \f[ \int_{t_0}^T u(t) dt \f]
+
+ @note The algorithm simply integrates the Lagrange-polynomial through up to three data points. For equidistant Data points this equals either the Trapezoidal (linear) or the Simpson's rule (quadratic)
+* @sa For an explanation of Simpson's rule: https://en.wikipedia.org/wiki/Simpson%27s_rule
+
+The class works by first calling the init function to set the left-side
+boundary and then adding values as they become available.
+* @snippet simpsons_t.cu docu
+* @copydoc hide_ContainerType
+* @ingroup time
+*/
 template<class ContainerType>
 struct SimpsonsRule
 {
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
-    SimpsonsRule() = default;
-    SimpsonsRule( const ContainerType& copyable):
-        m_counter(0), m_integral( copyable),
-        m_t(2,0.), m_u( 2, copyable)
+    /*! @brief Set integration order without initializing values
+     * @param order number of vectors to use for integration.
+         Choose 2 (linear) or 3 (parabola) integration.
+     */
+    SimpsonsRule( unsigned order = 3): m_counter(0), m_order(order)
     {
+        set_order(order);
     }
-    const ContainerType& copyable() const{ return m_integral;}
+    ///@copydoc SimpsonsRule(unsigned)
+    void set_order( unsigned order){
+        m_order=order;
+        m_t.resize( order-1);
+        m_u.resize(order-1);
+        if( !(order == 2 || order == 3))
+            throw dg::Error(dg::Message()<<"Integration order must be either 2 or 3!");
+    }
+    ///Access current integration order
+    unsigned get_order() const{return m_order;}
+    /*! @brief Initialize the left-side boundary of the integration
+     * @param t0 left-side time
+     * @param u0 left-side value
+     */
     void init(  value_type t0, const ContainerType& u0) {
-        dg::blas1::copy( u0, m_u.front());
+        m_integral = u0;
+        for( auto& u: m_u)
+            u = u0;
+        for( auto& t: m_t)
+            t = 0;
         m_t.front() = t0;
         flush();
     }
+    /*! @brief Reset the integral to zero and the last (t,u) pair in the add function as the new left-side
+     */
     void flush() {
         m_counter = 0;
         dg::blas1::scal( m_integral, 0.);
     }
+    /*! @brief Add a new (t,u) pair to the time integral
+     * @param t_new time
+     * @param u_new value (must have the same size as \c u0 in the init function)
+     * @attention the \c init function must be called before you can add values to the integral
+     */
     void add( value_type t_new, const ContainerType& u_new){
         if( t_new < m_t.front())
             throw dg::Error(dg::Message()<<"New time must be strictly larger than old time!");
@@ -39,7 +83,7 @@ struct SimpsonsRule
         auto pu0 = m_u.begin();
         auto pu1 = std::next( pu0);
         value_type t0 = *pt1, t1 = *pt0, t2 = t_new;
-        if( m_counter % 2 == 0)
+        if( m_counter % 2 == 0 || m_order == 2)
         {
             //Trapezoidal rule
             dg::blas1::axpbypgz( 0.5*(t2 - t1), u_new,
@@ -53,24 +97,29 @@ struct SimpsonsRule
             value_type pre2 = (t0-3.*t1+2.*t2)*(t0-t2)/(6.*(t1-t2));
 
             dg::blas1::axpby( pre2, u_new, 1., m_integral);
-            //Also subtract old Trapezoidal rule
             dg::blas1::axpbypgz(
-                pre1-0.5*(t1-t0), *pu0,
+                pre1-0.5*(t1-t0), *pu0, //subtract last Trapezoidal step
                 pre0-0.5*(t1-t0), *pu1,
                 1., m_integral);
         }
-        m_t.splice( pt0, m_t, pt1, m_t.end());
+        //splice does not copy or move anything, only the internal pointers of the list nodes are re-pointed
+        m_t.splice( pt0, m_t, pt1, m_t.end());//permute elements
         m_u.splice( pu0, m_u, pu1, m_u.end());
-        m_t.front() = t_new; //and now remove zeroth element (it is moved now!)
+        m_t.front() = t_new; //and now remove zeroth element
         dg::blas1::copy( u_new, m_u.front());
         m_counter++;
     }
-    const ContainerType& get_sum() const{
+
+    /*! @brief Access the current value of the time integral
+     * @return the current integral
+     */
+    const ContainerType& get_integral() const{
         return m_integral;
     }
     private:
-    unsigned m_counter;
+    unsigned m_counter, m_order;
     ContainerType m_integral;
+    //we use a list here to avoid explicitly calling the swap function
     std::list<value_type> m_t;
     std::list<ContainerType> m_u;
 };
diff --git a/inc/dg/simpsons_t.cu b/inc/dg/simpsons_t.cu
index 5f1ecc493..0d2bf84bb 100644
--- a/inc/dg/simpsons_t.cu
+++ b/inc/dg/simpsons_t.cu
@@ -6,35 +6,44 @@
 #include "topology/evaluation.h"
 #include "topology/geometry.h"
 
-double sine( double t ) { return sin(t);}
-double cosine( double t ) { return cos(t);}
-
 
 int main()
 {
     std::cout << "Program to test the Simpson's rule quadrature algorithm\n";
+    //![docu]
+    // Let us construct an artificial list of data points
     unsigned N=20;
     dg::Grid1d g1d( 0, M_PI/2., 3, N );
     dg::HVec times = dg::evaluate( dg::cooX1d, g1d);
-    dg::HVec values = dg::evaluate( cosine, g1d);
+    dg::HVec values = dg::evaluate( cos, g1d);
+
+    dg::SimpsonsRule<double> simpsons;
 
-    dg::SimpsonsRule<double> simpsons( values[0]);
+    //Init the left side boundary
     simpsons.init( 0., 1.);
+
+    //Now add (time, value) pairs to the integral
     for ( unsigned i=0; i<g1d.size(); i++)
         simpsons.add( times[i], values[i]);
+
+    //Add a last point since the dg grid is cell-centered
     simpsons.add( M_PI/2., 0.);
-    double integral = simpsons.get_sum();
-    std::cout << "Integral Error is "<<fabs(integral-1.)<<std::endl;
+
+    //Ask for the integral
+    double integral = simpsons.get_integral();
+    std::cout << "Error Simpsons is "<<fabs(integral-1.)<<std::endl;
+    //![docu]
 
     g1d = dg::Grid1d( M_PI/2., M_PI, 3, N );
     times = dg::evaluate( dg::cooX1d, g1d);
-    values = dg::evaluate( cosine, g1d);
+    values = dg::evaluate( cos, g1d);
     simpsons.flush();
+    simpsons.set_order(2);
     for ( unsigned i=0; i<g1d.size(); i++)
         simpsons.add( times[i], values[i]);
     simpsons.add( M_PI, -1.);
-    integral = simpsons.get_sum();
-    std::cout << "Integral Error is "<<fabs(integral+1.)<<std::endl;
+    integral = simpsons.get_integral();
+    std::cout << "Error Trapezoidal is "<<fabs(integral+1.)<<std::endl;
 
 
     return 0;
-- 
GitLab


From 1e8b138411343a3adedac6f52bcf22b4c6b3fa72 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 27 Jun 2019 15:41:29 +0200
Subject: [PATCH 098/540] In the middle of restructuring feltor diagnostics

Feltor does not work right now
---
 inc/dg/simpsons.h                 |   5 +-
 src/feltor/feltor.cu              |  25 ++--
 src/feltor/feltor.cuh             | 202 +++------------------------
 {diag => src/feltor}/feltordiag.h | 221 ++++++++++++++++--------------
 4 files changed, 156 insertions(+), 297 deletions(-)
 rename {diag => src/feltor}/feltordiag.h (79%)

diff --git a/inc/dg/simpsons.h b/inc/dg/simpsons.h
index 9440c677a..3d232ae2d 100644
--- a/inc/dg/simpsons.h
+++ b/inc/dg/simpsons.h
@@ -54,6 +54,7 @@ struct SimpsonsRule
     /*! @brief Initialize the left-side boundary of the integration
      * @param t0 left-side time
      * @param u0 left-side value
+     * @note The integral itself is initialized to zero
      */
     void init(  value_type t0, const ContainerType& u0) {
         m_integral = u0;
@@ -71,7 +72,9 @@ struct SimpsonsRule
         dg::blas1::scal( m_integral, 0.);
     }
     /*! @brief Add a new (t,u) pair to the time integral
-     * @param t_new time
+     *
+     * The times in subsequent calls must strictly increase
+     * @param t_new time (must be strictly larger than in the previous call)
      * @param u_new value (must have the same size as \c u0 in the init function)
      * @attention the \c init function must be called before you can add values to the integral
      */
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 62ff80690..2c32aa4b7 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -9,6 +9,13 @@
 #include "feltor.cuh"
 #include "implicit.h"
 
+using HVec = dg::HVec;
+using DVec = dg::DVec;
+using DMatrix = dg::DMatrix;
+using IDMatrix = dg::IDMatrix;
+using IHMatrix = dg::IHMatrix;
+using Geometry = dg::CylindricalGrid3d;
+
 int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
@@ -51,9 +58,9 @@ int main( int argc, char* argv[])
 
     //create RHS
     std::cout << "Constructing Explicit...\n";
-    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, mag);
+    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
     std::cout << "Constructing Implicit...\n";
-    feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> im( grid, p, mag);
+    feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
@@ -89,16 +96,16 @@ int main( int argc, char* argv[])
             ntilde = dg::pullback( init0, grid);
         else if( p.initne == "blob")//rounds =3 ->2*3-1
         {
-            dg::geo::Fieldaligned<dg::CylindricalGrid3d, dg::IHMatrix,
-                dg::HVec> fieldaligned( mag, grid, p.bcxN, p.bcyN,
+            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                fieldaligned( mag, grid, p.bcxN, p.bcyN,
                 dg::geo::NoLimiter(), p.rk4eps, 5, 5);
             //evaluate should always be used with mx,my > 1
             ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 3);
         }
         else if( p.initne == "straight blob")//rounds =1 ->2*1-1
         {
-            dg::geo::Fieldaligned<dg::CylindricalGrid3d, dg::IHMatrix,
-                dg::HVec> fieldaligned( mag, grid, p.bcxN, p.bcyN,
+            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                fieldaligned( mag, grid, p.bcxN, p.bcyN,
                 dg::geo::NoLimiter(), p.rk4eps, 5, 5);
             //evaluate should always be used with mx,my > 1
             ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
@@ -112,8 +119,8 @@ int main( int argc, char* argv[])
             ntilde = dg::pullback( init0, grid);
         else
         {
-            dg::geo::Fieldaligned<dg::CylindricalGrid3d, dg::IHMatrix,
-                dg::HVec> fieldaligned( mag, grid, p.bcxN, p.bcyN,
+            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                fieldaligned( mag, grid, p.bcxN, p.bcyN,
                 dg::geo::NoLimiter(), p.rk4eps, 5, 5);
             //evaluate should always be used with mx,my > 1
             ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
@@ -153,7 +160,7 @@ int main( int argc, char* argv[])
     unsigned step = 0;
     dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
         feltor::FeltorSpecialSolver<
-            dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
+            Geometry, IDMatrix, DMatrix, DVec>
         > karniadakis( grid, p, mag);
     karniadakis.init( feltor, im, time, y0, p.dt);
     std::cout << "Done!" << std::endl;
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index e0d595b83..218d16615 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -141,15 +141,6 @@ struct ComputePsi{
         GammaPhi = GammaPhi - 0.5*HdxPhi;
     }
 };
-struct ComputeDiss{
-    ComputeDiss( double z, double mu, double tau):m_z(z), m_mu(mu), m_tau(tau){}
-    DG_DEVICE
-    void operator()( double& energy, double logN, double phi, double U) const{
-        energy = m_z*(m_tau*(1.+logN) + phi + 0.5*m_mu*U*U);
-    }
-    private:
-    double m_z, m_mu, m_tau;
-};
 struct ComputeLogN{
     DG_DEVICE
     void operator()( double tilde_n, double& npe, double& logn) const{
@@ -216,20 +207,6 @@ struct Explicit
         const std::array<std::array<Container,2>,2>& y,
         std::array<std::array<Container,2>,2>& yp);
 
-    // update quantities to the state of the last call to operator()
-    // This is to possibly save some computation time in the timestepper
-    void update_quantities() {
-        // set energy quantities in m_q, --- needs m_apar, m_logn and m_UE2
-        compute_energies( m_fields);
-
-        // remaining of m_q --- needs Delta_par U, N, m_logn
-        compute_dissipation( m_fields);
-    }
-
-    // get a link to the internal storage for quantities
-    const Quantities& quantities( ) const{
-        return m_q;
-    }
     const std::array<std::array<Container,2>,2>& fields() const{
         return m_fields;
     }
@@ -238,51 +215,9 @@ struct Explicit
     }
 
     /// ///////////////////DIAGNOSTIC MEMBERS //////////////////////
-    // Set the internal fields and derivatives (for diagnostics)
-    void set_fields( double time, const Container& ne, const Container& Ni,
-        const Container& ue, const Container& Ui, const Container& potential,
-        const Container& induction){
-
-        dg::blas1::copy( potential, m_phi[0]);
-
-        // set m_phi[1], m_d*phi[0], m_d*phi[1] and m_UE2 --- needs m_phi[0]
-        compute_psi( time);
-
-        dg::blas1::copy( ne, m_fields[0][0]);
-        dg::blas1::copy( Ni, m_fields[0][1]);
-        dg::blas1::transform( m_fields[0], m_logn, dg::LN<double>());
-
-        dg::blas1::copy( ue, m_fields[1][0]);
-        dg::blas1::copy( Ui, m_fields[1][1]);
-        if( m_p.beta != 0)
-        {
-            dg::blas1::copy( induction, m_apar);
-            //----------Compute Derivatives----------------------------//
-            dg::blas2::symv( m_dx_U, m_apar, m_dA[0]);
-            dg::blas2::symv( m_dy_U, m_apar, m_dA[1]);
-            if(!m_p.symmetric) dg::blas2::symv( m_dz, m_apar, m_dA[2]);
-        }
-        dg::blas1::axpby( 1., ne, -1., 1., m_temp0);
-        dg::blas2::symv( m_dx_N, m_temp0, m_dN[0][0]);
-        dg::blas2::symv( m_dy_N, m_temp0, m_dN[0][1]);
-        if(!m_p.symmetric) dg::blas2::symv( m_dz, m_temp0, m_dN[0][2]);
-        m_ds_N.centered( m_temp0, m_dsN[0]);
-        dg::blas1::axpby( 1., Ni, -1., 1., m_temp0);
-        dg::blas2::symv( m_dx_N, m_temp0, m_dN[1][0]);
-        dg::blas2::symv( m_dy_N, m_temp0, m_dN[1][1]);
-        if(!m_p.symmetric) dg::blas2::symv( m_dz, m_temp0, m_dN[1][2]);
-        m_ds_N.centered( m_temp0, m_dsN[1]);
-
-        for( unsigned i=0; i<2; i++)
-        {
-            dg::blas2::symv( m_dx_U, m_fields[1][i], m_dU[i][0]);
-            dg::blas2::symv( m_dy_U, m_fields[1][i], m_dU[i][1]);
-            if(!m_p.symmetric) dg::blas2::symv( m_dz, m_fields[1][i], m_dU[i][2]);
-        }
-
-     }
-
-    const Container& uE2() const {return m_UE2;}
+    const Container& uE2() const {
+        return m_UE2;
+    }
     const Container& density(int i)const{
         return m_fields[0][i];
     }
@@ -305,48 +240,30 @@ struct Explicit
         return m_dsN[i];
     }
     const Container & dssN(int i) { //2nd fieldaligned derivative
-        dg::blas1::axpby( 1., m_fields[0][i], -1., 1., m_temp0);
-        m_ds_N.dss( m_temp0, m_temp1);
-        return m_temp1;
+        return m_dssN[i];
     }
-    const Container & dssP(int i) {
+    const Container & compute_dssP(int i) {
         m_ds_P.dss( m_phi[i], m_temp1);
         return m_temp1;
     }
     const Container & dssU(int i) {
-        m_ds_U.dss( m_fields[1][i], m_temp1);
-        return m_temp1;
+        return m_dssU[i];
     }
-    const Container & dppN(int i) { //2nd varphi derivative
+    const Container & compute_dppN(int i) { //2nd varphi derivative
         dg::blas2::symv( m_dz, m_fields[0][i], m_temp0);
         dg::blas2::symv( m_dz, m_temp0, m_temp1);
         return m_temp1;
     }
-    const Container & dppP(int i) {
+    const Container & compute_dppP(int i) {
         dg::blas2::symv( m_dz, m_phi[i], m_temp0);
         dg::blas2::symv( m_dz, m_temp0, m_temp1);
         return m_temp1;
     }
-    const Container & dppU(int i) {
+    const Container & compute_dppU(int i) {
         dg::blas2::symv( m_dz, m_fields[1][i], m_temp0);
         dg::blas2::symv( m_dz, m_temp0, m_temp1);
         return m_temp1;
     }
-    const Container& lapParallelN( int i){
-        dg::blas1::axpby( 1., m_fields[0][i], -1., 1., m_temp0);
-        m_ds_N.dss( m_temp0, m_temp1);
-        dg::blas1::pointwiseDot( 1., m_divb, m_dsN[i],
-                                 0., m_temp0);
-        dg::blas1::axpby( 1., m_temp1, 1., m_temp0);
-        return m_temp0;
-    }
-    const Container& lapParallelU( int i){
-        m_ds_N.dss( m_fields[1][i], m_temp1);
-        dg::blas1::pointwiseDot( 1., m_divb, m_dsN[i],
-                                 0., m_temp0);
-        dg::blas1::axpby( 1., m_temp1, 1., m_temp0);
-        return m_temp0;
-    }
     const dg::SparseTensor<Container>& projection() const{
         return m_hh;
     }
@@ -358,6 +275,7 @@ struct Explicit
     }
     const Container& bphi( ) const { return m_bphi; }
     const Container& binv( ) const { return m_binv; }
+    const Container& divb( ) const { return m_divb; }
     //bhat / sqrt{g} / B
     const std::array<Container, 3> & bhatgB () const {
         return m_b;
@@ -413,10 +331,6 @@ struct Explicit
   private:
     void compute_phi( double t, const std::array<Container,2>& y);
     void compute_psi( double t);
-    void compute_energies(
-        const std::array<std::array<Container,2>,2>& fields);
-    void compute_dissipation(
-        const std::array<std::array<Container,2>,2>& fields);
     void compute_perp( double t,
         const std::array<std::array<Container,2>,2>& y,
         const std::array<std::array<Container,2>,2>& fields,
@@ -446,7 +360,7 @@ struct Explicit
     Container m_vol3d;
 
     Container m_apar;
-    std::array<Container,2> m_phi, m_logn, m_dsN, m_dsU;
+    std::array<Container,2> m_phi, m_logn, m_dsN, m_dsU, m_dssN, m_dssU;
     std::array<Container,3> m_dA;
     std::array<std::array<Container,3>,2> m_dP, m_dN, m_dU;
     std::array<std::array<Container,2>,2> m_fields, m_s; //fields, sources
@@ -467,7 +381,6 @@ struct Explicit
     dg::SparseTensor<Container> m_hh;
 
     const feltor::Parameters m_p;
-    Quantities m_q;
     double m_omega_source = 0.;
 
 };
@@ -619,7 +532,7 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     m_apar = m_temp0;
 
     m_phi[0] = m_phi[1] = m_temp0;
-    m_dsN = m_dsU =  m_logn = m_phi;
+    m_dssN = m_dssU = m_dsN = m_dsU =  m_logn = m_phi;
     m_dA[0] = m_dA[1] = m_dA[2] = m_temp0;
     m_dP[0] = m_dP[1] = m_dA;
     m_dN = m_dU = m_dP;
@@ -884,8 +797,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         //density: + nu_par Delta_par N
         dg::blas1::pointwiseDot( m_p.nu_parallel, m_divb, m_dsN[i],
                                  1., yp[0][i]);
-        m_ds_N.dss( y[0][i], m_dsN[i]);
-        dg::blas1::axpby( m_p.nu_parallel, m_dsN[i], 1., yp[0][i]);
+        m_ds_N.dss( y[0][i], m_dssN[i]);
+        dg::blas1::axpby( m_p.nu_parallel, m_dssN[i], 1., yp[0][i]);
         //---------------------velocity-------------------------//
         // Burgers term: -0.5 ds U^2
         dg::blas1::pointwiseDot(fields[1][i], fields[1][i], m_temp1); //U^2
@@ -897,90 +810,9 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         // diffusion: + nu_par Delta_par U
         dg::blas1::pointwiseDot(m_p.nu_parallel, m_divb, m_dsU[i],
                                 1., yp[1][i]);
-        m_ds_U.dss( fields[1][i], m_dsU[i]);
-        dg::blas1::axpby( m_p.nu_parallel, m_dsU[i], 1., yp[1][i]);
-    }
-}
-
-template<class Geometry, class IMatrix, class Matrix, class Container>
-void Explicit<Geometry, IMatrix, Matrix, Container>::compute_energies(
-    const std::array<std::array<Container,2>,2>& fields)
-{
-    double z[2]    = {-1.0,1.0};
-    m_q.mass = dg::blas1::dot( m_vol3d, fields[0][0]);
-    for(unsigned i=0; i<2; i++)
-    {
-        m_q.S[i] = z[i]*m_p.tau[i]*dg::blas2::dot(
-            m_logn[i], m_vol3d, fields[0][i]);
-        dg::blas1::pointwiseDot( fields[1][i], fields[1][i], m_temp0); //U^2
-        m_q.Tpar[i] = z[i]*0.5*m_p.mu[i]*dg::blas2::dot(
-            fields[0][i], m_vol3d, m_temp0);
-    }
-    //= 0.5 beta^{-1} (grad_perp Apar)^2
-    if( m_p.beta != 0)
-    {
-        dg::tensor::multiply3d( m_hh, m_dA[0], m_dA[1], m_dA[2],
-            m_temp0, m_temp1, m_temp2);
-        dg::blas1::subroutine( routines::ComputePsi(),
-            m_temp0, m_dA[0], m_dA[1], m_dA[2],
-            m_temp0, m_temp1, m_temp2);
-        m_q.Apar = 0.5*dg::blas1::dot( m_vol3d, m_temp0)/m_p.beta;
-    }
-    //= 0.5 mu_i N_i u_E^2
-    m_q.Tperp = 0.5*m_p.mu[1]*dg::blas2::dot( fields[0][1], m_vol3d, m_UE2);
-    m_q.energy = m_q.S[0] + m_q.S[1] + m_q.Tperp + m_q.Apar
-                 + m_q.Tpar[0] + m_q.Tpar[1];
-}
-
-template<class Geometry, class IMatrix, class Matrix, class Container>
-void Explicit<Geometry, IMatrix, Matrix, Container>::compute_dissipation(
-    const std::array<std::array<Container,2>,2>& fields)
-{
-    //alignement: lnN * Delta_s N
-    m_q.aligned = dg::blas2::dot( m_logn[0], m_vol3d, m_dsN[0]);
-    /////////////////DISSIPATION TERMS//////////////////////////////
-    m_q.diff = m_p.nu_parallel*dg::blas1::dot( m_vol3d, m_dsN[0]);
-    m_q.diff += dg::blas1::dot( m_vol3d, m_s[0][0]); //particle sources
-    // energy dissipation through diffusion
-    double z[2] = {-1.0,1.0};
-    for( unsigned i=0; i<2;i++)
-    {
-        //Compute dissipation for N
-        // Z*(tau (1+lnN )+psi + 0.5 mu U^2)
-        dg::blas1::subroutine( routines::ComputeDiss(z[i], m_p.mu[i], m_p.tau[i]),
-                m_temp2, m_logn[i], m_phi[i], fields[1][i]);
-        // Dissipation through sink/source terms
-        m_q.source[i] = dg::blas2::dot( m_temp2, m_vol3d, m_s[0][i]);
-        // parallel dissipation for N: nu_parallel *(Delta_s N)
-        m_q.Dpar[i] = m_p.nu_parallel*dg::blas2::dot(
-                        m_temp2, m_vol3d, m_dsN[i]);
-        // perp dissipation for N: nu_perp Delta_p N or -nu_perp Delta_p**2 N
-        compute_diffusive_lapMperpN( fields[0][i], m_temp1, m_temp0);
-        if( i==0)
-            m_q.diff += -m_p.nu_perp*dg::blas1::dot( m_vol3d, m_temp0);
-        m_q.Dperp[i] = -m_p.nu_perp*dg::blas2::dot(
-                        m_temp2, m_vol3d, m_temp0);
-        //Compute dissipation for U
-        //Z*mu*N*U
-        dg::blas1::pointwiseDot( z[i]*m_p.mu[i], fields[0][i], fields[1][i],
-                0, m_temp2);
-        // Dissipation through sink/source terms
-        m_q.source[i+2] = dg::blas2::dot( m_temp2, m_vol3d, m_s[1][i]);
-        // parallel dissipation for U: nu_parallel *(Delta_s U)
-        m_q.Dpar[i+2] = m_p.nu_parallel*dg::blas2::dot(
-            m_temp2, m_vol3d, m_dsU[i]);
-        // perp dissipation for U: nu_perp Delta_p U or -nu_perp Delta_p**2 U
-        compute_diffusive_lapMperpU( fields[1][i], m_temp1, m_temp0);
-        m_q.Dperp[i+2] = -m_p.nu_perp *dg::blas2::dot(
-            m_temp2, m_vol3d, m_temp0);
+        m_ds_U.dss( fields[1][i], m_dssU[i]);
+        dg::blas1::axpby( m_p.nu_parallel, m_dssU[i], 1., yp[1][i]);
     }
-    // resistive energy (quadratic current): -C (n_e (U_i-u_e))**2
-    dg::blas1::pointwiseDot(1., fields[0][0], fields[1][1],
-        -1., fields[0][0], fields[1][0], 0., m_temp0);
-    m_q.Dres = -m_p.eta*dg::blas2::dot(m_temp0, m_vol3d, m_temp0);
-    m_q.ediff = m_q.Dres + m_q.source[0] + m_q.source[1]
-        + m_q.Dpar[0]+m_q.Dperp[0]+m_q.Dpar[1]+m_q.Dperp[1]
-        + m_q.Dpar[2]+m_q.Dperp[2]+m_q.Dpar[3]+m_q.Dperp[3];
 }
 
 /* y[0][0] := n_e - 1
@@ -1004,7 +836,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     // Transform n-1 to n and n to logn
     dg::blas1::subroutine( routines::ComputeLogN(), y[0], m_fields[0], m_logn);
 
-    // Compute Apar and m_U if necessary --- reads and updates m_fields
+    // Compute Apar and m_U if necessary --- reads and updates m_fields[1]
     dg::blas1::copy( y[1], m_fields[1]);
     if( m_p.beta != 0)
         compute_apar( t, m_fields);
diff --git a/diag/feltordiag.h b/src/feltor/feltordiag.h
similarity index 79%
rename from diag/feltordiag.h
rename to src/feltor/feltordiag.h
index 3063f9a64..4cf00fed3 100644
--- a/diag/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -7,18 +7,14 @@
 
 #include "feltor/feltor.cuh"
 #include "feltor/parameters.h"
+
 namespace feltor{
-struct Jacobian{
-    DG_DEVICE double operator()(
-        double d0P, double d1P, double d2P, //any three vectors
-        double d0S, double d1S, double d2S,
-        double b_0, double b_1, double b_2)
-    {
-        return      b_0*( d1P*d2S-d2P*d1S)+
-                    b_1*( d2P*d0S-d0P*d2S)+
-                    b_2*( d0P*d1S-d1P*d0S);
-    }
-};
+
+// This file constitutes the diagnostics module of feltor
+// You can register you own diagnositcs in one of the diagnostics lists further down
+// which will then be applied during a simulation
+
+namespace routines{
 
 struct RadialParticleFlux{
     RadialParticleFlux( double tau, double mu):
@@ -99,6 +95,7 @@ struct RadialEnergyFlux{
                     + m_z*m_tau*ne*ue* (A*curvKappaS + SA );
         return Je;
     }
+    //energy dissipation
     DG_DEVICE double operator()( double ne, double ue, double P,
         double lapMperpN, double lapMperpU){
         return m_z*(m_tau*(1+log(ne))+P+0.5*m_mu*ue*ue)*lapMperpN
@@ -116,6 +113,17 @@ void dot( const std::array<Container, 3>& v,
     dg::blas1::evaluate( result, dg::equals(), dg::PairSum(),
         v[0], w[0], v[1], w[1], v[2], w[2]);
 }
+struct Jacobian{
+    DG_DEVICE double operator()(
+        double d0P, double d1P, double d2P, //any three vectors
+        double d0S, double d1S, double d2S,
+        double b_0, double b_1, double b_2)
+    {
+        return      b_0*( d1P*d2S-d2P*d1S)+
+                    b_1*( d2P*d0S-d0P*d2S)+
+                    b_2*( d0P*d1S-d1P*d0S);
+    }
+};
 template<class Container>
 void jacobian(
           const std::array<Container, 3>& a,
@@ -126,148 +134,148 @@ void jacobian(
     dg::blas1::evaluate( result, dg::equals(), Jacobian(),
         a[0], a[1], a[2], b[0], b[1], b[2], c[0], c[1], c[2]);
 }
+}//namespace routines
 
-using Feltor = feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>;
-
+//Here, we neeed the typedefs
 struct Variables{
-    Feltor f;
+    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> f;
     feltor::Parameters p;
-    std::array<dg::DVec, 3> gradPsip;
-    std::array<dg::DVec, 3> tmp;
-    dg::DVec dvdpsip3d;
+    std::array<DVec, 3> gradPsip;
+    std::array<DVec, 3> tmp;
+    DVec dvdpsip3d;
 };
 
 struct Record{
     std::string name;
     std::string long_name;
-    std::function<void( dg::DVec&, Variables&)> function;
+    std::function<void( DVec&, Variables&)> function;
 };
 
-std::vector<Record> records3d_list = {
+std::vector<Record> diagnostics3d_list = {
     {"dppne", "2nd varphi derivative of electron density",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.dppN(0), result);
         }
     },
     {"dppphi", "2nd varphi derivative of electric potential",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.dppP(0), result);
         }
     },
     {"dppue", "2nd varphi derivative of electron velocity",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.dppU(0), result);
         }
     },
     {"dssne", "2nd fieldaligned derivative of electron density",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.dssN(0), result);
         }
     },
     {"dssphi", "2nd fieldaligned derivative of electric potential",
-        []( dg::DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.dssP(0), result);
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.compute_dssP(0), result);
         }
     },
     {"dssue", "2nd fieldaligned derivative of electron velocity",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.dssU(0), result);
         }
     }
 };
 
-std::vector<Record> records_list = {
+std::vector<Record> diagnostics2d_list = {
     {"electrons", "Electron density",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(0), result);
         }
     },
     {"ions", "Ion gyro-centre density",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(1), result);
         }
     },
     {"Ue", "Electron parallel velocity",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.velocity(0), result);
         }
     },
     {"Ui", "Ion parallel velocity",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.velocity(1), result);
         }
     },
     {"potential", "Electric potential",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.potential()[0], result);
         }
     },
     {"psi", "Ion potential psi",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.potential()[1], result);
         }
     },
     {"induction", "Magnetic potential",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.induction(), result);
         }
     },
     {"vorticity", "Minus Lap_perp of electric potential",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.lapMperpP(0), result);
         }
     },
     {"apar_vorticity", "Minus Lap_perp of magnetic potential",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.lapMperpA(), result);
         }
     },
     {"neue", "Product of electron density and velocity",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot(
                 v.f.density(0), v.f.velocity(0), result);
         }
     },
     {"niui", "Product of ion gyrocentre density and velocity",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot(
                 v.f.density(1), v.f.velocity(1), result);
         }
     },
     {"neuebphi", "Product of neue and covariant phi component of magnetic field unit vector",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1.,
                 v.f.density(0), v.f.velocity(0), v.f.bphi(), 0., result);
         }
     },
     {"niuibphi", "Product of NiUi and covariant phi component of magnetic field unit vector",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1.,
                 v.f.density(1), v.f.velocity(1), v.f.bphi(), 0., result);
         }
     },
     {"lperpinv", "Perpendicular density gradient length scale",
-        []( dg::DVec& result, Variables& v ) {
-            const std::array<dg::DVec, 3>& dN = v.f.gradN(0);
+        []( DVec& result, Variables& v ) {
+            const std::array<DVec, 3>& dN = v.f.gradN(0);
             dg::tensor::multiply3d( v.f.projection(), //grad_perp
                 dN[0], dN[1], dN[2], v.tmp[0], v.tmp[1], v.tmp[2]);
-            dot(dN, v.tmp, result);
+            routines::dot(dN, v.tmp, result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
             dg::blas1::transform( result, result, dg::SQRT<double>());
         }
     },
     {"perpaligned", "Perpendicular density alignement",
-        []( dg::DVec& result, Variables& v ) {
-            const std::array<dg::DVec, 3>& dN = v.f.gradN(0);
+        []( DVec& result, Variables& v ) {
+            const std::array<DVec, 3>& dN = v.f.gradN(0);
             dg::tensor::multiply3d( v.f.projection(), //grad_perp
                 dN[0], dN[1], dN[2], v.tmp[0], v.tmp[1], v.tmp[2]);
-            dot(dN, v.tmp, result);
+            routines::dot(dN, v.tmp, result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
     {"lparallelinv", "Parallel density gradient length scale",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot ( v.f.dsN(0), v.f.dsN(0), result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
@@ -275,14 +283,14 @@ std::vector<Record> records_list = {
         }
     },
     {"aligned", "Parallel density alignement",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot ( v.f.dsN(0), v.f.dsN(0), result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
     /// ------------------ Density terms ------------------------//
     {"jvne", "Radial electron particle flux without induction contribution",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
                 v.f.density(0), v.f.velocity(0),
@@ -296,7 +304,7 @@ std::vector<Record> records_list = {
         }
     },
     {"jvneA", "Radial electron particle flux: induction contribution",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
                 v.f.density(0), v.f.velocity(0), v.f.induction(),
@@ -309,32 +317,33 @@ std::vector<Record> records_list = {
         }
     },
     {"lneperp", "Perpendicular electron diffusion",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), v.tmp[0], result);
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
     {"lneparallel", "Parallel electron diffusion",
-        []( dg::DVec& result, Variables& v ) {
-            dg::blas1::copy( v.f.lapParallelN(0), result);
-            dg::blas1::scal( result, v.p.nu_parallel);
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(0),
+                                     0., result);
+            dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., result);
         }
     },
     /// ------------------- Energy terms ------------------------//
     {"nelnne", "Entropy electrons",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::transform( v.f.density(0), result, dg::LN<double>());
             dg::blas1::pointwiseDot( result, v.f.density(0), result);
         }
     },
     {"nilnni", "Entropy ions",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::transform( v.f.density(1), result, dg::LN<double>());
             dg::blas1::pointwiseDot( v.p.tau[1], result, v.f.density(1), 0., result);
         }
     },
     {"aperp2", "Magnetic energy",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             if( v.p.beta == 0)
             {
                 dg::blas1::scal( result, 0.);
@@ -344,35 +353,35 @@ std::vector<Record> records_list = {
                 dg::tensor::multiply3d( v.f.projection(), //grad_perp
                     v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
                     v.tmp[0], v.tmp[1], v.tmp[2]);
-                dot( v.tmp, v.f.gradA(), result);
+                routines::dot( v.tmp, v.f.gradA(), result);
                 dg::blas1::scal( result, 1./2./v.p.beta);
             }
         }
     },
     {"ue2", "ExB energy",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::tensor::multiply3d( v.f.projection(), //grad_perp
                 v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
                 v.tmp[0], v.tmp[1], v.tmp[2]);
-            dot( v.tmp, v.f.gradP(0), result);
+            routines::dot( v.tmp, v.f.gradP(0), result);
             dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
             dg::blas1::pointwiseDot( 0.5, v.f.density(1), result, 0., result);
         }
     },
     {"neue2", "Parallel electron energy",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( -0.5*v.p.mu[0], v.f.density(0),
                 v.f.velocity(0), v.f.velocity(0), 0., result);
         }
     },
     {"niui2", "Parallel ion energy",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 0.5*v.p.mu[1], v.f.density(1),
                 v.f.velocity(1), v.f.velocity(1), 0., result);
         }
     },
     {"resistivity", "Energy dissipation through resistivity",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::axpby( 1., v.f.velocity(1), -1., v.f.velocity(0), result);
             dg::blas1::pointwiseDot( result, v.f.density(0), result);
             dg::blas1::pointwiseDot( v.p.eta, result, result, 0., result);
@@ -380,7 +389,7 @@ std::vector<Record> records_list = {
     },
     /// ------------------ Energy flux terms ------------------------//
     {"jvee", "Radial electron energy flux without induction contribution",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
                 v.f.density(0), v.f.velocity(0), v.f.potential()[0],
@@ -394,7 +403,7 @@ std::vector<Record> records_list = {
         }
     },
     {"jveea", "Radial electron energy flux: induction contribution",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
                 v.f.density(0), v.f.velocity(0), v.f.potential()[0], v.f.induction(),
@@ -407,7 +416,7 @@ std::vector<Record> records_list = {
         }
     },
     {"jvei", "Radial ion energy flux without induction contribution",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
                 v.f.density(1), v.f.velocity(1), v.f.potential()[1],
@@ -421,7 +430,7 @@ std::vector<Record> records_list = {
         }
     },
     {"jveia", "Radial ion energy flux: induction contribution",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
                 v.f.density(1), v.f.velocity(1), v.f.potential()[1], v.f.induction(),
@@ -435,7 +444,7 @@ std::vector<Record> records_list = {
     },
     /// ------------------------ Energy dissipation terms ------------------//
     {"leeperp", "Perpendicular electron energy dissipation",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(0), result, v.tmp[1]);
             dg::blas1::evaluate( result, dg::times_equals(),
@@ -447,7 +456,7 @@ std::vector<Record> records_list = {
         }
     },
     {"leiperp", "Perpendicular ion energy dissipation",
-        []( dg::DVec& result, Variables& v ) {
+        []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(1), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(1), result, v.tmp[1]);
             dg::blas1::evaluate( result, dg::times_equals(),
@@ -459,9 +468,13 @@ std::vector<Record> records_list = {
         }
     },
     {"leeparallel", "Parallel electron energy dissipation",
-        []( dg::DVec& result, Variables& v ) {
-            dg::blas1::copy(v.f.lapParallelN(0), v.tmp[0]);
-            dg::blas1::copy(v.f.lapParallelU(0), v.tmp[1]);
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(0),
+                                     0., v.tmp[0]);
+            dg::blas1::axpby( 1., v.f.dssN(0), 1., v.tmp[0]);
+            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(0),
+                                     0., v.tmp[0]);
+            dg::blas1::axpby( 1., v.f.dssU(0), 1., v.tmp[1]);
             dg::blas1::evaluate( result, dg::times_equals(),
                 RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
                 v.f.density(0), v.f.velocity(0), v.f.potential()[0],
@@ -471,9 +484,13 @@ std::vector<Record> records_list = {
         }
     },
     {"leiparallel", "Parallel ion energy dissipation",
-        []( dg::DVec& result, Variables& v ) {
-            dg::blas1::copy(v.f.lapParallelN(0), v.tmp[0]);
-            dg::blas1::copy(v.f.lapParallelU(0), v.tmp[1]);
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(1),
+                                     0., v.tmp[0]);
+            dg::blas1::axpby( 1., v.f.dssN(1), 1., v.tmp[0]);
+            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(1),
+                                     0., v.tmp[0]);
+            dg::blas1::axpby( 1., v.f.dssU(1), 1., v.tmp[1]);
             dg::blas1::evaluate( result, dg::times_equals(),
                 RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
                 v.f.density(1), v.f.velocity(1), v.f.potential()[1],
@@ -484,43 +501,43 @@ std::vector<Record> records_list = {
     },
     /// ------------------------ Vorticity terms ---------------------------//
     {"oexbi", "ExB vorticity term with ion density",
-        []( dg::DVec& result, Variables& v){
-            dot( v.f.gradP(0), v.gradPsip, result);
+        []( DVec& result, Variables& v){
+            routines::dot( v.f.gradP(0), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
             dg::blas1::pointwiseDot( v.p.mu[1], result, v.f.density(1), 0., result);
         }
     },
     {"oexbe", "ExB vorticity term with electron density",
-        []( dg::DVec& result, Variables& v){
-            dot( v.f.gradP(0), v.gradPsip, result);
+        []( DVec& result, Variables& v){
+            routines::dot( v.f.gradP(0), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
             dg::blas1::pointwiseDot( v.p.mu[1], result, v.f.density(0), 0., result);
         }
     },
     {"odiai", "Diamagnetic vorticity term with ion density",
-        []( dg::DVec& result, Variables& v){
-            dot( v.f.gradN(1), v.gradPsip, result);
+        []( DVec& result, Variables& v){
+            routines::dot( v.f.gradN(1), v.gradPsip, result);
             dg::blas1::scal( result, v.p.mu[1]*v.p.tau[1]);
         }
     },
     {"odiae", "Diamagnetic vorticity term with electron density",
-        []( dg::DVec& result, Variables& v){
-            dot( v.f.gradN(0), v.gradPsip, result);
+        []( DVec& result, Variables& v){
+            routines::dot( v.f.gradN(0), v.gradPsip, result);
             dg::blas1::scal( result, v.p.mu[1]*v.p.tau[1]);
         }
     },
     /// --------------------- Vorticity flux terms ---------------------------//
     {"jvoexbi", "ExB vorticity flux term with ion density",
-        []( dg::DVec& result, Variables& v){
+        []( DVec& result, Variables& v){
             // - ExB Dot GradPsi
-            jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
+            routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
 
             // Omega
-            dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
+            routines::dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
             dg::blas1::pointwiseDot( 1., v.tmp[0], v.f.binv(), v.f.binv(), 0., v.tmp[0]);
             dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], v.f.density(1), 0., v.tmp[0]);
 
-            dot( v.f.gradN(1), v.gradPsip, v.tmp[1]);
+            routines::dot( v.f.gradN(1), v.gradPsip, v.tmp[1]);
             dg::blas1::axpby( v.p.mu[1]*v.p.tau[1], v.tmp[1], 1., v.tmp[0] );
 
             // Multiply everything
@@ -528,16 +545,16 @@ std::vector<Record> records_list = {
         }
     },
     {"jvoexbe", "ExB vorticity flux term with electron density",
-        []( dg::DVec& result, Variables& v){
+        []( DVec& result, Variables& v){
             // - ExB Dot GradPsi
-            jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
+            routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
 
             // Omega
-            dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
+            routines::dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
             dg::blas1::pointwiseDot( 1., v.tmp[0], v.f.binv(), v.f.binv(), 0., v.tmp[0]);
             dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], v.f.density(0), 0., v.tmp[0]);
 
-            dot( v.f.gradN(0), v.gradPsip, v.tmp[1]);
+            routines::dot( v.f.gradN(0), v.gradPsip, v.tmp[1]);
             dg::blas1::axpby( v.p.mu[1]*v.p.tau[1], v.tmp[1], 1., v.tmp[0] );
 
             // Multiply everything
@@ -545,41 +562,41 @@ std::vector<Record> records_list = {
         }
     },
     {"jvoapar", "A parallel vorticity flux term (Maxwell stress)",
-        []( dg::DVec& result, Variables& v){
+        []( DVec& result, Variables& v){
             if( v.p.beta == 0)
                 dg::blas1::scal( result, 0.);
             else
             {
                 // - AxB Dot GradPsi
-                jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradA(), result);
-                dot( v.f.gradA(), v.gradPsip, v.tmp[0]);
+                routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradA(), result);
+                routines::dot( v.f.gradA(), v.gradPsip, v.tmp[0]);
                 dg::blas1::pointwiseDot( 1./v.p.beta, result, v.tmp[0], v.dvdpsip3d, 0., result);
             }
         }
     },
     /// --------------------- Vorticity source terms ---------------------------//
     {"socurve", "Vorticity source term electron curvature",
-        []( dg::DVec& result, Variables& v) {
-            dot( v.f.curv(), v.gradPsip, result);
+        []( DVec& result, Variables& v) {
+            routines::dot( v.f.curv(), v.gradPsip, result);
             dg::blas1::pointwiseDot( -v.p.tau[0], v.f.density(0), result, 0., result);
         }
     },
     {"socurvi", "Vorticity source term ion curvature",
-        []( dg::DVec& result, Variables& v) {
-            dot( v.f.curv(), v.gradPsip, result);
+        []( DVec& result, Variables& v) {
+            routines::dot( v.f.curv(), v.gradPsip, result);
             dg::blas1::pointwiseDot( v.p.tau[1], v.f.density(1), result, 0., result);
         }
     },
     {"socurvkappae", "Vorticity source term electron kappa curvature",
-        []( dg::DVec& result, Variables& v) {
-            dot( v.f.curvKappa(), v.gradPsip, result);
+        []( DVec& result, Variables& v) {
+            routines::dot( v.f.curvKappa(), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., v.f.density(0), v.f.velocity(0), v.f.velocity(0), 0., v.tmp[0]);
             dg::blas1::pointwiseDot( -v.p.mu[0], v.tmp[0], result, 0., result);
         }
     },
     {"socurvkappai", "Vorticity source term ion kappa curvature",
-        []( dg::DVec& result, Variables& v) {
-            dot( v.f.curvKappa(), v.gradPsip, result);
+        []( DVec& result, Variables& v) {
+            routines::dot( v.f.curvKappa(), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., v.f.density(1), v.f.velocity(1), v.f.velocity(1), 0., v.tmp[0]);
             dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], result, 0., result);
         }
-- 
GitLab


From 6b07df786d2d02c732bae9a97cb04f457a3e5a3d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 27 Jun 2019 22:46:31 +0200
Subject: [PATCH 099/540] Further work on diagnostics restructuring

---
 diag/feltordiag.cu       |   8 +-
 src/feltor/feltor.cuh    |  39 ++++----
 src/feltor/feltor_hpc.cu | 204 ++++++++++++++++++++++++++-------------
 src/feltor/feltordiag.h  | 158 ++++++++++++++++--------------
 4 files changed, 246 insertions(+), 163 deletions(-)

diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
index 168371f01..dec259b10 100644
--- a/diag/feltordiag.cu
+++ b/diag/feltordiag.cu
@@ -17,7 +17,7 @@ void create_records_in_file( int ncid,
     std::map<std::string, int>& id3d)
 {
     file::NC_Error_Handle err;
-    for( auto record : feltor::records3d_list)
+    for( auto record& : feltor::records3d_list)
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
@@ -26,7 +26,7 @@ void create_records_in_file( int ncid,
         err = nc_put_att_text( ncid, id3d[name], "long_name", long_name.size(),
             long_name.data());
     }
-    for( auto record : feltor::records_list)
+    for( auto record& : feltor::records_list)
     {
         std::string name = record.name + "_ta2d";
         std::string long_name = record.long_name + " (Toroidal average)";
@@ -412,7 +412,7 @@ int main( int argc, char* argv[])
         if( i%10 == 0) //write 3d fields only every 10th timestep
         {
             start3d_out[0] = i/10;
-            for( auto record : feltor::records3d_list)
+            for( auto record& : feltor::records3d_list)
             {
                 record.function( t3d, var);
                 //toroidal average
@@ -423,7 +423,7 @@ int main( int argc, char* argv[])
             //write time for 3d fields
             err = nc_put_vara_double( ncid_out, tvar4dID, start3d_out, count3d, &time);
         }
-        for( auto record : feltor::records_list)
+        for( auto record& : feltor::records_list)
         {
             record.function( t3d, var);
             //toroidal average
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index 218d16615..bb0204994 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -191,13 +191,6 @@ struct Explicit
     Explicit( const Geometry& g, feltor::Parameters p,
         dg::geo::TokamakMagneticField mag);
 
-    //potential[0]: electron potential, potential[1]: ion potential
-    const std::array<Container,2>& potential( ) const {
-        return m_phi;
-    }
-    const Container& induction() const {
-        return m_apar;
-    }
     //Given N_i-1 initialize n_e-1 such that phi=0
     void initializene( const Container& ni, Container& ne);
     //Given n_e-1 initialize N_i-1 such that phi=0
@@ -215,6 +208,7 @@ struct Explicit
     }
 
     /// ///////////////////DIAGNOSTIC MEMBERS //////////////////////
+    //potential[0]: electron potential, potential[1]: ion potential
     const Container& uE2() const {
         return m_UE2;
     }
@@ -224,6 +218,12 @@ struct Explicit
     const Container& velocity(int i)const{
         return m_fields[1][i];
     }
+    const Container& potential(int i) const {
+        return m_phi[i];
+    }
+    const Container& induction() const {
+        return m_apar;
+    }
     const std::array<Container, 3> & gradN (int i) const {
         return m_dN[i];
     }
@@ -239,13 +239,12 @@ struct Explicit
     const Container & dsN (int i) const {
         return m_dsN[i];
     }
+    const Container & dsU (int i) const {
+        return m_dsU[i];
+    }
     const Container & dssN(int i) { //2nd fieldaligned derivative
         return m_dssN[i];
     }
-    const Container & compute_dssP(int i) {
-        m_ds_P.dss( m_phi[i], m_temp1);
-        return m_temp1;
-    }
     const Container & dssU(int i) {
         return m_dssU[i];
     }
@@ -291,15 +290,6 @@ struct Explicit
         return m_temp1;
     }
     /////////////////////////DIAGNOSTICS END////////////////////////////////
-
-    //source strength, profile - 1
-    void set_source( Container profile, double omega_source, Container source)
-    {
-        m_profne = profile;
-        m_omega_source = omega_source;
-        m_source = source;
-    }
-    void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
     void compute_diffusive_lapMperpN( const Container& density, Container& temp0, Container& result ){
         // compute the negative diffusion contribution -Lambda N
         // perp dissipation for N: nu_perp Delta_p N or -nu_perp Delta_p**2 N
@@ -328,6 +318,15 @@ struct Explicit
             dg::blas2::gemv( m_lapperpU, temp0, result); //!plus
         }
     }
+
+    //source strength, profile - 1
+    void set_source( Container profile, double omega_source, Container source)
+    {
+        m_profne = profile;
+        m_omega_source = omega_source;
+        m_source = source;
+    }
+    void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
   private:
     void compute_phi( double t, const std::array<Container,2>& y);
     void compute_psi( double t);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index d048cdcf8..bf4d6e666 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -130,7 +130,7 @@ int main( int argc, char* argv[])
         , comm
         #endif //FELTOR_MPI
         );
-    Geometry grid_out( Rmin, Rmax, Zmin, Zmax, 0, 2.*M_PI,
+    Geometry g3d_out( Rmin, Rmax, Zmin, Zmax, 0, 2.*M_PI,
         p.n_out, p.Nx_out, p.Ny_out, p.symmetric ? 1 : p.Nz_out, p.bcxN, p.bcyN, dg::PER
         #ifdef FELTOR_MPI
         , comm
@@ -277,8 +277,8 @@ int main( int argc, char* argv[])
         int dimsIN[3],  coordsIN[3];
         MPI_Cart_get( comm, 3, dimsIN, periods, coordsIN);
         size_t countIN[4] = {1, grid_IN.local().Nz(),
-            grid_out.n()*(grid_IN.local().Ny()),
-            grid_out.n()*(grid_IN.local().Nx())};
+            g3d_out.n()*(grid_IN.local().Ny()),
+            g3d_out.n()*(grid_IN.local().Nx())};
         size_t startIN[4] = {0, coordsIN[2]*countIN[1],
                                 coordsIN[1]*countIN[2],
                                 coordsIN[0]*countIN[3]};
@@ -375,18 +375,18 @@ int main( int argc, char* argv[])
     int dim_ids[4], tvarID;
 #ifdef FELTOR_MPI
     int coords[3];
-    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
-    size_t start[4] = {0, 0, 0, 0};
-    size_t count[4] = {1, grid_out.local().Nz(),
-        grid_out.n()*(grid_out.local().Ny()),
-        grid_out.n()*(grid_out.local().Nx())};
+    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out.global());
+    size_t start4d[4] = {0, 0, 0, 0};
+    size_t count4d[4] = {1, g3d_out.local().Nz(),
+        g3d_out.n()*(g3d_out.local().Ny()),
+        g3d_out.n()*(g3d_out.local().Nx())};
 #else //FELTOR_MPI
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
-    size_t start[4] = {0, 0, 0, 0};
-    size_t count[4] = {1, grid_out.Nz(), grid_out.n()*grid_out.Ny(),
-        grid_out.n()*grid_out.Nx()};
+    err = file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out);
+    size_t start4d[4] = {0, 0, 0, 0};
+    size_t count4d[4] = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(),
+        g3d_out.n()*g3d_out.Nx()};
 #endif //FELTOR_MPI
-    {   //output 3d variables into file
+    {   //output static 3d variables into file
         dg::geo::BFieldR fieldR(mag);
         dg::geo::BFieldZ fieldZ(mag);
         dg::geo::BFieldP fieldP(mag);
@@ -415,8 +415,8 @@ int main( int argc, char* argv[])
         v3d.emplace_back( "yc", &yc, "y-coordinate in Cartesian coordinate system");
         v3d.emplace_back( "zc", &zc, "z-coordinate in Cartesian coordinate system");
 
-        IHMatrix project = dg::create::projection( grid_out, grid);
-        HVec transferH( dg::evaluate(dg::zero, grid_out));
+        IHMatrix project = dg::create::projection( g3d_out, grid);
+        HVec transferH( dg::evaluate(dg::zero, g3d_out));
         for( auto tp : v3d)
         {
             int vecID;
@@ -432,22 +432,22 @@ int main( int argc, char* argv[])
                 for( int rrank=0; rrank<size; rrank++)
                 {
                     if(rrank!=0)
-                        MPI_Recv( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
-                              rrank, rrank, grid_out.communicator(), &status);
-                    MPI_Cart_coords( grid_out.communicator(), rrank, 3, coords);
-                    start[1] = coords[2]*count[1],
-                    start[2] = coords[1]*count[2],
-                    start[3] = coords[0]*count[3];
-                    err = nc_put_vara_double( ncid, vecID, &start[1], &count[1],
+                        MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                              rrank, rrank, g3d_out.communicator(), &status);
+                    MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
+                    start4d[1] = coords[2]*count4d[1],
+                    start4d[2] = coords[1]*count4d[2],
+                    start4d[3] = coords[0]*count4d[3];
+                    err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
                         transferH.data().data());
                 }
             }
             else
-                MPI_Send( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
-                          0, rank, grid_out.communicator());
-            MPI_Barrier( grid_out.communicator());
+                MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                          0, rank, g3d_out.communicator());
+            MPI_Barrier( g3d_out.communicator());
 #else
-            err = nc_put_vara_double( ncid, vecID, &start[1], &count[1],
+            err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
                 transferH.data());
 #endif // FELTOR_MPI
             MPI_OUT err = nc_redef(ncid);
@@ -457,16 +457,31 @@ int main( int argc, char* argv[])
     //field IDs
     int EtimeID, EtimevarID;
     MPI_OUT err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
-    std::map<std::string, int> id0d, id4d;
-    for( auto pair : v0d)
+    std::map<std::string, int> id3d, id4d;
+    for( auto record& : feltor::diagnostics3d_list)
     {
-        MPI_OUT err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 1,
-            &EtimeID, &id0d[pair.first]);
+        std::string name = record.name;
+        std::string long_name = record.long_name;
+        MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 4, dim_ids,
+            &id4d[name]);//creates a new id4d entry
+        MPI_OUT err = nc_put_att_text( ncid, id4d[name], "long_name", long_name.size(),
+            long_name.data());
     }
-    for( auto pair : v4d)
+    for( auto record& : feltor::diagnostics2d_list)
     {
-        MPI_OUT err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 4,
-            dim_ids, &id4d[pair.first]);
+        std::string name = record.name + "_ta2d";
+        std::string long_name = record.long_name + " (Toroidal average)";
+        MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id3d[name]);//creates a new id3d entry
+        MPI_OUT err = nc_put_att_text( ncid, id3d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record.name + "_2d";
+        long_name = record.long_name + " (Evaluated on phi = pi plane)";
+        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id3d[name]);
+        err = nc_put_att_text( ncid, id3d[name], "long_name", long_name.size(),
+            long_name.data());
     }
     MPI_OUT err = nc_enddef(ncid);
     ///////////////////////////////////first output/////////////////////////
@@ -487,12 +502,26 @@ int main( int argc, char* argv[])
     }
     MPI_OUT q.display(std::cout);
     double energy0 = q.energy, mass0 = q.mass, E0 = energy0, M0 = mass0;
-    DVec transferD( dg::evaluate(dg::zero, grid_out));
-    HVec transferH( dg::evaluate(dg::zero, grid_out));
-    IDMatrix project = dg::create::projection( grid_out, grid);
-    for( auto pair : v4d)
+    DVec transferD( dg::evaluate(dg::zero, g3d_out));
+    HVec transferH( dg::evaluate(dg::zero, g3d_out));
+    DVec transfer2dD = dg::evaluate( dg::zero, g2d_out);
+    HVec transfer2dH = dg::evaluate( dg::zero, g2d_out);
+    //fast_projection is an undocumented secret feltor function...
+    IDMatrix project = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
+    /// Construct feltor::Variables object for diagnostics
+    DVec result = dg::evaluate( dg::zero, grid);
+
+    std::array<DVec, 3> gradPsip;
+    gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
+    gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
+    gradPsip[2] =  result; //zero
+    feltor::Variables var = {
+        feltor, p, gradPsip, gradPsip
+    };
+    for( auto record& : diagnostics3d_list)
     {
-        dg::blas2::symv( project, *pair.second, transferD);
+        record.function( result, var);
+        dg::blas2::symv( project, result, transferD);
         dg::assign( transferD, transferH);
 #ifdef FELTOR_MPI
             if(rank==0)
@@ -500,27 +529,27 @@ int main( int argc, char* argv[])
                 for( int rrank=0; rrank<size; rrank++)
                 {
                     if(rrank!=0)
-                        MPI_Recv( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
-                              rrank, rrank, grid_out.communicator(), &status);
-                    MPI_Cart_coords( grid_out.communicator(), rrank, 3, coords);
-                    start[1] = coords[2]*count[1],
-                    start[2] = coords[1]*count[2],
-                    start[3] = coords[0]*count[3];
-                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start, count,
+                        MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                              rrank, rrank, g3d_out.communicator(), &status);
+                    MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
+                    start4d[1] = coords[2]*count4d[1],
+                    start4d[2] = coords[1]*count4d[2],
+                    start4d[3] = coords[0]*count4d[3];
+                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
                         transferH.data().data());
                 }
             }
             else
-                MPI_Send( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
-                          0, rank, grid_out.communicator());
-            MPI_Barrier( grid_out.communicator());
+                MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                          0, rank, g3d_out.communicator());
+            MPI_Barrier( g3d_out.communicator());
 #else
-            err = nc_put_vara_double( ncid, id4d.at(pair.first), start, count,
+            err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
                 transferH.data());
 #endif // FELTOR_MPI
     }
-    MPI_OUT err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-    MPI_OUT err = nc_put_vara_double( ncid, EtimevarID, start, count, &time);
+    MPI_OUT err = nc_put_vara_double( ncid, tvarID, start4d, count4d, &time);
+    MPI_OUT err = nc_put_vara_double( ncid, EtimevarID, start4d, count4d, &time);
 
     size_t Estart[] = {0};
     size_t Ecount[] = {1};
@@ -604,12 +633,13 @@ int main( int argc, char* argv[])
                     << ti.diff()/(double)p.itstp/(double)p.inner_loop<<"s";
         ti.tic();
         //////////////////////////write fields////////////////////////
-        start[0] = i;
+        start4d[0] = i;
         MPI_OUT err = nc_open(file_name.data(), NC_WRITE, &ncid);
-        MPI_OUT err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-        for( auto pair : v4d)
+        MPI_OUT err = nc_put_vara_double( ncid, tvarID, start4d, count4d, &time);
+        for( auto record& : diagnostics3d_list)
         {
-            dg::blas2::symv( project, *pair.second, transferD);
+            record.function( result, var);
+            dg::blas2::symv( project, result, transferD);
             dg::assign( transferD, transferH);
 #ifdef FELTOR_MPI
             if(rank==0)
@@ -617,22 +647,64 @@ int main( int argc, char* argv[])
                 for( int rrank=0; rrank<size; rrank++)
                 {
                     if(rrank!=0)
-                        MPI_Recv( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
-                              rrank, rrank, grid_out.communicator(), &status);
-                    MPI_Cart_coords( grid_out.communicator(), rrank, 3, coords);
-                    start[1] = coords[2]*count[1],
-                    start[2] = coords[1]*count[2],
-                    start[3] = coords[0]*count[3];
-                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start, count,
+                        MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                              rrank, rrank, g3d_out.communicator(), &status);
+                    MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
+                    start4d[1] = coords[2]*count4d[1],
+                    start4d[2] = coords[1]*count4d[2],
+                    start4d[3] = coords[0]*count4d[3];
+                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
+                        transferH.data().data());
+                }
+            }
+            else
+                MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                          0, rank, g3d_out.communicator());
+            MPI_Barrier( g3d_out.communicator());
+#else
+            err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
+                transferH.data());
+#endif // FELTOR_MPI
+        }
+        for( auto record& : diagnostics2d_list)
+        {
+            record.function( result, var);
+            dg::blas2::symv( project, result, transferD);
+            //toroidal average
+            toroidal_average( transferD, result, false);
+            dg::blas1::transfer( transfer2dD, transfer2dH);
+            err = nc_put_vara_double( ncid_out, id3d.at(record.name+"_ta2d"),
+                start2d, count2d, transfer2dH.data());
+
+            // 2d data of plane varphi = 0
+            unsigned kmp = g3d_out.Nz()/2;
+            dg::HVec t2d_mp(result.data().begin() + kmp*g2d_out.size(),
+                result.data().begin() + (kmp+1)*g2d_out.size() );
+            err = nc_put_vara_double( ncid_out, id3d.at(record.name+"_2d"),
+                start2d, count2d, t2d_mp.data() );
+
+#ifdef FELTOR_MPI
+            if(rank==0)
+            {
+                for( int rrank=0; rrank<size; rrank++)
+                {
+                    if(rrank!=0)
+                        MPI_Recv( transferH.data().data(), g2d_out.local().size(), MPI_DOUBLE,
+                              rrank, rrank, g2d_out.communicator(), &status);
+                    MPI_Cart_coords( g2d_out.communicator(), rrank, 3, coords);
+                    start4d[1] = coords[2]*count4d[1],
+                    start4d[2] = coords[1]*count4d[2],
+                    start4d[3] = coords[0]*count4d[3];
+                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
                         transferH.data().data());
                 }
             }
             else
-                MPI_Send( transferH.data().data(), grid_out.local().size(), MPI_DOUBLE,
-                          0, rank, grid_out.communicator());
-            MPI_Barrier( grid_out.communicator());
+                MPI_Send( transferH.data().data(), g2d_out.local().size(), MPI_DOUBLE,
+                          0, rank, g2d_out.communicator());
+            MPI_Barrier( g2d_out.communicator());
 #else
-            err = nc_put_vara_double( ncid, id4d.at(pair.first), start, count,
+            err = nc_put_vara_double( ncid, id3d.at(pair.first), start4d, count4d,
                 transferH.data());
 #endif // FELTOR_MPI
         }
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 4cf00fed3..b1f3d001e 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -11,7 +11,7 @@
 namespace feltor{
 
 // This file constitutes the diagnostics module of feltor
-// You can register you own diagnositcs in one of the diagnostics lists further down
+// You can register you own diagnostics in one of the diagnostics lists further down
 // which will then be applied during a simulation
 
 namespace routines{
@@ -138,123 +138,140 @@ void jacobian(
 
 //Here, we neeed the typedefs
 struct Variables{
-    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> f;
+    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
     feltor::Parameters p;
     std::array<DVec, 3> gradPsip;
     std::array<DVec, 3> tmp;
-    DVec dvdpsip3d;
 };
 
 struct Record{
     std::string name;
     std::string long_name;
+    bool integral;
     std::function<void( DVec&, Variables&)> function;
 };
 
+// Here are all 3d outputs we want to have
 std::vector<Record> diagnostics3d_list = {
-    {"dppne", "2nd varphi derivative of electron density",
+    {"electrons", "electron density", false,
         []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.dppN(0), result);
+             dg::blas1::copy(v.f.density(0), result);
         }
     },
-    {"dppphi", "2nd varphi derivative of electric potential",
+    {"ions", "ion density", false,
         []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.dppP(0), result);
+             dg::blas1::copy(v.f.density(1), result);
         }
     },
-    {"dppue", "2nd varphi derivative of electron velocity",
+    {"Ue", "parallel electron velocity", false,
         []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.dppU(0), result);
+             dg::blas1::copy(v.f.velocity(0), result);
         }
     },
-    {"dssne", "2nd fieldaligned derivative of electron density",
+    {"Ui", "parallel ion velocity", false,
         []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.dssN(0), result);
+             dg::blas1::copy(v.f.velocity(1), result);
         }
     },
-    {"dssphi", "2nd fieldaligned derivative of electric potential",
+    {"potential", "electric potential", false,
         []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.compute_dssP(0), result);
+             dg::blas1::copy(v.f.potential(0), result);
         }
     },
-    {"dssue", "2nd fieldaligned derivative of electron velocity",
+    {"induction", "parallel magnetic induction", false,
         []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.dssU(0), result);
+             dg::blas1::copy(v.f.induction(), result);
         }
     }
 };
 
+// and here are all the 2d outputs we want to produce
 std::vector<Record> diagnostics2d_list = {
-    {"electrons", "Electron density",
+    {"electrons", "Electron density", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(0), result);
         }
     },
-    {"ions", "Ion gyro-centre density",
+    {"ions", "Ion gyro-centre density", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(1), result);
         }
     },
-    {"Ue", "Electron parallel velocity",
+    {"Ue", "Electron parallel velocity", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.velocity(0), result);
         }
     },
-    {"Ui", "Ion parallel velocity",
+    {"Ui", "Ion parallel velocity", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.velocity(1), result);
         }
     },
-    {"potential", "Electric potential",
+    {"potential", "Electric potential", false,
         []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.potential()[0], result);
+             dg::blas1::copy(v.f.potential(0), result);
         }
     },
-    {"psi", "Ion potential psi",
+    {"psi", "Ion potential psi", false,
         []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.potential()[1], result);
+             dg::blas1::copy(v.f.potential(1), result);
         }
     },
-    {"induction", "Magnetic potential",
+    {"induction", "Magnetic potential", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.induction(), result);
         }
     },
-    {"vorticity", "Minus Lap_perp of electric potential",
+    {"vorticity", "Minus Lap_perp of electric potential", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.lapMperpP(0), result);
         }
     },
-    {"apar_vorticity", "Minus Lap_perp of magnetic potential",
+    {"dssue", "2nd parallel derivative of electron velocity", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.dssU(0), result);
+        }
+    },
+    {"dppue", "2nd varphi derivative of electron velocity", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.compute_dppU(0), result);
+        }
+    },
+    {"dpue2", "1st varphi derivative squared of electron velocity", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::pointwiseDot(v.f.gradU(0)[2], v.f.gradU(0)[2], result);
+        }
+    },
+    {"apar_vorticity", "Minus Lap_perp of magnetic potential", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.lapMperpA(), result);
         }
     },
-    {"neue", "Product of electron density and velocity",
+    {"neue", "Product of electron density and velocity", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot(
                 v.f.density(0), v.f.velocity(0), result);
         }
     },
-    {"niui", "Product of ion gyrocentre density and velocity",
+    {"niui", "Product of ion gyrocentre density and velocity", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot(
                 v.f.density(1), v.f.velocity(1), result);
         }
     },
-    {"neuebphi", "Product of neue and covariant phi component of magnetic field unit vector",
+    {"neuebphi", "Product of neue and covariant phi component of magnetic field unit vector", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1.,
                 v.f.density(0), v.f.velocity(0), v.f.bphi(), 0., result);
         }
     },
-    {"niuibphi", "Product of NiUi and covariant phi component of magnetic field unit vector",
+    {"niuibphi", "Product of NiUi and covariant phi component of magnetic field unit vector", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1.,
                 v.f.density(1), v.f.velocity(1), v.f.bphi(), 0., result);
         }
     },
-    {"lperpinv", "Perpendicular density gradient length scale",
+    {"lperpinv", "Perpendicular density gradient length scale", false,
         []( DVec& result, Variables& v ) {
             const std::array<DVec, 3>& dN = v.f.gradN(0);
             dg::tensor::multiply3d( v.f.projection(), //grad_perp
@@ -265,7 +282,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::transform( result, result, dg::SQRT<double>());
         }
     },
-    {"perpaligned", "Perpendicular density alignement",
+    {"perpaligned", "Perpendicular density alignement", false,
         []( DVec& result, Variables& v ) {
             const std::array<DVec, 3>& dN = v.f.gradN(0);
             dg::tensor::multiply3d( v.f.projection(), //grad_perp
@@ -274,7 +291,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
-    {"lparallelinv", "Parallel density gradient length scale",
+    {"lparallelinv", "Parallel density gradient length scale", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot ( v.f.dsN(0), v.f.dsN(0), result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
@@ -282,14 +299,14 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::transform( result, result, dg::SQRT<double>());
         }
     },
-    {"aligned", "Parallel density alignement",
+    {"aligned", "Parallel density alignement", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot ( v.f.dsN(0), v.f.dsN(0), result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
     /// ------------------ Density terms ------------------------//
-    {"jvne", "Radial electron particle flux without induction contribution",
+    {"jsne", "Radial electron particle flux without induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
@@ -300,10 +317,9 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
         }
     },
-    {"jvneA", "Radial electron particle flux: induction contribution",
+    {"jsneA", "Radial electron particle flux: induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
@@ -313,16 +329,15 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
         }
     },
-    {"lneperp", "Perpendicular electron diffusion",
+    {"lneperp", "Perpendicular electron diffusion", true,
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), v.tmp[0], result);
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
-    {"lneparallel", "Parallel electron diffusion",
+    {"lneparallel", "Parallel electron diffusion", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(0),
                                      0., result);
@@ -330,19 +345,19 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------- Energy terms ------------------------//
-    {"nelnne", "Entropy electrons",
+    {"nelnne", "Entropy electrons", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::transform( v.f.density(0), result, dg::LN<double>());
             dg::blas1::pointwiseDot( result, v.f.density(0), result);
         }
     },
-    {"nilnni", "Entropy ions",
+    {"nilnni", "Entropy ions", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::transform( v.f.density(1), result, dg::LN<double>());
             dg::blas1::pointwiseDot( v.p.tau[1], result, v.f.density(1), 0., result);
         }
     },
-    {"aperp2", "Magnetic energy",
+    {"aperp2", "Magnetic energy", false,
         []( DVec& result, Variables& v ) {
             if( v.p.beta == 0)
             {
@@ -358,7 +373,7 @@ std::vector<Record> diagnostics2d_list = {
             }
         }
     },
-    {"ue2", "ExB energy",
+    {"ue2", "ExB energy", false,
         []( DVec& result, Variables& v ) {
             dg::tensor::multiply3d( v.f.projection(), //grad_perp
                 v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
@@ -368,19 +383,20 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( 0.5, v.f.density(1), result, 0., result);
         }
     },
-    {"neue2", "Parallel electron energy",
+    {"neue2", "Parallel electron energy", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( -0.5*v.p.mu[0], v.f.density(0),
                 v.f.velocity(0), v.f.velocity(0), 0., result);
         }
     },
-    {"niui2", "Parallel ion energy",
+    {"niui2", "Parallel ion energy", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 0.5*v.p.mu[1], v.f.density(1),
                 v.f.velocity(1), v.f.velocity(1), 0., result);
         }
     },
-    {"resistivity", "Energy dissipation through resistivity",
+    /// ------------------- Energy dissipation ----------------------//
+    {"resistivity", "Energy dissipation through resistivity", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::axpby( 1., v.f.velocity(1), -1., v.f.velocity(0), result);
             dg::blas1::pointwiseDot( result, v.f.density(0), result);
@@ -388,7 +404,7 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------ Energy flux terms ------------------------//
-    {"jvee", "Radial electron energy flux without induction contribution",
+    {"jsee", "Radial electron energy flux without induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
@@ -399,10 +415,9 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
         }
     },
-    {"jveea", "Radial electron energy flux: induction contribution",
+    {"jseea", "Radial electron energy flux: induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
@@ -412,10 +427,9 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
         }
     },
-    {"jvei", "Radial ion energy flux without induction contribution",
+    {"jsei", "Radial ion energy flux without induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
@@ -426,10 +440,9 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
         }
     },
-    {"jveia", "Radial ion energy flux: induction contribution",
+    {"jseia", "Radial ion energy flux: induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
@@ -439,11 +452,10 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            dg::blas1::pointwiseDot( result, v.dvdpsip3d, result);
         }
     },
     /// ------------------------ Energy dissipation terms ------------------//
-    {"leeperp", "Perpendicular electron energy dissipation",
+    {"leeperp", "Perpendicular electron energy dissipation", true,
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(0), result, v.tmp[1]);
@@ -455,7 +467,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
-    {"leiperp", "Perpendicular ion energy dissipation",
+    {"leiperp", "Perpendicular ion energy dissipation", true,
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(1), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(1), result, v.tmp[1]);
@@ -467,7 +479,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
-    {"leeparallel", "Parallel electron energy dissipation",
+    {"leeparallel", "Parallel electron energy dissipation", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(0),
                                      0., v.tmp[0]);
@@ -483,7 +495,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::scal( result, v.p.nu_parallel);
         }
     },
-    {"leiparallel", "Parallel ion energy dissipation",
+    {"leiparallel", "Parallel ion energy dissipation", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(1),
                                      0., v.tmp[0]);
@@ -500,34 +512,34 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------------ Vorticity terms ---------------------------//
-    {"oexbi", "ExB vorticity term with ion density",
+    {"oexbi", "ExB vorticity term with ion density", false,
         []( DVec& result, Variables& v){
             routines::dot( v.f.gradP(0), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
             dg::blas1::pointwiseDot( v.p.mu[1], result, v.f.density(1), 0., result);
         }
     },
-    {"oexbe", "ExB vorticity term with electron density",
+    {"oexbe", "ExB vorticity term with electron density", false,
         []( DVec& result, Variables& v){
             routines::dot( v.f.gradP(0), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
             dg::blas1::pointwiseDot( v.p.mu[1], result, v.f.density(0), 0., result);
         }
     },
-    {"odiai", "Diamagnetic vorticity term with ion density",
+    {"odiai", "Diamagnetic vorticity term with ion density", false,
         []( DVec& result, Variables& v){
             routines::dot( v.f.gradN(1), v.gradPsip, result);
             dg::blas1::scal( result, v.p.mu[1]*v.p.tau[1]);
         }
     },
-    {"odiae", "Diamagnetic vorticity term with electron density",
+    {"odiae", "Diamagnetic vorticity term with electron density", false,
         []( DVec& result, Variables& v){
             routines::dot( v.f.gradN(0), v.gradPsip, result);
             dg::blas1::scal( result, v.p.mu[1]*v.p.tau[1]);
         }
     },
     /// --------------------- Vorticity flux terms ---------------------------//
-    {"jvoexbi", "ExB vorticity flux term with ion density",
+    {"jsoexbi", "ExB vorticity flux term with ion density", true,
         []( DVec& result, Variables& v){
             // - ExB Dot GradPsi
             routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
@@ -541,10 +553,10 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::axpby( v.p.mu[1]*v.p.tau[1], v.tmp[1], 1., v.tmp[0] );
 
             // Multiply everything
-            dg::blas1::pointwiseDot( 1., result, v.tmp[0], v.dvdpsip3d, 0., result);
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
         }
     },
-    {"jvoexbe", "ExB vorticity flux term with electron density",
+    {"jsoexbe", "ExB vorticity flux term with electron density", true,
         []( DVec& result, Variables& v){
             // - ExB Dot GradPsi
             routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
@@ -558,10 +570,10 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::axpby( v.p.mu[1]*v.p.tau[1], v.tmp[1], 1., v.tmp[0] );
 
             // Multiply everything
-            dg::blas1::pointwiseDot( 1., result, v.tmp[0], v.dvdpsip3d, 0., result);
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
         }
     },
-    {"jvoapar", "A parallel vorticity flux term (Maxwell stress)",
+    {"jsoapar", "A parallel vorticity flux term (Maxwell stress)", true,
         []( DVec& result, Variables& v){
             if( v.p.beta == 0)
                 dg::blas1::scal( result, 0.);
@@ -570,31 +582,31 @@ std::vector<Record> diagnostics2d_list = {
                 // - AxB Dot GradPsi
                 routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradA(), result);
                 routines::dot( v.f.gradA(), v.gradPsip, v.tmp[0]);
-                dg::blas1::pointwiseDot( 1./v.p.beta, result, v.tmp[0], v.dvdpsip3d, 0., result);
+                dg::blas1::pointwiseDot( 1./v.p.beta, result, v.tmp[0], 0., result);
             }
         }
     },
     /// --------------------- Vorticity source terms ---------------------------//
-    {"socurve", "Vorticity source term electron curvature",
+    {"socurve", "Vorticity source term electron curvature", true,
         []( DVec& result, Variables& v) {
             routines::dot( v.f.curv(), v.gradPsip, result);
             dg::blas1::pointwiseDot( -v.p.tau[0], v.f.density(0), result, 0., result);
         }
     },
-    {"socurvi", "Vorticity source term ion curvature",
+    {"socurvi", "Vorticity source term ion curvature", true,
         []( DVec& result, Variables& v) {
             routines::dot( v.f.curv(), v.gradPsip, result);
             dg::blas1::pointwiseDot( v.p.tau[1], v.f.density(1), result, 0., result);
         }
     },
-    {"socurvkappae", "Vorticity source term electron kappa curvature",
+    {"socurvkappae", "Vorticity source term electron kappa curvature", true,
         []( DVec& result, Variables& v) {
             routines::dot( v.f.curvKappa(), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., v.f.density(0), v.f.velocity(0), v.f.velocity(0), 0., v.tmp[0]);
             dg::blas1::pointwiseDot( -v.p.mu[0], v.tmp[0], result, 0., result);
         }
     },
-    {"socurvkappai", "Vorticity source term ion kappa curvature",
+    {"socurvkappai", "Vorticity source term ion kappa curvature", true,
         []( DVec& result, Variables& v) {
             routines::dot( v.f.curvKappa(), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., v.f.density(1), v.f.velocity(1), v.f.velocity(1), 0., v.tmp[0]);
-- 
GitLab


From 7a3f8f6cbef6e97f05866abd2bbc9122c942e8d1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 28 Jun 2019 16:15:07 +0200
Subject: [PATCH 100/540] HOTFIX: fix bug in fast_projection

for uneven divide fast_projection yields wrong projection because
explicit zeros were removed in creation
---
 inc/dg/topology/fast_interpolation.h | 24 +++++++-----
 inc/dg/topology/projection.h         |  1 +
 inc/dg/topology/projection_t.cu      | 55 +++++++++++++++-------------
 3 files changed, 45 insertions(+), 35 deletions(-)

diff --git a/inc/dg/topology/fast_interpolation.h b/inc/dg/topology/fast_interpolation.h
index fe15c2267..7a15e9aec 100644
--- a/inc/dg/topology/fast_interpolation.h
+++ b/inc/dg/topology/fast_interpolation.h
@@ -106,7 +106,7 @@ MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > fast
     unsigned n=t.n();
     dg::RealGrid1d<real_type> g_old( -1., 1., n, 1);
     dg::RealGrid1d<real_type> g_new( -1., 1., n, multiply);
-    dg::IHMatrix interpolX = dg::create::interpolation( g_new, g_old);
+    cusp::coo_matrix<int, real_type, cusp::host_memory> interpolX = dg::create::interpolation( g_new, g_old);
     EllSparseBlockMat<real_type> iX( multiply*t.N(), t.N(), 1, multiply, t.n());
     for( unsigned  k=0; k<multiply; k++)
     for( unsigned  i=0; i<n; i++)
@@ -127,18 +127,22 @@ MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > fast
 {
     unsigned n=t.n();
     if( t.N()%divide != 0) throw Error( Message(_ping_)<< "Nx and divide don't match: Nx: " << t.N()<< " divide "<< (unsigned)divide);
-    dg::RealGrid1d<real_type> g_oldX( -1., 1., n, divide);
-    dg::RealGrid1d<real_type> g_new(  -1., 1., n, 1);
-    dg::IHMatrix projectX;
-    if(no == normed)
-        projectX = dg::create::projection( g_new, g_oldX);
-    else
-        projectX = dg::create::interpolationT( g_new, g_oldX);
+    dg::RealGrid1d<real_type> g_old( -1., 1., n, divide);
+    dg::RealGrid1d<real_type> g_new( -1., 1., n, 1);
+    dg::HVec w1d = dg::create::weights( g_old);
+    dg::HVec v1d = dg::create::inv_weights( g_new);
+    cusp::coo_matrix<int, real_type, cusp::host_memory> projectX;
+    //Here, we cannot use create::projection because that would remove explicit zeros!!
+    projectX = dg::create::interpolationT( g_new, g_old);
     EllSparseBlockMat<real_type> pX( t.N()/divide, t.N(), divide, divide, t.n());
+    for( unsigned k=0; k<divide; k++)
     for( unsigned i=0; i<n; i++)
     for( unsigned j=0; j<n; j++)
-        for( unsigned k=0; k<divide; k++)
-            pX.data[(k*n+i)*n+j] = projectX.values[i*divide*n +k*n+j];
+    {
+        pX.data[(k*n+i)*n+j] = projectX.values[(i*divide +k)*n+j];
+        if( no == normed)
+            pX.data[(k*n+i)*n+j] *= v1d[i]*w1d[j];
+    }
     for( unsigned i=0; i<t.N()/divide; i++)
         for( unsigned d=0; d<divide; d++)
         {
diff --git a/inc/dg/topology/projection.h b/inc/dg/topology/projection.h
index c40b7843b..f0cce22da 100644
--- a/inc/dg/topology/projection.h
+++ b/inc/dg/topology/projection.h
@@ -134,6 +134,7 @@ cusp::coo_matrix< int, real_type, cusp::host_memory> projection( const RealGrid1
         Vc.values[i] = v_c[i];
     }
     cusp::coo_matrix<int, real_type, cusp::host_memory> A = interpolationT( g_new, g_old), temp;
+    //!!! cusp::multiply removes explicit zeros in the output
     cusp::multiply( A, Wf, temp);
     cusp::multiply( Vc, temp, A);
     A.sort_by_row_and_column();
diff --git a/inc/dg/topology/projection_t.cu b/inc/dg/topology/projection_t.cu
index ad8baba4f..11bd4d985 100644
--- a/inc/dg/topology/projection_t.cu
+++ b/inc/dg/topology/projection_t.cu
@@ -17,23 +17,26 @@ int main()
     std::cin >> n_old >> N_old;
     std::cout << "Type n and N of new (coarser) grid!\n";
     std::cin >> n_new >> N_new;
-    dg::Grid1d go ( 0, M_PI, n_old, N_old);
-    dg::Grid1d gn ( 0, M_PI, n_new, N_new);
-    cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::transformation( gn, go);
-    cusp::coo_matrix<int, double, cusp::host_memory> inte = dg::create::interpolation( gn, go);
-    thrust::host_vector<double> v = dg::evaluate( sine, go);
-    thrust::host_vector<double> w1do = dg::create::weights( go);
-    thrust::host_vector<double> w1dn = dg::create::weights( gn);
-    dg::HVec oneo( go.size(), 1.);
-    dg::HVec onen( gn.size(), 1.);
-    thrust::host_vector<double> w( gn.size());
-    dg::blas2::gemv( proj, v, w);
+    dg::Grid1d go ( 0, M_PI/2., n_old, N_old);
+    dg::Grid1d gn ( 0, M_PI/2., n_new, N_new);
+    //cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::projection( gn, go);
+    //cusp::coo_matrix<int, double, cusp::host_memory> inte = dg::create::interpolation( go, gn);
+    dg::MultiMatrix< dg::DMatrix, dg::DVec > proj = dg::create::fast_projection( go,  N_old/N_new);
+    dg::MultiMatrix< dg::DMatrix, dg::DVec > inte = dg::create::fast_interpolation( gn, N_old/N_new);
+    dg::DVec v = dg::evaluate( sine, go);
+    dg::DVec w1do = dg::create::weights( go);
+    dg::DVec w1dn = dg::create::weights( gn);
+    dg::DVec oneo( go.size(), 1.);
+    dg::DVec onen( gn.size(), 1.);
+    dg::DVec w( gn.size());
+    dg::blas2::symv( proj, v, w);
     std::cout << "Original vector  "<<dg::blas2::dot( oneo, w1do, v) << "\n";
     std::cout << "Projected vector "<<dg::blas2::dot( onen, w1dn, w) << "\n";
     std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << "\n"<<std::endl;
-    dg::blas2::gemv( inte, v, w);
-    std::cout << "Original vector  "<<dg::blas2::dot( oneo, w1do, v) << "\n";
-    std::cout << "Interpolated vec "<<dg::blas2::dot( onen, w1dn, w) << "\n";
+    w = dg::evaluate( sine, gn);
+    dg::blas2::symv( inte, w, v);
+    std::cout << "Original vector  "<<dg::blas2::dot( onen, w1dn, w) << "\n";
+    std::cout << "Interpolated vec "<<dg::blas2::dot( oneo, w1do, v) << "\n";
     std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << "\n"<<std::endl;
 
     std::cout << "TEST 2D and 3D\n";
@@ -42,14 +45,16 @@ int main()
     dg::Grid3d g2n (0, M_PI, 0, M_PI, 0,1, n_new, N_new, N_new, 4);
     //cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::transformation( g2n, g2o);
     cusp::coo_matrix<int, double, cusp::host_memory> inte2d = dg::create::interpolation( g2n, g2o);
-    dg::MultiMatrix< dg::HMatrix, std::vector<thrust::host_vector<double>> > proj2d;
-    proj2d.construct( dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new), 2);
+    //dg::MultiMatrix< dg::HMatrix, std::vector<thrust::host_vector<double>> > proj2d;
+    //proj2d.construct( dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new), 2);
+    //dg::IHMatrix proj2d = dg::create::projection( g2n, g2o);
+    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > proj2d = dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new);
     dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > fast_inte2d = dg::create::fast_interpolation( g2n, N_old/N_new, N_old/N_new);
-    const std::vector<dg::HVec> sinO(2, dg::evaluate( sine, g2o)),
-                                sinN(2, dg::evaluate( sine, g2n));
+    const dg::HVec sinO( dg::evaluate( sine, g2o)),
+                                sinN( dg::evaluate( sine, g2n));
     dg::HVec w2do = dg::create::weights( g2o);
     dg::HVec w2dn = dg::create::weights( g2n);
-    std::vector<dg::HVec> sinP( sinN), sinI(sinO);
+    dg::HVec sinP( sinN), sinI(sinO);
     dg::blas2::gemv( proj2d, sinO, sinP); //FAST PROJECTION
     std::cout << "Original vector     "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) << "\n";
     std::cout << "Projected vector    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
@@ -57,12 +62,12 @@ int main()
     std::cout << "Difference between projection and evaluation      ";
     dg::blas1::axpby( 1., sinN, -1., sinP);
     std::cout << sqrt(dg::blas2::dot( sinP, w2dn, sinP)/dg::blas2::dot(sinN, w2dn, sinN))<<"\n";
-    dg::blas2::gemv( inte2d, sinO[0], sinP[0]);
-    std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinP[0], w2dn, sinP[0])) << "\n";
-    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO[0], w2do, sinO[0])) - sqrt(dg::blas2::dot( sinP[0], w2dn, sinP[0])) << "\n" << std::endl;
-    dg::blas2::gemv( fast_inte2d, sinN[0], sinI[0]);
-    std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinI[0], w2do, sinI[0])) << "\n";
-    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinI[0], w2do, sinI[0])) - sqrt(dg::blas2::dot( sinN[0], w2dn, sinN[0])) << "\n" << std::endl;
+    dg::blas2::gemv( inte2d, sinO, sinP);
+    std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
+    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
+    dg::blas2::gemv( fast_inte2d, sinN, sinI);
+    std::cout << "Fast Interpolated vec "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) << "\n";
+    std::cout << "Difference in Norms   "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) - sqrt(dg::blas2::dot( sinN, w2dn, sinN)) << "\n" << std::endl;
 
     return 0;
 }
-- 
GitLab


From aa76466ce1957e8f007ac5a199ad97f984bef385 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 28 Jun 2019 16:17:13 +0200
Subject: [PATCH 101/540] Further modularize feltor

now with initialization in separate file and also static 3d vecs
in feltordiag
---
 src/feltor/feltor.cu     |  99 ++------------
 src/feltor/feltor_hpc.cu | 289 +++++++--------------------------------
 src/feltor/feltordiag.h  |  81 +++++++++++
 src/feltor/init.h        | 208 ++++++++++++++++++++++++++++
 4 files changed, 348 insertions(+), 329 deletions(-)
 create mode 100644 src/feltor/init.h

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 2c32aa4b7..b2129470d 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -16,6 +16,9 @@ using IDMatrix = dg::IDMatrix;
 using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
 
+#include "init.h"
+#include "feltordiag.h"
+
 int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
@@ -64,93 +67,13 @@ int main( int argc, char* argv[])
     std::cout << "Done!\n";
 
     /////////////////////The initial field///////////////////////////////////////////
-    //First the profile and the source (on the host since we want to output those)
-    dg::HVec profile = dg::pullback( dg::geo::Compose<dg::LinearX>( mag.psip(),
-        p.nprofamp/mag.psip()(mag.R0(), 0.), 0.), grid);
-    dg::HVec xpoint_damping = dg::evaluate( dg::one, grid);
-    if( gp.hasXpoint() )
-        xpoint_damping = dg::pullback(
-            dg::geo::ZCutter(-1.1*gp.elongation*gp.a), grid);
-    dg::HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-        //first change coordinate from psi to (psi_0 - psip)/psi_0
-        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift
-        p.rho_source, p.alpha, -1), grid);
-    dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
-
-    dg::HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
-        mag.psip(), -1*p.alpha, 1*p.alpha, -1), grid);
-    dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
-    dg::blas1::pointwiseDot( profile_damping, profile, profile);
-
-    feltor.set_source( profile, p.omega_source, source_damping);
-
-
-    //Now perturbation
-    dg::HVec ntilde = dg::evaluate(dg::zero,grid);
-    if( p.initne == "blob" || p.initne == "straight blob")
-    {
-        dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-        dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
-        if( p.symmetric)
-            ntilde = dg::pullback( init0, grid);
-        else if( p.initne == "blob")//rounds =3 ->2*3-1
-        {
-            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-            //evaluate should always be used with mx,my > 1
-            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 3);
-        }
-        else if( p.initne == "straight blob")//rounds =1 ->2*1-1
-        {
-            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-            //evaluate should always be used with mx,my > 1
-            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
-        }
-    }
-    else if( p.initne == "turbulence")
-    {
-        dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-        dg::BathRZ init0(16,16,Rmin,Zmin, 30.,2.,p.amp);
-        if( p.symmetric)
-            ntilde = dg::pullback( init0, grid);
-        else
-        {
-            dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-            //evaluate should always be used with mx,my > 1
-            ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
-        }
-        dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
-    }
-    else if( p.initne == "zonal")
-    {
-        dg::geo::ZonalFlow init0(mag.psip(), p.amp, 0., p.k_psi);
-        ntilde = dg::pullback( init0, grid);
-        dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
-    }
-    else
-        std::cerr <<"WARNING: Unknown initial condition!\n";
-    std::array<std::array<dg::DVec,2>,2> y0;
-    y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<dg::DVec>(profile);
-    dg::blas1::axpby( 1., dg::construct<dg::DVec>(ntilde), 1., y0[0][0]);
-    std::cout << "initialize ni" << std::endl;
-    feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
-    double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
-    std::cout << "Minimum Ni value "<<minimalni+1<<std::endl;
-    if( minimalni <= -1)
-    {
-        std::cerr << "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n";
-        std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
-        return -1;
-    }
-
-    dg::blas1::copy( 0., y0[1][0]); //set we = 0
-    dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
+    std::array<std::array<DVec,2>,2> y0;
+    feltor::Initialize init( grid, p, mag);
+    if( argc == 4)
+        y0 = init.init_from_parameters();
+    if( argc == 5)
+        y0 = init.init_from_file(argv[4]);
+    feltor.set_source( init.profile(), p.omega_source, init.source_damping());
     std::cout << "Initialize Timestepper" << std::endl;
 
     ////////////////////////create timer and timestepper
@@ -243,7 +166,7 @@ int main( int argc, char* argv[])
             else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
             {
                 dg::assign( *pair.second, hvisual);
-                dg::blas1::axpby( 1., hvisual, -1., profile, hvisual);
+                dg::blas1::axpby( 1., hvisual, -1., init.profile(), hvisual);
             }
             else
                 dg::assign( *pair.second, hvisual);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index bf4d6e666..27e15166a 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -32,6 +32,9 @@ using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #endif //FELTOR_MPI
 
+#include "init.h"
+#include "feltordiag.h"
+
 #ifdef FELTOR_MPI
 //ATTENTION: in slurm should be used with --signal=SIGINT@30 (<signal>@<time in seconds>)
 void sigterm_handler(int signal)
@@ -148,186 +151,16 @@ int main( int argc, char* argv[])
     feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
-    //!///////////////////The profiles///////////////////////////////////////////
-    //First the profile and the source (on the host since we want to output those)
-    HVec profile = dg::pullback( dg::geo::Compose<dg::LinearX>( mag.psip(),
-        p.nprofamp/mag.psip()(mag.R0(), 0.), 0.), grid);
-    HVec xpoint_damping = dg::evaluate( dg::one, grid);
-    if( gp.hasXpoint() )
-        xpoint_damping = dg::pullback(
-            dg::geo::ZCutter(-1.1*gp.elongation*gp.a), grid);
-    HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-        //first change coordinate from psi to (psi_0 - psip)/psi_0
-        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift
-        p.rho_source, p.alpha, -1), grid);
-    HVec damping_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-        //first change coordinate from psi to (psi_0 - psip)/psi_0
-        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift
-        p.rho_damping, p.alpha, +1), grid);
-    dg::blas1::pointwiseDot( xpoint_damping, source_damping, source_damping);
-
-    HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
-        mag.psip(), -p.alpha, p.alpha, -1), grid);
-    dg::blas1::pointwiseDot( xpoint_damping, profile_damping, profile_damping);
-    dg::blas1::pointwiseDot( profile_damping, profile, profile);
-
-    feltor.set_source( profile, p.omega_source, source_damping);
-
     //!///////////////////The initial field///////////////////////////////////////////
     double time = 0;
     std::array<std::array<DVec,2>,2> y0;
+    feltor::Initialize init( grid, p, mag);
     if( argc == 4)
-    {
-        //Now perturbation
-        HVec ntilde = dg::evaluate(dg::zero,grid);
-        if( p.initne == "blob" || p.initne == "straight blob")
-        {
-            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-            dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
-            if( p.symmetric)
-                ntilde = dg::pullback( init0, grid);
-            else if( p.initne == "blob")//rounds =3 ->2*3-1
-            {
-                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-                //evaluate should always be used with mx,my > 1
-                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 3);
-            }
-            else if( p.initne == "straight blob")//rounds =1 ->2*1-1
-            {
-                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-                //evaluate should always be used with mx,my > 1
-                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
-            }
-        }
-        else if( p.initne == "turbulence")
-        {
-            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-            dg::BathRZ init0(16,16,Rmin,Zmin, 30.,2.,p.amp);
-            if( p.symmetric)
-                ntilde = dg::pullback( init0, grid);
-            else
-            {
-                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-                //evaluate should always be used with mx,my > 1
-                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
-            }
-            dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
-        }
-        else if( p.initne == "zonal")
-        {
-            dg::geo::ZonalFlow init0(mag.psip(), p.amp, 0., p.k_psi);
-            ntilde = dg::pullback( init0, grid);
-            dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
-        }
-        else
-            MPI_OUT std::cerr <<"WARNING: Unknown initial condition!\n";
-        y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile);
-        dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-        MPI_OUT std::cout << "initialize ni" << std::endl;
-        feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
-        double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
-        MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
-        if( minimalni <= -1)
-        {
-            MPI_OUT std::cerr << "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n";
-            MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
-            return -1;
-        }
-
-        dg::blas1::copy( 0., y0[1][0]); //set we = 0
-        dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
-    }
+        y0 = init.init_from_parameters();
     if( argc == 5)
-    {
-        ///////////////////read in and show inputfile
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( argv[4], NC_NOWRITE, &ncidIN);
-        size_t lengthIN;
-        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
-        std::string inputIN( lengthIN, 'x');
-        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
-
-        Json::Value jsIN;
-        std::stringstream is(inputIN);
-        parseFromStream( parser, is, &jsIN, &errs); //read input without comments
-        const feltor::Parameters pIN(  jsIN);
-        MPI_OUT std::cout << "RESTART from file "<<argv[4]<< std::endl;
-        MPI_OUT std::cout << " file parameters:" << std::endl;
-        MPI_OUT pIN.display( std::cout);
-
-        // Now read in last timestep
-        Geometry grid_IN( Rmin, Rmax, Zmin, Zmax, 0, 2.*M_PI,
-            pIN.n_out, pIN.Nx_out, pIN.Ny_out, pIN.symmetric ? 1 : pIN.Nz_out, pIN.bcxN, pIN.bcyN, dg::PER
-            #ifdef FELTOR_MPI
-            , comm
-            #endif //FELTOR_MPI
-            );
-        IHMatrix interpolateIN = dg::create::interpolation( grid, grid_IN);
-
-        #ifdef FELTOR_MPI
-        int dimsIN[3],  coordsIN[3];
-        MPI_Cart_get( comm, 3, dimsIN, periods, coordsIN);
-        size_t countIN[4] = {1, grid_IN.local().Nz(),
-            g3d_out.n()*(grid_IN.local().Ny()),
-            g3d_out.n()*(grid_IN.local().Nx())};
-        size_t startIN[4] = {0, coordsIN[2]*countIN[1],
-                                coordsIN[1]*countIN[2],
-                                coordsIN[0]*countIN[3]};
-        #else //FELTOR_MPI
-        size_t startIN[4] = {0, 0, 0, 0};
-        size_t countIN[4] = {1, grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
-            grid_IN.n()*grid_IN.Nx()};
-        #endif //FELTOR_MPI
-        std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
-        HVec transferINH( dg::evaluate(dg::zero, grid_IN));
-
-        std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
+        y0 = init.init_from_file(argv[4]);
+    feltor.set_source( init.profile(), p.omega_source, init.source_damping());
 
-        int timeIDIN;
-        /////////////////////Get time length and initial data///////////////////////////
-        errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
-        errIN = nc_inq_dimlen(ncidIN, timeIDIN, &startIN[0]);
-        startIN[0] -= 1;
-        errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
-        errIN = nc_get_vara_double( ncidIN, timeIDIN, startIN, countIN, &time);
-        MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
-        for( unsigned i=0; i<5; i++)
-        {
-            int dataID;
-            errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataID);
-            errIN = nc_get_vara_double( ncidIN, dataID, startIN, countIN,
-                #ifdef FELTOR_MPI
-                    transferINH.data().data()
-                #else //FELTOR_MPI
-                    transferINH.data()
-                #endif //FELTOR_MPI
-                );
-            dg::blas2::gemv( interpolateIN, transferINH, transferINHvec[i]);
-        }
-        errIN = nc_close(ncidIN);
-        /// ///////////////Now Construct initial fields
-        //
-        //Convert to N-1 and W
-        dg::blas1::plus( transferINHvec[0], -1.);
-        dg::blas1::plus( transferINHvec[1], -1.);
-        dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
-        dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
-
-        dg::assign( transferINHvec[0], y0[0][0]); //ne-1
-        dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
-        dg::assign( transferINHvec[2], y0[1][0]); //We
-        dg::assign( transferINHvec[3], y0[1][1]); //Wi
-
-    }
     ////////////map quantities to output/////////////////
     //since we map pointers we don't need to update those later
     std::map<std::string, const DVec* > v4d;
@@ -386,72 +219,47 @@ int main( int argc, char* argv[])
     size_t count4d[4] = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(),
         g3d_out.n()*g3d_out.Nx()};
 #endif //FELTOR_MPI
-    {   //output static 3d variables into file
-        dg::geo::BFieldR fieldR(mag);
-        dg::geo::BFieldZ fieldZ(mag);
-        dg::geo::BFieldP fieldP(mag);
-
-        HVec vecR = dg::pullback( fieldR, grid);
-        HVec vecZ = dg::pullback( fieldZ, grid);
-        HVec vecP = dg::pullback( fieldP, grid);
-        HVec psip = dg::pullback( mag.psip(), grid);
-        HVec xc = dg::evaluate( dg::cooX3d, grid);
-        HVec yc = dg::evaluate( dg::cooY3d, grid);
-        HVec zc = dg::evaluate( dg::cooZ3d, grid);
-        dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
 
-        std::vector<std::tuple<std::string, const HVec*, std::string> > v3d;
-        v3d.emplace_back( "BR", &vecR,
-            "R-component of magnetic field in cylindrical coordinates");
-        v3d.emplace_back( "BZ", &vecZ,
-            "Z-component of magnetic field in cylindrical coordinates");
-        v3d.emplace_back( "BP", &vecP,
-            "P-component of magnetic field in cylindrical coordinates");
-        v3d.emplace_back( "Psip", &psip, "Flux-function psi");
-        v3d.emplace_back( "Nprof", &profile, "Density profile");
-        v3d.emplace_back( "Source", &source_damping, "Source region");
-        v3d.emplace_back( "Damping", &profile_damping, "Damping region for initial profile");
-        v3d.emplace_back( "xc", &xc, "x-coordinate in Cartesian coordinate system");
-        v3d.emplace_back( "yc", &yc, "y-coordinate in Cartesian coordinate system");
-        v3d.emplace_back( "zc", &zc, "z-coordinate in Cartesian coordinate system");
-
-        IHMatrix project = dg::create::projection( g3d_out, grid);
-        HVec transferH( dg::evaluate(dg::zero, g3d_out));
-        for( auto tp : v3d)
-        {
-            int vecID;
-            MPI_OUT err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 3,
-                &dim_ids[1], &vecID);
-            MPI_OUT err = nc_put_att_text( ncid, vecID,
-                "long_name", std::get<2>(tp).size(), std::get<2>(tp).data());
-            MPI_OUT err = nc_enddef( ncid);
-            dg::blas2::symv( project, *std::get<1>(tp), transferH);
+    //output static 3d variables into file
+    dg::MultiMatrix<IHMatrix,HVec> projectH = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
+    dg::MultiMatrix<IDMatrix,DVec> projectD = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
+    HVec transferH( dg::evaluate(dg::zero, g3d_out));
+    HVec resultH(dg::evaluate( dg::zero, grid));
+    for ( auto record& diagnostics3d_static_list)
+    {
+        int vecID;
+        MPI_OUT err = nc_def_var( ncid, record.name.data(), NC_DOUBLE, 3,
+            &dim_ids[1], &vecID);
+        MPI_OUT err = nc_put_att_text( ncid, vecID,
+            "long_name", record.long_name.size(), record.long_name.data());
+        MPI_OUT err = nc_enddef( ncid);
+        record.function( resultH, var, grid, p, gp, mag);
+        dg::blas2::symv( projectH, resultH, transferH);
 #ifdef FELTOR_MPI
-            if(rank==0)
+        if(rank==0)
+        {
+            for( int rrank=0; rrank<size; rrank++)
             {
-                for( int rrank=0; rrank<size; rrank++)
-                {
-                    if(rrank!=0)
-                        MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                              rrank, rrank, g3d_out.communicator(), &status);
-                    MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
-                    start4d[1] = coords[2]*count4d[1],
-                    start4d[2] = coords[1]*count4d[2],
-                    start4d[3] = coords[0]*count4d[3];
-                    err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
-                        transferH.data().data());
-                }
+                if(rrank!=0)
+                    MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                          rrank, rrank, g3d_out.communicator(), &status);
+                MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
+                start4d[1] = coords[2]*count4d[1],
+                start4d[2] = coords[1]*count4d[2],
+                start4d[3] = coords[0]*count4d[3];
+                err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
+                    transferH.data().data());
             }
-            else
-                MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                          0, rank, g3d_out.communicator());
-            MPI_Barrier( g3d_out.communicator());
+        }
+        else
+            MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                      0, rank, g3d_out.communicator());
+        MPI_Barrier( g3d_out.communicator());
 #else
-            err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
-                transferH.data());
+        err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
+            transferH.data());
 #endif // FELTOR_MPI
-            MPI_OUT err = nc_redef(ncid);
-        }
+        MPI_OUT err = nc_redef(ncid);
     }
 
     //field IDs
@@ -506,8 +314,6 @@ int main( int argc, char* argv[])
     HVec transferH( dg::evaluate(dg::zero, g3d_out));
     DVec transfer2dD = dg::evaluate( dg::zero, g2d_out);
     HVec transfer2dH = dg::evaluate( dg::zero, g2d_out);
-    //fast_projection is an undocumented secret feltor function...
-    IDMatrix project = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
     /// Construct feltor::Variables object for diagnostics
     DVec result = dg::evaluate( dg::zero, grid);
 
@@ -521,7 +327,7 @@ int main( int argc, char* argv[])
     for( auto record& : diagnostics3d_list)
     {
         record.function( result, var);
-        dg::blas2::symv( project, result, transferD);
+        dg::blas2::symv( projectD, result, transferD);
         dg::assign( transferD, transferH);
 #ifdef FELTOR_MPI
             if(rank==0)
@@ -639,7 +445,7 @@ int main( int argc, char* argv[])
         for( auto record& : diagnostics3d_list)
         {
             record.function( result, var);
-            dg::blas2::symv( project, result, transferD);
+            dg::blas2::symv( projectD, result, transferD);
             dg::assign( transferD, transferH);
 #ifdef FELTOR_MPI
             if(rank==0)
@@ -669,10 +475,10 @@ int main( int argc, char* argv[])
         for( auto record& : diagnostics2d_list)
         {
             record.function( result, var);
-            dg::blas2::symv( project, result, transferD);
+            dg::blas2::symv( projectD, result, transferD);
             //toroidal average
-            toroidal_average( transferD, result, false);
-            dg::blas1::transfer( transfer2dD, transfer2dH);
+            toroidal_average( transferD, transfer2dD, false);
+            dg::assign( transfer2dD, transfer2dH);
             err = nc_put_vara_double( ncid_out, id3d.at(record.name+"_ta2d"),
                 start2d, count2d, transfer2dH.data());
 
@@ -683,6 +489,7 @@ int main( int argc, char* argv[])
             err = nc_put_vara_double( ncid_out, id3d.at(record.name+"_2d"),
                 start2d, count2d, t2d_mp.data() );
 
+    void output( int ncid, HVec& transferH, Geometry grid, int varID)
 #ifdef FELTOR_MPI
             if(rank==0)
             {
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index b1f3d001e..d438c0eda 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -8,6 +8,8 @@
 #include "feltor/feltor.cuh"
 #include "feltor/parameters.h"
 
+#include "feltor/init.h"
+
 namespace feltor{
 
 // This file constitutes the diagnostics module of feltor
@@ -151,6 +153,85 @@ struct Record{
     std::function<void( DVec&, Variables&)> function;
 };
 
+struct Record_static{
+    std::string name;
+    std::string long_name;
+    bool integral;
+    std::function<void( HVec&, Variables&, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag)> function;
+};
+
+//Here is a list of static (time-independent) 3d variables that go into the output
+std::vector<Record_static> dianostics3d_static_list = {
+    { "BR", "R-component of magnetic field in cylindrical coordinates",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+            dg::geo::BFieldR fieldR(mag);
+            result = dg::pullback( fieldR, grid);
+        }
+    },
+    { "BZ", "Z-component of magnetic field in cylindrical coordinates",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+            dg::geo::BFieldZ fieldZ(mag);
+            result = dg::pullback( fieldZ, grid);
+        }
+    },
+    { "BP", "Contravariant P-component of magnetic field in cylindrical coordinates",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+            dg::geo::BFieldP fieldP(mag);
+            result = dg::pullback( fieldP, grid);
+        }
+    },
+    { "Psip", "Flux-function psi",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+             result = dg::pullback( mag.psip(), grid);
+        }
+    },
+    { "Nprof", "Density profile",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+            Initialize init;
+            result = init.profile();
+        }
+    },
+    { "Source", "Source region",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+            Initialize init;
+            result = init.source_damping();
+        }
+    },
+    { "Damping", "Damping region for initial profile",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+            Initialize init;
+            result = init.profile_damping();
+        }
+    },
+    { "xc", "x-coordinate in Cartesian coordinate system",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+            HVec xc = dg::evaluate( dg::cooX3d, grid);
+            HVec yc = dg::evaluate( dg::cooY3d, grid);
+            HVec zc = dg::evaluate( dg::cooZ3d, grid);
+            dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
+            result = xc;
+        }
+    },
+    { "yc", "y-coordinate in Cartesian coordinate system",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+            HVec xc = dg::evaluate( dg::cooX3d, grid);
+            HVec yc = dg::evaluate( dg::cooY3d, grid);
+            HVec zc = dg::evaluate( dg::cooZ3d, grid);
+            dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
+            result = yc;
+        }
+    },
+    { "zc", "z-coordinate in Cartesian coordinate system",
+        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+            HVec xc = dg::evaluate( dg::cooX3d, grid);
+            HVec yc = dg::evaluate( dg::cooY3d, grid);
+            HVec zc = dg::evaluate( dg::cooZ3d, grid);
+            dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
+            result = zc;
+        }
+    },
+}
+
 // Here are all 3d outputs we want to have
 std::vector<Record> diagnostics3d_list = {
     {"electrons", "electron density", false,
diff --git a/src/feltor/init.h b/src/feltor/init.h
new file mode 100644
index 000000000..b2d63a88f
--- /dev/null
+++ b/src/feltor/init.h
@@ -0,0 +1,208 @@
+#pragma once
+
+namespace feltor
+{
+//We use the typedefs and MPI_OUT
+struct Initialize
+{
+    Initialize( const Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp,
+        dg::geo::TokamakMagneticField mag) : p(p), gp(gp)
+    {
+    }
+    HVec profile()const{
+        //First the profile and the source (on the host since we want to output those)
+        HVec profile = dg::pullback( dg::geo::Compose<dg::LinearX>( mag.psip(),
+            p.nprofamp/mag.psip()(mag.R0(), 0.), 0.), grid);
+        dg::blas1::pointwiseDot( profile_damping(), profile, profile);
+        return profile;
+    }
+    HVec xpoint_damping()const{
+        HVec xpoint_damping = dg::evaluate( dg::one, grid);
+        if( gp.hasXpoint() )
+            xpoint_damping = dg::pullback(
+                dg::geo::ZCutter(-1.1*gp.elongation*gp.a), grid);
+        return xpoint_damping;
+    }
+    HVec source_damping()const{
+        HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
+            //first change coordinate from psi to (psi_0 - psip)/psi_0
+            dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
+            //then shift
+            p.rho_source, p.alpha, -1), grid);
+        dg::blas1::pointwiseDot( xpoint_damping(), source_damping, source_damping);
+        return source_damping;
+    }
+    HVec damping_damping()const{
+        HVec damping_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
+            //first change coordinate from psi to (psi_0 - psip)/psi_0
+            dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
+            //then shift
+            p.rho_damping, p.alpha, +1), grid);
+        return damping_damping;
+    }
+    HVec profile_damping()const{
+        HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
+            mag.psip(), -p.alpha, p.alpha, -1), grid);
+        dg::blas1::pointwiseDot( xpoint_damping(), profile_damping, profile_damping);
+        return profile_damping;
+    }
+    std::array<std::array<DVec,2>,2> init_from_parameters(){
+        std::array<std::array<DVec,2>,2> y0;
+        //Now perturbation
+        HVec ntilde = dg::evaluate(dg::zero,grid);
+        if( p.initne == "blob" || p.initne == "straight blob")
+        {
+            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
+            dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
+            if( p.symmetric)
+                ntilde = dg::pullback( init0, grid);
+            else if( p.initne == "blob")//rounds =3 ->2*3-1
+            {
+                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+                //evaluate should always be used with mx,my > 1
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 3);
+            }
+            else if( p.initne == "straight blob")//rounds =1 ->2*1-1
+            {
+                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+                //evaluate should always be used with mx,my > 1
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+            }
+        }
+        else if( p.initne == "turbulence")
+        {
+            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
+            dg::BathRZ init0(16,16,Rmin,Zmin, 30.,2.,p.amp);
+            if( p.symmetric)
+                ntilde = dg::pullback( init0, grid);
+            else
+            {
+                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+                //evaluate should always be used with mx,my > 1
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+            }
+            dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
+        }
+        else if( p.initne == "zonal")
+        {
+            dg::geo::ZonalFlow init0(mag.psip(), p.amp, 0., p.k_psi);
+            ntilde = dg::pullback( init0, grid);
+            dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
+        }
+        else
+            MPI_OUT std::cerr <<"WARNING: Unknown initial condition!\n";
+        y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile);
+        dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
+        MPI_OUT std::cout << "initialize ni" << std::endl;
+        feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
+        double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
+        MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
+        if( minimalni <= -1)
+        {
+            MPI_OUT std::cerr << "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n";
+            MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
+            return -1;
+        }
+
+        dg::blas1::copy( 0., y0[1][0]); //set we = 0
+        dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
+        return y0;
+    }
+
+    std::array<std::array<DVec,2>,2> init_from_file( std::string file_name){
+        std::array<std::array<DVec,2>,2> y0;
+        ///////////////////read in and show inputfile
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
+        size_t lengthIN;
+        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
+        std::string inputIN( lengthIN, 'x');
+        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
+
+        Json::Value jsIN;
+        std::stringstream is(inputIN);
+        parseFromStream( parser, is, &jsIN, &errs); //read input without comments
+        const feltor::Parameters pIN(  jsIN);
+        MPI_OUT std::cout << "RESTART from file "<<argv[4]<< std::endl;
+        MPI_OUT std::cout << " file parameters:" << std::endl;
+        MPI_OUT pIN.display( std::cout);
+
+        // Now read in last timestep
+        Geometry grid_IN( Rmin, Rmax, Zmin, Zmax, 0, 2.*M_PI,
+            pIN.n_out, pIN.Nx_out, pIN.Ny_out, pIN.symmetric ? 1 : pIN.Nz_out, pIN.bcxN, pIN.bcyN, dg::PER
+            #ifdef FELTOR_MPI
+            , comm
+            #endif //FELTOR_MPI
+            );
+        IHMatrix interpolateIN = dg::create::interpolation( grid, grid_IN);
+
+        #ifdef FELTOR_MPI
+        int dimsIN[3],  coordsIN[3];
+        MPI_Cart_get( comm, 3, dimsIN, periods, coordsIN);
+        size_t countIN[4] = {1, grid_IN.local().Nz(),
+            g3d_out.n()*(grid_IN.local().Ny()),
+            g3d_out.n()*(grid_IN.local().Nx())};
+        size_t startIN[4] = {0, coordsIN[2]*countIN[1],
+                                coordsIN[1]*countIN[2],
+                                coordsIN[0]*countIN[3]};
+        #else //FELTOR_MPI
+        size_t startIN[4] = {0, 0, 0, 0};
+        size_t countIN[4] = {1, grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
+            grid_IN.n()*grid_IN.Nx()};
+        #endif //FELTOR_MPI
+        std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
+        HVec transferINH( dg::evaluate(dg::zero, grid_IN));
+
+        std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
+
+        int timeIDIN;
+        /////////////////////Get time length and initial data///////////////////////////
+        errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
+        errIN = nc_inq_dimlen(ncidIN, timeIDIN, &startIN[0]);
+        startIN[0] -= 1;
+        errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
+        errIN = nc_get_vara_double( ncidIN, timeIDIN, startIN, countIN, &time);
+        MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
+        for( unsigned i=0; i<5; i++)
+        {
+            int dataID;
+            errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataID);
+            errIN = nc_get_vara_double( ncidIN, dataID, startIN, countIN,
+                #ifdef FELTOR_MPI
+                    transferINH.data().data()
+                #else //FELTOR_MPI
+                    transferINH.data()
+                #endif //FELTOR_MPI
+                );
+            dg::blas2::gemv( interpolateIN, transferINH, transferINHvec[i]);
+        }
+        errIN = nc_close(ncidIN);
+        /// ///////////////Now Construct initial fields
+        //
+        //Convert to N-1 and W
+        dg::blas1::plus( transferINHvec[0], -1.);
+        dg::blas1::plus( transferINHvec[1], -1.);
+        dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
+        dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
+
+        dg::assign( transferINHvec[0], y0[0][0]); //ne-1
+        dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
+        dg::assign( transferINHvec[2], y0[1][0]); //We
+        dg::assign( transferINHvec[3], y0[1][1]); //Wi
+        return y0;
+    }
+
+    private:
+    feltor::Parameters p;
+    dg::geo::solovev::Parameters gp;
+};
+
+
+} //namespace feltor
-- 
GitLab


From 3198ffef9c0ab0792c5ffab604df92caa74da828 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 28 Jun 2019 22:55:42 +0200
Subject: [PATCH 102/540] Add access to boundaries in Simpsons class

---
 inc/dg/simpsons.h    | 16 ++++++++++++++--
 inc/dg/simpsons_t.cu |  5 +++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/inc/dg/simpsons.h b/inc/dg/simpsons.h
index 3d232ae2d..dc795317b 100644
--- a/inc/dg/simpsons.h
+++ b/inc/dg/simpsons.h
@@ -37,7 +37,7 @@ struct SimpsonsRule
      * @param order number of vectors to use for integration.
          Choose 2 (linear) or 3 (parabola) integration.
      */
-    SimpsonsRule( unsigned order = 3): m_counter(0), m_order(order)
+    SimpsonsRule( unsigned order = 3): m_counter(0), m_order(order), m_t0(0)
     {
         set_order(order);
     }
@@ -64,12 +64,14 @@ struct SimpsonsRule
             t = 0;
         m_t.front() = t0;
         flush();
+        m_t0 = t0;
     }
     /*! @brief Reset the integral to zero and the last (t,u) pair in the add function as the new left-side
      */
     void flush() {
-        m_counter = 0;
+        m_counter = 0; //since the counter becomes zero we do not need to touch m_u and m_t since the next add triggers the Trapezoidal rule
         dg::blas1::scal( m_integral, 0.);
+        m_t0 = m_t.front();
     }
     /*! @brief Add a new (t,u) pair to the time integral
      *
@@ -119,12 +121,22 @@ struct SimpsonsRule
     const ContainerType& get_integral() const{
         return m_integral;
     }
+    /*! @brief Access the left and right boundary in time
+     *
+     * associated with the current value of the integral
+     * @return the current integral boundaries
+     */
+    std::array<value_type,2> get_boundaries() const{
+        std::array<value_type,2> times{ m_t0, m_t.front()};
+        return times;
+    }
     private:
     unsigned m_counter, m_order;
     ContainerType m_integral;
     //we use a list here to avoid explicitly calling the swap function
     std::list<value_type> m_t;
     std::list<ContainerType> m_u;
+    value_type m_t0;
 };
 
 }//namespace dg
diff --git a/inc/dg/simpsons_t.cu b/inc/dg/simpsons_t.cu
index 0d2bf84bb..33d7270d2 100644
--- a/inc/dg/simpsons_t.cu
+++ b/inc/dg/simpsons_t.cu
@@ -33,6 +33,9 @@ int main()
     double integral = simpsons.get_integral();
     std::cout << "Error Simpsons is "<<fabs(integral-1.)<<std::endl;
     //![docu]
+    //
+    std::array<double,2> boundaries = simpsons.get_boundaries();
+    std::cout << "Integrated from "<<boundaries[0]<<" (0) to "<<boundaries[1]<<" ("<<M_PI/2.<<") "<<std::endl;
 
     g1d = dg::Grid1d( M_PI/2., M_PI, 3, N );
     times = dg::evaluate( dg::cooX1d, g1d);
@@ -44,6 +47,8 @@ int main()
     simpsons.add( M_PI, -1.);
     integral = simpsons.get_integral();
     std::cout << "Error Trapezoidal is "<<fabs(integral+1.)<<std::endl;
+    boundaries = simpsons.get_boundaries();
+    std::cout << "Integrated from "<<boundaries[0]<<" ("<<M_PI/2.<<") to "<<boundaries[1]<<" ("<<M_PI<<") "<<std::endl;
 
 
     return 0;
-- 
GitLab


From 340e71476a65ddb69b943e5bbe47d079614c5994 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 28 Jun 2019 22:56:28 +0200
Subject: [PATCH 103/540] More work on restructuring feltor

---
 src/feltor/feltor.cu     |  93 +++++-----------
 src/feltor/feltor.cuh    |  28 +----
 src/feltor/feltor_hpc.cu | 231 +++++++++++++++++++++++----------------
 src/feltor/feltordiag.h  |  22 ++--
 src/feltor/init.h        |   5 +-
 5 files changed, 180 insertions(+), 199 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index b2129470d..9441e87e5 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -15,6 +15,7 @@ using DMatrix = dg::DMatrix;
 using IDMatrix = dg::IDMatrix;
 using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
+#define MPI_OUT
 
 #include "init.h"
 #include "feltordiag.h"
@@ -70,7 +71,7 @@ int main( int argc, char* argv[])
     std::array<std::array<DVec,2>,2> y0;
     feltor::Initialize init( grid, p, mag);
     if( argc == 4)
-        y0 = init.init_from_parameters();
+        y0 = init.init_from_parameters(feltor);
     if( argc == 5)
         y0 = init.init_from_file(argv[4]);
     feltor.set_source( init.profile(), p.omega_source, init.source_damping());
@@ -79,7 +80,7 @@ int main( int argc, char* argv[])
     ////////////////////////create timer and timestepper
     //
     dg::Timer t;
-    double time = 0, dt_new = p.dt;//, dt =0;
+    double time = 0.;
     unsigned step = 0;
     dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
         feltor::FeltorSpecialSolver<
@@ -88,45 +89,12 @@ int main( int argc, char* argv[])
     karniadakis.init( feltor, im, time, y0, p.dt);
     std::cout << "Done!" << std::endl;
 
-    //dg::Adaptive< dg::ERKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
-    //    "Bogacki-Shampine-4-2-3", y0);
-    //adaptive.stepper().ignore_fsal();//necessary for splitting
-    //dg::ImplicitRungeKutta<std::array<std::array<dg::DVec,2>,2>,
-    //    feltor::FeltorSpecialSolver<
-    //    dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>> dirk(
-    //        "Trapezoidal-2-2", grid, p, mag);
-    //since we map pointers we don't need to update those later
-
     std::map<std::string, const dg::DVec* > v4d;
     v4d["ne-1 / "] = &y0[0][0],               v4d["ni-1 / "] = &y0[0][1];
     v4d["Ue / "]   = &feltor.fields()[1][0],  v4d["Ui / "]   = &feltor.fields()[1][1];
     v4d["Ome / "] = &feltor.potential()[0]; v4d["Apar / "] = &feltor.induction();
-    const feltor::Quantities& q = feltor.quantities();
-    double dEdt = 0, accuracy = 0, dMdt = 0, accuracyM  = 0;
-    std::map<std::string, const double*> v0d{
-        {"energy", &q.energy}, {"ediff", &q.ediff},
-        {"mass", &q.mass}, {"diff", &q.diff}, {"Apar", &q.Apar},
-        {"Se", &q.S[0]}, {"Si", &q.S[1]}, {"Uperp", &q.Tperp},
-        {"Upare", &q.Tpar[0]}, {"Upari", &q.Tpar[1]},
-        {"dEdt", &dEdt}, {"accuracy", &accuracy},
-        {"aligned", &q.aligned}
-    };
-
-    //first, update quantities in feltor
-
-    {
-        std::array<std::array<dg::DVec,2>,2> y1(y0);
-        try{
-            feltor( time, y0, y1);
-        } catch( dg::Fail& fail) {
-            std::cerr << "CG failed to converge in first step to "
-                      << fail.epsilon()<<"\n";
-            return -1;
-        }
-        feltor.update_quantities();
-    }
-    q.display( std::cout );
-    double energy0 = q.energy, mass0 = q.mass, E0 = energy0, M0 = mass0;
+    double dEdt = 0, accuracy = 0;
+    double E0 = 0.;
     /////////////////////////set up transfer for glfw
     dg::DVec dvisual( grid.size(), 0.);
     dg::HVec hvisual( grid.size(), 0.), visual(hvisual), avisual(hvisual);
@@ -151,6 +119,8 @@ int main( int argc, char* argv[])
     dg::Average<dg::HVec> toroidal_average( grid, dg::coo3d::z);
     title << std::setprecision(2) << std::scientific;
     //unsigned failed_counter = 0;
+    std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
+    std::vector<std::string> energy_diff = { "resistivity", "leeperp", "leiperp", "leeparallel", "leiparallel"};
     while ( !glfwWindowShouldClose( w ))
     {
         title << std::fixed;
@@ -208,22 +178,6 @@ int main( int argc, char* argv[])
             {
                 try{
                     karniadakis.step( feltor, im, time, y0);
-                    //do
-                    //{
-                    //    //Strang splitting
-                    //    dt = dt_new;
-                    //    dirk.step( im, time, y0, time, y0, dt/2.);
-                    //    adaptive.step( feltor, time-dt/2., y0, time, y0, dt_new,
-                    //        dg::pid_control, dg::l2norm, p.rtol, 1e-10);
-                    //    if( adaptive.failed())
-                    //    {
-                    //        failed_counter++;
-                    //        std::cout << "FAILED STEP # "<<failed_counter<<" ! REPEAT!\n";
-                    //        time -= dt; // time has to be reset here
-                    //        // in case of failure diffusion is applied twice?
-                    //    }
-                    //}while ( adaptive.failed());
-                    //dirk.step( im, time-dt/2., y0, time, y0, dt/2.);
                 }
                 catch( dg::Fail& fail) {
                     std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
@@ -234,22 +188,29 @@ int main( int argc, char* argv[])
                 step++;
             }
             double deltat = time - previous_time;
-            feltor.update_quantities();
-            std::cout << "Timestep "<<dt_new<<"\n";
-            dEdt = (*v0d["energy"] - E0)/deltat, dMdt = (*v0d["mass"] - M0)/deltat;
-            E0 = *v0d["energy"], M0 = *v0d["mass"];
-            accuracy  = 2.*fabs( (dEdt - *v0d["ediff"])/( dEdt + *v0d["ediff"]));
-            accuracyM = 2.*fabs( (dMdt - *v0d["diff"])/( dMdt + *v0d["diff"]));
+            double energy = 0, ediff = 0.;
+            for( auto record& : diagnostics2d_list)
+            {
+                if( std::find( energies.begin(), energies.end(), record.name) != energies.end())
+                {
+                    record.function( result, var);
+                    energy += dg::blas1::dot( result, feltor.vol3d());
+                }
+                if( std::find( energy_diff.begin(), energy_diff.end(), record.name) != energy_diff.end())
+                {
+                    record.function( result, var);
+                    ediff += dg::blas1::dot( result, feltor.vol3d());
+                }
+
+            }
+            dEdt = (energy - E0)/deltat;
+            E0 = energy;
+            accuracy  = 2.*fabs( (dEdt - ediff)/( dEdt + ediff));
 
-            q.display(std::cout);
-            std::cout << "(m_tot-m_0)/m_0: "<< (*v0d["mass"]-mass0)/mass0<<"\t";
-            std::cout << "(E_tot-E_0)/E_0: "<< (*v0d["energy"]-energy0)/energy0<<"\t";
+            std::cout << "Time "<<time<<"\n";
             std::cout <<" d E/dt = " << dEdt
-              <<" Lambda = " << *v0d["ediff"]
+              <<" Lambda = " << ediff
               <<" -> Accuracy: " << accuracy << "\n";
-            std::cout <<" d M/dt = " << dMdt
-                      <<" Lambda = " << *v0d["diff"]
-                      <<" -> Accuracy: " << accuracyM << "\n";
             //----------------Test if induction equation holds
             if( p.beta != 0)
             {
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.cuh
index bb0204994..03aeb6458 100644
--- a/src/feltor/feltor.cuh
+++ b/src/feltor/feltor.cuh
@@ -158,33 +158,6 @@ struct ComputeSource{
 };
 }//namespace routines
 
-
-struct Quantities
-{
-    double mass = 0, diff = 0; //mass and mass diffusion
-    double energy = 0, ediff = 0; //total energy and energy diffusion
-    //entropy parallel and perp energies
-    double S[2] = {0,0}, Tpar[2] = {0,0}, Tperp = 0, Apar = 0;
-    //resisitive and diffusive terms
-    double Dres = 0, Dpar[4] = {0,0,0,0}, Dperp[4] = {0,0,0,0};
-    double aligned = 0; //alignment parameter
-    double source[4] = {0,0,0,0}; //source terms
-    void display( std::ostream& os = std::cout ) const
-    {
-        os << "Quantities: \n"
-           << "    Mass: "<<std::setw(11)<< mass  <<" Mass diffusion   "<<diff<<"\n"
-           << "  Energy: "<<std::setw(11)<<energy <<" Energy diffusion "<<ediff<<"\n"
-           << "       S: ["<<S[0]<<", "<<S[1]<<"]\n"
-           << "   Tperp: "<<Tperp<<"  Apar: "<<Apar<<"\n"
-           << "    Tpar: ["<<Tpar[0]<<", "<<Tpar[1]<<"]\n"
-           << "    Dres: "<<Dres<<"\n"
-           << "    Dpar: ["<<Dpar[0]<<", "<<Dpar[1]<<", "<<Dpar[2]<<", "<<Dpar[3]<<"]\n"
-           << "   Dperp: ["<<Dperp[0]<<", "<<Dperp[1]<<", "<<Dperp[2]<<", "<<Dperp[3]<<"]\n"
-           << " Sources: ["<<source[0]<<", "<<source[1]<<", "<<source[2]<<", "<<source[3]<<"]\n"
-           << " aligned: "<<aligned<<"\n";
-    }
-};
-
 template< class Geometry, class IMatrix, class Matrix, class Container >
 struct Explicit
 {
@@ -275,6 +248,7 @@ struct Explicit
     const Container& bphi( ) const { return m_bphi; }
     const Container& binv( ) const { return m_binv; }
     const Container& divb( ) const { return m_divb; }
+    const Container& vol3d() const { return m_vol3d;}
     //bhat / sqrt{g} / B
     const std::array<Container, 3> & bhatgB () const {
         return m_b;
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 27e15166a..e4bc86c3b 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -21,6 +21,7 @@ using DMatrix = dg::MDMatrix;
 using IDMatrix = dg::MIDMatrix;
 using IHMatrix = dg::MIHMatrix;
 using Geometry = dg::CylindricalMPIGrid3d;
+using Geometry2d = dg::CartesianMPIGrid2d;
 #define MPI_OUT if(rank==0)
 #else //FELTOR_MPI
 using HVec = dg::HVec;
@@ -29,6 +30,7 @@ using DMatrix = dg::DMatrix;
 using IDMatrix = dg::IDMatrix;
 using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
+using Geometry2d = dg::CartesianGrid2d;
 #define MPI_OUT
 #endif //FELTOR_MPI
 
@@ -139,8 +141,8 @@ int main( int argc, char* argv[])
         , comm
         #endif //FELTOR_MPI
         );
-    DVec vol3d = dg::create::volume( grid);
-    DVec temp(vol3d); //needed for Apar test
+    std::unique_ptr<Geometry2d> g2d_out_ptr = g3d_out.perp_grid();
+
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
@@ -151,34 +153,40 @@ int main( int argc, char* argv[])
     feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
+    // helper variables for various stuff
+    std::array<DVec, 3> gradPsip;
+    gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
+    gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
+    gradPsip[2] =  result; //zero
+    feltor::Variables var = {
+        feltor, p, gradPsip, gradPsip
+    };
+    std::map<std::string, dg::Simpsons<DVec>> time_integrals;
+    dg::Average<DVec> toroidal_average( g3d_out, dg::coo3d::z);
+    dg::MultiMatrix<IHMatrix,HVec> projectH = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
+    dg::MultiMatrix<IDMatrix,DVec> projectD = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
+    HVec transferH( dg::evaluate(dg::zero, g3d_out));
+    DVec transferD( dg::evaluate(dg::zero, g3d_out));
+    DVec transfer2dD = dg::evaluate( dg::zero, g2d_out);
+    HVec transfer2dH = dg::evaluate( dg::zero, g2d_out);
+    /// Construct feltor::Variables object for diagnostics
+    DVec result = dg::evaluate( dg::zero, grid);
+    HVec resultH( dg::evaluate( dg::zero, grid));
+
+    double dEdt = 0, accuracy = 0;
+    double E0 = 0.;
+
     //!///////////////////The initial field///////////////////////////////////////////
     double time = 0;
     std::array<std::array<DVec,2>,2> y0;
     feltor::Initialize init( grid, p, mag);
     if( argc == 4)
-        y0 = init.init_from_parameters();
+        y0 = init.init_from_parameters(feltor);
     if( argc == 5)
         y0 = init.init_from_file(argv[4]);
     feltor.set_source( init.profile(), p.omega_source, init.source_damping());
 
-    ////////////map quantities to output/////////////////
-    //since we map pointers we don't need to update those later
-    std::map<std::string, const DVec* > v4d;
-    v4d["electrons"] = &feltor.fields()[0][0], v4d["ions"] = &feltor.fields()[0][1];
-    v4d["Ue"] = &feltor.fields()[1][0],        v4d["Ui"] = &feltor.fields()[1][1];
-    v4d["potential"] = &feltor.potential()[0];
-    v4d["induction"] = &feltor.induction();
-    const feltor::Quantities& q = feltor.quantities();
-    double dEdt = 0, accuracy = 0, dMdt = 0, accuracyM  = 0;
-    std::map<std::string, const double*> v0d{
-        {"energy", &q.energy}, {"ediff", &q.ediff},
-        {"mass", &q.mass}, {"diff", &q.diff}, {"Apar", &q.Apar},
-        {"Se", &q.S[0]}, {"Si", &q.S[1]}, {"Uperp", &q.Tperp},
-        {"Upare", &q.Tpar[0]}, {"Upari", &q.Tpar[1]},
-        {"dEdt", &dEdt}, {"accuracy", &accuracy},
-        {"aligned", &q.aligned}
-    };
-    /////////////////////////////set up netcdf/////////////////////////////////////
+    /// //////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
     std::string file_name = argv[3];
     int ncid=-1;
@@ -205,6 +213,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_put_att_text( ncid, NC_GLOBAL,
             pair.first.data(), pair.second.size(), pair.second.data());
 
+    // Define dimensions (t,z,y,x)
     int dim_ids[4], tvarID;
 #ifdef FELTOR_MPI
     int coords[3];
@@ -220,11 +229,7 @@ int main( int argc, char* argv[])
         g3d_out.n()*g3d_out.Nx()};
 #endif //FELTOR_MPI
 
-    //output static 3d variables into file
-    dg::MultiMatrix<IHMatrix,HVec> projectH = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
-    dg::MultiMatrix<IDMatrix,DVec> projectD = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
-    HVec transferH( dg::evaluate(dg::zero, g3d_out));
-    HVec resultH(dg::evaluate( dg::zero, grid));
+    //create & output static 3d variables into file
     for ( auto record& diagnostics3d_static_list)
     {
         int vecID;
@@ -233,7 +238,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_put_att_text( ncid, vecID,
             "long_name", record.long_name.size(), record.long_name.data());
         MPI_OUT err = nc_enddef( ncid);
-        record.function( resultH, var, grid, p, gp, mag);
+        record.function( resultH, var, grid, gp, mag);
         dg::blas2::symv( projectH, resultH, transferH);
 #ifdef FELTOR_MPI
         if(rank==0)
@@ -262,9 +267,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_redef(ncid);
     }
 
-    //field IDs
-    int EtimeID, EtimevarID;
-    MPI_OUT err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    //Create field IDs
     std::map<std::string, int> id3d, id4d;
     for( auto record& : feltor::diagnostics3d_list)
     {
@@ -279,6 +282,10 @@ int main( int argc, char* argv[])
     {
         std::string name = record.name + "_ta2d";
         std::string long_name = record.long_name + " (Toroidal average)";
+        if( record.integral){
+            name += "_tt";
+            long_name+= " (Time average)";
+        }
         MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
             &id3d[name]);//creates a new id3d entry
         MPI_OUT err = nc_put_att_text( ncid, id3d[name], "long_name", long_name.size(),
@@ -286,6 +293,10 @@ int main( int argc, char* argv[])
 
         name = record.name + "_2d";
         long_name = record.long_name + " (Evaluated on phi = pi plane)";
+        if( record.integral){
+            name += "_tt";
+            long_name+= " (Time average)";
+        }
         err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
             &id3d[name]);
         err = nc_put_att_text( ncid, id3d[name], "long_name", long_name.size(),
@@ -293,9 +304,8 @@ int main( int argc, char* argv[])
     }
     MPI_OUT err = nc_enddef(ncid);
     ///////////////////////////////////first output/////////////////////////
-    double dt_new = p.dt;//, dt =0;
     MPI_OUT std::cout << "First output ... \n";
-    //first, update quantities in feltor
+    //first, update feltor (to get potential ....)
     {
         std::array<std::array<DVec,2>,2> y1(y0);
         try{
@@ -306,24 +316,8 @@ int main( int argc, char* argv[])
             MPI_OUT err = nc_close(ncid);
             return -1;
         }
-        feltor.update_quantities();
     }
-    MPI_OUT q.display(std::cout);
-    double energy0 = q.energy, mass0 = q.mass, E0 = energy0, M0 = mass0;
-    DVec transferD( dg::evaluate(dg::zero, g3d_out));
-    HVec transferH( dg::evaluate(dg::zero, g3d_out));
-    DVec transfer2dD = dg::evaluate( dg::zero, g2d_out);
-    HVec transfer2dH = dg::evaluate( dg::zero, g2d_out);
-    /// Construct feltor::Variables object for diagnostics
-    DVec result = dg::evaluate( dg::zero, grid);
 
-    std::array<DVec, 3> gradPsip;
-    gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
-    gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
-    gradPsip[2] =  result; //zero
-    feltor::Variables var = {
-        feltor, p, gradPsip, gradPsip
-    };
     for( auto record& : diagnostics3d_list)
     {
         record.function( result, var);
@@ -352,16 +346,39 @@ int main( int argc, char* argv[])
 #else
             err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
                 transferH.data());
+#endif // FELTOR_MPI
+    }
+    for( auto record& : diagnostics2d_list)
+    {
+        record.function( result, var);
+        dg::blas2::symv( projectD, result, transferD);
+        dg::assign( transferD, transferH);
+#ifdef FELTOR_MPI
+            if(rank==0)
+            {
+                for( int rrank=0; rrank<size; rrank++)
+                {
+                    if(rrank!=0)
+                        MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                              rrank, rrank, g3d_out.communicator(), &status);
+                    MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
+                    start4d[1] = coords[2]*count4d[1],
+                    start4d[2] = coords[1]*count4d[2],
+                    start4d[3] = coords[0]*count4d[3];
+                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
+                        transferH.data().data());
+                }
+            }
+            else
+                MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
+                          0, rank, g3d_out.communicator());
+            MPI_Barrier( g3d_out.communicator());
+#else
+            err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
+                transferH.data());
 #endif // FELTOR_MPI
     }
     MPI_OUT err = nc_put_vara_double( ncid, tvarID, start4d, count4d, &time);
-    MPI_OUT err = nc_put_vara_double( ncid, EtimevarID, start4d, count4d, &time);
-
-    size_t Estart[] = {0};
-    size_t Ecount[] = {1};
-    for( auto pair : v0d)
-        MPI_OUT err = nc_put_vara_double( ncid, id0d.at(pair.first),
-            Estart, Ecount, pair.second);
     MPI_OUT err = nc_close(ncid);
     MPI_OUT std::cout << "First write successful!\n";
     ///////////////////////////////////////Timeloop/////////////////////////////////
@@ -373,7 +390,8 @@ int main( int argc, char* argv[])
     dg::Timer t;
     t.tic();
     unsigned step = 0;
-    MPI_OUT q.display(std::cout);
+    std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
+    std::vector<std::string> energy_diff = { "resistivity", "leeperp", "leiperp", "leeparallel", "leiparallel"};
     for( unsigned i=1; i<=p.maxout; i++)
     {
 
@@ -394,31 +412,47 @@ int main( int argc, char* argv[])
                 }
                 step++;
             }
+            Timer tti;
+            tti.tic();
             double deltat = time - previous_time;
-            feltor.update_quantities();
-            MPI_OUT std::cout << "Time "<<time<<" Current timestep "<<dt_new<<"\n";
-            dEdt = (*v0d["energy"] - E0)/deltat, dMdt = (*v0d["mass"] - M0)/deltat;
-            E0 = *v0d["energy"], M0 = *v0d["mass"];
-            accuracy  = 2.*fabs( (dEdt - *v0d["ediff"])/( dEdt + *v0d["ediff"]));
-            accuracyM = 2.*fabs( (dMdt - *v0d["diff"])/( dMdt + *v0d["diff"]));
-            MPI_OUT err = nc_open(file_name.data(), NC_WRITE, &ncid);
-            Estart[0]++;
-            MPI_OUT err = nc_put_vara_double( ncid, EtimevarID,
-                Estart, Ecount, &time);
-            for( auto pair : v0d)
-                MPI_OUT err = nc_put_vara_double( ncid, id0d.at(pair.first),
-                    Estart, Ecount, pair.second);
-            MPI_OUT err = nc_close(ncid);
+            double energy = 0, ediff = 0.;
+            for( auto record& : diagnostics2d_list)
+            {
+                if( std::find( energies.begin(), energies.end(), record.name) != energies.end())
+                {
+                    record.function( result, var);
+                    energy += dg::blas1::dot( result, feltor.vol3d());
+                }
+                if( std::find( energy_diff.begin(), energy_diff.end(), record.name) != energy_diff.end())
+                {
+                    record.function( result, var);
+                    ediff += dg::blas1::dot( result, feltor.vol3d());
+                }
+                if( record.integral)
+                {
+                    record.function( result, var);
+                    dg::blas2::symv( projectD, result, transferD);
+                    //toroidal average and add to time integral
+                    toroidal_average( transferD, transfer2dD);
+                    time_integrals.at(record.name+"_ta2d_tt").add( time, transfer2dD);
 
-            MPI_OUT q.display(std::cout);
-            MPI_OUT std::cout << "(m_tot-m_0)/m_0: "<< (*v0d["mass"]-mass0)/mass0<<"\t";
-            MPI_OUT std::cout << "(E_tot-E_0)/E_0: "<< (*v0d["energy"]-energy0)/energy0<<"\t";
+                    // 2d data of plane varphi = 0
+                    unsigned kmp = g3d_out.Nz()/2;
+                    dg::HVec t2d_mp(result.data().begin() + kmp*g2d_out.size(),
+                        result.data().begin() + (kmp+1)*g2d_out.size() );
+                    time_integrals.at(record.name+"_2d_tt").add( time, transfer2dD);
+                }
+
+            }
+
+            dEdt = (energy - E0)/deltat;
+            E0 = energy;
+            accuracy  = 2.*fabs( (dEdt - ediff)/( dEdt + ediff));
+
+            MPI_OUT std::cout << "Time "<<time<<"\n";
             MPI_OUT std::cout <<" d E/dt = " << dEdt
-                      <<" Lambda = " << *v0d["ediff"]
+                      <<" Lambda = " << ediff
                       <<" -> Accuracy: " << accuracy << "\n";
-            MPI_OUT std::cout <<" d M/dt = " << dMdt
-                      <<" Lambda = " << *v0d["diff"]
-                      <<" -> Accuracy: " << accuracyM << "\n";
             //----------------Test if induction equation holds
             if( p.beta != 0)
             {
@@ -426,11 +460,13 @@ int main( int argc, char* argv[])
                     feltor.density(0), feltor.velocity(0), temp);
                 dg::blas1::pointwiseDot( p.beta,
                     feltor.density(1), feltor.velocity(1), -p.beta, temp);
-                double norm  = dg::blas2::dot( temp, vol3d, temp);
+                double norm  = dg::blas2::dot( temp, feltor.vol3d(), temp);
                 dg::blas1::axpby( -1., feltor.lapMperpA(), 1., temp);
-                double error = dg::blas2::dot( temp, vol3d, temp);
+                double error = dg::blas2::dot( temp, feltor.vol3d(), temp);
                 MPI_OUT std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
             }
+            tti.tic();
+            std::cout << " Time for internal diagnostics "<<tti.diff()<<"s\n";
         }
         ti.toc();
         MPI_OUT std::cout << "\n\t Step "<<step <<" of "
@@ -474,22 +510,29 @@ int main( int argc, char* argv[])
         }
         for( auto record& : diagnostics2d_list)
         {
-            record.function( result, var);
-            dg::blas2::symv( projectD, result, transferD);
-            //toroidal average
-            toroidal_average( transferD, transfer2dD, false);
-            dg::assign( transfer2dD, transfer2dH);
-            err = nc_put_vara_double( ncid_out, id3d.at(record.name+"_ta2d"),
-                start2d, count2d, transfer2dH.data());
-
-            // 2d data of plane varphi = 0
-            unsigned kmp = g3d_out.Nz()/2;
-            dg::HVec t2d_mp(result.data().begin() + kmp*g2d_out.size(),
-                result.data().begin() + (kmp+1)*g2d_out.size() );
-            err = nc_put_vara_double( ncid_out, id3d.at(record.name+"_2d"),
-                start2d, count2d, t2d_mp.data() );
+            if(!record.integral)
+            {
+                record.function( result, var);
+                dg::blas2::symv( projectD, result, transferD);
+                //toroidal average
+                toroidal_average( transferD, transfer2dD);
+                dg::assign( transfer2dD, transfer2dH);
 
-    void output( int ncid, HVec& transferH, Geometry grid, int varID)
+                // 2d data of plane varphi = 0
+                unsigned kmp = g3d_out.Nz()/2;
+                dg::HVec t2d_mp(result.data().begin() + kmp*g2d_out.size(),
+                    result.data().begin() + (kmp+1)*g2d_out.size() );
+            }
+            else //manage the time integrators
+            {
+                transfer2dD = time_integrals.at(record.name+"_ta2d_tt").get_integral( );
+                std::array<double,2> bb = time_integrals.at(record.name+"_ta2d_tt").get_boundaries( );
+                dg::scal( transfer2dD, 1./(bb[1]-bb[0]));
+                time_integrals.at(record.name+"_ta2d_tt").flush( );
+                transfer2dD = time_integrals.at(record.name+"_2d_tt").get_integral( );
+                std::array<double,2> bb = time_integrals.at(record.name+"_2d_tt").get_boundaries( );
+                dg::scal( transfer2dD, 1./(bb[1]-bb[0]));
+                time_integrals.at(record.name+"_2d_tt").flush( );
 #ifdef FELTOR_MPI
             if(rank==0)
             {
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index d438c0eda..2e4d4a8a2 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -157,54 +157,54 @@ struct Record_static{
     std::string name;
     std::string long_name;
     bool integral;
-    std::function<void( HVec&, Variables&, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag)> function;
+    std::function<void( HVec&, Variables&, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag)> function;
 };
 
 //Here is a list of static (time-independent) 3d variables that go into the output
 std::vector<Record_static> dianostics3d_static_list = {
     { "BR", "R-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
             dg::geo::BFieldR fieldR(mag);
             result = dg::pullback( fieldR, grid);
         }
     },
     { "BZ", "Z-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
             dg::geo::BFieldZ fieldZ(mag);
             result = dg::pullback( fieldZ, grid);
         }
     },
     { "BP", "Contravariant P-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
             dg::geo::BFieldP fieldP(mag);
             result = dg::pullback( fieldP, grid);
         }
     },
     { "Psip", "Flux-function psi",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
              result = dg::pullback( mag.psip(), grid);
         }
     },
     { "Nprof", "Density profile",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
             Initialize init;
             result = init.profile();
         }
     },
     { "Source", "Source region",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
             Initialize init;
             result = init.source_damping();
         }
     },
     { "Damping", "Damping region for initial profile",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
             Initialize init;
             result = init.profile_damping();
         }
     },
     { "xc", "x-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -213,7 +213,7 @@ std::vector<Record_static> dianostics3d_static_list = {
         }
     },
     { "yc", "y-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -222,7 +222,7 @@ std::vector<Record_static> dianostics3d_static_list = {
         }
     },
     { "zc", "z-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
diff --git a/src/feltor/init.h b/src/feltor/init.h
index b2d63a88f..e7b7a52ec 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -46,7 +46,10 @@ struct Initialize
         dg::blas1::pointwiseDot( xpoint_damping(), profile_damping, profile_damping);
         return profile_damping;
     }
-    std::array<std::array<DVec,2>,2> init_from_parameters(){
+    template<class Feltor>
+    std::array<std::array<DVec,2>,2> init_from_parameters(Feltor& feltor){
+        HVec profile = profile();
+        HVec profile_damping = profile_damping();
         std::array<std::array<DVec,2>,2> y0;
         //Now perturbation
         HVec ntilde = dg::evaluate(dg::zero,grid);
-- 
GitLab


From 80e1deca4fac8f7b83e3d2bd87cd98dadcc1f953 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 29 Jun 2019 13:06:42 +0200
Subject: [PATCH 104/540] Factor out serial netcdf output into separate file

and further work on cleaning feltor
---
 src/feltor/feltor_hpc.cu   | 196 ++++++++-----------------------------
 src/feltor/feltordiag.h    |   2 +
 src/feltor/init.h          |   6 +-
 src/feltor/serial_netcdf.h | 140 ++++++++++++++++++++++++++
 4 files changed, 185 insertions(+), 159 deletions(-)
 create mode 100644 src/feltor/serial_netcdf.h

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index e4bc86c3b..302abb443 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -21,7 +21,6 @@ using DMatrix = dg::MDMatrix;
 using IDMatrix = dg::MIDMatrix;
 using IHMatrix = dg::MIHMatrix;
 using Geometry = dg::CylindricalMPIGrid3d;
-using Geometry2d = dg::CartesianMPIGrid2d;
 #define MPI_OUT if(rank==0)
 #else //FELTOR_MPI
 using HVec = dg::HVec;
@@ -30,7 +29,6 @@ using DMatrix = dg::DMatrix;
 using IDMatrix = dg::IDMatrix;
 using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
-using Geometry2d = dg::CartesianGrid2d;
 #define MPI_OUT
 #endif //FELTOR_MPI
 
@@ -62,7 +60,6 @@ int main( int argc, char* argv[])
 #endif
     int periods[3] = {false, false, true}; //non-, non-, periodic
     int rank, size;
-    MPI_Status status;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     MPI_Comm_size( MPI_COMM_WORLD, &size);
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
@@ -141,7 +138,6 @@ int main( int argc, char* argv[])
         , comm
         #endif //FELTOR_MPI
         );
-    std::unique_ptr<Geometry2d> g2d_out_ptr = g3d_out.perp_grid();
 
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
@@ -167,8 +163,8 @@ int main( int argc, char* argv[])
     dg::MultiMatrix<IDMatrix,DVec> projectD = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
     HVec transferH( dg::evaluate(dg::zero, g3d_out));
     DVec transferD( dg::evaluate(dg::zero, g3d_out));
-    DVec transfer2dD = dg::evaluate( dg::zero, g2d_out);
-    HVec transfer2dH = dg::evaluate( dg::zero, g2d_out);
+    DVec transferD2d = dg::evaluate( dg::zero, g2d_out);
+    HVec transferH2d = dg::evaluate( dg::zero, g2d_out);
     /// Construct feltor::Variables object for diagnostics
     DVec result = dg::evaluate( dg::zero, grid);
     HVec resultH( dg::evaluate( dg::zero, grid));
@@ -191,6 +187,7 @@ int main( int argc, char* argv[])
     std::string file_name = argv[3];
     int ncid=-1;
     MPI_OUT err = nc_create( file_name.data(), NC_NETCDF4|NC_CLOBBER, &ncid);
+    feltor::ManageOutput output(g3d_out);
     /// Set global attributes
     std::map<std::string, std::string> att;
     att["title"] = "Output file of feltor/src/feltor_hpc.cu";
@@ -216,17 +213,9 @@ int main( int argc, char* argv[])
     // Define dimensions (t,z,y,x)
     int dim_ids[4], tvarID;
 #ifdef FELTOR_MPI
-    int coords[3];
     MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out.global());
-    size_t start4d[4] = {0, 0, 0, 0};
-    size_t count4d[4] = {1, g3d_out.local().Nz(),
-        g3d_out.n()*(g3d_out.local().Ny()),
-        g3d_out.n()*(g3d_out.local().Nx())};
 #else //FELTOR_MPI
     err = file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out);
-    size_t start4d[4] = {0, 0, 0, 0};
-    size_t count4d[4] = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(),
-        g3d_out.n()*g3d_out.Nx()};
 #endif //FELTOR_MPI
 
     //create & output static 3d variables into file
@@ -240,30 +229,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_enddef( ncid);
         record.function( resultH, var, grid, gp, mag);
         dg::blas2::symv( projectH, resultH, transferH);
-#ifdef FELTOR_MPI
-        if(rank==0)
-        {
-            for( int rrank=0; rrank<size; rrank++)
-            {
-                if(rrank!=0)
-                    MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                          rrank, rrank, g3d_out.communicator(), &status);
-                MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
-                start4d[1] = coords[2]*count4d[1],
-                start4d[2] = coords[1]*count4d[2],
-                start4d[3] = coords[0]*count4d[3];
-                err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
-                    transferH.data().data());
-            }
-        }
-        else
-            MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                      0, rank, g3d_out.communicator());
-        MPI_Barrier( g3d_out.communicator());
-#else
-        err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
-            transferH.data());
-#endif // FELTOR_MPI
+        output.output_static3d( ncid, vecID, transferH);
         MPI_OUT err = nc_redef(ncid);
     }
 
@@ -305,7 +271,7 @@ int main( int argc, char* argv[])
     MPI_OUT err = nc_enddef(ncid);
     ///////////////////////////////////first output/////////////////////////
     MPI_OUT std::cout << "First output ... \n";
-    //first, update feltor (to get potential ....)
+    //first, update feltor (to get potential etc.)
     {
         std::array<std::array<DVec,2>,2> y1(y0);
         try{
@@ -318,67 +284,29 @@ int main( int argc, char* argv[])
         }
     }
 
+    size_t start = 0, count = 1;
     for( auto record& : diagnostics3d_list)
     {
         record.function( result, var);
         dg::blas2::symv( projectD, result, transferD);
         dg::assign( transferD, transferH);
-#ifdef FELTOR_MPI
-            if(rank==0)
-            {
-                for( int rrank=0; rrank<size; rrank++)
-                {
-                    if(rrank!=0)
-                        MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                              rrank, rrank, g3d_out.communicator(), &status);
-                    MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
-                    start4d[1] = coords[2]*count4d[1],
-                    start4d[2] = coords[1]*count4d[2],
-                    start4d[3] = coords[0]*count4d[3];
-                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
-                        transferH.data().data());
-                }
-            }
-            else
-                MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                          0, rank, g3d_out.communicator());
-            MPI_Barrier( g3d_out.communicator());
-#else
-            err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
-                transferH.data());
-#endif // FELTOR_MPI
+        output.output_dynamic3d( ncid, id4d.at(record.name), start, transferH);
     }
     for( auto record& : diagnostics2d_list)
     {
         record.function( result, var);
         dg::blas2::symv( projectD, result, transferD);
-        dg::assign( transferD, transferH);
-#ifdef FELTOR_MPI
-            if(rank==0)
-            {
-                for( int rrank=0; rrank<size; rrank++)
-                {
-                    if(rrank!=0)
-                        MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                              rrank, rrank, g3d_out.communicator(), &status);
-                    MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
-                    start4d[1] = coords[2]*count4d[1],
-                    start4d[2] = coords[1]*count4d[2],
-                    start4d[3] = coords[0]*count4d[3];
-                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
-                        transferH.data().data());
-                }
-            }
-            else
-                MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                          0, rank, g3d_out.communicator());
-            MPI_Barrier( g3d_out.communicator());
-#else
-            err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
-                transferH.data());
-#endif // FELTOR_MPI
+        //toroidal average
+        toroidal_average( transferD, transfer2dD);
+        dg::assign( transfer2dD, transfer2dH);
+
+        // 2d data of plane varphi = 0
+        dg::HVec t2d_mp(result.data().begin(),
+            result.data().begin() + g2d_out.size() );
+        //compute toroidal average and extract 2d field
+        output.output_dynamic2d( ncid, id3d.at(name), start, transferH2d);
     }
-    MPI_OUT err = nc_put_vara_double( ncid, tvarID, start4d, count4d, &time);
+    MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
     MPI_OUT err = nc_close(ncid);
     MPI_OUT std::cout << "First write successful!\n";
     ///////////////////////////////////////Timeloop/////////////////////////////////
@@ -390,8 +318,6 @@ int main( int argc, char* argv[])
     dg::Timer t;
     t.tic();
     unsigned step = 0;
-    std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
-    std::vector<std::string> energy_diff = { "resistivity", "leeperp", "leiperp", "leeparallel", "leiparallel"};
     for( unsigned i=1; i<=p.maxout; i++)
     {
 
@@ -418,12 +344,12 @@ int main( int argc, char* argv[])
             double energy = 0, ediff = 0.;
             for( auto record& : diagnostics2d_list)
             {
-                if( std::find( energies.begin(), energies.end(), record.name) != energies.end())
+                if( std::find( feltor::energies.begin(), feltor::energies.end(), record.name) != feltor::energies.end())
                 {
                     record.function( result, var);
                     energy += dg::blas1::dot( result, feltor.vol3d());
                 }
-                if( std::find( energy_diff.begin(), energy_diff.end(), record.name) != energy_diff.end())
+                if( std::find( feltor::energy_diff.begin(), feltor::energy_diff.end(), record.name) != feltor::energy_diff.end())
                 {
                     record.function( result, var);
                     ediff += dg::blas1::dot( result, feltor.vol3d());
@@ -433,13 +359,12 @@ int main( int argc, char* argv[])
                     record.function( result, var);
                     dg::blas2::symv( projectD, result, transferD);
                     //toroidal average and add to time integral
-                    toroidal_average( transferD, transfer2dD);
-                    time_integrals.at(record.name+"_ta2d_tt").add( time, transfer2dD);
+                    toroidal_average( transferD, transferD2d);
+                    time_integrals.at(record.name+"_ta2d_tt").add( time, transferD2d);
 
                     // 2d data of plane varphi = 0
-                    unsigned kmp = g3d_out.Nz()/2;
-                    dg::HVec t2d_mp(result.data().begin() + kmp*g2d_out.size(),
-                        result.data().begin() + (kmp+1)*g2d_out.size() );
+                    dg::HVec t2d_mp(result.data().begin(),
+                        result.data().begin() + g2d_out.size() );
                     time_integrals.at(record.name+"_2d_tt").add( time, transfer2dD);
                 }
 
@@ -483,30 +408,7 @@ int main( int argc, char* argv[])
             record.function( result, var);
             dg::blas2::symv( projectD, result, transferD);
             dg::assign( transferD, transferH);
-#ifdef FELTOR_MPI
-            if(rank==0)
-            {
-                for( int rrank=0; rrank<size; rrank++)
-                {
-                    if(rrank!=0)
-                        MPI_Recv( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                              rrank, rrank, g3d_out.communicator(), &status);
-                    MPI_Cart_coords( g3d_out.communicator(), rrank, 3, coords);
-                    start4d[1] = coords[2]*count4d[1],
-                    start4d[2] = coords[1]*count4d[2],
-                    start4d[3] = coords[0]*count4d[3];
-                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
-                        transferH.data().data());
-                }
-            }
-            else
-                MPI_Send( transferH.data().data(), g3d_out.local().size(), MPI_DOUBLE,
-                          0, rank, g3d_out.communicator());
-            MPI_Barrier( g3d_out.communicator());
-#else
-            err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
-                transferH.data());
-#endif // FELTOR_MPI
+            output.output_dynamic3d( ncid, id4d.at(record.name), start, transferH);
         }
         for( auto record& : diagnostics2d_list)
         {
@@ -519,44 +421,26 @@ int main( int argc, char* argv[])
                 dg::assign( transfer2dD, transfer2dH);
 
                 // 2d data of plane varphi = 0
-                unsigned kmp = g3d_out.Nz()/2;
-                dg::HVec t2d_mp(result.data().begin() + kmp*g2d_out.size(),
-                    result.data().begin() + (kmp+1)*g2d_out.size() );
+                dg::HVec t2d_mp(result.data().begin(),
+                    result.data().begin() + g2d_out.size() );
             }
             else //manage the time integrators
             {
-                transfer2dD = time_integrals.at(record.name+"_ta2d_tt").get_integral( );
-                std::array<double,2> bb = time_integrals.at(record.name+"_ta2d_tt").get_boundaries( );
+                std::string name = record.name+"_ta2d_tt";
+                transfer2dD = time_integrals.at(name).get_integral( );
+                std::array<double,2> bb = time_integrals.at(name).get_boundaries( );
                 dg::scal( transfer2dD, 1./(bb[1]-bb[0]));
-                time_integrals.at(record.name+"_ta2d_tt").flush( );
-                transfer2dD = time_integrals.at(record.name+"_2d_tt").get_integral( );
-                std::array<double,2> bb = time_integrals.at(record.name+"_2d_tt").get_boundaries( );
-                dg::scal( transfer2dD, 1./(bb[1]-bb[0]));
-                time_integrals.at(record.name+"_2d_tt").flush( );
-#ifdef FELTOR_MPI
-            if(rank==0)
-            {
-                for( int rrank=0; rrank<size; rrank++)
-                {
-                    if(rrank!=0)
-                        MPI_Recv( transferH.data().data(), g2d_out.local().size(), MPI_DOUBLE,
-                              rrank, rrank, g2d_out.communicator(), &status);
-                    MPI_Cart_coords( g2d_out.communicator(), rrank, 3, coords);
-                    start4d[1] = coords[2]*count4d[1],
-                    start4d[2] = coords[1]*count4d[2],
-                    start4d[3] = coords[0]*count4d[3];
-                    err = nc_put_vara_double( ncid, id4d.at(pair.first), start4d, count4d,
-                        transferH.data().data());
-                }
-            }
-            else
-                MPI_Send( transferH.data().data(), g2d_out.local().size(), MPI_DOUBLE,
-                          0, rank, g2d_out.communicator());
-            MPI_Barrier( g2d_out.communicator());
-#else
-            err = nc_put_vara_double( ncid, id3d.at(pair.first), start4d, count4d,
-                transferH.data());
-#endif // FELTOR_MPI
+                time_integrals.at(name).flush( );
+                dg::assign( transferD2d, transferH2d);
+                output.output_dynamic2d( ncid, id3d.at(name), start, transferH2d);
+
+                name = record.name+"_2d_tt";
+                transfer2dD = time_integrals.at(name).get_integral( );
+                std::array<double,2> bb = time_integrals.at(name).get_boundaries( );
+                dg::scal( transferD2d, 1./(bb[1]-bb[0]));
+                time_integrals.at(name).flush( );
+                dg::assign( transferD2d, transferH2d);
+                output.output_dynamic2d( ncid, id3d.at(name), start, transferH2d);
         }
         MPI_OUT err = nc_close(ncid);
         ti.toc();
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 2e4d4a8a2..0c0a3748c 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -698,4 +698,6 @@ std::vector<Record> diagnostics2d_list = {
 
 };
 
+std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
+std::vector<std::string> energy_diff = { "resistivity", "leeperp", "leiperp", "leeparallel", "leiparallel"};
 }//namespace feltor
diff --git a/src/feltor/init.h b/src/feltor/init.h
index e7b7a52ec..6befb9ce3 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -65,7 +65,7 @@ struct Initialize
                     fieldaligned( mag, grid, p.bcxN, p.bcyN,
                     dg::geo::NoLimiter(), p.rk4eps, 5, 5);
                 //evaluate should always be used with mx,my > 1
-                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 3);
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 3);
             }
             else if( p.initne == "straight blob")//rounds =1 ->2*1-1
             {
@@ -73,7 +73,7 @@ struct Initialize
                     fieldaligned( mag, grid, p.bcxN, p.bcyN,
                     dg::geo::NoLimiter(), p.rk4eps, 5, 5);
                 //evaluate should always be used with mx,my > 1
-                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
         }
         else if( p.initne == "turbulence")
@@ -88,7 +88,7 @@ struct Initialize
                     fieldaligned( mag, grid, p.bcxN, p.bcyN,
                     dg::geo::NoLimiter(), p.rk4eps, 5, 5);
                 //evaluate should always be used with mx,my > 1
-                ntilde = fieldaligned.evaluate( init0, gaussianZ, (unsigned)p.Nz/2, 1);
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
             dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
         }
diff --git a/src/feltor/serial_netcdf.h b/src/feltor/serial_netcdf.h
new file mode 100644
index 000000000..bd732aec2
--- /dev/null
+++ b/src/feltor/serial_netcdf.h
@@ -0,0 +1,140 @@
+#pragma once
+
+namespace feltor
+{
+
+struct ManageOutput
+{
+
+    ManageOutput( Geometry& g3d_out):
+    {
+        std::unique_ptr<Geometry2d> g2d_out_ptr = g3d_out.perp_grid();
+        m_start4d = {0, 0, 0, 0};
+        m_start3d = {0, 0, 0};
+        m_local_size3d = g3d_out.size();
+        m_local_size2d = g2d_out_ptr->size();
+#ifdef FELTOR_MPI
+        m_comm3d = g3d_out.communicator();
+        m_comm2d = g2d_out_ptr->communicator();
+        MPI_Comm_rank( m_comm2d, &m_rank2d);
+        MPI_Comm_size( m_comm2d, &m_size2d);
+        MPI_Comm_rank( m_comm3d, &m_rank3d);
+        MPI_Comm_size( m_comm3d, &m_size3d);
+
+        m_coords2d.resize(m_size2d*2);
+        m_coords.resize(m_size3d*3);
+        for( int rrank=0; rrank<m_size2d; rrank++)
+            MPI_Cart_coords( m_comm2d, rrank, 2, &m_coords2d[2*rrank]);
+        for( int rrank=0; rrank<m_size3d; rrank++)
+            MPI_Cart_coords( m_comm3d, rrank, 3, &m_coords[3*rrank]);
+        m_count4d = {1, g3d_out.local().Nz(),
+            g3d_out.n()*(g3d_out.local().Ny()),
+            g3d_out.n()*(g3d_out.local().Nx())};
+        m_count3d = {1,
+            g3d_out.n()*(g3d_out.local().Ny()),
+            g3d_out.n()*(g3d_out.local().Nx())};
+#else //FELTOR_MPI
+        m_count4d = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(),
+            g3d_out.n()*g3d_out.Nx()};
+        m_count3d = {1, g3d_out.n()*g3d_out.Ny(),
+            g3d_out.n()*g3d_out.Nx()};
+#endif //FELTOR_MPI
+    }
+    //must enddef first
+    void output_static3d(int ncid, int vecID, HVec& transferH) const
+    {
+#ifdef FELTOR_MPI
+        if(m_rank3d==0)
+        {
+            for( int rrank=0; rrank<m_size3d; rrank++)
+            {
+                if(rrank!=0)
+                    MPI_Recv( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
+                          rrank, rrank, m_comm3d, &m_status);
+                m_start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
+                m_start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
+                m_start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
+                m_err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
+                    transferH.data().data());
+            }
+        }
+        else
+            MPI_Send( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
+                      0, m_rank3d, m_comm3d);
+        MPI_Barrier( m_comm3d);
+#else
+        m_err = nc_put_vara_double( ncid, vecID, &m_start4d[1], &m_count4d[1],
+            transferH.data());
+#endif // FELTOR_MPI
+    }
+    void output_dynamic3d(int ncid, int vecID, int start, HVec& transferH) const
+    {
+        m_start4d[0] = start;
+#ifdef FELTOR_MPI
+        if(m_rank3d==0)
+        {
+            for( int rrank=0; rrank<m_size3d; rrank++)
+            {
+                if(rrank!=0)
+                    MPI_Recv( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
+                          rrank, rrank, m_comm3d, &m_status);
+                m_start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
+                m_start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
+                m_start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
+                m_err = nc_put_vara_double( ncid, vecID, start4d, count4d,
+                    transferH.data().data());
+            }
+        }
+        else
+            MPI_Send( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
+                      0, m_rank3d, m_comm3d);
+        MPI_Barrier( m_comm3d);
+#else
+        m_err = nc_put_vara_double( ncid, vecID, m_start4d, m_count4d,
+            transferH.data());
+#endif // FELTOR_MPI
+    }
+
+//all send to their rank2d 0 but only rank3d 0 writes into file
+    void output_dynamic2d(int ncid, int vecID, int start, HVec& transferH2d) const
+    {
+        m_start3d[0] = start;
+#ifdef FELTOR_MPI
+        if(m_rank2d==0)
+        {
+            for( int rrank=0; rrank<m_size2d; rrank++)
+            {
+                if(rrank!=0)
+                    MPI_Recv( transferH.data().data(), m_local_size2d, MPI_DOUBLE,
+                          rrank, rrank, m_comm2d, &m_status);
+                m_start3d[1] = m_coords2d[2*rrank+1]*m_count3d[1],
+                m_start3d[2] = m_coords2d[2*rrank+0]*m_count3d[2];
+                m_err = nc_put_vara_double( ncid, vecID, start3d, count3d,
+                    transferH2d.data().data());
+            }
+        }
+        else
+            MPI_Send( transferH2d.data().data(), m_local_size2d, MPI_DOUBLE,
+                      0, m_rank2d, m_comm2d);
+        MPI_Barrier( m_comm2d);
+        MPI_Barrier( m_comm3d); //all processes synchronize
+#else
+        m_err = nc_put_vara_double( ncid, vecID, m_start3d, m_count3d,
+            transferH2d.data());
+#endif // FELTOR_MPI
+    }
+    private
+    unsigned m_local_size2d;
+    unsigned m_local_size3d;
+    size_t m_start3d[3], m_count3d[3];
+    size_t m_start4d[4], m_count4d[4];
+    file::NC_Error_Handle m_err;
+    int m_rank3d, m_size3d, m_rank2d, m_size2d;
+#ifdef FELTOR_MPI
+    MPI_Status m_status;
+    MPI_Comm m_comm2d, m_comm3d;
+    std::vector<int> m_coords, m_coords2d;
+#endif //FELTOR_MPI
+};
+
+}//namespace feltor
-- 
GitLab


From af664d62e340f31271cc8405534668d1b2783681 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 29 Jun 2019 15:30:36 +0200
Subject: [PATCH 105/540] Rename feltor.cuh to feltor.h

---
 src/feltor/Makefile                 | 8 ++++----
 src/feltor/feltor.cu                | 2 +-
 src/feltor/{feltor.cuh => feltor.h} | 0
 src/feltor/feltor_hpc.cu            | 2 +-
 src/feltor/feltordiag.h             | 2 +-
 src/feltor/manufactured.cu          | 2 +-
 6 files changed, 8 insertions(+), 8 deletions(-)
 rename src/feltor/{feltor.cuh => feltor.h} (100%)

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index bbcacab85..e03058ce5 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -10,16 +10,16 @@ INCLUDE+= -I../../inc   # other project libraries
 
 all: feltor_hpc manufactured
 
-manufactured: manufactured.cu manufactured.h feltor.cuh
+manufactured: manufactured.cu manufactured.h feltor.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
 
-feltor: feltor.cu feltor.cuh
+feltor: feltor.cu feltor.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
-feltor_hpc: feltor_hpc.cu feltor.cuh
+feltor_hpc: feltor_hpc.cu feltor.h
 	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
 
-feltor_mpi: feltor_hpc.cu feltor.cuh
+feltor_mpi: feltor_hpc.cu feltor.h
 	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DFELTOR_MPI -DDG_BENCHMARK
 
 .PHONY: clean
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 9441e87e5..2cf216949 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -6,7 +6,7 @@
 
 #include "draw/host_window.h"
 
-#include "feltor.cuh"
+#include "feltor.h"
 #include "implicit.h"
 
 using HVec = dg::HVec;
diff --git a/src/feltor/feltor.cuh b/src/feltor/feltor.h
similarity index 100%
rename from src/feltor/feltor.cuh
rename to src/feltor/feltor.h
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 302abb443..10e5ab268 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -11,7 +11,7 @@
 #endif //FELTOR_MPI
 
 #include "dg/file/nc_utilities.h"
-#include "feltor.cuh"
+#include "feltor.h"
 #include "implicit.h"
 
 #ifdef FELTOR_MPI
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 0c0a3748c..5e0eb67c0 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -5,7 +5,7 @@
 #include "dg/algorithm.h"
 #include "dg/geometries/geometries.h"
 
-#include "feltor/feltor.cuh"
+#include "feltor/feltor.h"
 #include "feltor/parameters.h"
 
 #include "feltor/init.h"
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 2de1f69c3..1e261855e 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -12,7 +12,7 @@
 #define FELTORPERP 1
 
 #include "manufactured.h"
-#include "feltor.cuh"
+#include "feltor.h"
 #include "implicit.h"
 
 int main( int argc, char* argv[])
-- 
GitLab


From dbe5034dd29fe8352219ea10174b94f060321726 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 29 Jun 2019 17:01:53 +0200
Subject: [PATCH 106/540] Debug feltor.cu which runs again

Now test if it does the same as before...
---
 src/feltor/feltor.cu     | 43 +++++++++++++-----------
 src/feltor/feltor_hpc.cu | 23 +++++++------
 src/feltor/feltordiag.h  | 72 ++++++++++++++++++++--------------------
 src/feltor/init.h        | 57 ++++++++++++++++---------------
 4 files changed, 102 insertions(+), 93 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 2cf216949..e8d952996 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -56,7 +56,6 @@ int main( int argc, char* argv[])
     //Make grid
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
-    dg::DVec vol3d = dg::create::volume( grid);
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
@@ -67,32 +66,41 @@ int main( int argc, char* argv[])
     feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     std::cout << "Done!\n";
 
+    DVec result = dg::evaluate( dg::zero, grid);
+    /// Construct feltor::Variables object for diagnostics
+    std::array<DVec, 3> gradPsip;
+    gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
+    gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
+    gradPsip[2] =  result; //zero
+    feltor::Variables var = {
+        feltor, p, gradPsip, gradPsip
+    };
+
+
+
     /////////////////////The initial field///////////////////////////////////////////
+    double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
-    feltor::Initialize init( grid, p, mag);
-    if( argc == 4)
-        y0 = init.init_from_parameters(feltor);
-    if( argc == 5)
-        y0 = init.init_from_file(argv[4]);
-    feltor.set_source( init.profile(), p.omega_source, init.source_damping());
-    std::cout << "Initialize Timestepper" << std::endl;
+    feltor::Initialize init( p, gp, mag);
+    y0 = init.init_from_parameters(feltor, grid);
+    feltor.set_source( init.profile(grid), p.omega_source, init.source_damping(grid));
 
     ////////////////////////create timer and timestepper
     //
     dg::Timer t;
-    double time = 0.;
     unsigned step = 0;
     dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
         feltor::FeltorSpecialSolver<
             Geometry, IDMatrix, DMatrix, DVec>
         > karniadakis( grid, p, mag);
+    std::cout << "Initialize Timestepper" << std::endl;
     karniadakis.init( feltor, im, time, y0, p.dt);
     std::cout << "Done!" << std::endl;
 
     std::map<std::string, const dg::DVec* > v4d;
     v4d["ne-1 / "] = &y0[0][0],               v4d["ni-1 / "] = &y0[0][1];
     v4d["Ue / "]   = &feltor.fields()[1][0],  v4d["Ui / "]   = &feltor.fields()[1][1];
-    v4d["Ome / "] = &feltor.potential()[0]; v4d["Apar / "] = &feltor.induction();
+    v4d["Ome / "] = &feltor.potential(0); v4d["Apar / "] = &feltor.induction();
     double dEdt = 0, accuracy = 0;
     double E0 = 0.;
     /////////////////////////set up transfer for glfw
@@ -119,8 +127,6 @@ int main( int argc, char* argv[])
     dg::Average<dg::HVec> toroidal_average( grid, dg::coo3d::z);
     title << std::setprecision(2) << std::scientific;
     //unsigned failed_counter = 0;
-    std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
-    std::vector<std::string> energy_diff = { "resistivity", "leeperp", "leiperp", "leeparallel", "leiparallel"};
     while ( !glfwWindowShouldClose( w ))
     {
         title << std::fixed;
@@ -136,7 +142,7 @@ int main( int argc, char* argv[])
             else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
             {
                 dg::assign( *pair.second, hvisual);
-                dg::blas1::axpby( 1., hvisual, -1., init.profile(), hvisual);
+                dg::blas1::axpby( 1., hvisual, -1., init.profile(grid), hvisual);
             }
             else
                 dg::assign( *pair.second, hvisual);
@@ -189,14 +195,14 @@ int main( int argc, char* argv[])
             }
             double deltat = time - previous_time;
             double energy = 0, ediff = 0.;
-            for( auto record& : diagnostics2d_list)
+            for( auto& record : feltor::diagnostics2d_list)
             {
-                if( std::find( energies.begin(), energies.end(), record.name) != energies.end())
+                if( std::find( feltor::energies.begin(), feltor::energies.end(), record.name) != feltor::energies.end())
                 {
                     record.function( result, var);
                     energy += dg::blas1::dot( result, feltor.vol3d());
                 }
-                if( std::find( energy_diff.begin(), energy_diff.end(), record.name) != energy_diff.end())
+                if( std::find( feltor::energy_diff.begin(), feltor::energy_diff.end(), record.name) != feltor::energy_diff.end())
                 {
                     record.function( result, var);
                     ediff += dg::blas1::dot( result, feltor.vol3d());
@@ -218,9 +224,9 @@ int main( int argc, char* argv[])
                     feltor.density(0), feltor.velocity(0), dvisual);
                 dg::blas1::pointwiseDot( p.beta,
                     feltor.density(1), feltor.velocity(1), -p.beta, dvisual);
-                double norm  = dg::blas2::dot( dvisual, vol3d, dvisual);
+                double norm  = dg::blas2::dot( dvisual, feltor.vol3d(), dvisual);
                 dg::blas1::axpby( -1., feltor.lapMperpA(), 1., dvisual);
-                double error = dg::blas2::dot( dvisual, vol3d, dvisual);
+                double error = dg::blas2::dot( dvisual, feltor.vol3d(), dvisual);
                 std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
             }
 
@@ -228,7 +234,6 @@ int main( int argc, char* argv[])
         t.toc();
         std::cout << "\n\t Step "<<step << " at time  "<<time;
         std::cout << "\n\t Average time for one step: "<<t.diff()/(double)p.itstp/(double)p.inner_loop<<"\n\n";
-        //std::cout << "\n\t Total # of failed steps:   "<<failed_counter<<"\n\n";
     }
     glfwTerminate();
     ////////////////////////////////////////////////////////////////////
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 10e5ab268..8803c984a 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -150,13 +150,6 @@ int main( int argc, char* argv[])
     MPI_OUT std::cout << "Done!\n";
 
     // helper variables for various stuff
-    std::array<DVec, 3> gradPsip;
-    gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
-    gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
-    gradPsip[2] =  result; //zero
-    feltor::Variables var = {
-        feltor, p, gradPsip, gradPsip
-    };
     std::map<std::string, dg::Simpsons<DVec>> time_integrals;
     dg::Average<DVec> toroidal_average( g3d_out, dg::coo3d::z);
     dg::MultiMatrix<IHMatrix,HVec> projectH = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
@@ -165,21 +158,29 @@ int main( int argc, char* argv[])
     DVec transferD( dg::evaluate(dg::zero, g3d_out));
     DVec transferD2d = dg::evaluate( dg::zero, g2d_out);
     HVec transferH2d = dg::evaluate( dg::zero, g2d_out);
-    /// Construct feltor::Variables object for diagnostics
     DVec result = dg::evaluate( dg::zero, grid);
     HVec resultH( dg::evaluate( dg::zero, grid));
 
+    /// Construct feltor::Variables object for diagnostics
+    std::array<DVec, 3> gradPsip;
+    gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
+    gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
+    gradPsip[2] =  result; //zero
+    feltor::Variables var = {
+        feltor, p, gradPsip, gradPsip
+    };
+
     double dEdt = 0, accuracy = 0;
     double E0 = 0.;
 
-    //!///////////////////The initial field///////////////////////////////////////////
-    double time = 0;
+    /// //////////////////The initial field///////////////////////////////////////////
+    double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
     feltor::Initialize init( grid, p, mag);
     if( argc == 4)
         y0 = init.init_from_parameters(feltor);
     if( argc == 5)
-        y0 = init.init_from_file(argv[4]);
+        y0 = init.init_from_file(argv[4], grid, time);
     feltor.set_source( init.profile(), p.omega_source, init.source_damping());
 
     /// //////////////////////////set up netcdf/////////////////////////////////////
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 5e0eb67c0..9d1595943 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -157,54 +157,54 @@ struct Record_static{
     std::string name;
     std::string long_name;
     bool integral;
-    std::function<void( HVec&, Variables&, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag)> function;
+    std::function<void( HVec&, Variables&, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag)> function;
 };
 
 //Here is a list of static (time-independent) 3d variables that go into the output
 std::vector<Record_static> dianostics3d_static_list = {
     { "BR", "R-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
             dg::geo::BFieldR fieldR(mag);
             result = dg::pullback( fieldR, grid);
         }
     },
     { "BZ", "Z-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
             dg::geo::BFieldZ fieldZ(mag);
             result = dg::pullback( fieldZ, grid);
         }
     },
     { "BP", "Contravariant P-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
             dg::geo::BFieldP fieldP(mag);
             result = dg::pullback( fieldP, grid);
         }
     },
     { "Psip", "Flux-function psi",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
              result = dg::pullback( mag.psip(), grid);
         }
     },
     { "Nprof", "Density profile",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
-            Initialize init;
-            result = init.profile();
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+            Initialize init(v.p, gp, mag);
+            result = init.profile(grid);
         }
     },
     { "Source", "Source region",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
-            Initialize init;
-            result = init.source_damping();
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+            Initialize init(v.p, gp, mag);
+            result = init.source_damping(grid);
         }
     },
     { "Damping", "Damping region for initial profile",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
-            Initialize init;
-            result = init.profile_damping();
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+            Initialize init(v.p, gp, mag);
+            result = init.profile_damping(grid);
         }
     },
     { "xc", "x-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -213,7 +213,7 @@ std::vector<Record_static> dianostics3d_static_list = {
         }
     },
     { "yc", "y-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -222,7 +222,7 @@ std::vector<Record_static> dianostics3d_static_list = {
         }
     },
     { "zc", "z-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -230,7 +230,7 @@ std::vector<Record_static> dianostics3d_static_list = {
             result = zc;
         }
     },
-}
+};
 
 // Here are all 3d outputs we want to have
 std::vector<Record> diagnostics3d_list = {
@@ -390,7 +390,7 @@ std::vector<Record> diagnostics2d_list = {
     {"jsne", "Radial electron particle flux without induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
-                RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
+                routines::RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
                 v.f.density(0), v.f.velocity(0),
                 v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
                 v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
@@ -403,7 +403,7 @@ std::vector<Record> diagnostics2d_list = {
     {"jsneA", "Radial electron particle flux: induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
-                RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
+                routines::RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
                 v.f.density(0), v.f.velocity(0), v.f.induction(),
                 v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
                 v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
@@ -488,8 +488,8 @@ std::vector<Record> diagnostics2d_list = {
     {"jsee", "Radial electron energy flux without induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
-                RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
-                v.f.density(0), v.f.velocity(0), v.f.potential()[0],
+                routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
+                v.f.density(0), v.f.velocity(0), v.f.potential(0),
                 v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
                 v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
@@ -501,8 +501,8 @@ std::vector<Record> diagnostics2d_list = {
     {"jseea", "Radial electron energy flux: induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
-                RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
-                v.f.density(0), v.f.velocity(0), v.f.potential()[0], v.f.induction(),
+                routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
+                v.f.density(0), v.f.velocity(0), v.f.potential(0), v.f.induction(),
                 v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
                 v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
@@ -513,8 +513,8 @@ std::vector<Record> diagnostics2d_list = {
     {"jsei", "Radial ion energy flux without induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
-                RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
-                v.f.density(1), v.f.velocity(1), v.f.potential()[1],
+                routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
+                v.f.density(1), v.f.velocity(1), v.f.potential(1),
                 v.f.gradP(1)[0], v.f.gradP(1)[1], v.f.gradP(1)[2],
                 v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
@@ -526,8 +526,8 @@ std::vector<Record> diagnostics2d_list = {
     {"jseia", "Radial ion energy flux: induction contribution", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
-                RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
-                v.f.density(1), v.f.velocity(1), v.f.potential()[1], v.f.induction(),
+                routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
+                v.f.density(1), v.f.velocity(1), v.f.potential(1), v.f.induction(),
                 v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
                 v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
@@ -541,8 +541,8 @@ std::vector<Record> diagnostics2d_list = {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(0), result, v.tmp[1]);
             dg::blas1::evaluate( result, dg::times_equals(),
-                RadialEnergyFlux( v.p.tau[0], v.p.mu[0], 1.),
-                v.f.density(0), v.f.velocity(0), v.f.potential()[0],
+                routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], 1.),
+                v.f.density(0), v.f.velocity(0), v.f.potential(0),
                 v.tmp[0], v.tmp[1]
             );
             dg::blas1::scal( result, -v.p.nu_perp);
@@ -553,8 +553,8 @@ std::vector<Record> diagnostics2d_list = {
             v.f.compute_diffusive_lapMperpN( v.f.density(1), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(1), result, v.tmp[1]);
             dg::blas1::evaluate( result, dg::times_equals(),
-                RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
-                v.f.density(1), v.f.velocity(1), v.f.potential()[1],
+                routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
+                v.f.density(1), v.f.velocity(1), v.f.potential(1),
                 v.tmp[0], v.tmp[1]
             );
             dg::blas1::scal( result, -v.p.nu_perp);
@@ -569,8 +569,8 @@ std::vector<Record> diagnostics2d_list = {
                                      0., v.tmp[0]);
             dg::blas1::axpby( 1., v.f.dssU(0), 1., v.tmp[1]);
             dg::blas1::evaluate( result, dg::times_equals(),
-                RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
-                v.f.density(0), v.f.velocity(0), v.f.potential()[0],
+                routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
+                v.f.density(0), v.f.velocity(0), v.f.potential(0),
                 v.tmp[0], v.tmp[1]
             );
             dg::blas1::scal( result, v.p.nu_parallel);
@@ -585,8 +585,8 @@ std::vector<Record> diagnostics2d_list = {
                                      0., v.tmp[0]);
             dg::blas1::axpby( 1., v.f.dssU(1), 1., v.tmp[1]);
             dg::blas1::evaluate( result, dg::times_equals(),
-                RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
-                v.f.density(1), v.f.velocity(1), v.f.potential()[1],
+                routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
+                v.f.density(1), v.f.velocity(1), v.f.potential(1),
                 v.tmp[0], v.tmp[1]
             );
             dg::blas1::scal( result, v.p.nu_parallel);
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 6befb9ce3..0ead24243 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -1,38 +1,39 @@
 #pragma once
+#include "dg/file/nc_utilities.h"
 
 namespace feltor
 {
 //We use the typedefs and MPI_OUT
 struct Initialize
 {
-    Initialize( const Geometry& grid, feltor::Parameters p, dg::geo::solovev::Parameters gp,
-        dg::geo::TokamakMagneticField mag) : p(p), gp(gp)
+    Initialize( feltor::Parameters p, dg::geo::solovev::Parameters gp,
+        dg::geo::TokamakMagneticField mag) : p(p), gp(gp), mag(mag)
     {
     }
-    HVec profile()const{
+    HVec profile(const Geometry& grid)const{
         //First the profile and the source (on the host since we want to output those)
         HVec profile = dg::pullback( dg::geo::Compose<dg::LinearX>( mag.psip(),
             p.nprofamp/mag.psip()(mag.R0(), 0.), 0.), grid);
-        dg::blas1::pointwiseDot( profile_damping(), profile, profile);
+        dg::blas1::pointwiseDot( profile_damping(grid), profile, profile);
         return profile;
     }
-    HVec xpoint_damping()const{
+    HVec xpoint_damping(const Geometry& grid)const{
         HVec xpoint_damping = dg::evaluate( dg::one, grid);
         if( gp.hasXpoint() )
             xpoint_damping = dg::pullback(
                 dg::geo::ZCutter(-1.1*gp.elongation*gp.a), grid);
         return xpoint_damping;
     }
-    HVec source_damping()const{
+    HVec source_damping(const Geometry& grid)const{
         HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
             //first change coordinate from psi to (psi_0 - psip)/psi_0
             dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
             //then shift
             p.rho_source, p.alpha, -1), grid);
-        dg::blas1::pointwiseDot( xpoint_damping(), source_damping, source_damping);
+        dg::blas1::pointwiseDot( xpoint_damping(grid), source_damping, source_damping);
         return source_damping;
     }
-    HVec damping_damping()const{
+    HVec damping_damping(const Geometry& grid)const{
         HVec damping_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
             //first change coordinate from psi to (psi_0 - psip)/psi_0
             dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
@@ -40,16 +41,14 @@ struct Initialize
             p.rho_damping, p.alpha, +1), grid);
         return damping_damping;
     }
-    HVec profile_damping()const{
+    HVec profile_damping(const Geometry& grid)const{
         HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
             mag.psip(), -p.alpha, p.alpha, -1), grid);
-        dg::blas1::pointwiseDot( xpoint_damping(), profile_damping, profile_damping);
+        dg::blas1::pointwiseDot( xpoint_damping(grid), profile_damping, profile_damping);
         return profile_damping;
     }
     template<class Feltor>
-    std::array<std::array<DVec,2>,2> init_from_parameters(Feltor& feltor){
-        HVec profile = profile();
-        HVec profile_damping = profile_damping();
+    std::array<std::array<DVec,2>,2> init_from_parameters(Feltor& feltor, const Geometry& grid){
         std::array<std::array<DVec,2>,2> y0;
         //Now perturbation
         HVec ntilde = dg::evaluate(dg::zero,grid);
@@ -79,7 +78,7 @@ struct Initialize
         else if( p.initne == "turbulence")
         {
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-            dg::BathRZ init0(16,16,Rmin,Zmin, 30.,2.,p.amp);
+            dg::BathRZ init0(16,16,grid.x0(),grid.y0(), 30.,2.,p.amp);
             if( p.symmetric)
                 ntilde = dg::pullback( init0, grid);
             else
@@ -90,17 +89,17 @@ struct Initialize
                 //evaluate should always be used with mx,my > 1
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
-            dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
+            dg::blas1::pointwiseDot( profile_damping(grid), ntilde, ntilde);
         }
         else if( p.initne == "zonal")
         {
             dg::geo::ZonalFlow init0(mag.psip(), p.amp, 0., p.k_psi);
             ntilde = dg::pullback( init0, grid);
-            dg::blas1::pointwiseDot( profile_damping, ntilde, ntilde);
+            dg::blas1::pointwiseDot( profile_damping(grid), ntilde, ntilde);
         }
         else
             MPI_OUT std::cerr <<"WARNING: Unknown initial condition!\n";
-        y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile);
+        y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid));
         dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
         MPI_OUT std::cout << "initialize ni" << std::endl;
         feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
@@ -108,9 +107,8 @@ struct Initialize
         MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
         if( minimalni <= -1)
         {
-            MPI_OUT std::cerr << "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n";
-            MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
-            return -1;
+            throw dg::Error(dg::Message()<< "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n"
+                << "Minimum Ni value "<<minimalni+1);
         }
 
         dg::blas1::copy( 0., y0[1][0]); //set we = 0
@@ -118,7 +116,8 @@ struct Initialize
         return y0;
     }
 
-    std::array<std::array<DVec,2>,2> init_from_file( std::string file_name){
+    //everyone reads their portion of the input data
+    std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Geometry& grid, double& time){
         std::array<std::array<DVec,2>,2> y0;
         ///////////////////read in and show inputfile
         file::NC_Error_Handle errIN;
@@ -131,27 +130,30 @@ struct Initialize
 
         Json::Value jsIN;
         std::stringstream is(inputIN);
+        Json::CharReaderBuilder parser;
+        parser["collectComments"] = false;
+        std::string errs;
         parseFromStream( parser, is, &jsIN, &errs); //read input without comments
         const feltor::Parameters pIN(  jsIN);
-        MPI_OUT std::cout << "RESTART from file "<<argv[4]<< std::endl;
+        MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
         MPI_OUT std::cout << " file parameters:" << std::endl;
         MPI_OUT pIN.display( std::cout);
 
         // Now read in last timestep
-        Geometry grid_IN( Rmin, Rmax, Zmin, Zmax, 0, 2.*M_PI,
+        Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
             pIN.n_out, pIN.Nx_out, pIN.Ny_out, pIN.symmetric ? 1 : pIN.Nz_out, pIN.bcxN, pIN.bcyN, dg::PER
             #ifdef FELTOR_MPI
-            , comm
+            , grid.communicator()
             #endif //FELTOR_MPI
             );
         IHMatrix interpolateIN = dg::create::interpolation( grid, grid_IN);
 
         #ifdef FELTOR_MPI
         int dimsIN[3],  coordsIN[3];
-        MPI_Cart_get( comm, 3, dimsIN, periods, coordsIN);
+        MPI_Cart_get( grid.communicator(), 3, dimsIN, periods, coordsIN);
         size_t countIN[4] = {1, grid_IN.local().Nz(),
-            g3d_out.n()*(grid_IN.local().Ny()),
-            g3d_out.n()*(grid_IN.local().Nx())};
+            grid_IN.n()*(grid_IN.local().Ny()),
+            grid_IN.n()*(grid_IN.local().Nx())};
         size_t startIN[4] = {0, coordsIN[2]*countIN[1],
                                 coordsIN[1]*countIN[2],
                                 coordsIN[0]*countIN[3]};
@@ -205,6 +207,7 @@ struct Initialize
     private:
     feltor::Parameters p;
     dg::geo::solovev::Parameters gp;
+    dg::geo::TokamakMagneticField mag;
 };
 
 
-- 
GitLab


From d635518a1d617ccf1378210a701b9a05a5d42ddd Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 29 Jun 2019 18:00:13 +0200
Subject: [PATCH 107/540] Debug feltordiag energy dissipation terms

---
 src/feltor/feltor.cu     | 15 ++++++++++-----
 src/feltor/feltor.tex    |  4 ++--
 src/feltor/feltor_hpc.cu |  6 +++---
 src/feltor/feltordiag.h  | 16 ++++++++--------
 4 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index e8d952996..9c3dfae4f 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -200,12 +200,17 @@ int main( int argc, char* argv[])
                 if( std::find( feltor::energies.begin(), feltor::energies.end(), record.name) != feltor::energies.end())
                 {
                     record.function( result, var);
-                    energy += dg::blas1::dot( result, feltor.vol3d());
+                    double norm = dg::blas1::dot( result, feltor.vol3d());
+                    energy += norm;
+                    std::cout << record.name<<" : "<<norm<<std::endl;
+
                 }
                 if( std::find( feltor::energy_diff.begin(), feltor::energy_diff.end(), record.name) != feltor::energy_diff.end())
                 {
                     record.function( result, var);
-                    ediff += dg::blas1::dot( result, feltor.vol3d());
+                    double norm = dg::blas1::dot( result, feltor.vol3d());
+                    ediff += norm;
+                    std::cout << record.name<<" : "<<norm<<std::endl;
                 }
 
             }
@@ -213,8 +218,8 @@ int main( int argc, char* argv[])
             E0 = energy;
             accuracy  = 2.*fabs( (dEdt - ediff)/( dEdt + ediff));
 
-            std::cout << "Time "<<time<<"\n";
-            std::cout <<" d E/dt = " << dEdt
+            std::cout << "\tTime "<<time<<"\n";
+            std::cout <<"\td E/dt = " << dEdt
               <<" Lambda = " << ediff
               <<" -> Accuracy: " << accuracy << "\n";
             //----------------Test if induction equation holds
@@ -227,7 +232,7 @@ int main( int argc, char* argv[])
                 double norm  = dg::blas2::dot( dvisual, feltor.vol3d(), dvisual);
                 dg::blas1::axpby( -1., feltor.lapMperpA(), 1., dvisual);
                 double error = dg::blas2::dot( dvisual, feltor.vol3d(), dvisual);
-                std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
+                std::cout << "\tRel. Error Induction "<<sqrt(error/norm) <<"\n";
             }
 
         }
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 119e9f53b..de14f5059 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -884,7 +884,7 @@ The terms of the energy theorem are
 \nabla \cdot \vec j_{\mathcal E}
 = \Lambda_{\mathcal E}
 +  S_{\mathcal E}
--  R_{\mathcal E}
++  R_{\mathcal E}
 \end{align}
 with ( $z_e=-1$ and $z_i=+1$)
 \begin{align} \label{eq:energy_conservation}
@@ -903,7 +903,7 @@ with ( $z_e=-1$ and $z_i=+1$)
 \nonumber \\
   S_{\mathcal E} =&  \sum_s  z\left[ \left(\tau\left( 1+\ln{N}\right) +\psi + \frac{1}{2} \mu U^2 \right)S_{N}  + \mu NU S_U\right]
 \nonumber \\
-  R_{\mathcal E} =&  \eta_\parallel  \left[ n_e(U_i-u_e)\right]^2.
+  R_{\mathcal E} =&  -\eta_\parallel  \left[ n_e(U_i-u_e)\right]^2.
 \end{align}
 where in the energy flux $\vec j_{\mathcal E}$
 we neglect terms  containing time derivatives
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 8803c984a..1ba931941 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -375,8 +375,8 @@ int main( int argc, char* argv[])
             E0 = energy;
             accuracy  = 2.*fabs( (dEdt - ediff)/( dEdt + ediff));
 
-            MPI_OUT std::cout << "Time "<<time<<"\n";
-            MPI_OUT std::cout <<" d E/dt = " << dEdt
+            MPI_OUT std::cout << "\tTime "<<time<<"\n";
+            MPI_OUT std::cout <<"\td E/dt = " << dEdt
                       <<" Lambda = " << ediff
                       <<" -> Accuracy: " << accuracy << "\n";
             //----------------Test if induction equation holds
@@ -389,7 +389,7 @@ int main( int argc, char* argv[])
                 double norm  = dg::blas2::dot( temp, feltor.vol3d(), temp);
                 dg::blas1::axpby( -1., feltor.lapMperpA(), 1., temp);
                 double error = dg::blas2::dot( temp, feltor.vol3d(), temp);
-                MPI_OUT std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
+                MPI_OUT std::cout << "\tRel. Error Induction "<<sqrt(error/norm) <<"\n";
             }
             tti.tic();
             std::cout << " Time for internal diagnostics "<<tti.diff()<<"s\n";
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 9d1595943..6f059c788 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -481,7 +481,7 @@ std::vector<Record> diagnostics2d_list = {
         []( DVec& result, Variables& v ) {
             dg::blas1::axpby( 1., v.f.velocity(1), -1., v.f.velocity(0), result);
             dg::blas1::pointwiseDot( result, v.f.density(0), result);
-            dg::blas1::pointwiseDot( v.p.eta, result, result, 0., result);
+            dg::blas1::pointwiseDot( -v.p.eta, result, result, 0., result);
         }
     },
     /// ------------------ Energy flux terms ------------------------//
@@ -540,8 +540,8 @@ std::vector<Record> diagnostics2d_list = {
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(0), result, v.tmp[1]);
-            dg::blas1::evaluate( result, dg::times_equals(),
-                routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], 1.),
+            dg::blas1::evaluate( result, dg::equals(),
+                routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
                 v.f.density(0), v.f.velocity(0), v.f.potential(0),
                 v.tmp[0], v.tmp[1]
             );
@@ -552,7 +552,7 @@ std::vector<Record> diagnostics2d_list = {
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(1), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(1), result, v.tmp[1]);
-            dg::blas1::evaluate( result, dg::times_equals(),
+            dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
                 v.f.density(1), v.f.velocity(1), v.f.potential(1),
                 v.tmp[0], v.tmp[1]
@@ -566,9 +566,9 @@ std::vector<Record> diagnostics2d_list = {
                                      0., v.tmp[0]);
             dg::blas1::axpby( 1., v.f.dssN(0), 1., v.tmp[0]);
             dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(0),
-                                     0., v.tmp[0]);
+                                     0., v.tmp[1]);
             dg::blas1::axpby( 1., v.f.dssU(0), 1., v.tmp[1]);
-            dg::blas1::evaluate( result, dg::times_equals(),
+            dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
                 v.f.density(0), v.f.velocity(0), v.f.potential(0),
                 v.tmp[0], v.tmp[1]
@@ -582,9 +582,9 @@ std::vector<Record> diagnostics2d_list = {
                                      0., v.tmp[0]);
             dg::blas1::axpby( 1., v.f.dssN(1), 1., v.tmp[0]);
             dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(1),
-                                     0., v.tmp[0]);
+                                     0., v.tmp[1]);
             dg::blas1::axpby( 1., v.f.dssU(1), 1., v.tmp[1]);
-            dg::blas1::evaluate( result, dg::times_equals(),
+            dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
                 v.f.density(1), v.f.velocity(1), v.f.potential(1),
                 v.tmp[0], v.tmp[1]
-- 
GitLab


From e481ee2bfde7f8c3bd3b18709e054456770410d4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 30 Jun 2019 01:13:34 +0200
Subject: [PATCH 108/540] Rename SimpsonsRule to Simpsons; debug feltor_hpc

There is still a segmentation fault in the 2d output though
---
 inc/dg/algorithm.h         |   1 +
 inc/dg/simpsons.h          |   4 +-
 inc/dg/simpsons_t.cu       |   2 +-
 src/feltor/feltor_hpc.cu   | 197 ++++++++++++++++++++++---------------
 src/feltor/feltordiag.h    |  26 ++---
 src/feltor/serial_netcdf.h | 122 ++++++++++++++---------
 6 files changed, 209 insertions(+), 143 deletions(-)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 755b63bd1..3b9741893 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -27,6 +27,7 @@
 #include "refined_elliptic.h"
 #include "arakawa.h"
 #include "poisson.h"
+#include "simpsons.h"
 #include "topology/average.h"
 #ifdef MPI_VERSION
 #include "topology/average_mpi.h"
diff --git a/inc/dg/simpsons.h b/inc/dg/simpsons.h
index dc795317b..af81d80c2 100644
--- a/inc/dg/simpsons.h
+++ b/inc/dg/simpsons.h
@@ -29,7 +29,7 @@ boundary and then adding values as they become available.
 * @ingroup time
 */
 template<class ContainerType>
-struct SimpsonsRule
+struct Simpsons
 {
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
@@ -37,7 +37,7 @@ struct SimpsonsRule
      * @param order number of vectors to use for integration.
          Choose 2 (linear) or 3 (parabola) integration.
      */
-    SimpsonsRule( unsigned order = 3): m_counter(0), m_order(order), m_t0(0)
+    Simpsons( unsigned order = 3): m_counter(0), m_order(order), m_t0(0)
     {
         set_order(order);
     }
diff --git a/inc/dg/simpsons_t.cu b/inc/dg/simpsons_t.cu
index 33d7270d2..4371be7e3 100644
--- a/inc/dg/simpsons_t.cu
+++ b/inc/dg/simpsons_t.cu
@@ -17,7 +17,7 @@ int main()
     dg::HVec times = dg::evaluate( dg::cooX1d, g1d);
     dg::HVec values = dg::evaluate( cos, g1d);
 
-    dg::SimpsonsRule<double> simpsons;
+    dg::Simpsons<double> simpsons;
 
     //Init the left side boundary
     simpsons.init( 0., 1.);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 1ba931941..d52fa0196 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -17,6 +17,7 @@
 #ifdef FELTOR_MPI
 using HVec = dg::MHVec;
 using DVec = dg::MDVec;
+using HMatrix = dg::MHMatrix;
 using DMatrix = dg::MDMatrix;
 using IDMatrix = dg::MIDMatrix;
 using IHMatrix = dg::MIHMatrix;
@@ -25,6 +26,7 @@ using Geometry = dg::CylindricalMPIGrid3d;
 #else //FELTOR_MPI
 using HVec = dg::HVec;
 using DVec = dg::DVec;
+using HMatrix = dg::HMatrix;
 using DMatrix = dg::DMatrix;
 using IDMatrix = dg::IDMatrix;
 using IHMatrix = dg::IHMatrix;
@@ -34,6 +36,7 @@ using Geometry = dg::CylindricalGrid3d;
 
 #include "init.h"
 #include "feltordiag.h"
+#include "serial_netcdf.h"
 
 #ifdef FELTOR_MPI
 //ATTENTION: in slurm should be used with --signal=SIGINT@30 (<signal>@<time in seconds>)
@@ -138,6 +141,12 @@ int main( int argc, char* argv[])
         , comm
         #endif //FELTOR_MPI
         );
+    std::unique_ptr<typename Geometry::perpendicular_grid> g2d_out_ptr  ( dynamic_cast<typename Geometry::perpendicular_grid*>( g3d_out.perp_grid()));
+#ifdef FELTOR_MPI
+    unsigned local_size2d = g2d_out_ptr->local().size();
+#else
+    unsigned local_size2d = g2d_out_ptr->size();
+#endif
 
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
@@ -149,26 +158,29 @@ int main( int argc, char* argv[])
     feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
-    // helper variables for various stuff
+    // helper variables for output computations
     std::map<std::string, dg::Simpsons<DVec>> time_integrals;
     dg::Average<DVec> toroidal_average( g3d_out, dg::coo3d::z);
-    dg::MultiMatrix<IHMatrix,HVec> projectH = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
-    dg::MultiMatrix<IDMatrix,DVec> projectD = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
+    dg::MultiMatrix<HMatrix,HVec> projectH = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
+    dg::MultiMatrix<DMatrix,DVec> projectD = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
     HVec transferH( dg::evaluate(dg::zero, g3d_out));
     DVec transferD( dg::evaluate(dg::zero, g3d_out));
-    DVec transferD2d = dg::evaluate( dg::zero, g2d_out);
-    HVec transferH2d = dg::evaluate( dg::zero, g2d_out);
-    DVec result = dg::evaluate( dg::zero, grid);
-    HVec resultH( dg::evaluate( dg::zero, grid));
+    HVec transferH2d = dg::evaluate( dg::zero, *g2d_out_ptr);
+    DVec transferD2d = dg::evaluate( dg::zero, *g2d_out_ptr);
+    HVec resultH = dg::evaluate( dg::zero, grid);
+    DVec resultD = dg::evaluate( dg::zero, grid);
+
+    feltor::ManageOutput output(g3d_out);
 
-    /// Construct feltor::Variables object for diagnostics
     std::array<DVec, 3> gradPsip;
     gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
-    gradPsip[2] =  result; //zero
+    gradPsip[2] =  resultD; //zero
     feltor::Variables var = {
         feltor, p, gradPsip, gradPsip
     };
+    // the vector ids
+    std::map<std::string, int> id3d, id4d;
 
     double dEdt = 0, accuracy = 0;
     double E0 = 0.;
@@ -176,19 +188,18 @@ int main( int argc, char* argv[])
     /// //////////////////The initial field///////////////////////////////////////////
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
-    feltor::Initialize init( grid, p, mag);
+    feltor::Initialize init( p, gp, mag);
     if( argc == 4)
-        y0 = init.init_from_parameters(feltor);
+        y0 = init.init_from_parameters(feltor, grid);
     if( argc == 5)
         y0 = init.init_from_file(argv[4], grid, time);
-    feltor.set_source( init.profile(), p.omega_source, init.source_damping());
+    feltor.set_source( init.profile(grid), p.omega_source, init.source_damping(grid));
 
     /// //////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
     std::string file_name = argv[3];
     int ncid=-1;
     MPI_OUT err = nc_create( file_name.data(), NC_NETCDF4|NC_CLOBBER, &ncid);
-    feltor::ManageOutput output(g3d_out);
     /// Set global attributes
     std::map<std::string, std::string> att;
     att["title"] = "Output file of feltor/src/feltor_hpc.cu";
@@ -218,9 +229,10 @@ int main( int argc, char* argv[])
 #else //FELTOR_MPI
     err = file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out);
 #endif //FELTOR_MPI
+    int dim_ids3d[3] = {dim_ids[0], dim_ids[2], dim_ids[3]};
 
     //create & output static 3d variables into file
-    for ( auto record& diagnostics3d_static_list)
+    for ( auto& record : feltor::diagnostics3d_static_list)
     {
         int vecID;
         MPI_OUT err = nc_def_var( ncid, record.name.data(), NC_DOUBLE, 3,
@@ -228,6 +240,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_put_att_text( ncid, vecID,
             "long_name", record.long_name.size(), record.long_name.data());
         MPI_OUT err = nc_enddef( ncid);
+        MPI_OUT std::cout << "Computing "<<record.name<<"\n";
         record.function( resultH, var, grid, gp, mag);
         dg::blas2::symv( projectH, resultH, transferH);
         output.output_static3d( ncid, vecID, transferH);
@@ -235,8 +248,7 @@ int main( int argc, char* argv[])
     }
 
     //Create field IDs
-    std::map<std::string, int> id3d, id4d;
-    for( auto record& : feltor::diagnostics3d_list)
+    for( auto& record : feltor::diagnostics3d_list)
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
@@ -245,7 +257,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_put_att_text( ncid, id4d[name], "long_name", long_name.size(),
             long_name.data());
     }
-    for( auto record& : feltor::diagnostics2d_list)
+    for( auto& record : feltor::diagnostics2d_list)
     {
         std::string name = record.name + "_ta2d";
         std::string long_name = record.long_name + " (Toroidal average)";
@@ -253,20 +265,20 @@ int main( int argc, char* argv[])
             name += "_tt";
             long_name+= " (Time average)";
         }
-        MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+        MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids3d,
             &id3d[name]);//creates a new id3d entry
-        MPI_OUT err = nc_put_att_text( ncid, id3d[name], "long_name", long_name.size(),
+        MPI_OUT err = nc_put_att_text( ncid, id3d.at(name), "long_name", long_name.size(),
             long_name.data());
 
         name = record.name + "_2d";
-        long_name = record.long_name + " (Evaluated on phi = pi plane)";
+        long_name = record.long_name + " (Evaluated on phi = 0 plane)";
         if( record.integral){
             name += "_tt";
             long_name+= " (Time average)";
         }
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+        MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids3d,
             &id3d[name]);
-        err = nc_put_att_text( ncid, id3d[name], "long_name", long_name.size(),
+        MPI_OUT err = nc_put_att_text( ncid, id3d.at(name), "long_name", long_name.size(),
             long_name.data());
     }
     MPI_OUT err = nc_enddef(ncid);
@@ -286,28 +298,47 @@ int main( int argc, char* argv[])
     }
 
     size_t start = 0, count = 1;
-    for( auto record& : diagnostics3d_list)
+    MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
+    for( auto& record : feltor::diagnostics3d_list)
     {
-        record.function( result, var);
-        dg::blas2::symv( projectD, result, transferD);
+        record.function( resultD, var);
+        dg::blas2::symv( projectD, resultD, transferD);
         dg::assign( transferD, transferH);
         output.output_dynamic3d( ncid, id4d.at(record.name), start, transferH);
     }
-    for( auto record& : diagnostics2d_list)
+    MPI_OUT err = nc_close(ncid);
+
+    std::cin >> count;
+    MPI_OUT err = nc_open(file_name.data(), NC_WRITE, &ncid);
+    for( auto& record : feltor::diagnostics2d_list)
     {
-        record.function( result, var);
-        dg::blas2::symv( projectD, result, transferD);
+        record.function( resultD, var);
+        dg::blas2::symv( projectD, resultD, transferD);
+
         //toroidal average
-        toroidal_average( transferD, transfer2dD);
-        dg::assign( transfer2dD, transfer2dH);
-
-        // 2d data of plane varphi = 0
-        dg::HVec t2d_mp(result.data().begin(),
-            result.data().begin() + g2d_out.size() );
-        //compute toroidal average and extract 2d field
-        output.output_dynamic2d( ncid, id3d.at(name), start, transferH2d);
+        std::string name = record.name + "_ta2d";
+        toroidal_average( transferD, transferD2d);
+        //create and init Simspons for time integrals
+        if( record.integral)
+        {
+            name += "_tt";
+            time_integrals[name].init( time, transferD2d);
+        }
+        std::cout << "Output "<<name<<"\n";
+        dg::assign( transferD2d, transferH2d);
+        output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+
+        // and a slice
+        name = record.name + "_2d";
+        feltor::slice_vector3d( transferD, transferD2d, local_size2d);
+        if( record.integral)
+        {
+            name += "_tt";
+            time_integrals[name].init( time, transferD2d);
+        }
+        dg::assign( transferD2d, transferH2d);
+        output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
     }
-    MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
     MPI_OUT err = nc_close(ncid);
     MPI_OUT std::cout << "First write successful!\n";
     ///////////////////////////////////////Timeloop/////////////////////////////////
@@ -339,34 +370,33 @@ int main( int argc, char* argv[])
                 }
                 step++;
             }
-            Timer tti;
+            dg::Timer tti;
             tti.tic();
             double deltat = time - previous_time;
             double energy = 0, ediff = 0.;
-            for( auto record& : diagnostics2d_list)
+            for( auto& record : feltor::diagnostics2d_list)
             {
                 if( std::find( feltor::energies.begin(), feltor::energies.end(), record.name) != feltor::energies.end())
                 {
-                    record.function( result, var);
-                    energy += dg::blas1::dot( result, feltor.vol3d());
+                    record.function( resultD, var);
+                    energy += dg::blas1::dot( resultD, feltor.vol3d());
                 }
                 if( std::find( feltor::energy_diff.begin(), feltor::energy_diff.end(), record.name) != feltor::energy_diff.end())
                 {
-                    record.function( result, var);
-                    ediff += dg::blas1::dot( result, feltor.vol3d());
+                    record.function( resultD, var);
+                    ediff += dg::blas1::dot( resultD, feltor.vol3d());
                 }
                 if( record.integral)
                 {
-                    record.function( result, var);
-                    dg::blas2::symv( projectD, result, transferD);
+                    record.function( resultD, var);
+                    dg::blas2::symv( projectD, resultD, transferD);
                     //toroidal average and add to time integral
                     toroidal_average( transferD, transferD2d);
                     time_integrals.at(record.name+"_ta2d_tt").add( time, transferD2d);
 
                     // 2d data of plane varphi = 0
-                    dg::HVec t2d_mp(result.data().begin(),
-                        result.data().begin() + g2d_out.size() );
-                    time_integrals.at(record.name+"_2d_tt").add( time, transfer2dD);
+                    feltor::slice_vector3d( transferD, transferD2d, local_size2d);
+                    time_integrals.at(record.name+"_2d_tt").add( time, transferD2d);
                 }
 
             }
@@ -383,15 +413,15 @@ int main( int argc, char* argv[])
             if( p.beta != 0)
             {
                 dg::blas1::pointwiseDot(
-                    feltor.density(0), feltor.velocity(0), temp);
+                    feltor.density(0), feltor.velocity(0), resultD);
                 dg::blas1::pointwiseDot( p.beta,
-                    feltor.density(1), feltor.velocity(1), -p.beta, temp);
-                double norm  = dg::blas2::dot( temp, feltor.vol3d(), temp);
-                dg::blas1::axpby( -1., feltor.lapMperpA(), 1., temp);
-                double error = dg::blas2::dot( temp, feltor.vol3d(), temp);
+                    feltor.density(1), feltor.velocity(1), -p.beta, resultD);
+                double norm  = dg::blas2::dot( resultD, feltor.vol3d(), resultD);
+                dg::blas1::axpby( -1., feltor.lapMperpA(), 1., resultD);
+                double error = dg::blas2::dot( resultD, feltor.vol3d(), resultD);
                 MPI_OUT std::cout << "\tRel. Error Induction "<<sqrt(error/norm) <<"\n";
             }
-            tti.tic();
+            tti.toc();
             std::cout << " Time for internal diagnostics "<<tti.diff()<<"s\n";
         }
         ti.toc();
@@ -401,47 +431,52 @@ int main( int argc, char* argv[])
                     << ti.diff()/(double)p.itstp/(double)p.inner_loop<<"s";
         ti.tic();
         //////////////////////////write fields////////////////////////
-        start4d[0] = i;
+        start = i;
         MPI_OUT err = nc_open(file_name.data(), NC_WRITE, &ncid);
-        MPI_OUT err = nc_put_vara_double( ncid, tvarID, start4d, count4d, &time);
-        for( auto record& : diagnostics3d_list)
+        MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
+        for( auto& record : feltor::diagnostics3d_list)
         {
-            record.function( result, var);
-            dg::blas2::symv( projectD, result, transferD);
+            record.function( resultD, var);
+            dg::blas2::symv( projectD, resultD, transferD);
             dg::assign( transferD, transferH);
             output.output_dynamic3d( ncid, id4d.at(record.name), start, transferH);
         }
-        for( auto record& : diagnostics2d_list)
+        for( auto& record : feltor::diagnostics2d_list)
         {
-            if(!record.integral)
+            if(record.integral) // we already computed the output...
             {
-                record.function( result, var);
-                dg::blas2::symv( projectD, result, transferD);
-                //toroidal average
-                toroidal_average( transferD, transfer2dD);
-                dg::assign( transfer2dD, transfer2dH);
+                std::string name = record.name+"_ta2d_tt";
+                transferD2d = time_integrals.at(name).get_integral();
+                std::array<double,2> tt = time_integrals.at(name).get_boundaries();
+                dg::blas1::scal( transferD2d, 1./(tt[1]-tt[0]));
+                time_integrals.at(name).flush();
+                dg::assign( transferD2d, transferH2d);
+                output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
 
-                // 2d data of plane varphi = 0
-                dg::HVec t2d_mp(result.data().begin(),
-                    result.data().begin() + g2d_out.size() );
+                name = record.name+"_2d_tt";
+                transferD2d = time_integrals.at(name).get_integral( );
+                tt = time_integrals.at(name).get_boundaries( );
+                dg::blas1::scal( transferD2d, 1./(tt[1]-tt[0]));
+                time_integrals.at(name).flush( );
+                dg::assign( transferD2d, transferH2d);
+                output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
             }
             else //manage the time integrators
             {
-                std::string name = record.name+"_ta2d_tt";
-                transfer2dD = time_integrals.at(name).get_integral( );
-                std::array<double,2> bb = time_integrals.at(name).get_boundaries( );
-                dg::scal( transfer2dD, 1./(bb[1]-bb[0]));
-                time_integrals.at(name).flush( );
+                record.function( resultD, var);
+                dg::blas2::symv( projectD, resultD, transferD);
+
+                std::string name = record.name+"_ta2d";
+                toroidal_average( transferD, transferD2d);
                 dg::assign( transferD2d, transferH2d);
-                output.output_dynamic2d( ncid, id3d.at(name), start, transferH2d);
+                output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
 
-                name = record.name+"_2d_tt";
-                transfer2dD = time_integrals.at(name).get_integral( );
-                std::array<double,2> bb = time_integrals.at(name).get_boundaries( );
-                dg::scal( transferD2d, 1./(bb[1]-bb[0]));
-                time_integrals.at(name).flush( );
+                // 2d data of plane varphi = 0
+                name = record.name+"_2d";
+                feltor::slice_vector3d( transferD, transferD2d, local_size2d);
                 dg::assign( transferD2d, transferH2d);
-                output.output_dynamic2d( ncid, id3d.at(name), start, transferH2d);
+                output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+            }
         }
         MPI_OUT err = nc_close(ncid);
         ti.toc();
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 6f059c788..e43241e3c 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -156,55 +156,54 @@ struct Record{
 struct Record_static{
     std::string name;
     std::string long_name;
-    bool integral;
-    std::function<void( HVec&, Variables&, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag)> function;
+    std::function<void( HVec&, Variables&, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag)> function;
 };
 
 //Here is a list of static (time-independent) 3d variables that go into the output
-std::vector<Record_static> dianostics3d_static_list = {
+std::vector<Record_static> diagnostics3d_static_list = {
     { "BR", "R-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
             dg::geo::BFieldR fieldR(mag);
             result = dg::pullback( fieldR, grid);
         }
     },
     { "BZ", "Z-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
             dg::geo::BFieldZ fieldZ(mag);
             result = dg::pullback( fieldZ, grid);
         }
     },
     { "BP", "Contravariant P-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
             dg::geo::BFieldP fieldP(mag);
             result = dg::pullback( fieldP, grid);
         }
     },
     { "Psip", "Flux-function psi",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
              result = dg::pullback( mag.psip(), grid);
         }
     },
     { "Nprof", "Density profile",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
             Initialize init(v.p, gp, mag);
             result = init.profile(grid);
         }
     },
     { "Source", "Source region",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
             Initialize init(v.p, gp, mag);
             result = init.source_damping(grid);
         }
     },
     { "Damping", "Damping region for initial profile",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
             Initialize init(v.p, gp, mag);
             result = init.profile_damping(grid);
         }
     },
     { "xc", "x-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -213,7 +212,7 @@ std::vector<Record_static> dianostics3d_static_list = {
         }
     },
     { "yc", "y-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -222,7 +221,7 @@ std::vector<Record_static> dianostics3d_static_list = {
         }
     },
     { "zc", "z-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, dg::geo::solovev::Parameters gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -698,6 +697,7 @@ std::vector<Record> diagnostics2d_list = {
 
 };
 
+// These two lists signify the quantities for accuracy computation
 std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
 std::vector<std::string> energy_diff = { "resistivity", "leeperp", "leiperp", "leeparallel", "leiparallel"};
 }//namespace feltor
diff --git a/src/feltor/serial_netcdf.h b/src/feltor/serial_netcdf.h
index bd732aec2..3be841f46 100644
--- a/src/feltor/serial_netcdf.h
+++ b/src/feltor/serial_netcdf.h
@@ -2,18 +2,32 @@
 
 namespace feltor
 {
-
+template<class Container>
+void slice_vector3d( const Container& transfer, Container& transfer2d, size_t local_size2d)
+{
+#ifdef FELTOR_MPI
+    thrust::copy(
+        transfer.data().begin(),
+        transfer.data().begin() + local_size2d,
+        transfer2d.data().begin()
+    );
+#else
+    thrust::copy(
+        transfer.begin(),
+        transfer.begin() + local_size2d,
+        transfer2d.begin()
+    );
+#endif
+}
+//Manages the communication for serial netcdf output
 struct ManageOutput
 {
-
-    ManageOutput( Geometry& g3d_out):
+    ManageOutput( const Geometry& g3d_out)
     {
-        std::unique_ptr<Geometry2d> g2d_out_ptr = g3d_out.perp_grid();
-        m_start4d = {0, 0, 0, 0};
-        m_start3d = {0, 0, 0};
-        m_local_size3d = g3d_out.size();
-        m_local_size2d = g2d_out_ptr->size();
+        std::unique_ptr<typename Geometry::perpendicular_grid> g2d_out_ptr  ( dynamic_cast<typename Geometry::perpendicular_grid*>( g3d_out.perp_grid()));
 #ifdef FELTOR_MPI
+        m_local_size3d = g3d_out.local().size();
+        m_local_size2d = g2d_out_ptr->local().size();
         m_comm3d = g3d_out.communicator();
         m_comm2d = g2d_out_ptr->communicator();
         MPI_Comm_rank( m_comm2d, &m_rank2d);
@@ -27,34 +41,43 @@ struct ManageOutput
             MPI_Cart_coords( m_comm2d, rrank, 2, &m_coords2d[2*rrank]);
         for( int rrank=0; rrank<m_size3d; rrank++)
             MPI_Cart_coords( m_comm3d, rrank, 3, &m_coords[3*rrank]);
-        m_count4d = {1, g3d_out.local().Nz(),
-            g3d_out.n()*(g3d_out.local().Ny()),
-            g3d_out.n()*(g3d_out.local().Nx())};
-        m_count3d = {1,
-            g3d_out.n()*(g3d_out.local().Ny()),
-            g3d_out.n()*(g3d_out.local().Nx())};
+        m_count4d[0] = 1;
+        m_count4d[1] = g3d_out.local().Nz();
+        m_count4d[2] = g3d_out.n()*(g3d_out.local().Ny());
+        m_count4d[3] = g3d_out.n()*(g3d_out.local().Nx());
+        m_count3d[0] = 1;
+        m_count3d[1] = g3d_out.n()*(g3d_out.local().Ny()),
+        m_count3d[2] = g3d_out.n()*(g3d_out.local().Nx());
 #else //FELTOR_MPI
-        m_count4d = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(),
-            g3d_out.n()*g3d_out.Nx()};
-        m_count3d = {1, g3d_out.n()*g3d_out.Ny(),
-            g3d_out.n()*g3d_out.Nx()};
+        m_count4d[0] = 1;
+        m_count4d[1] = g3d_out.Nz();
+        m_count4d[2] = g3d_out.n()*g3d_out.Ny();
+        m_count4d[3] = g3d_out.n()*g3d_out.Nx();
+        m_count3d[0] = 1;
+        m_count3d[1] = g3d_out.n()*g3d_out.Ny();
+        m_count3d[2] = g3d_out.n()*g3d_out.Nx();
+        m_local_size3d = g3d_out.size();
+        m_local_size2d = g2d_out_ptr->size();
 #endif //FELTOR_MPI
     }
     //must enddef first
-    void output_static3d(int ncid, int vecID, HVec& transferH) const
+    void output_static3d(int ncid, int vecID, const HVec& transferH) const
     {
+        file::NC_Error_Handle err;
+        size_t start4d[4] = {0,0,0,0};
 #ifdef FELTOR_MPI
+        MPI_Status status;
         if(m_rank3d==0)
         {
             for( int rrank=0; rrank<m_size3d; rrank++)
             {
                 if(rrank!=0)
                     MPI_Recv( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
-                          rrank, rrank, m_comm3d, &m_status);
-                m_start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
-                m_start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
-                m_start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
-                m_err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
+                          rrank, rrank, m_comm3d, &status);
+                start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
+                start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
+                start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
+                err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
                     transferH.data().data());
             }
         }
@@ -63,25 +86,27 @@ struct ManageOutput
                       0, m_rank3d, m_comm3d);
         MPI_Barrier( m_comm3d);
 #else
-        m_err = nc_put_vara_double( ncid, vecID, &m_start4d[1], &m_count4d[1],
+        err = nc_put_vara_double( ncid, vecID, &start4d[1], &m_count4d[1],
             transferH.data());
 #endif // FELTOR_MPI
     }
-    void output_dynamic3d(int ncid, int vecID, int start, HVec& transferH) const
+    void output_dynamic3d(int ncid, int vecID, unsigned start, const HVec& transferH) const
     {
-        m_start4d[0] = start;
+        size_t start4d[4] = {start, 0, 0, 0};
+        file::NC_Error_Handle err;
 #ifdef FELTOR_MPI
+        MPI_Status status;
         if(m_rank3d==0)
         {
             for( int rrank=0; rrank<m_size3d; rrank++)
             {
                 if(rrank!=0)
                     MPI_Recv( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
-                          rrank, rrank, m_comm3d, &m_status);
-                m_start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
-                m_start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
-                m_start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
-                m_err = nc_put_vara_double( ncid, vecID, start4d, count4d,
+                          rrank, rrank, m_comm3d, &status);
+                start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
+                start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
+                start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
+                err = nc_put_vara_double( ncid, vecID, start4d, count4d,
                     transferH.data().data());
             }
         }
@@ -90,26 +115,29 @@ struct ManageOutput
                       0, m_rank3d, m_comm3d);
         MPI_Barrier( m_comm3d);
 #else
-        m_err = nc_put_vara_double( ncid, vecID, m_start4d, m_count4d,
+        err = nc_put_vara_double( ncid, vecID, start4d, m_count4d,
             transferH.data());
 #endif // FELTOR_MPI
     }
 
 //all send to their rank2d 0 but only rank3d 0 writes into file
-    void output_dynamic2d(int ncid, int vecID, int start, HVec& transferH2d) const
+    void output_dynamic2d_slice(int ncid, int vecID, unsigned start, const HVec& transferH2d) const
     {
-        m_start3d[0] = start;
+        file::NC_Error_Handle err;
+        size_t start3d[3] = {start, 0, 0};
 #ifdef FELTOR_MPI
+        MPI_Status status;
+        // 2d data of plane varphi = 0
         if(m_rank2d==0)
         {
             for( int rrank=0; rrank<m_size2d; rrank++)
             {
                 if(rrank!=0)
                     MPI_Recv( transferH.data().data(), m_local_size2d, MPI_DOUBLE,
-                          rrank, rrank, m_comm2d, &m_status);
-                m_start3d[1] = m_coords2d[2*rrank+1]*m_count3d[1],
-                m_start3d[2] = m_coords2d[2*rrank+0]*m_count3d[2];
-                m_err = nc_put_vara_double( ncid, vecID, start3d, count3d,
+                          rrank, rrank, m_comm2d, &status);
+                start3d[1] = m_coords2d[2*rrank+1]*m_count3d[1],
+                start3d[2] = m_coords2d[2*rrank+0]*m_count3d[2];
+                err = nc_put_vara_double( ncid, vecID, start3d, count3d,
                     transferH2d.data().data());
             }
         }
@@ -119,19 +147,21 @@ struct ManageOutput
         MPI_Barrier( m_comm2d);
         MPI_Barrier( m_comm3d); //all processes synchronize
 #else
-        m_err = nc_put_vara_double( ncid, vecID, m_start3d, m_count3d,
-            transferH2d.data());
+        std::cout << " Size "<<m_count3d[0]*m_count3d[1]*m_count3d[2]<<" "<<transferH2d.size()<<std::endl;
+        std::cout << " Start"<<start3d[0]<<" "<<start3d[1]<<" "<<start3d[2]<<"\n";
+        std::cout << "ncid "<<ncid<<" vecId "<<vecID<<std::endl;
+        HVec test( 3304, 0.);
+        err = nc_put_vara_double( ncid, vecID, start3d, m_count3d,
+            test.data());
 #endif // FELTOR_MPI
     }
-    private
+    private:
     unsigned m_local_size2d;
     unsigned m_local_size3d;
-    size_t m_start3d[3], m_count3d[3];
-    size_t m_start4d[4], m_count4d[4];
-    file::NC_Error_Handle m_err;
+    size_t m_count3d[3];
+    size_t m_count4d[4];
     int m_rank3d, m_size3d, m_rank2d, m_size2d;
 #ifdef FELTOR_MPI
-    MPI_Status m_status;
     MPI_Comm m_comm2d, m_comm3d;
     std::vector<int> m_coords, m_coords2d;
 #endif //FELTOR_MPI
-- 
GitLab


From eb68e3eb8151f5e2dc01cc2b7a495bf6faa8371b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Sun, 30 Jun 2019 13:22:37 +0200
Subject: [PATCH 109/540] Debug mpi version of feltor_hpc

Segfault is still there unfortunately
---
 src/feltor/init.h          |  5 +++++
 src/feltor/serial_netcdf.h | 14 +++++++-------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/feltor/init.h b/src/feltor/init.h
index 0ead24243..90d612859 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -49,6 +49,8 @@ struct Initialize
     }
     template<class Feltor>
     std::array<std::array<DVec,2>,2> init_from_parameters(Feltor& feltor, const Geometry& grid){
+        int rank;
+        MPI_Comm_rank( MPI_COMM_WORLD, &rank);
         std::array<std::array<DVec,2>,2> y0;
         //Now perturbation
         HVec ntilde = dg::evaluate(dg::zero,grid);
@@ -118,6 +120,8 @@ struct Initialize
 
     //everyone reads their portion of the input data
     std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Geometry& grid, double& time){
+        int rank;
+        MPI_Comm_rank( MPI_COMM_WORLD, &rank);
         std::array<std::array<DVec,2>,2> y0;
         ///////////////////read in and show inputfile
         file::NC_Error_Handle errIN;
@@ -150,6 +154,7 @@ struct Initialize
 
         #ifdef FELTOR_MPI
         int dimsIN[3],  coordsIN[3];
+        int periods[3] = {false, false, true}; //non-, non-, periodic
         MPI_Cart_get( grid.communicator(), 3, dimsIN, periods, coordsIN);
         size_t countIN[4] = {1, grid_IN.local().Nz(),
             grid_IN.n()*(grid_IN.local().Ny()),
diff --git a/src/feltor/serial_netcdf.h b/src/feltor/serial_netcdf.h
index 3be841f46..1abd56137 100644
--- a/src/feltor/serial_netcdf.h
+++ b/src/feltor/serial_netcdf.h
@@ -61,7 +61,7 @@ struct ManageOutput
 #endif //FELTOR_MPI
     }
     //must enddef first
-    void output_static3d(int ncid, int vecID, const HVec& transferH) const
+    void output_static3d(int ncid, int vecID, HVec& transferH) const
     {
         file::NC_Error_Handle err;
         size_t start4d[4] = {0,0,0,0};
@@ -77,7 +77,7 @@ struct ManageOutput
                 start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
                 start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
                 start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
-                err = nc_put_vara_double( ncid, vecID, &start4d[1], &count4d[1],
+                err = nc_put_vara_double( ncid, vecID, &start4d[1], &m_count4d[1],
                     transferH.data().data());
             }
         }
@@ -90,7 +90,7 @@ struct ManageOutput
             transferH.data());
 #endif // FELTOR_MPI
     }
-    void output_dynamic3d(int ncid, int vecID, unsigned start, const HVec& transferH) const
+    void output_dynamic3d(int ncid, int vecID, unsigned start, HVec& transferH) const
     {
         size_t start4d[4] = {start, 0, 0, 0};
         file::NC_Error_Handle err;
@@ -106,7 +106,7 @@ struct ManageOutput
                 start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
                 start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
                 start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
-                err = nc_put_vara_double( ncid, vecID, start4d, count4d,
+                err = nc_put_vara_double( ncid, vecID, start4d, m_count4d,
                     transferH.data().data());
             }
         }
@@ -121,7 +121,7 @@ struct ManageOutput
     }
 
 //all send to their rank2d 0 but only rank3d 0 writes into file
-    void output_dynamic2d_slice(int ncid, int vecID, unsigned start, const HVec& transferH2d) const
+    void output_dynamic2d_slice(int ncid, int vecID, unsigned start, HVec& transferH2d) const
     {
         file::NC_Error_Handle err;
         size_t start3d[3] = {start, 0, 0};
@@ -133,11 +133,11 @@ struct ManageOutput
             for( int rrank=0; rrank<m_size2d; rrank++)
             {
                 if(rrank!=0)
-                    MPI_Recv( transferH.data().data(), m_local_size2d, MPI_DOUBLE,
+                    MPI_Recv( transferH2d.data().data(), m_local_size2d, MPI_DOUBLE,
                           rrank, rrank, m_comm2d, &status);
                 start3d[1] = m_coords2d[2*rrank+1]*m_count3d[1],
                 start3d[2] = m_coords2d[2*rrank+0]*m_count3d[2];
-                err = nc_put_vara_double( ncid, vecID, start3d, count3d,
+                err = nc_put_vara_double( ncid, vecID, start3d, m_count3d,
                     transferH2d.data().data());
             }
         }
-- 
GitLab


From 02cc9387a77c073aa4371a41f29c1fab90127653 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 30 Jun 2019 14:44:19 +0200
Subject: [PATCH 110/540] Update mpi average class with extend parameter

and update tests accordingly, but maybe the binary results should
be checked somewhere
---
 inc/dg/topology/average_mpi.h   | 55 +++++++++++++++++++++++++++------
 inc/dg/topology/average_mpit.cu | 47 +++++++++++++++-------------
 inc/dg/topology/average_t.cu    | 17 +++++-----
 3 files changed, 82 insertions(+), 37 deletions(-)

diff --git a/inc/dg/topology/average_mpi.h b/inc/dg/topology/average_mpi.h
index 0945dd2d8..4453f1260 100644
--- a/inc/dg/topology/average_mpi.h
+++ b/inc/dg/topology/average_mpi.h
@@ -30,12 +30,14 @@ struct Average<MPI_Vector<container> >
     {
         m_nx = g.local().Nx()*g.n(), m_ny = g.local().Ny()*g.n();
         m_w=dg::construct<MPI_Vector<container>>(dg::create::weights(g, direction));
-        m_temp1d = m_temp = m_w;
+        m_temp = m_w;
         int remain_dims[] = {false,false}; //true true false
         m_transpose = false;
+        unsigned size1d = 0;
         if( direction == dg::coo2d::x)
         {
             dg::blas1::scal( m_w, 1./g.lx());
+            size1d = m_ny;
             remain_dims[0] = true;
         }
         else
@@ -44,19 +46,30 @@ struct Average<MPI_Vector<container> >
             remain_dims[1] = true;
             dg::blas1::scal( m_temp, 1./g.ly());
             dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
+            size1d = m_nx;
         }
+
+        //Now get the reduction communicator
         MPI_Cart_sub( g.communicator(), remain_dims, &m_comm);
         exblas::mpi_reduce_communicator( m_comm, &m_comm_mod, &m_comm_mod_reduce);
+        // ... and the one perpendicular to it
+        for( unsigned i=0; i<2; i++)
+            remain_dims[i] = !remain_dims[i];
+        MPI_Comm comm2;
+        MPI_Cart_sub( g.communicator(), remain_dims, &comm2);
+        // with that construct the reduce mpi vec
+        thrust::host_vector<double> t1d( size1d);
+        m_temp1d = MPI_Vector<container>( dg::construct<container>( t1d), comm2);
     }
 
     ///@copydoc Average()
     Average( const aMPITopology3d& g, enum coo3d direction)
     {
         m_w = dg::construct<MPI_Vector<container>>(dg::create::weights(g, direction));
-        m_temp1d = m_temp = m_w;
+        m_temp = m_w;
         m_transpose = false;
         unsigned nx = g.n()*g.local().Nx(), ny = g.n()*g.local().Ny(), nz = g.local().Nz();
-        int remain_dims[] = {false,false,false}; //true true false
+        int remain_dims[] = {false,false,false};
         m_transpose = false;
         if( direction == dg::coo3d::x) {
             dg::blas1::scal( m_w, 1./g.lx());
@@ -84,8 +97,21 @@ struct Average<MPI_Vector<container> >
         }
         else
             std::cerr << "Warning: this direction is not implemented\n";
+        //Now get the reduction communicator
         MPI_Cart_sub( g.communicator(), remain_dims, &m_comm);
         exblas::mpi_reduce_communicator( m_comm, &m_comm_mod, &m_comm_mod_reduce);
+        // ... and the one perpendicular to it
+        for( unsigned i=0; i<3; i++)
+            remain_dims[i] = !remain_dims[i];
+        MPI_Comm comm2;
+        MPI_Cart_sub( g.communicator(), remain_dims, &comm2);
+        // with that construct the reduce mpi vec
+        thrust::host_vector<double> t1d;
+        if(!m_transpose)
+            t1d = thrust::host_vector<double>( m_ny,0.);
+        else
+            t1d = thrust::host_vector<double>( m_nx,0.);
+        m_temp1d = MPI_Vector<container>( dg::construct<container>( t1d), comm2);
     }
     /**
      * @brief Compute the average as configured in the constructor
@@ -95,22 +121,33 @@ struct Average<MPI_Vector<container> >
      *  - extend the lower dimensional result back to the original dimensionality
      *
      * @param src Source Vector (must have the same size and communicator as the grid given in the constructor)
-     * @param res result Vector (must have same size and communicator as src vector, may alias src)
+     * @param res result Vector
+     (if \c extend==true, \c res must have same size and communicator as \c src vector, else it gets properly resized, may alias \c src)
+     * @param extend if \c true the average is extended back to the original dimensionality and the communicator is the 3d communicator, if \c false, this step is skipped.
+     * In that case, each process has a result vector with reduced dimensionality and a Cartesian communicator only in the remaining dimensions. Note that in any case \b all processes get the result (since the underlying dot product distributes its result to all processes)
      */
-    void operator() (const MPI_Vector<container>& src, MPI_Vector<container>& res)
+    void operator() (const MPI_Vector<container>& src, MPI_Vector<container>& res, bool extend = true)
     {
         if( !m_transpose)
         {
-            dg::mpi_average( m_nx, m_ny, src.data(), m_w.data(), m_temp.data(), m_comm, m_comm_mod, m_comm_mod_reduce);
-            dg::extend_column( m_nx, m_ny, m_temp.data(), res.data());
+            //temp1d has size m_ny
+            dg::mpi_average( m_nx, m_ny, src.data(), m_w.data(),
+                m_temp1d.data(), m_comm, m_comm_mod, m_comm_mod_reduce);
+            if( extend )
+                dg::extend_column( m_nx, m_ny, m_temp1d.data(), res.data());
+            else
+                res = m_temp1d;
         }
         else
         {
+            //temp1d has size m_nx
             dg::transpose( m_nx, m_ny, src.data(), m_temp.data());
             dg::mpi_average( m_ny, m_nx, m_temp.data(), m_w.data(), m_temp1d.data(), m_comm, m_comm_mod, m_comm_mod_reduce);
-            dg::extend_line( m_nx, m_ny, m_temp1d.data(), res.data());
+            if( extend )
+                dg::extend_line( m_nx, m_ny, m_temp1d.data(), res.data());
+            else
+                res = m_temp1d;
         }
-
     }
   private:
     unsigned m_nx, m_ny;
diff --git a/inc/dg/topology/average_mpit.cu b/inc/dg/topology/average_mpit.cu
index d53e6f17e..360d53cbc 100644
--- a/inc/dg/topology/average_mpit.cu
+++ b/inc/dg/topology/average_mpit.cu
@@ -9,9 +9,10 @@
 
 const double lx = M_PI/2.;
 const double ly = M_PI;
-double function( double x, double y) {return cos(x)*sin(y);}
-double pol_average( double x, double y) {return cos(x)*2./M_PI;}
-double tor_average( double x, double y) {return sin(y)*2./M_PI;}
+const double lz = M_PI/2.;
+double function( double x, double y, double z) {return cos(x)*sin(z);}
+double z_average( double x, double y) {return cos(x)*2./M_PI;}
+double x_average( double x, double y, double z) {return sin(z)*2./M_PI;}
 
 int main(int argc, char* argv[])
 {
@@ -19,33 +20,37 @@ int main(int argc, char* argv[])
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 
-    if(rank==0)std::cout << "Program to test the average in x and y direction\n";
+    if(rank==0)std::cout << "Program to test the average in x and z direction\n";
     MPI_Comm comm;
-    mpi_init2d( dg::PER, dg::PER, comm);
-    unsigned n = 3, Nx = 32, Ny = 48;
+    mpi_init3d( dg::PER, dg::PER, dg::PER, comm);
+    unsigned n = 3, Nx = 32, Ny = 48, Nz = 48;
     //![doxygen]
-    dg::MPIGrid2d g( 0, lx, 0, ly, n, Nx, Ny, comm);
+    dg::MPIGrid3d g( 0, lx, 0, ly, 0, lz, n, Nx, Ny, Nz, comm);
 
-    dg::Average<dg::MDVec > pol(g, dg::coo2d::y);
+    dg::Average<dg::MDVec > avg(g, dg::coo3d::z);
 
     const dg::MDVec vector = dg::evaluate( function ,g);
-    dg::MDVec average_y( vector);
-    if(rank==0)std::cout << "Averaging y ... \n";
-    pol( vector, average_y);
+    dg::MDVec average_z;
+    if(rank==0)std::cout << "Averaging z ... \n";
+    avg( vector, average_z, false);
     //![doxygen]
-    const dg::MDVec w2d = dg::create::weights( g);
-    dg::MDVec solution = dg::evaluate( pol_average, g);
-    dg::blas1::axpby( 1., solution, -1., average_y);
+    dg::MPIGrid2d gxy( 0, lx, 0, ly, n, Nx, Ny, average_z.communicator());
+    const dg::MDVec w2d = dg::create::weights( gxy);
+    dg::MDVec solution = dg::evaluate( z_average, gxy);
+    dg::blas1::axpby( 1., solution, -1., average_z);
     int64_t binary[] = {4406193765905047925,4395311848786989976};
     exblas::udouble res;
-    res.d = sqrt( dg::blas2::dot( average_y, w2d, average_y));
-    if(rank==0)std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i-binary[0]<<std::endl;
+    res.d = sqrt( dg::blas2::dot( average_z, w2d, average_z));
+    if(rank==0)std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i<<std::endl;
+    if(rank==0)std::cout << "(Converges with 2nd order).\n";
     if(rank==0)std::cout << "Averaging x ... \n";
-    dg::Average< dg::MDVec> tor( g, dg::coo2d::x);
-    tor( vector, average_y);
-    solution = dg::evaluate( tor_average, g);
-    dg::blas1::axpby( 1., solution, -1., average_y);
-    res.d = sqrt( dg::blas2::dot( average_y, w2d, average_y));
+    dg::Average< dg::MDVec> tor( g, dg::coo3d::x);
+    average_z = vector;
+    tor( vector, average_z);
+    solution = dg::evaluate( x_average, g);
+    dg::blas1::axpby( 1., solution, -1., average_z);
+    const dg::MDVec w3d = dg::create::weights( g);
+    res.d = sqrt( dg::blas2::dot( average_z, w3d, average_z));
     if(rank==0)std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i-binary[1]<<std::endl;
     //if(rank==0)std::cout << "\n Continue with \n\n";
 
diff --git a/inc/dg/topology/average_t.cu b/inc/dg/topology/average_t.cu
index 2b9d1cfde..5b1419c0f 100644
--- a/inc/dg/topology/average_t.cu
+++ b/inc/dg/topology/average_t.cu
@@ -7,7 +7,7 @@
 const double lx = M_PI/2.;
 const double ly = M_PI;
 double function( double x, double y) {return cos(x)*sin(y);}
-double pol_average( double x, double y) {return cos(x)*2./M_PI;}
+double pol_average( double x) {return cos(x)*2./M_PI;}
 double tor_average( double x, double y) {return sin(y)*2./M_PI;}
 
 int main()
@@ -20,22 +20,25 @@ int main()
     dg::Average< dg::DVec > pol(g, dg::coo2d::y);
 
     const dg::DVec vector = dg::evaluate( function ,g);
-    dg::DVec average_y( vector);
+    dg::DVec average_y;
     std::cout << "Averaging y ... \n";
-    pol( vector, average_y);
+    pol( vector, average_y, false);
     //![doxygen]
-    const dg::DVec w2d = dg::create::weights( g);
-    dg::DVec solution = dg::evaluate( pol_average, g);
+    const dg::Grid1d gx( 0, lx, n, Nx);
+    const dg::DVec w1d = dg::create::weights( gx);
+    dg::DVec solution = dg::evaluate( pol_average, gx);
     dg::blas1::axpby( 1., solution, -1., average_y);
     int64_t binary[] = {4406193765905047925,4395311848786989976};
     exblas::udouble res;
-    res.d = sqrt( dg::blas2::dot( average_y, w2d, average_y));
-    std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i-binary[0]<<std::endl;
+    res.d = sqrt( dg::blas2::dot( average_y, w1d, average_y));
+    std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i<<std::endl;
     std::cout << "Averaging x ... \n";
     dg::Average< dg::DVec> tor( g, dg::coo2d::x);
+    average_y = vector;
     tor( vector, average_y);
     solution = dg::evaluate( tor_average, g);
     dg::blas1::axpby( 1., solution, -1., average_y);
+    const dg::DVec w2d = dg::create::weights( g);
     res.d = sqrt( dg::blas2::dot( average_y, w2d, average_y));
     std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i-binary[1]<<std::endl;
     //std::cout << "\n Continue with \n\n";
-- 
GitLab


From a9adc9d0e66475d670b1a2bb3697b0a452890c93 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 30 Jun 2019 15:08:33 +0200
Subject: [PATCH 111/540] With updated toroidal average feltor_hpc works

or seems to work.
More tests necessary and next step will be feltordiag
---
 src/feltor/feltor_hpc.cu   | 13 ++++---------
 src/feltor/init.h          |  4 ++++
 src/feltor/serial_netcdf.h |  6 +-----
 3 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index d52fa0196..0cfcef56c 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -306,10 +306,6 @@ int main( int argc, char* argv[])
         dg::assign( transferD, transferH);
         output.output_dynamic3d( ncid, id4d.at(record.name), start, transferH);
     }
-    MPI_OUT err = nc_close(ncid);
-
-    std::cin >> count;
-    MPI_OUT err = nc_open(file_name.data(), NC_WRITE, &ncid);
     for( auto& record : feltor::diagnostics2d_list)
     {
         record.function( resultD, var);
@@ -317,14 +313,13 @@ int main( int argc, char* argv[])
 
         //toroidal average
         std::string name = record.name + "_ta2d";
-        toroidal_average( transferD, transferD2d);
-        //create and init Simspons for time integrals
+        toroidal_average( transferD, transferD2d, false);
+        //create and init Simpsons for time integrals
         if( record.integral)
         {
             name += "_tt";
             time_integrals[name].init( time, transferD2d);
         }
-        std::cout << "Output "<<name<<"\n";
         dg::assign( transferD2d, transferH2d);
         output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
 
@@ -391,7 +386,7 @@ int main( int argc, char* argv[])
                     record.function( resultD, var);
                     dg::blas2::symv( projectD, resultD, transferD);
                     //toroidal average and add to time integral
-                    toroidal_average( transferD, transferD2d);
+                    toroidal_average( transferD, transferD2d, false);
                     time_integrals.at(record.name+"_ta2d_tt").add( time, transferD2d);
 
                     // 2d data of plane varphi = 0
@@ -467,7 +462,7 @@ int main( int argc, char* argv[])
                 dg::blas2::symv( projectD, resultD, transferD);
 
                 std::string name = record.name+"_ta2d";
-                toroidal_average( transferD, transferD2d);
+                toroidal_average( transferD, transferD2d, false);
                 dg::assign( transferD2d, transferH2d);
                 output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
 
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 90d612859..345df0512 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -49,8 +49,10 @@ struct Initialize
     }
     template<class Feltor>
     std::array<std::array<DVec,2>,2> init_from_parameters(Feltor& feltor, const Geometry& grid){
+#ifdef FELTOR_MPI
         int rank;
         MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+#endif
         std::array<std::array<DVec,2>,2> y0;
         //Now perturbation
         HVec ntilde = dg::evaluate(dg::zero,grid);
@@ -120,8 +122,10 @@ struct Initialize
 
     //everyone reads their portion of the input data
     std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Geometry& grid, double& time){
+#ifdef FELTOR_MPI
         int rank;
         MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+#endif
         std::array<std::array<DVec,2>,2> y0;
         ///////////////////read in and show inputfile
         file::NC_Error_Handle errIN;
diff --git a/src/feltor/serial_netcdf.h b/src/feltor/serial_netcdf.h
index 1abd56137..97c7c5ec7 100644
--- a/src/feltor/serial_netcdf.h
+++ b/src/feltor/serial_netcdf.h
@@ -147,12 +147,8 @@ struct ManageOutput
         MPI_Barrier( m_comm2d);
         MPI_Barrier( m_comm3d); //all processes synchronize
 #else
-        std::cout << " Size "<<m_count3d[0]*m_count3d[1]*m_count3d[2]<<" "<<transferH2d.size()<<std::endl;
-        std::cout << " Start"<<start3d[0]<<" "<<start3d[1]<<" "<<start3d[2]<<"\n";
-        std::cout << "ncid "<<ncid<<" vecId "<<vecID<<std::endl;
-        HVec test( 3304, 0.);
         err = nc_put_vara_double( ncid, vecID, start3d, m_count3d,
-            test.data());
+            transferH2d.data());
 #endif // FELTOR_MPI
     }
     private:
-- 
GitLab


From 8f573b084629689705dd358be7c0b4c14ec00f3b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 30 Jun 2019 15:44:19 +0200
Subject: [PATCH 112/540] Fix map access bug in feltor_hpc

---
 src/feltor/feltor_hpc.cu | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 0cfcef56c..00be43112 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -252,9 +252,10 @@ int main( int argc, char* argv[])
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
+        id4d[name] = 0;//creates a new id4d entry
         MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 4, dim_ids,
-            &id4d[name]);//creates a new id4d entry
-        MPI_OUT err = nc_put_att_text( ncid, id4d[name], "long_name", long_name.size(),
+            &id4d.at(name));
+        MPI_OUT err = nc_put_att_text( ncid, id4d.at(name), "long_name", long_name.size(),
             long_name.data());
     }
     for( auto& record : feltor::diagnostics2d_list)
@@ -265,8 +266,9 @@ int main( int argc, char* argv[])
             name += "_tt";
             long_name+= " (Time average)";
         }
+        id3d[name] = 0;
         MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids3d,
-            &id3d[name]);//creates a new id3d entry
+            &id3d.at(name));//creates a new id3d entry
         MPI_OUT err = nc_put_att_text( ncid, id3d.at(name), "long_name", long_name.size(),
             long_name.data());
 
@@ -276,8 +278,9 @@ int main( int argc, char* argv[])
             name += "_tt";
             long_name+= " (Time average)";
         }
+        id3d[name] = 0;
         MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids3d,
-            &id3d[name]);
+            &id3d.at(name));
         MPI_OUT err = nc_put_att_text( ncid, id3d.at(name), "long_name", long_name.size(),
             long_name.data());
     }
-- 
GitLab


From bc25c0bd8f818dc0c4add612404eafeb57ed84c0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 30 Jun 2019 18:34:36 +0200
Subject: [PATCH 113/540] Change diagnostics to serial in feltor_hpc

since on GPUs the toroidal average takes forever
---
 inc/dg/topology/average_mpib.cu |  8 +++---
 src/feltor/feltor_hpc.cu        | 50 ++++++++++++++++++++-------------
 src/feltor/serial_netcdf.h      |  5 ++--
 3 files changed, 37 insertions(+), 26 deletions(-)

diff --git a/inc/dg/topology/average_mpib.cu b/inc/dg/topology/average_mpib.cu
index 82c3b0547..9a16cda18 100644
--- a/inc/dg/topology/average_mpib.cu
+++ b/inc/dg/topology/average_mpib.cu
@@ -29,9 +29,9 @@ int main(int argc, char* argv[])
     dg::Timer t;
 
 
-    dg::Average<dg::MDVec > pol(g, dg::coo2d::y);
-    dg::MDVec vector = dg::evaluate( function ,g), average_y( vector);
-    const dg::MDVec solution = dg::evaluate( pol_average, g);
+    dg::Average<dg::MHVec > pol(g, dg::coo2d::y);
+    dg::MHVec vector = dg::evaluate( function ,g), average_y( vector);
+    const dg::MHVec solution = dg::evaluate( pol_average, g);
     t.tic();
     for( unsigned i=0; i<100; i++)
         pol( vector, average_y);
@@ -39,7 +39,7 @@ int main(int argc, char* argv[])
     if(rank==0)std::cout << "Assembly of average vector took:      "<<t.diff()/100.<<"s\n";
 
     dg::blas1::axpby( 1., solution, -1., average_y, vector);
-    dg::MDVec w2d = dg::create::weights(g);
+    dg::MHVec w2d = dg::create::weights(g);
     double norm = dg::blas2::dot(vector, w2d, vector);
     if(rank==0)std::cout << "Distance to solution is: "<<        sqrt(norm)<<std::endl;
 
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 00be43112..4510e235c 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -159,8 +159,8 @@ int main( int argc, char* argv[])
     MPI_OUT std::cout << "Done!\n";
 
     // helper variables for output computations
-    std::map<std::string, dg::Simpsons<DVec>> time_integrals;
-    dg::Average<DVec> toroidal_average( g3d_out, dg::coo3d::z);
+    std::map<std::string, dg::Simpsons<HVec>> time_integrals;
+    dg::Average<HVec> toroidal_average( g3d_out, dg::coo3d::z);
     dg::MultiMatrix<HMatrix,HVec> projectH = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
     dg::MultiMatrix<DMatrix,DVec> projectD = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
     HVec transferH( dg::evaluate(dg::zero, g3d_out));
@@ -252,7 +252,7 @@ int main( int argc, char* argv[])
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
-        id4d[name] = 0;//creates a new id4d entry
+        id4d[name] = 0;//creates a new id4d entry for all processes
         MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 4, dim_ids,
             &id4d.at(name));
         MPI_OUT err = nc_put_att_text( ncid, id4d.at(name), "long_name", long_name.size(),
@@ -266,9 +266,9 @@ int main( int argc, char* argv[])
             name += "_tt";
             long_name+= " (Time average)";
         }
-        id3d[name] = 0;
+        id3d[name] = 0;//creates a new id3d entry for all processes
         MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids3d,
-            &id3d.at(name));//creates a new id3d entry
+            &id3d.at(name));
         MPI_OUT err = nc_put_att_text( ncid, id3d.at(name), "long_name", long_name.size(),
             long_name.data());
 
@@ -311,31 +311,41 @@ int main( int argc, char* argv[])
     }
     for( auto& record : feltor::diagnostics2d_list)
     {
+        dg::Timer tti;
+        tti.tic();
         record.function( resultD, var);
         dg::blas2::symv( projectD, resultD, transferD);
 
         //toroidal average
         std::string name = record.name + "_ta2d";
-        toroidal_average( transferD, transferD2d, false);
+        dg::assign( transferD, transferH);
+        toroidal_average( transferH, transferH2d, false);
         //create and init Simpsons for time integrals
         if( record.integral)
         {
             name += "_tt";
-            time_integrals[name].init( time, transferD2d);
+            time_integrals[name].init( time, transferH2d);
         }
-        dg::assign( transferD2d, transferH2d);
+        tti.toc();
+        MPI_OUT std::cout<< name << " Computing average took "<<tti.diff()<<"\n";
+        tti.tic();
         output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+        tti.toc();
+        MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
+        tti.tic();
 
         // and a slice
         name = record.name + "_2d";
         feltor::slice_vector3d( transferD, transferD2d, local_size2d);
+        dg::assign( transferD2d, transferH2d);
         if( record.integral)
         {
             name += "_tt";
-            time_integrals[name].init( time, transferD2d);
+            time_integrals[name].init( time, transferH2d);
         }
-        dg::assign( transferD2d, transferH2d);
         output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+        tti.toc();
+        MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
     }
     MPI_OUT err = nc_close(ncid);
     MPI_OUT std::cout << "First write successful!\n";
@@ -389,12 +399,14 @@ int main( int argc, char* argv[])
                     record.function( resultD, var);
                     dg::blas2::symv( projectD, resultD, transferD);
                     //toroidal average and add to time integral
-                    toroidal_average( transferD, transferD2d, false);
-                    time_integrals.at(record.name+"_ta2d_tt").add( time, transferD2d);
+                    dg::assign( transferD, transferH);
+                    toroidal_average( transferH, transferH2d, false);
+                    time_integrals.at(record.name+"_ta2d_tt").add( time, transferH2d);
 
                     // 2d data of plane varphi = 0
                     feltor::slice_vector3d( transferD, transferD2d, local_size2d);
-                    time_integrals.at(record.name+"_2d_tt").add( time, transferD2d);
+                    dg::assign( transferD2d, transferH2d);
+                    time_integrals.at(record.name+"_2d_tt").add( time, transferH2d);
                 }
 
             }
@@ -420,7 +432,7 @@ int main( int argc, char* argv[])
                 MPI_OUT std::cout << "\tRel. Error Induction "<<sqrt(error/norm) <<"\n";
             }
             tti.toc();
-            std::cout << " Time for internal diagnostics "<<tti.diff()<<"s\n";
+            MPI_OUT std::cout << " Time for internal diagnostics "<<tti.diff()<<"s\n";
         }
         ti.toc();
         MPI_OUT std::cout << "\n\t Step "<<step <<" of "
@@ -444,19 +456,17 @@ int main( int argc, char* argv[])
             if(record.integral) // we already computed the output...
             {
                 std::string name = record.name+"_ta2d_tt";
-                transferD2d = time_integrals.at(name).get_integral();
+                transferH2d = time_integrals.at(name).get_integral();
                 std::array<double,2> tt = time_integrals.at(name).get_boundaries();
                 dg::blas1::scal( transferD2d, 1./(tt[1]-tt[0]));
                 time_integrals.at(name).flush();
-                dg::assign( transferD2d, transferH2d);
                 output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
 
                 name = record.name+"_2d_tt";
-                transferD2d = time_integrals.at(name).get_integral( );
+                transferH2d = time_integrals.at(name).get_integral( );
                 tt = time_integrals.at(name).get_boundaries( );
                 dg::blas1::scal( transferD2d, 1./(tt[1]-tt[0]));
                 time_integrals.at(name).flush( );
-                dg::assign( transferD2d, transferH2d);
                 output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
             }
             else //manage the time integrators
@@ -465,8 +475,8 @@ int main( int argc, char* argv[])
                 dg::blas2::symv( projectD, resultD, transferD);
 
                 std::string name = record.name+"_ta2d";
-                toroidal_average( transferD, transferD2d, false);
-                dg::assign( transferD2d, transferH2d);
+                dg::assign( transferD, transferH);
+                toroidal_average( transferH, transferH2d, false);
                 output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
 
                 // 2d data of plane varphi = 0
diff --git a/src/feltor/serial_netcdf.h b/src/feltor/serial_netcdf.h
index 97c7c5ec7..cce0c25e3 100644
--- a/src/feltor/serial_netcdf.h
+++ b/src/feltor/serial_netcdf.h
@@ -137,8 +137,9 @@ struct ManageOutput
                           rrank, rrank, m_comm2d, &status);
                 start3d[1] = m_coords2d[2*rrank+1]*m_count3d[1],
                 start3d[2] = m_coords2d[2*rrank+0]*m_count3d[2];
-                err = nc_put_vara_double( ncid, vecID, start3d, m_count3d,
-                    transferH2d.data().data());
+                if(m_rank3d==0)//only global rank 0 writes out
+                    err = nc_put_vara_double( ncid, vecID, start3d, m_count3d,
+                        transferH2d.data().data());
             }
         }
         else
-- 
GitLab


From 4f3b7580b759543c53fc81c0c8b2dd59c33b478d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 1 Jul 2019 17:41:11 +0200
Subject: [PATCH 114/540] Rewrite feltordiag

to compute fsa only
also mv to feltor/src/feltor
---
 diag/Makefile            |  15 +-
 diag/feltordiag.cu       | 495 ---------------------------------------
 src/feltor/Makefile      |   5 +-
 src/feltor/feltor_hpc.cu |  26 +-
 src/feltor/feltordiag.cu | 339 +++++++++++++++++++++++++++
 src/feltor/feltordiag.h  |  59 +++--
 6 files changed, 392 insertions(+), 547 deletions(-)
 delete mode 100644 diag/feltordiag.cu
 create mode 100644 src/feltor/feltordiag.cu

diff --git a/diag/Makefile b/diag/Makefile
index 4c00110c5..e56ba6f24 100644
--- a/diag/Makefile
+++ b/diag/Makefile
@@ -1,22 +1,19 @@
 device = omp
 
-#configure machine 
+#configure machine
 include ../config/default.mk
-include ../config/*.mk 
+include ../config/*.mk
 include ../config/devices/devices.mk
 
 INCLUDE+= -I../inc/# include files libs
 INCLUDE+= -I../src/# include files from source code
 
-TARGETS =  compare histdiag fftwdiag crosscoherencdiag feltorSesoldiag feltorShwdiag feltorSHdiag feltordiag vmaxnc feltorSHvmaxdiag toeflRdiag impRdiag feltorShwmerger feltorShwradstat feltorShwstat growthrate toeflEPdiag normdiag reco2Ddiag
+TARGETS =  compare histdiag fftwdiag crosscoherencdiag feltorSesoldiag feltorShwdiag feltorSHdiag vmaxnc feltorSHvmaxdiag toeflRdiag impRdiag feltorShwmerger feltorShwradstat feltorShwstat growthrate toeflEPdiag normdiag reco2Ddiag
 
 all: $(TARGETS)
 
-
-feltordiag: feltordiag.cu
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g 
 feltorSHdiag: feltorSHdiag.cu
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB)
 toeflRdiag: toeflRdiag.cu
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB)
 toeflEPdiag: toeflEPdiag.cu
@@ -26,7 +23,7 @@ impRdiag: impRdiag.cu
 feltorSHvmaxdiag: feltorSHvmaxdiag.cu
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB)
 vmaxnc: vmaxnc.cu
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS)
 fftwdiag: fftwdiag.cpp
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -I$(HOME)/include/spectral -lfftw3  -DTL_DEBUG -g
 growthrate: growthrate.cpp
@@ -46,7 +43,7 @@ feltorShwmerger: feltorShwmerger.cpp
 feltorShwstat: feltorShwstat.cpp
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g
 feltorShwradstat: feltorShwradstat.cpp
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g	
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g
 normdiag: normdiag.cu
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_DEBUG -g
 reco2Ddiag: reco2Ddiag.cu
diff --git a/diag/feltordiag.cu b/diag/feltordiag.cu
deleted file mode 100644
index dec259b10..000000000
--- a/diag/feltordiag.cu
+++ /dev/null
@@ -1,495 +0,0 @@
-#include <iostream>
-#include <fstream>
-#include <iomanip>
-#include <vector>
-#include <string>
-#include <functional>
-#include "json/json.h"
-
-#include "dg/file/nc_utilities.h"
-#include "feltordiag.h"
-
-void create_records_in_file( int ncid,
-    int dim_ids4d[4], int dim_ids[3], int dim_ids1d[2],
-    std::map<std::string, int>& id0d,
-    std::map<std::string, int>& id1d,
-    std::map<std::string, int>& id2d,
-    std::map<std::string, int>& id3d)
-{
-    file::NC_Error_Handle err;
-    for( auto record& : feltor::records3d_list)
-    {
-        std::string name = record.name;
-        std::string long_name = record.long_name;
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 4, dim_ids4d,
-            &id3d[name]);//creates a new id3d entry
-        err = nc_put_att_text( ncid, id3d[name], "long_name", long_name.size(),
-            long_name.data());
-    }
-    for( auto record& : feltor::records_list)
-    {
-        std::string name = record.name + "_ta2d";
-        std::string long_name = record.long_name + " (Toroidal average)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);//creates a new id2d entry
-        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
-            long_name.data());
-
-        name = record.name + "_2d";
-        long_name = record.long_name + " (Evaluated on phi = 0 plane)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
-            long_name.data());
-
-        name = record.name + "_fluc2d";
-        long_name = record.long_name + " (Fluctuations wrt fsa on phi = 0 plane)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
-            long_name.data());
-
-        name = record.name + "_fsa2d";
-        long_name = record.long_name + " (Flux surface average interpolated to 2d plane)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid, id2d[name], "long_name", long_name.size(),
-            long_name.data());
-
-        name = record.name + "_fsa";
-        long_name = record.long_name + " (Flux surface average)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 2, dim_ids1d,
-            &id1d[name]);
-        err = nc_put_att_text( ncid, id1d[name], "long_name", long_name.size(),
-            long_name.data());
-
-        name = record.name + "_ifs";
-        long_name = record.long_name + " (Integrated Flux surface average unless it is a current then it is the derived flux surface average)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 2, dim_ids1d,
-            &id1d[name]);
-        err = nc_put_att_text( ncid, id1d[name], "long_name", long_name.size(),
-            long_name.data());
-
-        name = record.name + "_ifs_lcfs";
-        long_name = record.long_name + " (Integrated Flux surface average evaluated on last closed flux surface unless it is a current then it is the fsa evaluated)";
-        err = nc_def_var( ncid, name.data(), NC_DOUBLE, 1, dim_ids,
-            &id0d[name]);
-        err = nc_put_att_text( ncid, id0d[name], "long_name", long_name.size(),
-            long_name.data());
-    }
-}
-
-int main( int argc, char* argv[])
-{
-    if( argc != 3)
-    {
-        std::cerr << "Usage: "<<argv[0]<<" [input.nc] [output.nc]\n";
-        return -1;
-    }
-    std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
-
-    //------------------------open input nc file--------------------------------//
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "geomfile", &length);
-    std::string geom( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "geomfile", &geom[0]);
-    err = nc_close(ncid);
-
-    //std::cout << "input "<<input<<std::endl;
-    //std::cout << "geome "<<geom <<std::endl;
-    Json::Value js,gs;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss( input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
-    ss.str( geom);
-    parseFromStream( parser, ss, &gs, &errs); //read input without comments
-    const feltor::Parameters p(js);
-    const dg::geo::solovev::Parameters gp(gs);
-    p.display();
-    gp.display();
-    std::vector<std::string> names_input{
-        "electrons", "ions", "Ue", "Ui", "potential", "induction"
-    };
-
-    //-----------------Create Netcdf output file with attributes----------//
-    int ncid_out;
-    err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
-
-    /// Set global attributes
-    std::map<std::string, std::string> att;
-    att["title"] = "Output file of feltor/diag/feltordiag.cu";
-    att["Conventions"] = "CF-1.7";
-    ///Get local time and begin file history
-    auto ttt = std::time(nullptr);
-    auto tm = *std::localtime(&ttt);
-    std::ostringstream oss;
-    ///time string  + program-name + args
-    oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
-    for( int i=0; i<argc; i++) oss << " "<<argv[i];
-    att["history"] = oss.str();
-    att["comment"] = "Find more info in feltor/src/feltor.tex";
-    att["source"] = "FELTOR";
-    att["references"] = "https://github.com/feltor-dev/feltor";
-    att["inputfile"] = input;
-    att["geomfile"] = geom;
-    for( auto pair : att)
-        err = nc_put_att_text( ncid, NC_GLOBAL,
-            pair.first.data(), pair.second.size(), pair.second.data());
-
-    //-------------------Construct grids-------------------------------------//
-
-    const double Rmin=gp.R_0-p.boxscaleRm*gp.a;
-    const double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
-    const double Rmax=gp.R_0+p.boxscaleRp*gp.a;
-    const double Zmax=p.boxscaleZp*gp.a*gp.elongation;
-
-    dg::CylindricalGrid3d g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
-        p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bcxN, p.bcyN, dg::PER);
-    dg::Grid2d   g2d_out( Rmin,Rmax, Zmin,Zmax,
-        p.n_out, p.Nx_out, p.Ny_out, p.bcxN, p.bcyN);
-
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    const double psip0 = mag.psip()(gp.R_0, 0);
-    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*psip0, p.alpha_mag);
-    dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
-    dg::HVec psipog3d = dg::evaluate( mag.psip(), g3d_out);
-    // Construct weights and temporaries
-
-    const dg::DVec w3d = dg::create::volume( g3d_out);
-
-    dg::HVec transfer3d = dg::evaluate(dg::zero,g3d_out);
-    dg::HVec transfer2d = dg::evaluate(dg::zero,g2d_out);
-
-    dg::DVec t2d = dg::evaluate( dg::zero, g2d_out);
-    dg::DVec t3d = dg::evaluate( dg::zero, g3d_out);
-    dg::DVec result(t3d);
-
-    std::array<dg::DVec, 3> gradPsip;
-    gradPsip[0] =  dg::evaluate( mag.psipR(), g3d_out);
-    gradPsip[1] =  dg::evaluate( mag.psipZ(), g3d_out);
-    gradPsip[2] =  t3d; //zero
-
-    ///--------------- Construct X-point grid ---------------------//
-    //Find O-point
-    double R_O = gp.R_0, Z_O = 0.;
-    dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
-    const double psipmin = mag.psip()(R_O, Z_O);
-
-
-    unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
-    std::cout << "Generate X-point flux-aligned grid!\n";
-    double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
-    double Z_X = -1.1*gp.elongation*gp.a;
-    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), R_X, Z_X) ;
-    dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, R_X, Z_X, mag.R0(), 0, 0, true);
-    double fx_0 = 1./8.;
-    double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
-    std::cout << "psi max is            "<<psipmax<<"\n";
-    psipmax = -fx_0/(1.-fx_0)*psipmin;
-    std::cout << "psi max in g1d_out is "<<psipmax<<"\n";
-    dg::geo::CurvilinearGridX2d gridX2d( generator, fx_0, 0., npsi, Npsi, 160, dg::DIR_NEU, dg::NEU);
-    std::cout << "DONE!\n";
-    //Create 1d grids, one for psi and one for x
-    dg::Grid1d g1d_out(psipmin, psipmax, 3, Npsi, dg::DIR_NEU); //inner value is always 0
-    const double f0 = ( gridX2d.x1() - gridX2d.x0() ) / ( psipmax - psipmin );
-    const dg::DVec w1d = dg::create::weights( g1d_out);
-    dg::HVec t1d = dg::evaluate( dg::zero, g1d_out), fsa1d( t1d);
-    dg::HVec transfer1d = dg::evaluate(dg::zero,g1d_out);
-    dg::HVec transfer2dX = dg::evaluate( dg::zero, gridX2d);
-
-
-
-    /// ------------------- Compute 1d flux labels ---------------------//
-
-    std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
-    /// Compute flux volume label
-    dg::Average<dg::HVec > poloidal_average( gridX2d.grid(), dg::coo2d::y);
-    dg::HVec dvdpsip;
-    //metric and map
-    dg::SparseTensor<dg::HVec> metricX = gridX2d.metric();
-    std::vector<dg::HVec > coordsX = gridX2d.map();
-    dg::HVec volX2d = dg::tensor::volume2d( metricX);
-    dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
-    poloidal_average( volX2d, dvdpsip, false);
-    dg::blas1::scal( dvdpsip, 4.*M_PI*M_PI*f0);
-    map1d.emplace_back( "dvdpsi", dvdpsip,
-        "Derivative of flux volume with respect to flux label psi");
-    dg::HVec X_psi_vol = dg::integrate( dvdpsip, g1d_out);
-    map1d.emplace_back( "psi_vol", X_psi_vol,
-        "Flux volume evaluated with X-point grid");
-
-    /// Compute flux area label
-    dg::HVec gradZetaX = metricX.value(0,0), X_psi_area;
-    dg::blas1::transform( gradZetaX, gradZetaX, dg::SQRT<double>());
-    dg::blas1::pointwiseDot( volX2d, gradZetaX, gradZetaX); //R\sqrt{g}|\nabla\zeta|
-    poloidal_average( gradZetaX, X_psi_area, false);
-    dg::blas1::scal( X_psi_area, 4.*M_PI*M_PI);
-    map1d.emplace_back( "psi_area", X_psi_area,
-        "Flux area evaluated with X-point grid");
-
-    dg::HVec rho = dg::evaluate( dg::cooX1d, g1d_out);
-    dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
-    map1d.emplace_back("rho", rho,
-        "Alternative flux label rho = -psi/psimin + 1");
-    dg::geo::SafetyFactor qprofile( mag);
-    map1d.emplace_back("q-profile", dg::evaluate( qprofile,   g1d_out),
-        "q-profile (Safety factor) using direct integration");
-    map1d.emplace_back("psi_psi",    dg::evaluate( dg::cooX1d, g1d_out),
-        "Flux label psi (same as coordinate)");
-
-
-    // interpolate from 2d grid to X-point points
-    dg::IHMatrix grid2gridX2d  = dg::create::interpolation(
-        coordsX[0], coordsX[1], g2d_out);
-    // interpolate fsa back to 2d or 3d grid
-    dg::IHMatrix fsa2rzmatrix = dg::create::interpolation(
-        psipog2d, g1d_out, dg::DIR_NEU);
-    dg::DVec dvdpsip3d;
-    {
-        dg::IHMatrix fsa2rzpmatrix = dg::create::interpolation(
-            psipog3d, g1d_out, dg::DIR_NEU);
-        dg::blas2::symv(fsa2rzpmatrix, dvdpsip, transfer3d);
-        dg::assign( transfer3d, dvdpsip3d);
-    };//save some storage by deleting matrix immediately
-
-    dg::HMatrix dpsi = dg::create::dx( g1d_out, dg::DIR_NEU);
-    /// Construct Feltor Variables object
-    feltor::Variables var = {
-        feltor::Feltor( g3d_out, p, mag), p, gradPsip, gradPsip, dvdpsip3d
-    };
-
-    // define 2d and 1d and 0d dimensions and variables
-    int dim_ids4d[4], tvar4dID, tvarID;
-    err = file::define_dimensions( ncid_out, dim_ids4d, &tvarID, g3d_out);
-    int dim_ids[3] = {dim_ids4d[0], dim_ids4d[2], dim_ids4d[3]};
-    int dim_ids1d[2] = {dim_ids[0], 0}; //time , psi
-    err = file::define_dimension( ncid_out, "psi", &dim_ids1d[1], g1d_out);
-    err = file::define_time( ncid_out, "time3d", &dim_ids4d[0], &tvar4dID);
-    //Write long description
-    std::string long_name = "Time at which 2d fields are written";
-    err = nc_put_att_text( ncid, tvarID, "long_name", long_name.size(),
-            long_name.data());
-    long_name = "Time at which 3d fields are written";
-    err = nc_put_att_text( ncid, tvar4dID, "long_name", long_name.size(),
-            long_name.data());
-    long_name = "Flux surface label";
-    err = nc_put_att_text( ncid, dim_ids1d[1], "long_name",
-        long_name.size(), long_name.data());
-
-    std::map<std::string, int> id0d, id1d, id2d, id3d;
-    /// Construct 0d names to output
-    std::vector<std::string> names_0d{
-        "correlationNPhi", "correlationNTildePhi"
-    };
-    std::map<std::string, double> v0d;
-    for( auto name : names_0d)
-    {
-        v0d[name] = 0.;
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
-            &id0d[name]);
-    }
-
-    // construct a vector for each name in the input
-    std::map<std::string, dg::DVec> v3d;
-    for( auto name : names_input)
-        v3d[name] = t3d;
-    //now create the variables in the netcdf file
-    create_records_in_file( ncid_out, dim_ids4d, dim_ids, dim_ids1d, id0d, id1d, id2d, id3d);
-
-    size_t count1d[2] = {1, g1d_out.n()*g1d_out.N()};
-    size_t start1d[2] = {0, 0};
-    size_t count2d[3] = {1, g2d_out.n()*g2d_out.Ny(), g2d_out.n()*g2d_out.Nx()};
-    size_t start2d[3] = {0, 0, 0};
-    size_t count3d[4] = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(), g3d_out.n()*g3d_out.Nx()};
-    size_t start3d[4] = {0, 0, 0, 0};
-    size_t start3d_out[4] = {0,0,0,0};
-
-    //write 1d static vectors (psi, q-profile, ...) into file
-    for( auto tp : map1d)
-    {
-        int vid;
-        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 1,
-            &dim_ids1d[1], &vid);
-        err = nc_put_att_text( ncid, vid, "long_name",
-            std::get<2>(tp).size(), std::get<2>(tp).data());
-        err = nc_enddef( ncid);
-        err = nc_put_var_double( ncid, vid, std::get<1>(tp).data());
-        err = nc_redef(ncid);
-    }
-    //and the Cartesian 3d coordinates
-    {
-        std::vector<std::tuple<std::string, const dg::HVec*, std::string> > c3d;
-        dg::HVec xc = dg::evaluate( dg::cooX3d, g3d_out);
-        dg::HVec yc = dg::evaluate( dg::cooY3d, g3d_out);
-        dg::HVec zc = dg::evaluate( dg::cooZ3d, g3d_out);
-        dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
-        c3d.emplace_back( "xc", &xc, "x-coordinate in Cartesian coordinate system");
-        c3d.emplace_back( "yc", &yc, "y-coordinate in Cartesian coordinate system");
-        c3d.emplace_back( "zc", &zc, "z-coordinate in Cartesian coordinate system");
-        for( auto tp : c3d)
-        {
-            int vecID;
-            err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 3,
-                &dim_ids4d[1], &vecID);
-            err = nc_put_att_text( ncid, vecID,
-                "long_name", std::get<2>(tp).size(), std::get<2>(tp).data());
-            err = nc_enddef( ncid);
-            err = nc_put_var_double( ncid, vecID, std::get<1>(tp)->data());
-            err = nc_redef(ncid);
-        }
-    }
-    err = nc_close(ncid_out);
-
-    /////////////////////////////////////////////////////////////////////////
-    int timeID;
-    double time=0.;
-
-    size_t steps;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
-    err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
-    err = nc_inq_dimlen( ncid, timeID, &steps);
-    err = nc_close( ncid); //close 3d file
-
-    dg::Average<dg::DVec> toroidal_average( g3d_out, dg::coo3d::z);
-
-    //steps = 3;
-    for( unsigned i=0; i<steps; i++)//timestepping
-    {
-        start3d[0] = i;
-        start2d[0] = i;
-        start1d[0] = i;
-        err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
-        err = nc_get_vara_double( ncid, timeID, start3d, count3d, &time);
-        std::cout << "Timestep = " << i << "  time = " << time << std::endl;
-        //read in Ne,Ni,Ue,Ui,Phi,Apar
-        for( auto name : names_input)
-        {
-            int dataID;
-            err = nc_inq_varid(ncid, name.data(), &dataID);
-            err = nc_get_vara_double( ncid, dataID, start3d, count3d,
-                transfer3d.data());
-            dg::assign( transfer3d, v3d.at(name));
-        }
-        err = nc_close(ncid);  //close 3d file
-        var.f.set_fields( time, v3d.at("electrons"), v3d.at("ions"), v3d.at("Ue"),
-            v3d.at("Ui"), v3d.at("potential"), v3d.at("induction") );
-
-        //----------------Test if induction equation holds
-        if( p.beta != 0)
-        {
-            dg::blas1::copy(var.f.lapMperpA(), result);
-            dg::blas1::pointwiseDot(
-                var.f.density(0), var.f.velocity(0), t3d);
-            dg::blas1::pointwiseDot( p.beta,
-                var.f.density(1), var.f.velocity(1), -p.beta, t3d);
-            double norm  = dg::blas2::dot( result, w3d, result);
-            dg::blas1::axpby( -1., result, 1., t3d);
-            double error = dg::blas2::dot( t3d, w3d, t3d);
-            std::cout << " Rel. Error Induction "<<sqrt(error/norm) <<"\n";
-        }
-
-        //------------------correlation------------//
-
-        dg::blas1::transform( v3d.at("potential"), t3d, dg::EXP<double>());
-        double norm1 = sqrt(dg::blas2::dot(t3d, w3d, t3d));
-        double norm2 = sqrt(dg::blas2::dot(v3d.at("electrons"), w3d, v3d.at("electrons")));
-        v0d.at("correlationNPhi") = dg::blas2::dot( t3d, w3d, v3d.at("electrons"))
-            /norm1/norm2;  //<e^phi, N>/||e^phi||/||N||
-        v0d.at("correlationNTildePhi") = dg::blas2::dot( t3d, w3d, v3d.at("electrons"))
-            /norm1/norm2;  //<phi, n-<n> >/||phi||/||n-<n>||
-
-
-        //now write out quantities
-        err = nc_open(argv[2], NC_WRITE, &ncid_out);
-        if( i%10 == 0) //write 3d fields only every 10th timestep
-        {
-            start3d_out[0] = i/10;
-            for( auto record& : feltor::records3d_list)
-            {
-                record.function( t3d, var);
-                //toroidal average
-                dg::blas1::transfer( t3d, transfer3d);
-                err = nc_put_vara_double( ncid_out, id3d.at(record.name),
-                    start3d_out, count3d, transfer3d.data());
-            }
-            //write time for 3d fields
-            err = nc_put_vara_double( ncid_out, tvar4dID, start3d_out, count3d, &time);
-        }
-        for( auto record& : feltor::records_list)
-        {
-            record.function( t3d, var);
-            //toroidal average
-            toroidal_average( t3d, t2d, false);
-            dg::blas1::transfer( t2d, transfer2d);
-            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_ta2d"),
-                start2d, count2d, transfer2d.data());
-
-            //flux surface average
-            dg::blas2::symv( grid2gridX2d, transfer2d, transfer2dX); //interpolate onto X-point grid
-            dg::blas1::pointwiseDot( transfer2dX, volX2d, transfer2dX); //multiply by sqrt(g)
-            poloidal_average( transfer2dX, t1d, false); //average over eta
-            dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
-            dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
-            err = nc_put_vara_double( ncid_out, id1d.at(record.name+"_fsa"),
-                start1d, count1d, fsa1d.data());
-
-
-            // 2d data of plane varphi = 0
-            unsigned kmp = g3d_out.Nz()/2;
-            dg::HVec t2d_mp(t3d.begin() + kmp*g2d_out.size(),
-                t3d.begin() + (kmp+1)*g2d_out.size() );
-            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_2d"),
-                start2d, count2d, t2d_mp.data() );
-
-            // fsa on 2d plane : <f>
-            dg::blas2::gemv(fsa2rzmatrix, fsa1d, transfer2d); //fsa on RZ grid
-            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_fsa2d"),
-                start2d, count2d, transfer2d.data() );
-
-            // delta f on midplane : df = f_mp - <f>
-            dg::blas1::axpby( 1.0, t2d_mp, -1.0, transfer2d);
-            err = nc_put_vara_double( ncid_out, id2d.at(record.name+"_fluc2d"),
-                start2d, count2d, transfer2d.data() );
-
-            //flux surface integral/derivative
-            if( record.name[0] == 'j') //j indicates a flux
-            {
-                v0d[record.name+"_ifs_lcfs"] = dg::interpolate( fsa1d, 0., g1d_out); //create a new entry
-                dg::blas2::symv( dpsi, fsa1d, t1d);
-                dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
-            }
-            else
-            {
-                t1d = dg::integrate( fsa1d, g1d_out);
-                dg::blas1::pointwiseDot( t1d, dvdpsip, transfer1d);
-                v0d[record.name+"_ifs_lcfs"] = dg::interpolate( transfer1d, 0., g1d_out); //create a new entry
-            }
-            err = nc_put_vara_double( ncid_out, id1d.at(record.name+"_ifs"),
-                start1d, count1d, transfer1d.data());
-            //flux surface integral/derivative on last closed flux surface
-
-        }
-        //and the 0d quantities
-        for( auto pair : v0d) //{name, double}
-        {
-            err = nc_put_vara_double( ncid_out, id0d.at(pair.first),
-                start2d, count2d, &pair.second );
-        }
-        //write time
-        err = nc_put_vara_double( ncid_out, tvarID, start2d, count2d, &time);
-        //and close file
-        err = nc_close(ncid_out);
-
-    } //end timestepping
-    //relative fluctuation amplitude(R,Z,phi) = delta n(R,Z,phi)/n0(psi)
-
-    return 0;
-}
diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index e03058ce5..384e91c1c 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -13,6 +13,9 @@ all: feltor_hpc manufactured
 manufactured: manufactured.cu manufactured.h feltor.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
 
+feltordiag: feltordiag.cu
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g
+
 feltor: feltor.cu feltor.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
@@ -25,4 +28,4 @@ feltor_mpi: feltor_hpc.cu feltor.h
 .PHONY: clean
 
 clean:
-	rm -f feltor feltor_hpc feltor_mpi manufactured
+	rm -f feltor feltor_hpc feltor_mpi manufactured feltordiag
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 4510e235c..ee7b83927 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -262,10 +262,6 @@ int main( int argc, char* argv[])
     {
         std::string name = record.name + "_ta2d";
         std::string long_name = record.long_name + " (Toroidal average)";
-        if( record.integral){
-            name += "_tt";
-            long_name+= " (Time average)";
-        }
         id3d[name] = 0;//creates a new id3d entry for all processes
         MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids3d,
             &id3d.at(name));
@@ -274,10 +270,6 @@ int main( int argc, char* argv[])
 
         name = record.name + "_2d";
         long_name = record.long_name + " (Evaluated on phi = 0 plane)";
-        if( record.integral){
-            name += "_tt";
-            long_name+= " (Time average)";
-        }
         id3d[name] = 0;
         MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids3d,
             &id3d.at(name));
@@ -321,11 +313,6 @@ int main( int argc, char* argv[])
         dg::assign( transferD, transferH);
         toroidal_average( transferH, transferH2d, false);
         //create and init Simpsons for time integrals
-        if( record.integral)
-        {
-            name += "_tt";
-            time_integrals[name].init( time, transferH2d);
-        }
         tti.toc();
         MPI_OUT std::cout<< name << " Computing average took "<<tti.diff()<<"\n";
         tti.tic();
@@ -338,11 +325,6 @@ int main( int argc, char* argv[])
         name = record.name + "_2d";
         feltor::slice_vector3d( transferD, transferD2d, local_size2d);
         dg::assign( transferD2d, transferH2d);
-        if( record.integral)
-        {
-            name += "_tt";
-            time_integrals[name].init( time, transferH2d);
-        }
         output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
@@ -401,12 +383,12 @@ int main( int argc, char* argv[])
                     //toroidal average and add to time integral
                     dg::assign( transferD, transferH);
                     toroidal_average( transferH, transferH2d, false);
-                    time_integrals.at(record.name+"_ta2d_tt").add( time, transferH2d);
+                    time_integrals.at(record.name+"_ta2d").add( time, transferH2d);
 
                     // 2d data of plane varphi = 0
                     feltor::slice_vector3d( transferD, transferD2d, local_size2d);
                     dg::assign( transferD2d, transferH2d);
-                    time_integrals.at(record.name+"_2d_tt").add( time, transferH2d);
+                    time_integrals.at(record.name+"_2d").add( time, transferH2d);
                 }
 
             }
@@ -455,14 +437,14 @@ int main( int argc, char* argv[])
         {
             if(record.integral) // we already computed the output...
             {
-                std::string name = record.name+"_ta2d_tt";
+                std::string name = record.name+"_ta2d";
                 transferH2d = time_integrals.at(name).get_integral();
                 std::array<double,2> tt = time_integrals.at(name).get_boundaries();
                 dg::blas1::scal( transferD2d, 1./(tt[1]-tt[0]));
                 time_integrals.at(name).flush();
                 output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
 
-                name = record.name+"_2d_tt";
+                name = record.name+"_2d";
                 transferH2d = time_integrals.at(name).get_integral( );
                 tt = time_integrals.at(name).get_boundaries( );
                 dg::blas1::scal( transferD2d, 1./(tt[1]-tt[0]));
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
new file mode 100644
index 000000000..709720796
--- /dev/null
+++ b/src/feltor/feltordiag.cu
@@ -0,0 +1,339 @@
+#include <iostream>
+#include <fstream>
+#include <iomanip>
+#include <vector>
+#include <string>
+#include <functional>
+#include "json/json.h"
+
+#include "dg/algorithm.h"
+#include "dg/geometries/geometries.h"
+#include "dg/file/nc_utilities.h"
+using HVec = dg::HVec;
+using DVec = dg::DVec;
+using DMatrix = dg::DMatrix;
+using IDMatrix = dg::IDMatrix;
+using IHMatrix = dg::IHMatrix;
+using Geometry = dg::CylindricalGrid3d;
+#define MPI_OUT
+#include "feltordiag.h"
+
+int main( int argc, char* argv[])
+{
+    if( argc != 3)
+    {
+        std::cerr << "Usage: "<<argv[0]<<" [input.nc] [output.nc]\n";
+        return -1;
+    }
+    std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
+
+    //------------------------open input nc file--------------------------------//
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input( length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "geomfile", &length);
+    std::string geom( length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "geomfile", &geom[0]);
+    err = nc_close(ncid);
+
+    //std::cout << "input "<<input<<std::endl;
+    //std::cout << "geome "<<geom <<std::endl;
+    Json::Value js,gs;
+    Json::CharReaderBuilder parser;
+    parser["collectComments"] = false;
+    std::string errs;
+    std::stringstream ss( input);
+    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    ss.str( geom);
+    parseFromStream( parser, ss, &gs, &errs); //read input without comments
+    const feltor::Parameters p(js);
+    const dg::geo::solovev::Parameters gp(gs);
+    p.display();
+    gp.display();
+    std::vector<std::string> names_input{
+        "electrons", "ions", "Ue", "Ui", "potential", "induction"
+    };
+
+    //-----------------Create Netcdf output file with attributes----------//
+    int ncid_out;
+    err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
+
+    /// Set global attributes
+    std::map<std::string, std::string> att;
+    att["title"] = "Output file of feltor/diag/feltordiag.cu";
+    att["Conventions"] = "CF-1.7";
+    ///Get local time and begin file history
+    auto ttt = std::time(nullptr);
+    auto tm = *std::localtime(&ttt);
+    std::ostringstream oss;
+    ///time string  + program-name + args
+    oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
+    for( int i=0; i<argc; i++) oss << " "<<argv[i];
+    att["history"] = oss.str();
+    att["comment"] = "Find more info in feltor/src/feltor.tex";
+    att["source"] = "FELTOR";
+    att["references"] = "https://github.com/feltor-dev/feltor";
+    att["inputfile"] = input;
+    att["geomfile"] = geom;
+    for( auto pair : att)
+        err = nc_put_att_text( ncid, NC_GLOBAL,
+            pair.first.data(), pair.second.size(), pair.second.data());
+
+    //-------------------Construct grids-------------------------------------//
+
+    const double Rmin=gp.R_0-p.boxscaleRm*gp.a;
+    const double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
+    const double Rmax=gp.R_0+p.boxscaleRp*gp.a;
+    const double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+
+    dg::Grid2d   g2d_out( Rmin,Rmax, Zmin,Zmax,
+        p.n_out, p.Nx_out, p.Ny_out, p.bcxN, p.bcyN);
+
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    const double psip0 = mag.psip()(gp.R_0, 0);
+    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*psip0, p.alpha_mag);
+    dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
+    // Construct weights and temporaries
+
+    dg::HVec transferH2d = dg::evaluate(dg::zero,g2d_out);
+    dg::HVec t2d_mp = dg::evaluate(dg::zero,g2d_out);
+
+
+    ///--------------- Construct X-point grid ---------------------//
+    //Find O-point
+    double R_O = gp.R_0, Z_O = 0.;
+    dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
+    const double psipmin = mag.psip()(R_O, Z_O);
+
+
+    unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
+    std::cout << "Generate X-point flux-aligned grid!\n";
+    double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
+    double Z_X = -1.1*gp.elongation*gp.a;
+    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), R_X, Z_X) ;
+    dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, R_X, Z_X, mag.R0(), 0, 0, true);
+    double fx_0 = 1./8.;
+    double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
+    std::cout << "psi max is            "<<psipmax<<"\n";
+    psipmax = -fx_0/(1.-fx_0)*psipmin;
+    std::cout << "psi max in g1d_out is "<<psipmax<<"\n";
+    dg::geo::CurvilinearGridX2d gridX2d( generator, fx_0, 0., npsi, Npsi, 160, dg::DIR_NEU, dg::NEU);
+    std::cout << "DONE!\n";
+    //Create 1d grid
+    dg::Grid1d g1d_out(psipmin, psipmax, 3, Npsi, dg::DIR_NEU); //inner value is always 0
+    const double f0 = ( gridX2d.x1() - gridX2d.x0() ) / ( psipmax - psipmin );
+    dg::HVec t1d = dg::evaluate( dg::zero, g1d_out), fsa1d( t1d);
+    dg::HVec transfer1d = dg::evaluate(dg::zero,g1d_out);
+
+    /// ------------------- Compute 1d flux labels ---------------------//
+
+    std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
+    /// Compute flux volume label
+    dg::Average<dg::HVec > poloidal_average( gridX2d.grid(), dg::coo2d::y);
+    dg::HVec dvdpsip;
+    //metric and map
+    dg::SparseTensor<dg::HVec> metricX = gridX2d.metric();
+    std::vector<dg::HVec > coordsX = gridX2d.map();
+    dg::HVec volX2d = dg::tensor::volume2d( metricX);
+    dg::HVec transferH2dX(volX2d);
+    dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
+    poloidal_average( volX2d, dvdpsip, false);
+    dg::blas1::scal( dvdpsip, 4.*M_PI*M_PI*f0);
+    map1d.emplace_back( "dvdpsi", dvdpsip,
+        "Derivative of flux volume with respect to flux label psi");
+    dg::HVec X_psi_vol = dg::integrate( dvdpsip, g1d_out);
+    map1d.emplace_back( "psi_vol", X_psi_vol,
+        "Flux volume evaluated with X-point grid");
+
+    /// Compute flux area label
+    dg::HVec gradZetaX = metricX.value(0,0), X_psi_area;
+    dg::blas1::transform( gradZetaX, gradZetaX, dg::SQRT<double>());
+    dg::blas1::pointwiseDot( volX2d, gradZetaX, gradZetaX); //R\sqrt{g}|\nabla\zeta|
+    poloidal_average( gradZetaX, X_psi_area, false);
+    dg::blas1::scal( X_psi_area, 4.*M_PI*M_PI);
+    map1d.emplace_back( "psi_area", X_psi_area,
+        "Flux area evaluated with X-point grid");
+
+    dg::HVec rho = dg::evaluate( dg::cooX1d, g1d_out);
+    dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
+    map1d.emplace_back("rho", rho,
+        "Alternative flux label rho = -psi/psimin + 1");
+    dg::geo::SafetyFactor qprofile( mag);
+    map1d.emplace_back("q-profile", dg::evaluate( qprofile,   g1d_out),
+        "q-profile (Safety factor) using direct integration");
+    map1d.emplace_back("psi_psi",    dg::evaluate( dg::cooX1d, g1d_out),
+        "Flux label psi (same as coordinate)");
+
+
+    // interpolate from 2d grid to X-point points
+    dg::IHMatrix grid2gridX2d  = dg::create::interpolation(
+        coordsX[0], coordsX[1], g2d_out);
+    // interpolate fsa back to 2d or 3d grid
+    dg::IHMatrix fsa2rzmatrix = dg::create::interpolation(
+        psipog2d, g1d_out, dg::DIR_NEU);
+
+    dg::HVec dvdpsip2d = dg::evaluate( dg::zero, g2d_out);
+    dg::blas2::symv( fsa2rzmatrix, dvdpsip, dvdpsip2d);
+    dg::HMatrix dpsi = dg::create::dx( g1d_out, dg::DIR_NEU);
+
+    // define 2d and 1d and 0d dimensions and variables
+    int dim_ids[3], tvarID;
+    err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
+    int dim_ids1d[2] = {dim_ids[0], 0}; //time , psi
+    err = file::define_dimension( ncid_out, "psi", &dim_ids1d[1], g1d_out);
+    //Write long description
+    std::string long_name = "Time at which 2d fields are written";
+    err = nc_put_att_text( ncid, tvarID, "long_name", long_name.size(),
+            long_name.data());
+    long_name = "Flux surface label";
+    err = nc_put_att_text( ncid, dim_ids1d[1], "long_name",
+        long_name.size(), long_name.data());
+
+    std::map<std::string, int> id0d, id1d, id2d;
+
+    size_t count1d[2] = {1, g1d_out.n()*g1d_out.N()};
+    size_t start1d[2] = {0, 0};
+    size_t count2d[3] = {1, g2d_out.n()*g2d_out.Ny(), g2d_out.n()*g2d_out.Nx()};
+    size_t start2d[3] = {0, 0, 0};
+
+    //write 1d static vectors (psi, q-profile, ...) into file
+    for( auto tp : map1d)
+    {
+        int vid;
+        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 1,
+            &dim_ids1d[1], &vid);
+        err = nc_put_att_text( ncid, vid, "long_name",
+            std::get<2>(tp).size(), std::get<2>(tp).data());
+        err = nc_enddef( ncid);
+        err = nc_put_var_double( ncid, vid, std::get<1>(tp).data());
+        err = nc_redef(ncid);
+    }
+    err = nc_close(ncid_out);
+
+    /////////////////////////////////////////////////////////////////////////
+    int timeID;
+    double time=0.;
+
+    size_t steps;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
+    err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
+    err = nc_inq_dimlen( ncid, timeID, &steps);
+    err = nc_close( ncid); //close 3d file
+
+    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
+    //read in Ne,Ni,Ue,Ui,Phi,Apar
+    for( auto& record : feltor::diagnostics2d_list)
+    {
+        std::string record_name = record.name;
+        if( record_name[0] == 'j')
+            record_name[1] = 'v';
+        std::string name = record_name + "_fluc2d";
+        long_name = record.long_name + " (Fluctuations wrt fsa on phi = 0 plane.)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid_out, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_fsa2d";
+        long_name = record.long_name + " (Flux surface average interpolated to 2d plane.)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid_out, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_fsa";
+        long_name = record.long_name + " (Flux surface average.)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        err = nc_put_att_text( ncid_out, id1d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_ifs";
+        long_name = record.long_name + " (wrt. vol integrated Flux surface average unless it is a current then it is the derived flux surface average)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        err = nc_put_att_text( ncid_out, id1d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_ifs_lcfs";
+        long_name = record.long_name + " (wrt. vol integrated Flux surface average evaluated on last closed flux surface unless it is a current then it is the fsa evaluated)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
+            &id0d[name]);
+        err = nc_put_att_text( ncid_out, id0d[name], "long_name", long_name.size(),
+            long_name.data());
+    }
+    //steps = 3;
+    for( unsigned i=0; i<steps; i++)//timestepping
+    {
+        err = nc_get_vara_double( ncid, timeID, start2d, count2d, &time);
+        std::cout << "Timestep = " << i << "  time = " << time << std::endl;
+        //write time
+        err = nc_put_vara_double( ncid_out, tvarID, start2d, count2d, &time);
+        start2d[0] = i;
+        start1d[0] = i;
+        for( auto& record : feltor::diagnostics2d_list)
+        {
+            std::string record_name = record.name;
+            if( record_name[0] == 'j')
+                record_name[1] = 'v';
+            //1. Read toroidal average
+            err = nc_get_vara_double( ncid, id2d.at(record_name+"_ta2d"),
+                start2d, count2d, transferH2d.data());
+            //2. Compute fsa and output fsa
+            dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
+            dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
+            poloidal_average( transferH2dX, t1d, false); //average over eta
+            dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
+            dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
+            if( record_name[0] == 'j')
+                dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d );
+            err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_fsa"),
+                start1d, count1d, fsa1d.data());
+            //3. Interpolate fsa on 2d plane : <f>
+            dg::blas2::gemv(fsa2rzmatrix, fsa1d, transferH2d); //fsa on RZ grid
+            err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fsa2d"),
+                start2d, count2d, transferH2d.data() );
+
+            //4. Read 2d variable and compute fluctuations
+            err = nc_get_vara_double( ncid, id2d.at(record.name+"_2d"), start2d, count2d,
+                t2d_mp.data());
+            if( record_name[0] == 'j')
+                dg::blas1::pointwiseDot( t2d_mp, dvdpsip2d, t2d_mp );
+            dg::blas1::axpby( 1.0, t2d_mp, -1.0, transferH2d);
+            err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
+                start2d, count2d, transferH2d.data() );
+
+            //5. flux surface integral/derivative
+            double result =0.;
+            if( record_name[0] == 'j') //j indicates a flux
+            {
+                dg::blas2::symv( dpsi, fsa1d, t1d);
+                dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
+
+                result = dg::interpolate( fsa1d, 0., g1d_out);
+            }
+            else
+            {
+                dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d);
+                transfer1d = dg::integrate( fsa1d, g1d_out);
+
+                result = dg::interpolate( transfer1d, 0., g1d_out);
+            }
+            err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
+                start1d, count1d, transfer1d.data());
+            //flux surface integral/derivative on last closed flux surface
+            err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
+                start2d, count2d, &result );
+
+        }
+
+
+    } //end timestepping
+    err = nc_close(ncid_out);
+
+    return 0;
+}
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index e43241e3c..994e36f9b 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -385,8 +385,27 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
+    /// ------------------ Correlation terms --------------------//
+    {"ne2", "Square of electron density", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot(
+                v.f.density(0), v.f.density(0), result);
+        }
+    },
+    {"phi2", "Square of electron potential", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot(
+                v.f.potential(0), v.f.potential(0), result);
+        }
+    },
+    {"nephi", "Product of electron potential and electron density", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot(
+                v.f.potential(0), v.f.density(0), result);
+        }
+    },
     /// ------------------ Density terms ------------------------//
-    {"jsne", "Radial electron particle flux without induction contribution", true,
+    {"jsne_tt", "Radial electron particle flux without induction contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
@@ -399,7 +418,7 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
-    {"jsneA", "Radial electron particle flux: induction contribution", true,
+    {"jsneA_tt", "Radial electron particle flux: induction contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
@@ -411,13 +430,13 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
-    {"lneperp", "Perpendicular electron diffusion", true,
+    {"lneperp_tt", "Perpendicular electron diffusion (Time average)", true,
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), v.tmp[0], result);
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
-    {"lneparallel", "Parallel electron diffusion", true,
+    {"lneparallel_tt", "Parallel electron diffusion (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(0),
                                      0., result);
@@ -476,7 +495,7 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------- Energy dissipation ----------------------//
-    {"resistivity", "Energy dissipation through resistivity", true,
+    {"resistivity_tt", "Energy dissipation through resistivity (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::axpby( 1., v.f.velocity(1), -1., v.f.velocity(0), result);
             dg::blas1::pointwiseDot( result, v.f.density(0), result);
@@ -484,7 +503,7 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------ Energy flux terms ------------------------//
-    {"jsee", "Radial electron energy flux without induction contribution", true,
+    {"jsee_tt", "Radial electron energy flux without induction contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
@@ -497,7 +516,7 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
-    {"jseea", "Radial electron energy flux: induction contribution", true,
+    {"jseea_tt", "Radial electron energy flux: induction contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
@@ -509,7 +528,7 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
-    {"jsei", "Radial ion energy flux without induction contribution", true,
+    {"jsei_tt", "Radial ion energy flux without induction contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
@@ -522,7 +541,7 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
-    {"jseia", "Radial ion energy flux: induction contribution", true,
+    {"jseia_tt", "Radial ion energy flux: induction contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
@@ -535,7 +554,7 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------------ Energy dissipation terms ------------------//
-    {"leeperp", "Perpendicular electron energy dissipation", true,
+    {"leeperp_tt", "Perpendicular electron energy dissipation (Time average)", true,
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(0), result, v.tmp[1]);
@@ -547,7 +566,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
-    {"leiperp", "Perpendicular ion energy dissipation", true,
+    {"leiperp_tt", "Perpendicular ion energy dissipation (Time average)", true,
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(1), result, v.tmp[0]);
             v.f.compute_diffusive_lapMperpU( v.f.velocity(1), result, v.tmp[1]);
@@ -559,7 +578,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
-    {"leeparallel", "Parallel electron energy dissipation", true,
+    {"leeparallel_tt", "Parallel electron energy dissipation (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(0),
                                      0., v.tmp[0]);
@@ -575,7 +594,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::scal( result, v.p.nu_parallel);
         }
     },
-    {"leiparallel", "Parallel ion energy dissipation", true,
+    {"leiparallel_tt", "Parallel ion energy dissipation (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(1),
                                      0., v.tmp[0]);
@@ -619,7 +638,7 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// --------------------- Vorticity flux terms ---------------------------//
-    {"jsoexbi", "ExB vorticity flux term with ion density", true,
+    {"jsoexbi_tt", "ExB vorticity flux term with ion density (Time average)", true,
         []( DVec& result, Variables& v){
             // - ExB Dot GradPsi
             routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
@@ -636,7 +655,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
         }
     },
-    {"jsoexbe", "ExB vorticity flux term with electron density", true,
+    {"jsoexbe_tt", "ExB vorticity flux term with electron density (Time average)", true,
         []( DVec& result, Variables& v){
             // - ExB Dot GradPsi
             routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
@@ -653,7 +672,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
         }
     },
-    {"jsoapar", "A parallel vorticity flux term (Maxwell stress)", true,
+    {"jsoapar_tt", "A parallel vorticity flux term (Maxwell stress) (Time average)", true,
         []( DVec& result, Variables& v){
             if( v.p.beta == 0)
                 dg::blas1::scal( result, 0.);
@@ -667,26 +686,26 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// --------------------- Vorticity source terms ---------------------------//
-    {"socurve", "Vorticity source term electron curvature", true,
+    {"socurve_tt", "Vorticity source term electron curvature (Time average)", true,
         []( DVec& result, Variables& v) {
             routines::dot( v.f.curv(), v.gradPsip, result);
             dg::blas1::pointwiseDot( -v.p.tau[0], v.f.density(0), result, 0., result);
         }
     },
-    {"socurvi", "Vorticity source term ion curvature", true,
+    {"socurvi_tt", "Vorticity source term ion curvature (Time average)", true,
         []( DVec& result, Variables& v) {
             routines::dot( v.f.curv(), v.gradPsip, result);
             dg::blas1::pointwiseDot( v.p.tau[1], v.f.density(1), result, 0., result);
         }
     },
-    {"socurvkappae", "Vorticity source term electron kappa curvature", true,
+    {"socurvkappae_tt", "Vorticity source term electron kappa curvature (Time average)", true,
         []( DVec& result, Variables& v) {
             routines::dot( v.f.curvKappa(), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., v.f.density(0), v.f.velocity(0), v.f.velocity(0), 0., v.tmp[0]);
             dg::blas1::pointwiseDot( -v.p.mu[0], v.tmp[0], result, 0., result);
         }
     },
-    {"socurvkappai", "Vorticity source term ion kappa curvature", true,
+    {"socurvkappai_tt", "Vorticity source term ion kappa curvature (Time average)", true,
         []( DVec& result, Variables& v) {
             routines::dot( v.f.curvKappa(), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., v.f.density(1), v.f.velocity(1), v.f.velocity(1), 0., v.tmp[0]);
-- 
GitLab


From 1a27259f4c81aef79b77daabfc0a9e6069efe8b9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Jul 2019 11:36:56 +0200
Subject: [PATCH 115/540] Fix forgotten init in feltor_hpc

---
 src/feltor/feltor.tex    | 10 +++++-----
 src/feltor/feltor_hpc.cu |  2 ++
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index de14f5059..f910ede2e 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1058,9 +1058,9 @@ compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing poin
 inner\_loop & integer & 2  & 1 & Number of time steps between each energy output \\
 itstp       & integer & 2  & - & Number of energy outputs for each fields outputs \\
 maxout      & integer & 10 & - & Total Number of fields outputs excluding first (The total number of time steps is maxout$\cdot$itstp$\cdot$inner\_loop) \\
-eps\_time   & float & 1e-9  & - & Accuracy of solver for implicit part in time-stepper (if too low, you'll see oscillations in ue) \\
+eps\_time   & float & 1e-7  & - & Accuracy of solver for implicit part in time-stepper (if too low, you'll see oscillations in $u_e$ and/or $\phi$) \\
 rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper \\
-eps\_pol    & float & 1e-5  & - &  Accuracy of residual of the inversion of polarisation and induction Eq. (should not be too far from eps\_time for $\beta\neq  0$ ) \\
+eps\_pol    & float & 1e-6  & - &  Accuracy of residual of the inversion of polarisation and induction Eq. (should not be more than a factor 10 from eps\_time for $\beta\neq  0$ ) \\
 jumpfactor  & float & 1 & 1 & Jumpfactor $\in \left[0.01,1\right]$ in Elliptic\\
 eps\_gamma  & float & 1e-6  & - & Accuracy of $\Gamma_1$  \\
 stages      & integer & 3 & 3 & number of stages in multigrid, $2^{\text{stages-1}}$
@@ -1075,7 +1075,7 @@ nu\_perp   & float &1e-3   & - & perpendicular viscosity $\nu_\perp$ \\
 perp\_diff & string & "viscous" & "viscous" & "viscous": $\Lambda_\perp\propto \nu_\perp\Delta_\perp$ , "hyperviscous": $\Lambda_\perp \propto -\nu_\perp\Delta_\perp^2$\\
 nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$ \\
 resistivity & float &1e-4  & - & parallel resistivity parameter Eq.~\eqref{eq:resistivity}\\
-curvmode  & string & "low beta"  & "toroidal"& curvature mode ("low beta", "true" no approximation, "toroidal": toroidal field approx) \\
+curvmode  & string & "low beta"  & "toroidal"& curvature mode ("low beta", "true": no approximation - requires significantly more resolution in Nz, "toroidal": toroidal field approx - elliptic equation does not need communication in z)  \\
 symmetric & bool & false & false & If true, initialize all quantities symmetric in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used to construct the parallel derivatives and then overwritten to $N_z\equiv 1$. \\
 bc & dict & & & Dictionary of boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
 \qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y for $n_e$ and $N_i$\\
@@ -1089,13 +1089,13 @@ perturbation $\tilde n$ in \eqref{eq:initial_ne}. "zonal" (Eq.~\eqref{eq:initial
     "straight blob" = straight blob simulation( 1 round fieldaligned),
     "turbulence" = turbulence simulations ( 1 round fieldaligned, Eq.~\eqref{eq:initial_turbulent})\\
 initphi   & string & "zero"  & "balance" & initial condition for $\phi$ and thus $N_i$ (Eq.~\eqref{eq:initphi}: "zero" : $\phi = 0$, vanishing
-electric potential, "balance": $n_e=N_i$, ExB vorticity equals ion diamagnetic vorticity (For $\tau_i =0 $ both are the same)
+electric potential, "balance": ExB vorticity equals ion diamagnetic vorticity (For $\tau_i =0 $ both are the same)
 \\
 amplitude  & float &0.01   & - & amplitude $A$ of initial perturbation (blob, turbulent bath or zonal flow)  \\
 sigma      & float &2      & - & blob variance in units of $\rho_s$ \\
 posX       & float &0.3    & - & blob R-position in units of $a$\\
 posY       & float &0.0    & - & blob Z-position in units of $a$ \\
-sigma\_z    & float &0.25   & - & variance in units of $R_0$  \\
+sigma\_z    & float &0.25   & - & variance in units of $R_0$ of the fieldline-following initialization \\
 k\_psi     & float &0    & - & zonal mode wave number  \\
 nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} \\
 alpha\_mag   & float & 0.05 & - & Width $\alpha$ of the Heaviside in the modified $\psi_p$ function \eqref{eq:modified_psip}\\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index ee7b83927..c4b38d117 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -313,6 +313,7 @@ int main( int argc, char* argv[])
         dg::assign( transferD, transferH);
         toroidal_average( transferH, transferH2d, false);
         //create and init Simpsons for time integrals
+        if( record.integral) time_integrals[name].init( time, transferH2d);
         tti.toc();
         MPI_OUT std::cout<< name << " Computing average took "<<tti.diff()<<"\n";
         tti.tic();
@@ -325,6 +326,7 @@ int main( int argc, char* argv[])
         name = record.name + "_2d";
         feltor::slice_vector3d( transferD, transferD2d, local_size2d);
         dg::assign( transferD2d, transferH2d);
+        if( record.integral) time_integrals[name].init( time, transferH2d);
         output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
-- 
GitLab


From 2b9088a45129218b08f55bb8f1bb00ca3b734b1c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Jul 2019 11:55:52 +0200
Subject: [PATCH 116/540] Debug feltordiag

at least it runs through
---
 src/feltor/feltordiag.cu | 47 ++++++++++++++++++++--------------------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 709720796..5d828f738 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -64,7 +64,7 @@ int main( int argc, char* argv[])
 
     /// Set global attributes
     std::map<std::string, std::string> att;
-    att["title"] = "Output file of feltor/diag/feltordiag.cu";
+    att["title"] = "Output file of feltor/src/feltor/feltordiag.cu";
     att["Conventions"] = "CF-1.7";
     ///Get local time and begin file history
     auto ttt = std::time(nullptr);
@@ -80,7 +80,7 @@ int main( int argc, char* argv[])
     att["inputfile"] = input;
     att["geomfile"] = geom;
     for( auto pair : att)
-        err = nc_put_att_text( ncid, NC_GLOBAL,
+        err = nc_put_att_text( ncid_out, NC_GLOBAL,
             pair.first.data(), pair.second.size(), pair.second.data());
 
     //-------------------Construct grids-------------------------------------//
@@ -187,10 +187,10 @@ int main( int argc, char* argv[])
     err = file::define_dimension( ncid_out, "psi", &dim_ids1d[1], g1d_out);
     //Write long description
     std::string long_name = "Time at which 2d fields are written";
-    err = nc_put_att_text( ncid, tvarID, "long_name", long_name.size(),
+    err = nc_put_att_text( ncid_out, tvarID, "long_name", long_name.size(),
             long_name.data());
     long_name = "Flux surface label";
-    err = nc_put_att_text( ncid, dim_ids1d[1], "long_name",
+    err = nc_put_att_text( ncid_out, dim_ids1d[1], "long_name",
         long_name.size(), long_name.data());
 
     std::map<std::string, int> id0d, id1d, id2d;
@@ -204,28 +204,15 @@ int main( int argc, char* argv[])
     for( auto tp : map1d)
     {
         int vid;
-        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 1,
+        err = nc_def_var( ncid_out, std::get<0>(tp).data(), NC_DOUBLE, 1,
             &dim_ids1d[1], &vid);
-        err = nc_put_att_text( ncid, vid, "long_name",
+        err = nc_put_att_text( ncid_out, vid, "long_name",
             std::get<2>(tp).size(), std::get<2>(tp).data());
         err = nc_enddef( ncid);
-        err = nc_put_var_double( ncid, vid, std::get<1>(tp).data());
-        err = nc_redef(ncid);
+        err = nc_put_var_double( ncid_out, vid, std::get<1>(tp).data());
+        err = nc_redef(ncid_out);
     }
-    err = nc_close(ncid_out);
-
-    /////////////////////////////////////////////////////////////////////////
-    int timeID;
-    double time=0.;
 
-    size_t steps;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
-    err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
-    err = nc_inq_dimlen( ncid, timeID, &steps);
-    err = nc_close( ncid); //close 3d file
-
-    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
-    //read in Ne,Ni,Ue,Ui,Phi,Apar
     for( auto& record : feltor::diagnostics2d_list)
     {
         std::string record_name = record.name;
@@ -266,6 +253,14 @@ int main( int argc, char* argv[])
         err = nc_put_att_text( ncid_out, id0d[name], "long_name", long_name.size(),
             long_name.data());
     }
+    /////////////////////////////////////////////////////////////////////////
+    int timeID;
+    double time=0.;
+
+    size_t steps;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
+    err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
+    err = nc_inq_dimlen( ncid, timeID, &steps);
     //steps = 3;
     for( unsigned i=0; i<steps; i++)//timestepping
     {
@@ -281,7 +276,9 @@ int main( int argc, char* argv[])
             if( record_name[0] == 'j')
                 record_name[1] = 'v';
             //1. Read toroidal average
-            err = nc_get_vara_double( ncid, id2d.at(record_name+"_ta2d"),
+            int dataID =0;
+            err = nc_inq_varid(ncid, (record.name+"_ta2d").data(), &dataID);
+            err = nc_get_vara_double( ncid, dataID,
                 start2d, count2d, transferH2d.data());
             //2. Compute fsa and output fsa
             dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
@@ -299,7 +296,10 @@ int main( int argc, char* argv[])
                 start2d, count2d, transferH2d.data() );
 
             //4. Read 2d variable and compute fluctuations
-            err = nc_get_vara_double( ncid, id2d.at(record.name+"_2d"), start2d, count2d,
+            err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
+            err = nc_get_vara_double( ncid, dataID,
+                start2d, count2d, transferH2d.data());
+            err = nc_get_vara_double( ncid, dataID, start2d, count2d,
                 t2d_mp.data());
             if( record_name[0] == 'j')
                 dg::blas1::pointwiseDot( t2d_mp, dvdpsip2d, t2d_mp );
@@ -333,6 +333,7 @@ int main( int argc, char* argv[])
 
 
     } //end timestepping
+    err = nc_close(ncid);
     err = nc_close(ncid_out);
 
     return 0;
-- 
GitLab


From f204aa62d9ab7d9b2be753012b8c13f801e86926 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Jul 2019 16:21:27 +0200
Subject: [PATCH 117/540] Bring feltor documentation up to date

---
 src/feltor/feltor.tex | 69 +++++++++++++++----------------------------
 1 file changed, 23 insertions(+), 46 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index f910ede2e..1faa9bc04 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -421,6 +421,10 @@ In the DG framework the left-hand side
 of Eq.~\eqref{eq:dirac_delta} can thus readily be computed
 via Gauss-Legendre quadrature, which we propse as a first method to compute area
 integrals even if our coordinate system is not aligned to the area.
+Note: in order for this to work the Delta function needs to be numerically
+resolved and cannot be made arbitrarily small.
+This introduces a smoothing effect
+over neighboring contour lines which is given by the grid distance.
 
 Furthermore, recall the {\bf co-area formula}
 \begin{align} \label{eq:coarea}
@@ -711,7 +715,6 @@ Let us define a flux-aligned density profile as
   n_{\text{prof}}(R,Z)=
       n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_p(R_0,0)}\Theta_{\alpha}(\psi_p(R, Z)+\alpha) H(Z-Z_X)
 \end{align}
-%NOTE: this profile is slightly below n_0 outside the LCFS! (not sure whether this does anything for nabla_parallel)
 The second Heaviside is multiplied only if the equilibrium $\psi_p$ has an
 X-point and avoids a profile in the private flux region. The factor $\alpha$ provides a smooth transition
 zone that avoids numerical oscillations.
@@ -721,9 +724,9 @@ We have two possibilities to initialize the ion density
 \begin{align} \label{eq:initphi}
   N_i = \Gamma_{1,i}^{-1} n_e \text{ or } N_i = \Gamma_{1,i}n_e\approx \left(1+\frac{1}{2}\tau_i\mu_i\Delta_\perp\right)n_e
 \end{align}
-In the first case the potential $\phi= 0$ while in the second (prefered)
-case
+In the first case the potential $\phi= 0$ while in the second case
 the $E\times B$ and ion diamagnetic vorticity coincide $\Delta_\perp N_i \propto \Delta_\perp \phi$ in the long-wavelength limit.
+Note that $\alpha$ must not be too small to avoid $N_i < 0$.
 We can choose between several initial conditions for $\tilde n$:
 
 \subsubsection{Blob and Straight blob}
@@ -757,14 +760,14 @@ wavelength $k_\psi$ aligned with the magnetic flux surfaces.
 \end{align}
 
 \subsection{Sinks and sources} \label{sec:sources}
-The idea for the source terms $S_N$ is to fix the profile $n_{prof}$ in the
+The idea for the source terms $S_N$ is to fix the profile $n_{\text{prof}}$ in the
 core of our domain, where our model does not apply.
 We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
   S_{n_e}(R,Z,\varphi, t) &= \omega_s
-    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta( \rho_{s} -\rho(R,Z)) H(Z-Z_X)\\
+    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta_\alpha( \rho_{s} -\rho(R,Z)) H(Z-Z_X)\\
     \rho(R,Z) &:= \frac{\psi_{p,\min}- \psi_p(R,Z) }{\psi_{p,\min}},\\
-    \psi_p(R,Z)&:= (1-\rho(R,Z))\psi_{p,\min},\quad \text{In general we have }\psi_{p,\min} = \psi_p(R_O, Z_O) \neq\psi_{p}(R_0,0)
+    \psi_p(R,Z)&:= (1-\rho(R,Z))\psi_{p,\min},\quad \text{In general }\psi_{p,\min} = \psi_p(R_O, Z_O) \neq\psi_{p}(R_0,0)
 \end{align}
 with $0 < \rho_{s}<1$
 where $\omega_s$ is the source strength parameter and $R_O$, $Z_O$ are the coordinates of the O-point.
@@ -981,11 +984,11 @@ continuity equation
 In order to test the implementation we manufacture a solution to Eqs.~\eqref{eq:Egyrofluid} and \eqref{eq:elliptic} of the form
 \begin{align*}
 n_e(R,Z,\varphi, t) &:= 1 + 0.5\sin(\pi(R-R_0))\sin(\pi Z)\sin(\varphi)\sin(\pi t) \\
-N_i(R,Z,\varphi, t) &:= n_e(R,Z,\varphi,t) = \Gamma N_i  \\
+N_i(R,Z,\varphi, t) &:= n_e(R,Z,\varphi,t) = \gamma_{ N_i}  \\
 u_e(R,Z,\varphi, t) &:= \sin(2\pi(R-R_0))\sin(2\pi Z)\sin(2\varphi)\sin(2\pi t)/(3\sqrt{-\mu_e}) \\
 U_i(R,Z,\varphi, t) &:= \sqrt{-\mu_e}u_e(R,Z,\varphi,t) \\
 \phi(R,Z,\varphi,t) &:= \sin(3\pi(R-R_0))\sin(3\pi Z)\sin(3\varphi)\sin(3\pi t)/5; \\
-\psi(R,Z,\varphi,t) &:= \phi(R,Z,\varphi, t) = \Gamma \phi \\
+\psi(R,Z,\varphi,t) &:= \phi(R,Z,\varphi, t) = \gamma_{\phi} \\
 A_\parallel( R,Z,\varphi,t) &:= \beta\sin(4\pi(R-R_0))\sin(4\pi Z)\sin(4\varphi)\sin(4\pi t)/4;
 \end{align*}
 We choose circular flux surfaces of the form
@@ -1019,10 +1022,11 @@ time & Multistep "Karniadakis" & \\
 \end{longtable}
 
 \section{Usage}
+
 Compilation:\\
 \texttt{make feltor device=\{gpu,omp\}} Compile \texttt{feltor.cu} (only shared memory)\\
-\texttt{make feltor\_hpc device=\{gpu,omp\}} Compile \texttt{feltor\_hpc.cu} for shared memory system.\\
-\texttt{make feltor\_mpi device=\{gpu,omp,skl,knl\}} Compile \texttt{feltor\_hpc.cu} for distributed memory systems.\\
+\texttt{make feltor\_hpc device=\{gpu,omp\}} Compile \texttt{feltor\_hpc.cu} for shared memory system. Needs {\it serial netcdf} \\
+\texttt{make feltor\_mpi device=\{gpu,omp,skl,knl\}} Compile \texttt{feltor\_hpc.cu} for distributed memory systems. Also needs {\it serial netcdf}\\
 Usage:\\
 \texttt{./feltor\_hpc input.json geometry.json output.nc [initial.nc]} \\
 \texttt{./feltor\_mpi input.json geometry.json output.nc [initial.nc]} \\
@@ -1055,8 +1059,8 @@ dt     & integer &1e-2& - &initial time stepsize in units of $c_s/\rho_s$ \\
 compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing points in x and y: output contains n*Nx/c[0] points in x,
     (has to divde Nx evenly), and n*Ny/c[1] points in y,
     (has to divde Ny evenly)\\
-inner\_loop & integer & 2  & 1 & Number of time steps between each energy output \\
-itstp       & integer & 2  & - & Number of energy outputs for each fields outputs \\
+inner\_loop & integer & 2  & 1 & Number of time steps between diagnostics updates \\
+itstp       & integer & 2  & - & Number of updates for one output \\
 maxout      & integer & 10 & - & Total Number of fields outputs excluding first (The total number of time steps is maxout$\cdot$itstp$\cdot$inner\_loop) \\
 eps\_time   & float & 1e-7  & - & Accuracy of solver for implicit part in time-stepper (if too low, you'll see oscillations in $u_e$ and/or $\phi$) \\
 rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper \\
@@ -1142,14 +1146,13 @@ x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, co
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 z                & Coord. Var. & 1 (z) & $\varphi$-coordinate (computational space, size: $N_z$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
-energy\_time     & Coord. Var. & 1 (energy\_time) & timesteps at which 1d variables are written (variable size: itstp$\cdot$maxout$+1$, dimension size: unlimited ) \\
 xc           & Dataset & 3 (z,y,x) & Cartesian x-coordinate $x=R\sin(\varphi)$ \\
 yc           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
 zc           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 Psip             & Dataset & 3 (z,y,x) & Flux function $\psi_p(R,Z)$ \\
 Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ \\
 Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta_\alpha(\rho_{s} - \rho(R,Z)) H(Z-Z_X)$\\
-%Damping          & Dataset & 3 (z,y,x) & Damping profile $\Theta(\rho(R,Z) - \rho_{d} )$\\
+Damping          & Dataset & 3 (z,y,x) & Damping profile for initial profile \\
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
@@ -1159,19 +1162,9 @@ Ue               & Dataset & 4 (time, z, y, x) & electron velocity $u_e$ \\
 Ui               & Dataset & 4 (time, z, y, x) & ion velocity $U_i$ \\
 potential        & Dataset & 4 (time, z, y, x) & electric potential $\phi$ \\
 induction        & Dataset & 4 (time, z, y, x) & parallel vector potential $A_\parallel$ \\
-mass      & Dataset & 1 (energy\_time) & total mass integral Eq.~\eqref{eq:mass_conservation} \\
-diff      & Dataset & 1 (energy\_time) & total mass integral diffusion Eq.~\eqref{eq:mass_conservation} \\
-energy    & Dataset & 1 (energy\_time) & total energy integral Eq.~\eqref{eq:energy_conservation} \\
-ediff     & Dataset & 1 (energy\_time) & total energy integral diffusion Eq.~\eqref{eq:energy_conservation} \\
-Se        & Dataset & 1 (energy\_time) & total electron entropy integral Eq.~\eqref{eq:energy_conservation} \\
-Si        & Dataset & 1 (energy\_time) & total ion entropy integral Eq.~\eqref{eq:energy_conservation} \\
-Upare        & Dataset & 1 (energy\_time) & total electron parallel energy integral Eq.~\eqref{eq:energy_conservation} \\
-Upari        & Dataset & 1 (energy\_time) & total ion parallel energy integral Eq.~\eqref{eq:energy_conservation} \\
-Uperp     & Dataset & 1 (energy\_time) & total perpendicular energy integral Eq.~\eqref{eq:energy_conservation} \\
-Apar     & Dataset & 1 (energy\_time) & total magnetic energy integral Eq.~\eqref{eq:energy_conservation} \\
-dEdt      & Dataset & 1 (energy\_time) & change of energy per time  \\
-accuracy  & Dataset & 1 (energy\_time) & accuracy of energy theorem in time  \\
-aligned   & Dataset & 1 (energy\_time) & Alignment parameter $\int\dV \ln n_e\Delta_\parallel n_e$\\
+X\_2d            & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
+X\_ta2d          & Dataset & 3 (time,y,x) & Toroidal average $\langle X
+    \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
 \bottomrule
 \end{longtable}
 \subsection{Restart file} \label{sec:restart_file}
@@ -1209,31 +1202,15 @@ y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, co
 z                & Coord. Var. & 1 (z) & $\varphi$-coordinate (computational space, compressed size: $N_z$)\\
 psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
-time3d           & Coord. Var. & 1 (time3d)& time at which 3d fields are written (variable size: maxout$/10+1$, dimension size: unlimited) \\
 xc           & Dataset & 3 (z,y,x) & Cartesian x-coordinate $x=R\sin(\varphi)$ \\
 yc           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
 zc           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
-dssne        & Dataset & 4 (time3d,z,y,x) & $\nabla_\parallel^2n_e$ \\
-dssue        & Dataset & 4 (time3d,z,y,x) & $\nabla_\parallel^2u_e$ \\
-dssphi       & Dataset & 4 (time3d,z,y,x) & $\nabla_\parallel^2\phi$ \\
-dppne        & Dataset & 4 (time3d,z,y,x) & $\partial_\varphi^2n_e$ \\
-dppue        & Dataset & 4 (time3d,z,y,x) & $\partial_\varphi^2u_e$ \\
-dppphi       & Dataset & 4 (time3d,z,y,x) & $\partial_\varphi^2\phi$ \\
-correlationNTildePhi  & Dataset & 1 (time) & $ \int\d v \langle \tilde \phi \tilde n_e\rangle / ||\tilde \phi||||\tilde n_e||$
-with $\tilde n_e \equiv (n_e - \langle n_e \rangle)/\langle n_e\rangle$, 
-$\tilde\phi = \phi - \langle\phi\rangle$
-and $||\phi|| = \left( \int \d v \langle \phi^2 \rangle\right)^{1/2}$
-\\
-covarianceNTildePhi  & Dataset & 1 (time) & $ \int\d v \langle \tilde \phi \tilde n_e\rangle $
-\\
-correlationNPhi  & Dataset & 1 (time) & $\int\d v \langle e^\phi n_e\rangle/ ||e^\phi||||n_e||$\\
 q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} \\
-psip1d           & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
+psi\_psi         & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
 rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
 psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\nabla\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
-X\_2d            & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
-X\_ta2d          & Dataset & 3 (time,y,x) & Toroidal average $\langle X
-    \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
+psi\_vol         & Dataset & 1 (psi) & The volume enclosed by the flux surfaces $v(\psi_p) = \int_{\psi_p} \dV $ \\
+dvdpsip          & Dataset & 1 (psi) & $\d v/\d\psi_p$ \\
 X\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,0) - \langle X\rangle_{\psi_{p}}$ \\
 X\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\langle X\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
 X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
-- 
GitLab


From c71701353de3eb4d4bd8305001a2a5ce9ac2b483 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Jul 2019 22:05:23 +0200
Subject: [PATCH 118/540] Further update feltor.tex docu

with description of output quantities
---
 src/feltor/feltor.tex | 159 +++++++++++++++++++++++++-----------------
 1 file changed, 95 insertions(+), 64 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 1faa9bc04..629fc358e 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1134,6 +1134,12 @@ the program exits with an error message.
 
 \subsection{Output} \label{sec:output_file}
 Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Dataset with the same name as a dimension.
+We follow CF Conventions CF-1.7
+\url{http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html}
+and write according attributes into the file.
+The command \texttt{ncdump -h output.nc} gives a full list of what a file contains.
+Here, we list the content without attributes
+since the internal netcdf information does not display equations.
 %
 %Name | Type | Dimensionality | Description
 %---|---|---|---|
@@ -1162,11 +1168,90 @@ Ue               & Dataset & 4 (time, z, y, x) & electron velocity $u_e$ \\
 Ui               & Dataset & 4 (time, z, y, x) & ion velocity $U_i$ \\
 potential        & Dataset & 4 (time, z, y, x) & electric potential $\phi$ \\
 induction        & Dataset & 4 (time, z, y, x) & parallel vector potential $A_\parallel$ \\
-X\_2d            & Dataset & 3 (time,y,x) & Selected plane ($\varphi=0$) \\
+X\_2d            & Dataset & 3 (time,y,x) & Selected plane $X(\varphi=0)$ \\
 X\_ta2d          & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
+Y\_tt\_2d        & Dataset & 3 (time,y,x) & Time averaged selected plane
+$\langle Y(\varphi=0) \rangle_{\Delta t}$
+where $\Delta t = $dt*inner\_loop*itstp \\
+Y\_tt\_ta2d      & Dataset & 3 (time,y,x) & Time averaged toroidal average
+$\langle\langle Y \rangle_\varphi\rangle_{\Delta t}$
+where $\Delta t = $dt*inner\_loop*itstp, Eq.~\eqref{eq:phi_average}\\
 \bottomrule
 \end{longtable}
+where X $\in$
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} & \textbf{Name} &  \textbf{Equation}\\
+\midrule
+    electrons &$n_e$ &
+    ions &$N_i$ \\
+    Ue &$u_e$ &
+    Ui &$U_i$ \\
+    potential &$\phi$ &
+    psi &$\psi$ \\
+    induction &$A_\parallel$ &
+    vorticity &$-\Delta_\perp\phi$ \\
+    dssue & $\nabla_\parallel^2 u_e$&
+    dppue & $\partial_\varphi^2 u_e$\\
+    dpue2 & $(\partial_\varphi u_e)^2$&
+    apar\_vorticity &$-\Delta_\perp A_\parallel$ \\
+    neue &$n_e u_e$ &
+    niui &$N_i U_i$ \\
+    neuebphi &$n_eu_eb_\varphi$ &
+    niuibphi &$N_iU_ib_\varphi$ \\
+    lperpinv &$L_\perp^{-1} := |\vec\nabla_\perp n_e|/n_e$ &
+    perpaligned &$(\vec\nabla_\perp n_e)^2/n_e$ \\
+    lparallelinv &$L_\parallel^{-1} := |\nabla_\parallel n_e|/n_e$ &
+    aligned &$ (\nabla_\parallel n_e)^2/n_e$ \\
+    ne2 & $n_e^2$ &
+    phi2 & $\phi^2$ \\
+    nephi & $n_e\phi$ &
+     & \\
+    nelnne &$ z_e\tau_e n_e \ln n_e$ &
+    nilnni &$ z_i\tau_i N_i \ln N_i$ \\
+    aperp2 &$ (\nabla_\perp A_\parallel)^2/2/\beta$ &
+    ue2   &$z_i\mu_i N_i u_E^2 /2$ \\
+    neue2 &$ z_e\mu_e n_e u_e^2/2$ &
+    niui2 &$ z_i\mu_i N_i U_i^2/2$ \\
+    oexbe &$\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2$ &
+    oexbi &$\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2$ \\
+    odiae &$\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e$ &
+    odiai &$\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i$ \\
+\bottomrule
+\end{longtable}
+and the sources and currents Y $\in$
+\begin{longtable}{ll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}\\
+\midrule
+    jsne &$ n_e (\vec v_E + \vec v_K + \vec v_C )\cdot\nabla \psi_p$ \\
+    jsneA &$ n_e u_e \vec{\tilde b}_\perp  \cdot\nabla \psi_p$ \\
+    lneperp &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ \\
+    lneparallel &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
+    resistivity &-$\eta_\parallel n_e^2 (U_i-u_e)^2$ \\
+    jsee &$z_e(\tau_e \ln n_e + \mu_e u_e^2/2 + \phi)n_e(\vec v_E + \vec v_C + \vec v_K)\cdot\nabla \psi_p
+        + z_e \tau_e n_e u_e^2 \vec K_{\nabla\times\bhat}\cdot\nabla \psi_p$ \\
+    jsei &$z_i(\tau_i \ln N_i + \mu_i U_i^2/2 + \psi_i)N_i(\vec v_E^i + \vec v_C + \vec v_K)\cdot\nabla \psi_p
+        + z_i \tau_i N_i U_i^2 \vec K_{\nabla\times\bhat}\cdot\nabla \psi_p$ \\
+    jseea &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec {\tilde b}_\perp\cdot\nabla \psi_p
+        + z_e \tau_e n_e u_e \vec{\tilde b}_\perp \cdot \nabla \psi_p $ \\
+    jseia &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec {\tilde b}_\perp\cdot\nabla \psi_p
+        + z_i \tau_i N_i U_i \vec{\tilde b}_\perp \cdot \nabla \psi_p $ \\
+    leeperp &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_e \nu_\perp \Delta_\perp u_e$ \\
+    leiperp &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_i \nu_\perp \Delta_\perp U_i$ \\
+    leeparallel &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_e \nu_\parallel \Delta_\parallel u_e$ \\
+    leiparallel &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_i \nu_\parallel \Delta_\parallel U_i$ \\
+    jsoexbi &$-(\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i)  \vec u_E\cdot\nabla \psi_p$ \\
+    jsoexbe &$-(\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e) \vec u_E\cdot\nabla \psi_p$ \\
+    jsoapar &$ \nabla\psi_p\cdot\nabla A_\parallel \bhat\times\nabla A_\parallel/B/\beta$ \\
+    socurve &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
+    socurvi &$z_i\tau_i N_i \mathcal K(\psi_p)$ \\
+    socurvkappae &$z_e\mu_e n_eu_e^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
+    socurvkappai &$z_i\mu_i N_iU_i^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
+\bottomrule
+\end{longtable}
+The time averages are computed with the help of Simpson's rule.
 \subsection{Restart file} \label{sec:restart_file}
 The program \texttt{feltor\_hpc.cu} has the possibility to initialize time and the fields with
 the results of a previous simulation. This behaviour is enabled by giving an additional file \texttt{initial.nc}
@@ -1211,71 +1296,17 @@ rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\
 psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\nabla\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
 psi\_vol         & Dataset & 1 (psi) & The volume enclosed by the flux surfaces $v(\psi_p) = \int_{\psi_p} \dV $ \\
 dvdpsip          & Dataset & 1 (psi) & $\d v/\d\psi_p$ \\
-X\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta X := X(R,Z,0) - \langle X\rangle_{\psi_{p}}$ \\
-X\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\langle X\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
-X\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle X\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
-X\_ifs           & Dataset & 2 (time, psi) & Integrated flux surface average $\int\d v\langle X\rangle_{\psi_p}$ unless X is a current, then it is the derived flux-surface average $\partial_v \langle X\rangle_{\psi_p}$ \\
-X\_ifs\_lcfs     & Dataset & 1 (time) & Integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle X\rangle_{\psi_p}$ unless X is a current, then it is the fsa evaluated \\
-\bottomrule
-\end{longtable}
-where X $\in$
-\begin{longtable}{ll}
-\toprule
-\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}\\
-\midrule
-    electrons &$n_e$ \\
-    ions &$N_i$ \\
-    Ue &$u_e$ \\
-    Ui &$U_i$ \\
-    potential &$\phi$ \\
-    psi &$\psi$ \\
-    induction &$A_\parallel$ \\
-    vorticity &$-\Delta_\perp\phi$ \\
-    apar\_vorticity &$-\Delta_\perp A_\parallel$ \\
-    neue &$n_e u_e$ \\
-    niui &$N_i U_i$ \\
-    neuebphi &$n_eu_eb_\varphi$ \\
-    niuibphi &$N_iU_ib_\varphi$ \\
-    lperpinv &$L_\perp^{-1} := |\vec\nabla_\perp n_e|/n_e$ \\
-    perpaligned &$(\vec\nabla_\perp n_e)^2/n_e$ \\
-    lparallelinv &$L_\parallel^{-1} := |\nabla_\parallel n_e|/n_e$ \\
-    aligned &$ (\nabla_\parallel n_e)^2/n_e$ \\
-    jvne &$ n_e (\vec v_E + \vec v_K + \vec v_C )\cdot\nabla v$ \\
-    jvneA &$ n_e u_e \vec{\tilde b}_\perp  \cdot\nabla v$ \\
-    lneperp &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ \\
-    lneparallel &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
-    nelnne &$ z_e\tau_e n_e \ln n_e$ \\
-    nilnni &$ z_i\tau_i N_i \ln N_i$ \\
-    aperp2 &$ (\nabla_\perp A_\parallel)^2/2/\beta$ \\
-    ue2   &$z_i\mu_i N_i u_E^2 /2$ \\
-    neue2 &$ z_e\mu_e n_e u_e^2/2$ \\
-    niui2 &$ z_i\mu_i N_i U_i^2/2$ \\
-    resistivity &$\eta_\parallel n_e^2 (U_i-u_e)^2$ \\
-    jvee &$z_e(\tau_e \ln n_e + \mu_e u_e^2/2 + \phi)n_e(\vec v_E + \vec v_C + \vec v_K)\cdot\nabla v
-        + z_e \tau_e n_e u_e^2 \vec K_{\nabla\times\bhat}\cdot\nabla v$ \\
-    jvei &$z_i(\tau_i \ln N_i + \mu_i U_i^2/2 + \psi_i)N_i(\vec v_E^i + \vec v_C + \vec v_K)\cdot\nabla v
-        + z_i \tau_i N_i U_i^2 \vec K_{\nabla\times\bhat}\cdot\nabla v$ \\
-    jveea &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec {\tilde b}_\perp\cdot\nabla v
-        + z_e \tau_e n_e u_e \vec{\tilde b}_\perp \cdot \nabla v $ \\
-    jveia &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec {\tilde b}_\perp\cdot\nabla v
-        + z_i \tau_i N_i U_i \vec{\tilde b}_\perp \cdot \nabla v $ \\
-    leeperp &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_e \nu_\perp \Delta_\perp u_e$ \\
-    leiperp &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_i \nu_\perp \Delta_\perp U_i$ \\
-    leeparallel &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_e \nu_\parallel \Delta_\parallel u_e$ \\
-    leiparallel &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_i \nu_\parallel \Delta_\parallel U_i$ \\
-    oexbi &$\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2$ \\
-    oexbe &$\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2$ \\
-    odiai &$\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i$ \\
-    odiae &$\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e$ \\
-    jvoexbi &$-(\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i)  \vec u_E\cdot\nabla v$ \\
-    jvoexbe &$-(\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e) \vec u_E\cdot\nabla v$ \\
-    jvoapar &$ \nabla\psi_p\cdot\nabla A_\parallel \bhat\times\nabla A_\parallel/B/\beta$ \\
-    socurve &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
-    socurvi &$z_i\tau_i N_i \mathcal K(\psi_p)$ \\
-    socurvkappae &$z_e\mu_e n_eu_e^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
-    socurvkappai &$z_i\mu_i N_iU_i^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
+Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \langle Z\rangle_{\psi_{p}}$ \\
+Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\langle Z\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
+Z\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle Z\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
+Z\_ifs           & Dataset & 2 (time, psi) & Integrated flux surface average $\int\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the derived flux-surface average $\partial_v \langle Z\rangle_{\psi_p}$ \\
+Z\_ifs\_lcfs     & Dataset & 1 (time) & Integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the fsa evaluated \\
 \bottomrule
 \end{longtable}
+where Z $\in$ \{X, Y\_tt\}
+Note that feltoridag converts all $jsX$ quantities into $jvX$
+by multiplying $\d v/\d \psi_p$
+in the sense that $\vec j\cdot \nabla v  = \vec j \cdot \nabla \psi_p \d v/\d\psi_p$.
 
 
 
-- 
GitLab


From fe860491ae4e67c8597ad29ff403877fd29fbf31 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 3 Jul 2019 16:49:13 +0200
Subject: [PATCH 119/540] Fix accuracy bug in feltor_hpc

- also update feltordiag with input parameters
- Change time average to time integral
---
 src/feltor/feltor.tex    | 16 ++++++++--------
 src/feltor/feltor_hpc.cu |  4 ----
 src/feltor/feltordiag.cu | 12 +++++++-----
 src/feltor/feltordiag.h  |  2 +-
 4 files changed, 16 insertions(+), 18 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 629fc358e..4c46c48b1 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1171,15 +1171,16 @@ induction        & Dataset & 4 (time, z, y, x) & parallel vector potential $A_\p
 X\_2d            & Dataset & 3 (time,y,x) & Selected plane $X(\varphi=0)$ \\
 X\_ta2d          & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
-Y\_tt\_2d        & Dataset & 3 (time,y,x) & Time averaged selected plane
-$\langle Y(\varphi=0) \rangle_{\Delta t}$
-where $\Delta t = $dt*inner\_loop*itstp \\
-Y\_tt\_ta2d      & Dataset & 3 (time,y,x) & Time averaged toroidal average
-$\langle\langle Y \rangle_\varphi\rangle_{\Delta t}$
-where $\Delta t = $dt*inner\_loop*itstp, Eq.~\eqref{eq:phi_average}\\
+Y\_tt\_2d        & Dataset & 3 (time,y,x) & Time integrated (between two outputs) selected plane
+$\int_{t_0}^{t_1}\d t Y(\varphi=0) $
+where $t_1 - t_0 = $dt*inner\_loop*itstp\\
+Y\_tt\_ta2d      & Dataset & 3 (time,y,x) & Time integrated (between two outputs) toroidal average (Eq.~\eqref{eq:phi_average})
+$\int_{t_0}^{t_1}\d t \langle Y \rangle_\varphi$
+where $t_1 - t_0 = $dt*inner\_loop*itstp\\
 \bottomrule
 \end{longtable}
-where X $\in$
+where the time integrals are computed with the help of Simpson's rule
+and X $\in$
 \begin{longtable}{llll}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} & \textbf{Name} &  \textbf{Equation}\\
@@ -1251,7 +1252,6 @@ and the sources and currents Y $\in$
     socurvkappai &$z_i\mu_i N_iU_i^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
 \bottomrule
 \end{longtable}
-The time averages are computed with the help of Simpson's rule.
 \subsection{Restart file} \label{sec:restart_file}
 The program \texttt{feltor\_hpc.cu} has the possibility to initialize time and the fields with
 the results of a previous simulation. This behaviour is enabled by giving an additional file \texttt{initial.nc}
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index c4b38d117..cc9c60e77 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -441,15 +441,11 @@ int main( int argc, char* argv[])
             {
                 std::string name = record.name+"_ta2d";
                 transferH2d = time_integrals.at(name).get_integral();
-                std::array<double,2> tt = time_integrals.at(name).get_boundaries();
-                dg::blas1::scal( transferD2d, 1./(tt[1]-tt[0]));
                 time_integrals.at(name).flush();
                 output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
 
                 name = record.name+"_2d";
                 transferH2d = time_integrals.at(name).get_integral( );
-                tt = time_integrals.at(name).get_boundaries( );
-                dg::blas1::scal( transferD2d, 1./(tt[1]-tt[0]));
                 time_integrals.at(name).flush( );
                 output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
             }
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 5d828f738..b94777528 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -110,7 +110,9 @@ int main( int argc, char* argv[])
     const double psipmin = mag.psip()(R_O, Z_O);
 
 
-    unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
+    std::cout << "Type X-point grid resolution (n(3), Npsi(64), Neta(160)) Must be divisible by 8\n";
+    unsigned npsi = 3, Npsi = 32, Neta = 320;//set number of psivalues (NPsi % 8 == 0)
+    std::cin >> npsi >> Npsi >> Neta;
     std::cout << "Generate X-point flux-aligned grid!\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
@@ -121,7 +123,7 @@ int main( int argc, char* argv[])
     std::cout << "psi max is            "<<psipmax<<"\n";
     psipmax = -fx_0/(1.-fx_0)*psipmin;
     std::cout << "psi max in g1d_out is "<<psipmax<<"\n";
-    dg::geo::CurvilinearGridX2d gridX2d( generator, fx_0, 0., npsi, Npsi, 160, dg::DIR_NEU, dg::NEU);
+    dg::geo::CurvilinearGridX2d gridX2d( generator, fx_0, 0., npsi, Npsi, Neta, dg::DIR_NEU, dg::NEU);
     std::cout << "DONE!\n";
     //Create 1d grid
     dg::Grid1d g1d_out(psipmin, psipmax, 3, Npsi, dg::DIR_NEU); //inner value is always 0
@@ -264,12 +266,12 @@ int main( int argc, char* argv[])
     //steps = 3;
     for( unsigned i=0; i<steps; i++)//timestepping
     {
+        start2d[0] = i;
+        start1d[0] = i;
+        // read and write time
         err = nc_get_vara_double( ncid, timeID, start2d, count2d, &time);
         std::cout << "Timestep = " << i << "  time = " << time << std::endl;
-        //write time
         err = nc_put_vara_double( ncid_out, tvarID, start2d, count2d, &time);
-        start2d[0] = i;
-        start1d[0] = i;
         for( auto& record : feltor::diagnostics2d_list)
         {
             std::string record_name = record.name;
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 994e36f9b..ed16263e1 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -718,5 +718,5 @@ std::vector<Record> diagnostics2d_list = {
 
 // These two lists signify the quantities for accuracy computation
 std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
-std::vector<std::string> energy_diff = { "resistivity", "leeperp", "leiperp", "leeparallel", "leiparallel"};
+std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt"};
 }//namespace feltor
-- 
GitLab


From da702e6493f4b26b83788c62a7dd62f24c05812e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 3 Jul 2019 19:04:10 +0200
Subject: [PATCH 120/540] Added norm of quantities to feltordiag

in order to check importance of vorticity terms
---
 src/feltor/feltor.tex    |  7 +++---
 src/feltor/feltordiag.cu | 46 +++++++++++++++++++++++++++++++++++-----
 2 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 4c46c48b1..43e985851 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1245,7 +1245,7 @@ and the sources and currents Y $\in$
     leiparallel &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_i \nu_\parallel \Delta_\parallel U_i$ \\
     jsoexbi &$-(\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i)  \vec u_E\cdot\nabla \psi_p$ \\
     jsoexbe &$-(\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e) \vec u_E\cdot\nabla \psi_p$ \\
-    jsoapar &$ \nabla\psi_p\cdot\nabla A_\parallel \bhat\times\nabla A_\parallel/B/\beta$ \\
+    jsoapar &$ \nabla\psi_p\cdot\nabla A_\parallel \bhat\times\nabla A_\parallel\cdot\nabla \psi_p/B/\beta$ \\
     socurve &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
     socurvi &$z_i\tau_i N_i \mathcal K(\psi_p)$ \\
     socurvkappae &$z_e\mu_e n_eu_e^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
@@ -1299,8 +1299,9 @@ dvdpsip          & Dataset & 1 (psi) & $\d v/\d\psi_p$ \\
 Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \langle Z\rangle_{\psi_{p}}$ \\
 Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\langle Z\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
 Z\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle Z\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
-Z\_ifs           & Dataset & 2 (time, psi) & Integrated flux surface average $\int\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the derived flux-surface average $\partial_v \langle Z\rangle_{\psi_p}$ \\
-Z\_ifs\_lcfs     & Dataset & 1 (time) & Integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the fsa evaluated \\
+Z\_ifs           & Dataset & 2 (time, psi) & Volume integrated flux surface average $\int\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the volume derived flux-surface average $\partial_v \langle Z\rangle_{\psi_p}$ \\
+Z\_ifs\_lcfs     & Dataset & 1 (time) & Volume integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the fsa evaluated $\langle j_v\rangle_{\psi_p}(0)$ \\
+Z\_ifs\_norm     & Dataset & 1 (time) & Volume integrated square flux surface average, unless Z is a current, then it is the square derivative of the flux surface average \\
 \bottomrule
 \end{longtable}
 where Z $\in$ \{X, Y\_tt\}
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index b94777528..0748321e6 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -110,9 +110,10 @@ int main( int argc, char* argv[])
     const double psipmin = mag.psip()(R_O, Z_O);
 
 
-    std::cout << "Type X-point grid resolution (n(3), Npsi(64), Neta(160)) Must be divisible by 8\n";
+    std::cout << "Type X-point grid resolution (n(3), Npsi(32), Neta(640)) Must be divisible by 8\n";
     unsigned npsi = 3, Npsi = 32, Neta = 320;//set number of psivalues (NPsi % 8 == 0)
     std::cin >> npsi >> Npsi >> Neta;
+    std::cout << "You typed "<<npsi<<" x "<<Npsi<<" x "<<Neta<<"\n";
     std::cout << "Generate X-point flux-aligned grid!\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
@@ -242,14 +243,27 @@ int main( int argc, char* argv[])
             long_name.data());
 
         name = record_name + "_ifs";
-        long_name = record.long_name + " (wrt. vol integrated Flux surface average unless it is a current then it is the derived flux surface average)";
+        long_name = record.long_name + " (wrt. vol integrated flux surface average)";
+        if( record_name[0] == 'j')
+            long_name = record.long_name + " (wrt. vol derivative of the flux surface average)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
             &id1d[name]);
         err = nc_put_att_text( ncid_out, id1d[name], "long_name", long_name.size(),
             long_name.data());
 
         name = record_name + "_ifs_lcfs";
-        long_name = record.long_name + " (wrt. vol integrated Flux surface average evaluated on last closed flux surface unless it is a current then it is the fsa evaluated)";
+        long_name = record.long_name + " (wrt. vol integrated flux surface average evaluated on last closed flux surface)";
+        if( record_name[0] == 'j')
+            long_name = record.long_name + " (flux surface average evaluated on the last closed flux surface)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
+            &id0d[name]);
+        err = nc_put_att_text( ncid_out, id0d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_ifs_norm";
+        long_name = record.long_name + " (wrt. vol integrated square flux surface average from 0 to lcfs)";
+        if( record_name[0] == 'j')
+            long_name = record.long_name + " (wrt. vol integrated square derivative of the flux surface average from 0 to lcfs)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
             &id0d[name]);
         err = nc_put_att_text( ncid_out, id0d[name], "long_name", long_name.size(),
@@ -320,8 +334,8 @@ int main( int argc, char* argv[])
             }
             else
             {
-                dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d);
-                transfer1d = dg::integrate( fsa1d, g1d_out);
+                dg::blas1::pointwiseDot( fsa1d, dvdpsip, t1d);
+                transfer1d = dg::integrate( t1d, g1d_out);
 
                 result = dg::interpolate( transfer1d, 0., g1d_out);
             }
@@ -330,6 +344,28 @@ int main( int argc, char* argv[])
             //flux surface integral/derivative on last closed flux surface
             err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
                 start2d, count2d, &result );
+            //6. Compute norm of time-integral terms to get relative importance
+            if( record_name[0] == 'j') //j indicates a flux
+            {
+                dg::blas2::symv( dpsi, fsa1d, t1d);
+                dg::blas1::pointwiseDivide( t1d, dvdpsip, t1d); //dvjv
+                dg::blas1::pointwiseDot( t1d, t1d, t1d);//dvjv2
+                dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);//dvjv2
+                transfer1d = dg::integrate( t1d, g1d_out);
+                result = dg::interpolate( transfer1d, 0., g1d_out);
+                result = sqrt(result);
+            }
+            else
+            {
+                dg::blas1::pointwiseDot( fsa1d, fsa1d, t1d);
+                dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);
+                transfer1d = dg::integrate( t1d, g1d_out);
+
+                result = dg::interpolate( transfer1d, 0., g1d_out);
+                result = sqrt(result);
+            }
+            err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
+                start2d, count2d, &result );
 
         }
 
-- 
GitLab


From d318bbd2d7f3b9903369ec59d88f3a40081d8be8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Jul 2019 12:22:09 +0200
Subject: [PATCH 121/540] Fix bug in feltordiag and geometry_diag

- also first implementation of Torpex source term
---
 inc/geometries/geometry_diag.cu |  9 +++++++--
 src/feltor/feltordiag.cu        |  3 +--
 src/feltor/init.h               | 14 ++++++++++++++
 3 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 08a3d7e81..700362de7 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -145,8 +145,13 @@ int main( int argc, char* argv[])
     //Test coefficients
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(), 0.), p.alpha_mag);
-    const double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
-    const double Z_X = -1.1*gp.elongation*gp.a;
+    double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
+    double Z_X = -1.1*gp.elongation*gp.a;
+    if( gp.hasXpoint())
+    {
+        dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
+        std::cout <<  "X-point found at "<<R_X << " "<<Z_X<<"\n";
+    }
     const double R_H = gp.R_0-gp.triangularity*gp.a;
     const double Z_H = gp.elongation*gp.a;
     const double alpha_ = asin(gp.triangularity);
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 0748321e6..54595cd69 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -117,6 +117,7 @@ int main( int argc, char* argv[])
     std::cout << "Generate X-point flux-aligned grid!\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
+    dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
     dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), R_X, Z_X) ;
     dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, R_X, Z_X, mag.R0(), 0, 0, true);
     double fx_0 = 1./8.;
@@ -313,8 +314,6 @@ int main( int argc, char* argv[])
 
             //4. Read 2d variable and compute fluctuations
             err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
-            err = nc_get_vara_double( ncid, dataID,
-                start2d, count2d, transferH2d.data());
             err = nc_get_vara_double( ncid, dataID, start2d, count2d,
                 t2d_mp.data());
             if( record_name[0] == 'j')
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 345df0512..c55adbc38 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -3,6 +3,20 @@
 
 namespace feltor
 {
+
+struct TorpexSource
+{
+    TorpexSource( double R0, double Z0, double a, double b, double c):m_R0(R0), m_Z0(Z0), m_a(a), m_b(b), m_c(c){}
+    DG_DEVICE
+    double operator()( double R, double Z) const{
+        if( R < m_R0)
+            return exp( - (R-m_R0)*(R-m_R0)/m_a/m_a - (Z-m_Z0)*(Z-m_Z0)/m_b/m_b);
+        return 0.5*exp( - (R-m_R0)*(R-m_R0)/m_a/m_a +2.*m_c*(R-m_R0)*(Z-m_Z0)- (Z-m_Z0)*(Z-m_Z0)/m_b/m_b );
+    }
+    private:
+    double m_R0, m_Z0, m_a,m_b,m_c;
+};
+
 //We use the typedefs and MPI_OUT
 struct Initialize
 {
-- 
GitLab


From c05961859391fd97bfd07495a1f3df41ef63aa4b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 21 Aug 2019 15:24:39 +0200
Subject: [PATCH 122/540] Change and document Solovev coefficients

- add B coefficient on inhomogeneous solution
- remove qampl
- correct torpex source
---
 inc/geometries/average.h                     |  2 +-
 inc/geometries/geometry_diag.cu              | 34 +++++---
 inc/geometries/solovev.h                     | 88 +++++++-------------
 inc/geometries/solovev_parameters.h          | 37 ++++----
 src/feltor/feltor.tex                        | 11 ++-
 src/feltor/geometry/geometry_paramsXpoint.js |  7 --
 src/feltor/init.h                            |  5 +-
 7 files changed, 82 insertions(+), 102 deletions(-)

diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index ffa03dbe0..c39753f42 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -257,7 +257,6 @@ struct SafetyFactorAverage
      * @brief Construct from a field and a grid
      * @param g2d 2d grid
      * @param c contains psip, psipR and psipZ and Ipol
-     * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point)
      * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
      */
     SafetyFactorAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, double width_factor = 1.) :
@@ -268,6 +267,7 @@ struct SafetyFactorAverage
         dg::blas1::pointwiseDivide( alpha, R, alpha);
         m_fsi.set_left( alpha);
     }
+    /// Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point)
     void set_weights( const thrust::host_vector<double>& weights){
         m_fsi.set_right( weights);
     }
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 700362de7..353077c06 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -23,7 +23,7 @@
 
 struct Parameters
 {
-    unsigned n, Nx, Ny, Nz;
+    unsigned n, Nx, Ny, Nz, Npsi;
     double boxscaleRm, boxscaleRp;
     double boxscaleZm, boxscaleZp;
     double amp, k_psi, bgprofamp, nprofileamp;
@@ -34,6 +34,7 @@ struct Parameters
         Nx = js.get("Nx",100).asUInt();
         Ny = js.get("Ny",100).asUInt();
         Nz = js.get("Nz", 1).asUInt();
+        Npsi = js.get("Npsi", 16).asUInt();
         boxscaleRm = js.get("boxscaleRm", 1.1).asDouble();
         boxscaleRp = js.get("boxscaleRp", 1.1).asDouble();
         boxscaleZm = js.get("boxscaleZm", 1.2).asDouble();
@@ -55,6 +56,7 @@ struct Parameters
             <<" Nx            = "<<Nx<<"\n"
             <<" Ny            = "<<Ny<<"\n"
             <<" Nz            = "<<Nz<<"\n"
+            <<" Npsi          = "<<Npsi<<"\n"
             <<" boxscaleRm    = "<<boxscaleRm<<"\n"
             <<" boxscaleRp    = "<<boxscaleRp<<"\n"
             <<" boxscaleZm    = "<<boxscaleZm<<"\n"
@@ -144,7 +146,7 @@ int main( int argc, char* argv[])
 
     //Test coefficients
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(), 0.), p.alpha_mag);
+    //mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(), 0.), p.alpha_mag);
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
     if( gp.hasXpoint())
@@ -204,8 +206,8 @@ int main( int argc, char* argv[])
         {"LnB", "Natural logarithm of Bmodule", dg::geo::LnB(mag)},
         {"GradLnB", "The parallel derivative of LnB", dg::geo::GradLnB(mag)},
         {"Divb", "The divergence of the magnetic unit vector", dg::geo::Divb(mag)},
-        {"BR", "Derivative of Bmodule in R", dg::geo::BR(mag)},
-        {"BZ", "Derivative of Bmodule in Z", dg::geo::BZ(mag)},
+        {"B_R", "Derivative of Bmodule in R", dg::geo::BR(mag)},
+        {"B_Z", "Derivative of Bmodule in Z", dg::geo::BZ(mag)},
         {"CurvatureNablaBR",  "R-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBR(mag)},
         {"CurvatureNablaBZ",  "Z-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBZ(mag)},
         {"CurvatureKappaR",   "R-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaR(mag)},
@@ -247,7 +249,7 @@ int main( int argc, char* argv[])
     dg::DVec psipog2d   = dg::evaluate( mag.psip(), grid2d);
     std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
     ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
-    unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
+    unsigned npsi = 3, Npsi = p.Npsi;//set number of psivalues (NPsi % 8 == 0)
     double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     double volumeXGrid;
     /// -------  Elements for fsa of curvature operators ----------------
@@ -336,16 +338,26 @@ int main( int argc, char* argv[])
     }
 
     ///////////////////Compute flux average////////////////////
+    std::cout << "Compute flux averages\n";
     dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
     if( gp.hasXpoint() )
         dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
-    dg::geo::SafetyFactor     qprof( mag);
     dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, mag, psipog2d, xpoint_weights);
     dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
     map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
         "Flux surface average of psi with delta function");
-    map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
-        "q-profile (Safety factor) using direct integration");
+    if( gp.equilibrium == "solovev")
+    {
+        dg::geo::SafetyFactor     qprof( mag);
+        map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
+            "q-profile (Safety factor) using direct integration");
+    }
+    else
+    {
+        dg::geo::SafetyFactorAverage     qprof( grid2d, mag);
+        map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
+            "q-profile (Safety factor) using average integration");
+    }
     map1d.emplace_back("psip1d",    dg::evaluate( dg::cooX1d, grid1d),
         "Flux label psi evaluated on grid1d");
     dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
@@ -462,9 +474,9 @@ int main( int argc, char* argv[])
         "Z-component of the magnetic field vector (3d version of BFieldZ)",
         "Contravariant Phi-component of the magnetic field vector (3d version of BFieldP)"};
     int vecID[3];
-    err = nc_def_var( ncid, "B_R", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[0]);
-    err = nc_def_var( ncid, "B_Z", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[1]);
-    err = nc_def_var( ncid, "B_P", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[2]);
+    err = nc_def_var( ncid, "BR", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[0]);
+    err = nc_def_var( ncid, "BZ", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[1]);
+    err = nc_def_var( ncid, "BP", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[2]);
     for(int i=0; i<3; i++)
     {
         err = nc_put_att_text( ncid, vecID[i], "long_name", vec_long[i].size(), vec_long[i].data());
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 05ec43c66..dcc77a991 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -34,10 +34,10 @@ namespace solovev
  * @brief \f[ \hat{\psi}_p  \f]
  *
  * \f[ \hat{\psi}_p(R,Z) =
-      \hat{R}_0\Bigg\{\bar{R}^4/8 + A \left[ 1/2 \bar{R}^2  \ln{(\bar{R}   )}-(\bar{R}^4 )/8\right]
+      \hat{R}_0\Bigg\{B\bar{R}^4/8 + A \left[ 1/2 \bar{R}^2  \ln{(\bar{R}   )}-(\bar{R}^4 )/8\right]
       + \sum_{i=1}^{12} c_i\bar \psi_{pi}(\bar R, \bar Z) \Bigg\}
       =
-      \hat{R}_0\Bigg\{A \left[ 1/2 \bar{R}^2  \ln{(\bar{R}   )}-(\bar{R}^4 )/8\right]
+      \hat{R}_0\Bigg\{B\bar{R}^4/8 + A \left[ 1/2 \bar{R}^2  \ln{(\bar{R}   )}-(\bar{R}^4 )/8\right]
       +c_1+
       c_2 \bar{R}^2 +
       c_3 \left[  \bar{Z}^2-\bar{R}^2  \ln{(\bar{R}   )} \right] +
@@ -64,14 +64,14 @@ struct Psip: public aCylindricalFunctor<Psip>
      *
      * @param gp geometric parameters
      */
-    Psip( Parameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
+    Psip( Parameters gp ): R_0_(gp.R_0), A_(gp.A), B_(gp.B), c_(gp.c) {}
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,Zn5,Zn6,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn4 = Rn2*Rn2;
         Zn = Z/R_0_; Zn2 = Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2; Zn5 = Zn3*Zn2; Zn6 = Zn3*Zn3;
         lgRn= log(Rn);
-        return   R_0_*( c_[12]*Rn4/8.+ A_ * ( 1./2.* Rn2* lgRn-(Rn4)/8.)  //c_[12] is to make fieldlines straight
+        return   R_0_*( B_*Rn4/8.+ A_ * ( 1./2.* Rn2* lgRn-(Rn4)/8.)
                       + c_[0]  //c_[0] entspricht c_1
               + c_[1]  *Rn2
               + c_[2]  *(Zn2 - Rn2 * lgRn )
@@ -88,32 +88,7 @@ struct Psip: public aCylindricalFunctor<Psip>
                       );
     }
   private:
-    double psi_horner(double R, double Z) const
-    {
-        //qampl is missing!!
-        const double Rn = R/R_0_, Zn = Z/R_0_;
-        const double lgRn = log(Rn);
-        double a0,a1, b0,b1, d0, d1;
-        a0 = 8.*c_[11]+8*c_[6]*Zn;
-        a1 = 2.*c_[4] +Zn*a0;
-        a0 = c_[9] + Zn*a1;
-        a1 = c_[2] + Zn*a0;
-        a0 = c_[7] + Zn*a1;
-        a1 = c_[0] + Zn*a0; //////
-        b0 = -12.*c_[5] + 75.*c_[6] + 180.*c_[6]*lgRn;
-        b1 = 3.*c_[10] - 45.*c_[11]+60.*c_[11]*lgRn + Zn*b0;
-        b0 = 1./8.-A_/8. + c_[3] + 3.*c_[4]*lgRn + Rn*Rn*(c_[5] - 15.*c_[6]*lgRn)+Zn*b1;
-        b1 = c_[1] + 0.5*A_*lgRn - c_[2]*lgRn + Rn*Rn*b0;
-        b0 = Rn*Rn*b1;//////
-        d0 = 8.*c_[5] - 140.*c_[6]-120.*c_[6]*lgRn;
-        d1 = -4.*c_[10] - 80.*c_[11]*lgRn + Zn*d0;
-        d0 = -4.*c_[3]-9.*c_[4]-12.*c_[4]*lgRn + Zn*d1;
-        d1 = c_[8] - 3.*c_[9]*lgRn + Zn*d0;
-        d0 = Rn*Rn*Zn*d1; /////
-
-        return  R_0_*(a1 + b0 + d0);
-    }
-    double R_0_, A_;
+    double R_0_, A_, B_;
     std::vector<double> c_;
 };
 
@@ -123,7 +98,7 @@ struct Psip: public aCylindricalFunctor<Psip>
  * \f[ \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}} =
       \Bigg\{ 2 c_2 \bar{R} +(\bar{R}^3 )/2+2 c_9 \bar{R}  \bar{Z}
       +c_4 (4 \bar{R}^3 -8 \bar{R}  \bar{Z}^2)+c_{11}
-      (12 \bar{R}^3  \bar{Z}-8 \bar{R}  \bar{Z}^3
+      (12 B \bar{R}^3  \bar{Z}-8 \bar{R}  \bar{Z}^3
       +c_6 (6 \bar{R}^5 -48 \bar{R}^3  \bar{Z}^2+16 \bar{R}  \bar{Z}^4)+c_3 (-\bar{R} -2 \bar{R}  \ln{(\bar{R}   )})+
       A ((\bar{R} )/2-(\bar{R}^3 )/2+\bar{R}  \ln{(\bar{R}   )})
       +c_{10} (-3 \bar{R}  \bar{Z}-6 \bar{R}  \bar{Z} \ln{(\bar{R}   )})+c_5 (3 \bar{R}^3 -30 \bar{R}
@@ -138,14 +113,14 @@ struct Psip: public aCylindricalFunctor<Psip>
 struct PsipR: public aCylindricalFunctor<PsipR>
 {
     ///@copydoc Psip::Psip()
-    PsipR( Parameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
+    PsipR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), B_(gp.B), c_(gp.c) {}
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn3,Rn5,Zn,Zn2,Zn3,Zn4,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn3 = Rn2*Rn;  Rn5 = Rn3*Rn2;
         Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
         lgRn= log(Rn);
-        return   (Rn3/2.*c_[12] + (Rn/2. - Rn3/2. + Rn*lgRn)* A_ +
+        return   (Rn3/2.*B_ + (Rn/2. - Rn3/2. + Rn*lgRn)* A_ +
         2.* Rn* c_[1] + (-Rn - 2.* Rn*lgRn)* c_[2] + (4.*Rn3 - 8.* Rn *Zn2)* c_[3] +
         (3. *Rn3 - 30.* Rn *Zn2 + 12. *Rn3*lgRn -  24.* Rn *Zn2*lgRn)* c_[4]
         + (6 *Rn5 - 48 *Rn3 *Zn2 + 16.* Rn *Zn4)*c_[5]
@@ -156,14 +131,14 @@ struct PsipR: public aCylindricalFunctor<PsipR>
           );
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, B_;
     std::vector<double> c_;
 };
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}\f]
  *
  * \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}=
-     \hat{R}_0^{-1} \Bigg\{ 2 c_2 +(3 \hat{\bar{R}}^2 )/2+2 c_9  \bar{Z}+c_4 (12 \bar{R}^2 -8  \bar{Z}^2)+c_{11}
+     \hat{R}_0^{-1} \Bigg\{ 2 c_2 +B(3 \hat{\bar{R}}^2 )/2+2 c_9  \bar{Z}+c_4 (12 \bar{R}^2 -8  \bar{Z}^2)+c_{11}
       (36 \bar{R}^2  \bar{Z}-8  \bar{Z}^3)
       +c_6 (30 \bar{R}^4 -144 \bar{R}^2  \bar{Z}^2+16  \bar{Z}^4)+c_3 (-3 -2  \ln{(\bar{R}
       )})+
@@ -179,14 +154,14 @@ struct PsipR: public aCylindricalFunctor<PsipR>
 struct PsipRR: public aCylindricalFunctor<PsipRR>
 {
     ///@copydoc Psip::Psip()
-    PsipRR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) {}
+    PsipRR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), B_(gp.B), c_(gp.c) {}
     double do_compute(double R, double Z) const
     {
        double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,lgRn;
        Rn = R/R_0_; Rn2 = Rn*Rn;  Rn4 = Rn2*Rn2;
        Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
        lgRn= log(Rn);
-       return   1./R_0_*( (3.* Rn2)/2.*c_[12] + (3./2. - (3. *Rn2)/2. +lgRn) *A_ +  2.* c_[1] + (-3. - 2.*lgRn)* c_[2] + (12. *Rn2 - 8. *Zn2) *c_[3] +
+       return   1./R_0_*( (3.* Rn2)/2.*B_ + (3./2. - (3. *Rn2)/2. +lgRn) *A_ +  2.* c_[1] + (-3. - 2.*lgRn)* c_[2] + (12. *Rn2 - 8. *Zn2) *c_[3] +
          (21. *Rn2 - 54. *Zn2 + 36. *Rn2*lgRn - 24. *Zn2*lgRn)* c_[4]
          + (30. *Rn4 - 144. *Rn2 *Zn2 + 16.*Zn4)*c_[5] + (-165. *Rn4 + 2160. *Rn2 *Zn2 - 640. *Zn4 - 450. *Rn4*lgRn +
       2160. *Rn2 *Zn2*lgRn - 240. *Zn4*lgRn)* c_[6] +
@@ -195,7 +170,7 @@ struct PsipRR: public aCylindricalFunctor<PsipRR>
  +   (-120. *Rn2* Zn - 240. *Zn3 + 720. *Rn2* Zn*lgRn - 160. *Zn3*lgRn)* c_[11]);
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, B_;
     std::vector<double> c_;
 };
 /**
@@ -309,14 +284,14 @@ struct PsipRZ: public aCylindricalFunctor<PsipRZ>
 struct Ipol: public aCylindricalFunctor<Ipol>
 {
     ///@copydoc Psip::Psip()
-    Ipol(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp) { }
+    Ipol(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), psip_(gp) { }
     double do_compute(double R, double Z) const
     {
         //sign before A changed to -
-        return qampl_*sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.);
+        return sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.);
     }
   private:
-    double R_0_, A_,qampl_;
+    double R_0_, A_;
     Psip psip_;
 };
 /**
@@ -325,13 +300,13 @@ struct Ipol: public aCylindricalFunctor<Ipol>
 struct IpolR: public aCylindricalFunctor<IpolR>
 {
     ///@copydoc Psip::Psip()
-    IpolR(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipR_(gp) { }
+    IpolR(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), psip_(gp), psipR_(gp) { }
     double do_compute(double R, double Z) const
     {
-        return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipR_(R,Z)/R_0_);
+        return -1./sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipR_(R,Z)/R_0_);
     }
   private:
-    double R_0_, A_,qampl_;
+    double R_0_, A_;
     Psip psip_;
     PsipR psipR_;
 };
@@ -341,13 +316,13 @@ struct IpolR: public aCylindricalFunctor<IpolR>
 struct IpolZ: public aCylindricalFunctor<IpolZ>
 {
     ///@copydoc Psip::Psip()
-    IpolZ(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp), psipZ_(gp) { }
+    IpolZ(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), psip_(gp), psipZ_(gp) { }
     double do_compute(double R, double Z) const
     {
-        return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipZ_(R,Z)/R_0_);
+        return -1./sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipZ_(R,Z)/R_0_);
     }
   private:
-    double R_0_, A_,qampl_;
+    double R_0_, A_;
     Psip psip_;
     PsipZ psipZ_;
 };
@@ -481,37 +456,36 @@ struct PsipRZ: public aCylindricalFunctor<PsipRZ>
 
 struct Ipol: public aCylindricalFunctor<Ipol>
 {
-    Ipol(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp, psi0, alpha) { }
+    Ipol(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), psip_(gp, psi0, alpha) { }
     double do_compute(double R, double Z) const
     {
-        //sign before A changed to -
-        return qampl_*sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.);
+        return sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.);
     }
   private:
-    double R_0_, A_,qampl_;
+    double R_0_, A_;
     mod::Psip psip_;
 };
 struct IpolR: public aCylindricalFunctor<IpolR>
 {
-    IpolR(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp, psi0, alpha), psipR_(gp, psi0, alpha) { }
+    IpolR(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), psip_(gp, psi0, alpha), psipR_(gp, psi0, alpha) { }
     double do_compute(double R, double Z) const
     {
-        return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipR_(R,Z)/R_0_);
+        return -1./sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipR_(R,Z)/R_0_);
     }
   private:
-    double R_0_, A_,qampl_;
+    double R_0_, A_;
     mod::Psip psip_;
     mod::PsipR psipR_;
 };
 struct IpolZ: public aCylindricalFunctor<IpolZ>
 {
-    IpolZ(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), qampl_(gp.qampl), psip_(gp, psi0, alpha), psipZ_(gp, psi0, alpha) { }
+    IpolZ(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), psip_(gp, psi0, alpha), psipZ_(gp, psi0, alpha) { }
     double do_compute(double R, double Z) const
     {
-        return -qampl_/sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipZ_(R,Z)/R_0_);
+        return -1./sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipZ_(R,Z)/R_0_);
     }
   private:
-    double R_0_, A_,qampl_;
+    double R_0_, A_;
     mod::Psip psip_;
     mod::PsipZ psipZ_;
 };
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index 96d4a9b3a..98ff9c751 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -18,7 +18,7 @@ namespace solovev
  */
 struct Parameters
 {
-    double A, //!< A
+    double A,B, //!< A and B coefficients, B is not there originally but is added for more flexibility
            R_0, //!< central tokamak radius
            a,  //!<  little tokamak radius
            elongation, //!< elongation of the magnetic surfaces
@@ -28,25 +28,23 @@ struct Parameters
            psipmin, //!< for source
            psipmax, //!< for profile
            psipmaxcut, //!< for cutting
-           psipmaxlim,  //!< for limiter
-           qampl; //!< scales grad-shafranov q factor
-    std::vector<double> c;  //!< coefficients for the solovev equilibrium
+           psipmaxlim;  //!< for limiter
+    std::vector<double> c;  //!< 12 coefficients for the solovev equilibrium;
     std::string equilibrium;
 #ifdef JSONCPP_VERSION_STRING
     /**
      * @brief Construct from Json dataset
-     * @param js Must contain the variables "A", "c", "R_0", "inverseaspectratio", "elongation", "triangularity", "alpha", "rk4eps" (1e-5), "psip_min" (0), "psip_max" (0), "psip_max_cut" (0), "psip_max_lim" (1e10), "qampl" (1), "equilibrium" ("solovev")
+     * @param js Must contain the variables "A", "c", "R_0", "inverseaspectratio", "elongation", "triangularity", "alpha", "rk4eps" (1e-5), "psip_min" (0), "psip_max" (0), "psip_max_cut" (0), "psip_max_lim" (1e10), "equilibrium" ("solovev")
      * @note the default values in brackets are taken if the variables are not found in the input
      * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
      */
     Parameters( const Json::Value& js) {
         A  = js.get("A", 0).asDouble();
-        c.resize(13);//there are only 12 originially c[12] is to make fieldlines straight
-        for (unsigned i=0;i<12;i++) c[i] = js["c"][i].asDouble();
-        c[12] = 0;
-        if( A!=0) c[12] = 1;
-        for( unsigned i=0; i<12; i++)
-            if(c[i]!=0) c[12] = 1.;
+        B  = js.get("B", 1).asDouble();
+        c.resize(12);
+        for (unsigned i=0;i<12;i++)
+            c[i] = js["c"][i].asDouble();
+
         R_0  = js["R_0"].asDouble();
         a  = R_0*js["inverseaspectratio"].asDouble();
         elongation=js["elongation"].asDouble();
@@ -57,7 +55,6 @@ struct Parameters
         psipmax= js.get("psip_max",0).asDouble();
         psipmaxcut= js.get("psip_max_cut",0).asDouble();
         psipmaxlim= js.get("psip_max_lim",1e10).asDouble();
-        qampl = js.get("qampl", 1.).asDouble();
         equilibrium = js.get( "equilibrium", "solovev").asString();
     }
     /**
@@ -70,6 +67,7 @@ struct Parameters
     {
         Json::Value js;
         js["A"] = A;
+        js["B"] = B;
         for (unsigned i=0;i<12;i++) js["c"][i] = c[i];
         js["R_0"] = R_0;
         js["inverseaspectratio"] = a/R_0;
@@ -81,15 +79,14 @@ struct Parameters
         js["psip_max"] = psipmax;
         js["psip_max_cut"] = psipmaxcut;
         js["psip_max_lim"] = psipmaxlim;
-        js["qampl"] = qampl;
         js[ "equilibrium"] = equilibrium;
         return js;
     }
 #endif // JSONCPP_VERSION_STRING
     /**
-    * @brief True if Psip has an Xpoint
+    * @brief True if all coefficients \c c_i==0 with \c 7<=i<12
     *
-    * The Xpoint is situated at
+    * The Xpoint is situated close to
      <tt> R_X = R_0-1.1*triangularity*a</tt>
      <tt> Z_X = -1.1*elongation*a</tt>
     *
@@ -98,7 +95,7 @@ struct Parameters
     bool hasXpoint( ) const{
         bool Xpoint = false;
         for( int i=7; i<12; i++)
-            if( c[i] != 0)
+            if( fabs(c[i]) >= 1e-10)
                 Xpoint = true;
         return Xpoint;
     }
@@ -106,8 +103,9 @@ struct Parameters
     void display( std::ostream& os = std::cout ) const
     {
         os << "Geometrical parameters are: \n"
-            <<" A             = "<<A<<"\n";
-        for( unsigned i=0; i<13; i++)
+            <<" A               = "<<A<<"\n"
+            <<" B [optional]    = "<<B<<"\n";
+        for( unsigned i=0; i<12; i++)
             os<<" c"<<i+1<<"\t\t = "<<c[i]<<"\n";
 
         os  <<" R0            = "<<R_0<<"\n"
@@ -119,8 +117,7 @@ struct Parameters
             <<" psipmin       = "<<psipmin<<"\n"
             <<" psipmax       = "<<psipmax<<"\n"
             <<" psipmaxcut    = "<<psipmaxcut<<"\n"
-            <<" psipmaxlim    = "<<psipmaxlim<<"\n"
-            <<" qampl    = "<<qampl<<"\n";
+            <<" psipmaxlim    = "<<psipmaxlim<<"\n";
         os << std::flush;
 
     }
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 43e985851..9770327e7 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -182,8 +182,9 @@ By integration over \(\psi_p\) we find
 Now, we introduce \(\bar{R} \equiv \frac{R}{R_0}\) and \(\bar{Z} \equiv\frac{Z}{R_0}\)
 and solve Equations~\eqref{eq:GSEdimless} and~\eqref{eq:solovevassumption} to obtain
 \begin{align}\label{eq:solovev}
- \psi_p (R,Z) &= R_0 \left[ \frac{1}{8}\bar{R}^4 + A\left( \frac{1}{2} \bar{R}^2 \ln{\bar{R}} 
-   - \frac{1}{8}\bar{R}^4\right) + \sum_{i=1}^{12} c_{i}  \bar{\psi}_{pi}\right],
+ \psi_p (R,Z) &= R_0 \left[ A\left( \frac{1}{2} \bar{R}^2 \ln{\bar{R}}
+   - \frac{1}{8}\bar{R}^4\right)+ B\frac{1}{8}\bar{R}^4 
+   + \sum_{i=1}^{12} c_{i}  \bar{\psi}_{pi}\right],
 \end{align}
 with
 \rowcolors{2}{gray!25}{white}
@@ -218,6 +219,7 @@ $\bar{\psi}_{p11}=3 \bar{Z}\bar{R}^4 - 4\bar{Z}^3\bar{R}^2$\\
 \bottomrule
 \end{longtable}
 The choice of the coefficients \(c_{i}\) and \(A\) determines the actual form of the magnetic field, while $R_0$ appears as an artificial scaling factor (note here that a change in $\rho_s$ changes $R_0$ but not the form or size of the dimensional equilibrium magnetic field).
+Note that $B$ is not part of the original work ($B=1$ per default) and has been added for additional flexibility in fitting the coefficients to an existing experimental equilibrium.
 Eq.~\eqref{eq:solovev} allows axisymmetric equilibria with e.g. single and asymmetric double X-point configurations, force-free states,
 field reversed configurations and low and high beta tokamak equilbria. This casts this simple analytical equilibrium to the ideal choice in order to study geometric effects (e.g. inverse aspect ratio, elongation and triangularity) in magnetised plasmas.
 
@@ -1120,8 +1122,9 @@ File format: json
 \begin{longtable}{llll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
-    A      & float & 0 &  - & Solovev parameter in Eq.~\eqref{eq:solovev} \\
-    C      & float[12] &  - & - & Solovev coefficients in Eq.~\eqref{eq:solovev}  \\
+    A      & float & 0 &  0 & Solovev parameter in Eq.~\eqref{eq:solovev} \\
+    B      & float & 1 &  1 & Solovev parameter in Eq.~\eqref{eq:solovev} \\
+    c      & float[12] &  - & - & Solovev coefficients in Eq.~\eqref{eq:solovev} \\
     R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ in Eq.~\eqref{eq:solovev} (This is the only geometry quantity to change if $\rho_s$ changes)\\
     elongation    & float & 1 & - & Elongation $e$ \\
     triangularity & float & 0 & - & Triangularity $\delta$ \\
diff --git a/src/feltor/geometry/geometry_paramsXpoint.js b/src/feltor/geometry/geometry_paramsXpoint.js
index 90ee9b27a..651059ae0 100644
--- a/src/feltor/geometry/geometry_paramsXpoint.js
+++ b/src/feltor/geometry/geometry_paramsXpoint.js
@@ -5,7 +5,6 @@
 {
 	"A" : 0,
 	"R_0" : 91.884233908188605,
-	"alpha" : 0.02,
 	"c" :
 	[
 		0.58136855188608583,
@@ -24,11 +23,5 @@
 	"elongation" : 1.7,
 	"equilibrium" : "solovev",
 	"inverseaspectratio" : 0.16666666666666669,
-	"psip_max" : 0,
-	"psip_max_cut" : 10000000000,
-	"psip_max_lim" : 10000000000,
-	"psip_min" : -1.1,
-	"qampl" : 1,
-	"rk4eps" : 0.01,
 	"triangularity" : 0.33000000000000002
 }
diff --git a/src/feltor/init.h b/src/feltor/init.h
index c55adbc38..a163e0c23 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -9,9 +9,10 @@ struct TorpexSource
     TorpexSource( double R0, double Z0, double a, double b, double c):m_R0(R0), m_Z0(Z0), m_a(a), m_b(b), m_c(c){}
     DG_DEVICE
     double operator()( double R, double Z) const{
-        if( R < m_R0)
+        if( R > m_R0)
             return exp( - (R-m_R0)*(R-m_R0)/m_a/m_a - (Z-m_Z0)*(Z-m_Z0)/m_b/m_b);
-        return 0.5*exp( - (R-m_R0)*(R-m_R0)/m_a/m_a +2.*m_c*(R-m_R0)*(Z-m_Z0)- (Z-m_Z0)*(Z-m_Z0)/m_b/m_b );
+        return 0.5*exp( - (R-m_R0)*(R-m_R0)/m_a/m_a -2.*m_c*(R-m_R0)*(Z-m_Z0)- (Z-m_Z0)*(Z-m_Z0)/m_b/m_b );
+              +0.5*exp( - (R-m_R0)*(R-m_R0)/m_a/m_a +2.*m_c*(R-m_R0)*(Z-m_Z0)- (Z-m_Z0)*(Z-m_Z0)/m_b/m_b );
     }
     private:
     double m_R0, m_Z0, m_a,m_b,m_c;
-- 
GitLab


From 8611d2a15b9df5a683b03f69a32003cb01f8fc2a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 21 Aug 2019 17:04:18 +0200
Subject: [PATCH 123/540] Extend the source capabilities of feltor

- with torpex and profile influx
- and one additional input parameter source_type
---
 src/feltor/feltor.h     |  7 +++++--
 src/feltor/feltor.tex   | 28 ++++++++++++++++++++++++----
 src/feltor/init.h       |  4 ++--
 src/feltor/parameters.h |  2 ++
 4 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 03aeb6458..481425312 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -835,8 +835,11 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     //Add source terms
     if( m_omega_source != 0)
     {
-        dg::blas1::subroutine( routines::ComputeSource(), m_s[0][0], y[0][0],
-            m_profne, m_source, m_omega_source);
+        if( m_p.source_type == "profile")
+            dg::blas1::subroutine( routines::ComputeSource(), m_s[0][0], y[0][0],
+                m_profne, m_source, m_omega_source);
+        else
+            dg::blas1::axpby( m_omega_source, m_source, 0., m_s[0][0]);
         //compute FLR correction
         dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
         dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 9770327e7..a9abf340d 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -768,13 +768,30 @@ We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
   S_{n_e}(R,Z,\varphi, t) &= \omega_s
     (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta_\alpha( \rho_{s} -\rho(R,Z)) H(Z-Z_X)\\
-    \rho(R,Z) &:= \frac{\psi_{p,\min}- \psi_p(R,Z) }{\psi_{p,\min}},\\
-    \psi_p(R,Z)&:= (1-\rho(R,Z))\psi_{p,\min},\quad \text{In general }\psi_{p,\min} = \psi_p(R_O, Z_O) \neq\psi_{p}(R_0,0)
+    \rho(R,Z) &:= \frac{\psi_{p,\min}- \psi_p(R,Z) }{\psi_{p,\min}}, \quad
+    \psi_p(R,Z):= (1-\rho(R,Z))\psi_{p,\min},\\ \text{In general }\psi_{p,\min} &= \psi_p(R_O, Z_O) \neq\psi_{p}(R_0,0)
 \end{align}
 with $0 < \rho_{s}<1$
 where $\omega_s$ is the source strength parameter and $R_O$, $Z_O$ are the coordinates of the O-point.
-This will result in exponential adaption of the core and wall
+This will result in exponential adaption of the core
 density profile of the form $n_e \propto n_{prof}+(n_{prof}-n_{e,0})e^{-\omega_st}$.
+
+As an alternative we can also choose a constant influx
+\begin{align} \label{eq:electron_source_influx}
+  S_{n_e}(R,Z,\varphi, t) &= \omega_s\Theta_\alpha( \rho_{s} -\rho(R,Z)) H(Z-Z_X)
+\end{align}
+or a Torpex inspired source profile
+\begin{align} \label{eq:electron_source_torpex}
+  S_{n_e}(R,Z,\varphi, t) &= \omega_s
+  \begin{cases}
+    \exp\left( - \frac{(R-R_0)^2}{a^2 }- \frac{(Z-Z_0)^2}{b^2}\right) \text{ if} R > R_0 \\
+    \frac{1}{2}\exp\left( - \frac{(R-R_0)^2}{a^2} -2c(R-R_0)(Z-Z_0)- \frac{(Z-Z_0)^2}{b^2} \right) \\
+  +\frac{1}{2}\exp\left( - \frac{(R-R_0)^2}{a^2} +2c(R-R_0)(Z-Z_0)- \frac{(Z-Z_0)^2}{b^2} \right) \text{ else}
+              \end{cases}
+\end{align}
+with $a=0.0335$m, $b=0.05$m, $c=565m^{-2}$, $R_0=0.98$m and $Z_0=-0.02$m.
+
+
 For ions we use
 \begin{align}
     S_{N_i} = \Gamma_{1,i}^{-1} S_{n_e} = \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S_{n_e}
@@ -1107,7 +1124,10 @@ nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq
 alpha\_mag   & float & 0.05 & - & Width $\alpha$ of the Heaviside in the modified $\psi_p$ function \eqref{eq:modified_psip}\\
 alpha       & float & 0.2 & - & Width $\alpha$ of the Heaviside Eq.~\eqref{eq:approx_heaviside} in the density and source profiles (should be small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes) \\
 source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
-rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source}  \\
+source\_type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
+"influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx}, "torpex": Torpex inspired
+source profile Eq.~\eqref{eq:electron_source_torpex}\\
+rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
 %damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
 rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0$ in Eq.~\eqref{eq:modified_psip}  \\
 \bottomrule
diff --git a/src/feltor/init.h b/src/feltor/init.h
index a163e0c23..c8884f3b7 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -11,11 +11,11 @@ struct TorpexSource
     double operator()( double R, double Z) const{
         if( R > m_R0)
             return exp( - (R-m_R0)*(R-m_R0)/m_a/m_a - (Z-m_Z0)*(Z-m_Z0)/m_b/m_b);
-        return 0.5*exp( - (R-m_R0)*(R-m_R0)/m_a/m_a -2.*m_c*(R-m_R0)*(Z-m_Z0)- (Z-m_Z0)*(Z-m_Z0)/m_b/m_b );
+        return 0.5*exp( - (R-m_R0)*(R-m_R0)/m_a/m_a -2.*m_c*(R-m_R0)*(Z-m_Z0)- (Z-m_Z0)*(Z-m_Z0)/m_b/m_b )
               +0.5*exp( - (R-m_R0)*(R-m_R0)/m_a/m_a +2.*m_c*(R-m_R0)*(Z-m_Z0)- (Z-m_Z0)*(Z-m_Z0)/m_b/m_b );
     }
     private:
-    double m_R0, m_Z0, m_a,m_b,m_c;
+    double m_R0, m_Z0, m_a, m_b, m_c;
 };
 
 //We use the typedefs and MPI_OUT
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 64c1bcc92..8a44d070a 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -46,6 +46,7 @@ struct Parameters
 
     enum dg::bc bcxN, bcyN, bcxU, bcyU, bcxP, bcyP;
     std::string initne, initphi, curvmode, perp_diff;
+    std::string source_type;
     bool symmetric;
     Parameters() = default;
     Parameters( const Json::Value& js) {
@@ -91,6 +92,7 @@ struct Parameters
 
         nprofamp  = js["nprofileamp"].asDouble();
         omega_source  = js.get("source", 0.).asDouble();
+        source_type  = js.get("source_type", "profile").asString();
         omega_damping = js.get("damping", 0.).asDouble();
         alpha_mag    = js.get("alpha_mag", 0.05).asDouble();
         alpha        = js.get("alpha", 0.2).asDouble();
-- 
GitLab


From 218490d5821baad0577d27d154b9e11d96c5f677 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 26 Aug 2019 16:57:35 +0200
Subject: [PATCH 124/540] Add new nonlinear solver Anderson Acceleration

courtesy of Aslak Poulsen
- and BDF
- still needs the interface adapted
---
 inc/dg/andersonacc.h          | 221 ++++++++++++++++++++++++++++++++++
 inc/dg/cg.h                   |   2 +-
 inc/dg/implicit.h             |  56 +++++++++
 inc/dg/multistep.h            | 152 +++++++++++++++++++++++
 src/feltor/feltor.tex         |   2 +-
 src/feltor/input/default.json |   1 -
 6 files changed, 431 insertions(+), 3 deletions(-)
 create mode 100644 inc/dg/andersonacc.h

diff --git a/inc/dg/andersonacc.h b/inc/dg/andersonacc.h
new file mode 100644
index 000000000..d3944aa13
--- /dev/null
+++ b/inc/dg/andersonacc.h
@@ -0,0 +1,221 @@
+#pragma once
+
+#include <functional>
+#include "blas.h"
+
+namespace dg{
+///@cond
+//const double m_EPS = 2.2204460492503131e-16;
+namespace detail{
+
+template<class ContainerType, class Mat>
+void QRdelete1(std::vector<ContainerType>& Q,Mat& R, unsigned m)
+{
+    using value_type = dg::get_value_type<ContainerType>;
+    for(unsigned i = 0; i<m-1;i++){
+        value_type temp = sqrt(R[i][i+1]*R[i][i+1]+R[i+1][i+1]*R[i+1][i+1]);
+        value_type c = R[i][i+1]/temp;
+        value_type s = R[i+1][i+1]/temp;
+        R[i][i+1] = temp;
+        R[i+1][i+1] = 0;
+        if (i < m-2) {
+            for (unsigned j = i+2; j < m; j++){
+                temp = c * R[i][j] + s * R[i+1][j];
+                R[i+1][j] = - s * R[i][j] + c * R[i+1][j];
+                R[i][j] = temp;
+            }
+        }
+        //Collapse look into blas1 routines
+        dg::blas1::axpby(-s,Q[i],c,Q[i+1]); // Q(i + 1) = s ∗ Q(ℓ, i) + c ∗ Q(ℓ, i + 1).
+        dg::blas1::axpbypgz(c,Q[i],s,Q[i+1],0.,Q[i]); //Q(i) = c ∗ Q(ℓ, i) + s ∗ Q(ℓ, i + 1).
+    } //Check for error in keeping the last row.!!!
+    for(int i = 0; i<(int)m-1;i++)
+        for(int j = 0; j < (int)m-1; j++)
+            R[i][j] = R[i][j+1];
+    return;
+}
+}//namespace detail
+///@endcond
+
+
+
+/*!@brief Anderson Acceleration of Fixed Point Iteration for \f[ f(x) = 0\f]
+ *
+ * This class implements the Anderson acceleration of the fixed point iteration algorithm for the problem
+ * \f[
+ *  f(x) = 0
+ *  \f]
+ *  described by https://users.wpi.edu/~walker/Papers/Walker-Ni,SINUM,V49,1715-1735.pdf
+ * @copydoc hide_ContainerType
+ */
+template<class ContainerType>
+struct AndersonAcceleration
+{
+    using container_type = ContainerType;
+    using value_type = get_value_type<ContainerType>;
+    AndersonAcceleration(){}
+//mMax worth trying something between 3 and 10
+//AAstart 0
+//damping Fixed Point damping
+//restart: restart the iteration
+    AndersonAcceleration(const ContainerType& copyable, unsigned mMax ):
+        m_gval( copyable), m_g_old( m_gval), m_fval( m_gval), m_f_old(m_gval),
+        m_df( m_gval), m_DG( mMax, copyable), m_Q( m_DG),
+        m_gamma( mMax, 0.), m_Ry( m_gamma),
+        m_R( mMax, m_gamma)
+    {}
+
+    template<class BinarySubroutine>
+    unsigned solve( BinarySubroutine& f, ContainerType& x, const ContainerType& b, const ContainerType& weights,
+        value_type rtol, value_type atol, value_type beta, unsigned AAstart, unsigned max_iter,
+        value_type damping, unsigned restart, bool verbose);
+
+    private:
+    ContainerType m_gval, m_g_old, m_fval, m_f_old, m_df;
+    std::vector<ContainerType> m_DG, m_Q;
+    std::vector<value_type> m_gamma, m_Ry;
+    std::vector<std::vector<value_type>> m_R;
+
+    value_type m_tol;
+    unsigned m_max_iter, m_mMax;
+};
+
+/*
+template<class ContainerType>
+template<class BinarySubroutine>
+unsigned AndersonAcceleration<ContainerType>::solve(
+    BinarySubroutine& func, ContainerType& x, const ContainerType& b, const ContainerType& weights,
+    value_type rtol, value_type atol, value_type beta, unsigned AAstart, unsigned max_iter,
+    value_type damping, unsigned restart, bool verbose )
+{
+    if (m_mMax == 0){
+        if(verbose)std::cout<< "No acceleration will occur" << std::endl;
+    }
+
+    unsigned mAA = 0;
+
+    for(unsigned iter=0;iter < max_iter; iter++)
+    {
+
+        // Restart if a certain number of iterations are reached. Does not need to be mMax... Just seems to work nicely right now.
+        if (iter % (restart) == 0) {
+            if(verbose)std::cout << "Iter = " << iter << std::endl;
+            mAA = 0;
+        }
+
+        func(x,m_fval);
+        value_type res_norm = sqrt(dg::blas2::dot(m_fval,weights,m_fval));          //l2norm(m_fval)
+
+        dg::blas1::axpby(1.,x,-damping,m_fval,m_gval);                      // m_gval = x - damping*m_fval
+
+        if(iter == 0){
+            m_tol = std::max(atol*sqrt(numu),rtol*res_norm);
+            if(verbose)std::cout << "tol = " << m_tol << std::endl;
+        }
+        if(verbose)std::cout << "res_norm = " << res_norm << " Against tol = " << m_tol << std::endl;
+        // Test for stopping
+        if (res_norm <= m_tol){
+            if(verbose)std::cout << "Terminate with residual norm = " << res_norm << std::endl;
+            break;
+        }
+
+        if (m_mMax == 0){
+            // Without acceleration, update x <- g(x) to obtain the next approximate solution.
+
+            dg::blas1::copy(m_gval,x);                                    //x = m_gval;
+
+        } else {
+            // Apply Anderson acceleration.
+
+            if(iter > AAstart){                                         // Update the m_df vector and the m_DG array.t,
+
+                dg::blas1::axpby(1.,m_fval,-1.,m_f_fold,m_df);                 //m_df = m_fval-m_f_fold;
+
+                if (mAA < m_mMax) {
+
+                    dg::blas1::axpby(1.,m_gval,-1.,m_g_old,m_DG[mAA]);        //Update m_DG = [m_DG   m_gval-m_g_old];
+
+                } else {
+
+                    std::rotate(m_DG.begin(), m_DG.begin() + 1, m_DG.end());  //Rotate to the left hopefully this works... otherwise for i = 0 .. mMax-2 m_DG[i] = m_DG[i+1], and finally m_DG[mMax-1] = update...
+                    dg::blas1::axpby(1.,m_gval,-1.,m_g_old,m_DG[m_mMax-1]);     //Update last m_DG entry
+
+                }
+                mAA = mAA + 1;
+            }
+
+            dg::blas1::copy(m_fval,m_f_fold);                                //m_f_fold = m_fval;
+
+            dg::blas1::copy(m_gval,m_g_old);                                //m_g_old = m_gval;
+
+            if(mAA==0.){
+
+                dg::blas1::copy(m_gval,x);                                // If mAA == 0, update x <- g(x) to obtain the next approximate solution.
+
+            } else {                                                    // If mAA > 0, solve the least-squares problem and update the solution.
+
+                if (mAA == 1) {                                         // If mAA == 1, form the initial QR decomposition.
+
+                    R[0][0] = sqrt(dg::blas2::dot(m_df,weights, m_df));
+                    dg::blas1::axpby(1./R[0][0],m_df,0.,Q[0]);
+
+                } else {                                                // If mAA > 1, update the QR decomposition.
+
+                    if ((mAA > m_mMax)) {                                 // If the column dimension of Q is mMax, delete the first column and update the decomposition.
+
+
+                        mAA = mAA - 1;
+                        QRdelete1(Q,R,mAA);
+
+                    }
+                    // Now update the QR decomposition to incorporate the new column.
+                    for (unsigned j = 1; j < mAA; j++) {
+                        R[j-1][mAA-1] = dg::blas2::dot(Q[j-1],weights,m_df);      //Q(:,j)’*m_df; //Changed mAA -> mAA-1
+
+                        dg::blas1::axpby(-R[j-1][mAA-1],Q[j-1],1.,m_df);  //m_df = m_df - R(j,mAA)*Q(:,j);
+                    }
+                    R[mAA-1][mAA-1] = sqrt(dg::blas2::dot(m_df,weights,m_df));
+                    dg::blas1::axpby(1./R[mAA-1][mAA-1],m_df,0.,Q[mAA-1]);
+                }
+
+                //Calculate condition number of R to figure whether to keep going or call QR delete to reduce Q and R.
+                //value_type condDF = cond(R,mAA);
+                //Here should be the check for whether to proceed.
+
+                //Solve least squares problem.
+                for(int i = (int)mAA-1; i>=0; i--){
+                    m_gamma[i] = dg::blas2::dot(Q[i],weights,m_fval);
+                    for(int j = i + 1; j < mAA; j++){
+                        m_gamma[i] -= R[i][j]*m_gamma[j];
+                    }
+                    m_gamma[i] /= R[i][i];
+                }
+
+                //Update new approximate solution x = m_gval - m_DG*gamma
+                dg::blas1::copy(m_gval,x);
+                for (unsigned i = 0; i < mAA; i++) {
+                    dg::blas1::axpby(-m_gamma[i],m_DG[i],1.,x);
+                }
+
+                //In the paper there is a damping terms included.
+                if ((abs(beta) > 0) && (beta != 1)){
+                    for(unsigned i = 0; i < mAA; i++) {
+                        m_Ry[i] = 0;
+                        for(unsigned j = i; j < mAA; j++) {
+                            m_Ry[i] += R[i][j]*m_gamma[j];                  //Check correctness
+                        }
+                    }
+                    dg::blas1::axpby(-(1.-beta),m_fval,1,x);              //x = x -(1-beta)*m_fval
+                    for(unsigned i = 0; i < mAA; i++) {
+                        dg::blas1::axpby((1.0-beta)*m_Ry[i],Q[i],1.,x);   // x = x - (1-beta)*(-1*Q*R*gamma) = x + (1-beta)*Q*R*gamma = x + (1-beta)*Q*Ry
+                    }
+                }
+            }//Should all mAA
+        }
+
+    }
+    return max_iter;
+
+}
+*/
+}//namespace dg
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 60dd3a7b6..61801ed78 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -95,7 +95,7 @@ class CG
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
      * @param P The preconditioner to be used
-     * @param S Weights used to compute the norm for the error condition
+     * @param S (Inverse) Weights used to compute the norm for the error condition
      * @param eps The relative error to be respected
      * @param nrmb_correction the absolute error \c C in units of \c eps to be respected
      *
diff --git a/inc/dg/implicit.h b/inc/dg/implicit.h
index a9b793441..e377df19b 100644
--- a/inc/dg/implicit.h
+++ b/inc/dg/implicit.h
@@ -1,5 +1,6 @@
 #pragma once
 #include "cg.h"
+#include "andersonacc.h"
 
 namespace dg{
 ///@cond
@@ -187,4 +188,59 @@ struct FixedPointSolver
     value_type m_eps;
     unsigned m_max_iter;
 };
+
+/*!@brief Fixed Point iterator with Anderson Acceleration for solving \f[ (y+\alpha\hat I(t,y)) = \rho\f]
+ *
+ * for given t, alpha and rho.
+ * @copydoc hide_ContainerType
+ * @sa AndersonAcceleration Karniadakis ARKStep DIRKStep
+ * @ingroup invert
+ */
+template<class ContainerType>
+struct AndersonSolver
+{
+    using container_type = ContainerType;
+    using value_type = get_value_type<ContainerType>;//!< value type of vectors
+    ///No memory allocation
+    AndersonSolver(){}
+    /*!
+    * @param copyable vector of the size that is later used in \c solve (
+     it does not matter what values \c copyable contains, but its size is important;
+     the \c solve method can only be called with vectors of the same size)
+    * @param max_iter maimum iteration number in cg
+    * @param eps accuracy parameter for cg
+    */
+    AndersonSolver( const ContainerType& copyable, unsigned mMax, value_type eps):
+        m_acc(copyable, mMax), m_rhs( copyable), m_eps(eps)
+        {}
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_rhs;}
+
+    template< class Implicit> //going to be a reference type
+    void solve( value_type alpha, Implicit im, value_type t, ContainerType& y, const ContainerType& rhs)
+    {
+        detail::Implicit<Implicit, ContainerType> implicit( alpha, t, im);
+#ifdef DG_BENCHMARK
+#ifdef MPI_VERSION
+        int rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+#endif//MPI
+        Timer ti;
+        ti.tic();
+        unsigned number = m_acc( implicit, y, m_rhs, im.precond(), im.inv_weights(), m_eps);
+        ti.toc();
+#ifdef MPI_VERSION
+        if(rank==0)
+#endif//MPI
+        std::cout << "# of pcg iterations time solver: "<<number<<"/"<<m_pcg.get_max()<<" took "<<ti.diff()<<"s\n";
+#else
+        m_pcg( implicit, y, m_rhs, im.precond(), im.inv_weights(), m_eps);
+#endif //DG_BENCHMARK
+    }
+    private:
+    AndersonAcceleration< ContainerType> m_acc;
+    ContainerType m_rhs;
+    value_type m_eps;
+};
 }//namespace dg
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 077ecb4dc..81a39ea25 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -306,6 +306,158 @@ void Karniadakis<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, valu
     f(t_, m_u[0], m_f[0]); //call f on new point
 }
 ///@endcond
+/*
+template<class ContainerType,class RHS> // , class RHS
+struct BDF{
+    using value_type = dg::get_value_type<ContainerType>;
+    using container_type = ContainerType;
+
+    BDF(){}
+
+    //template<class RHS>
+    BDF( unsigned order, const ContainerType& copyable, RHS& rhs, value_type dt):rhs_(rhs){ //
+        construct( order, copyable, rhs, dt); //rhs, 
+    }
+
+    //template<class RHS>
+    void construct(unsigned order, const ContainerType& copyable, RHS& rhs, value_type dt){ // 
+        //RHS& rhs_ = rhs;
+        //rhs_ = rhs;
+        m_k = order;
+        u_.assign( order, copyable);
+        f_ = copyable;
+        sum_prev_x = copyable;
+        dt_ = dt;
+        //Construct Newton
+        
+        switch (order){
+            case 1: 
+            m_bdf = {1.}; 
+            f_factor = 1.;
+            break;
+            case 2: 
+            m_bdf = {4./3., -1./3.}; 
+            f_factor = 2./3.;
+            break;
+            case 3: 
+            m_bdf = { 18./11., -9./11., 2./11.}; 
+            f_factor = 6./11.;
+            break;
+            case 4: 
+            m_bdf = {48./25., -36./25., 16./25., -3./25.}; 
+            f_factor = 12./25.;
+            break;
+            case 5: 
+            m_bdf = { 300./137., -300./137., 200./137., -75./137., 12./137.}; 
+            f_factor = 60/137.;
+            break;
+            case 6: 
+            m_bdf = { 360./147., -450./147., 400./147., -225./147., 172./147., -10./147.}; 
+            f_factor = 60/147.;
+            break;
+            default: throw dg::Error(dg::Message()<<"Order not implemented in BDF!");
+        }
+        std::cout << "Hello, i'm the BDF constructor :) " << std::endl;
+    }
+
+    void func(const ContainerType& uin, ContainerType& uout);           //Function to find root of.
+
+    void init(value_type t0, const ContainerType& u0);                  //Initialise
+
+    void set_mMax(int input_mMax){
+        mMax = input_mMax;
+    }
+
+    void set_itmax(int input_itmax){
+        itmax = input_itmax;
+    }
+
+    void set_AAstart(int input_AAstart){
+        AAstart = input_AAstart;
+    }    
+
+    void step(value_type t0, value_type& t1, container_type& u, value_type rtol, value_type atol, int solver, value_type damping, value_type beta);
+    private:
+        int mMax = 10;
+        int itmax = 100;
+        int AAstart = 0;
+        RHS& rhs_;
+        value_type tu_, dt_, f_factor;
+        std::vector<ContainerType> u_;
+        ContainerType f_;
+        ContainerType sum_prev_x;
+        std::vector<value_type> m_bdf;
+        unsigned m_k;
+};
+
+template< class ContainerType, class RHS>
+void BDF<ContainerType, RHS>::func(const ContainerType& x, ContainerType& fval){ //Function to pass to JFNK
+    rhs_(tu_,x,fval);                                                   //fval = rhs(x,t) //tu_ needs to be set
+    dg::blas1::scal(fval,dt_);                                          //fval = fx*dt
+    dg::blas1::axpby(1.,sum_prev_x,f_factor,fval);                      //fval = sum_x_prev + f_factor*fval where sum_x_prev = sum_order x_order
+    dg::blas1::axpby(1.,x,-1.,fval);                                    //fx = x-fx... 
+}
+
+template< class ContainerType, class RHS>
+void BDF<ContainerType, RHS>::init(value_type t0, const ContainerType& u0){
+    //Perform a number of backward euler steps
+    std::cout << "Initialising method" << std::endl;
+    ContainerType fout(u0);
+    dg::blas1::copy(u0, u_[0]);
+    std::cout << "l2norm of u0 = " << std::setprecision(15) << dg::l2norm(u0) << std::endl;
+    for (unsigned i = 0; i<m_bdf.size()-1; i++){
+        rhs_(t0-i*dt_,u_[i],fout);
+        dg::blas1::axpby(-dt_,fout,1.,u_[i],u_[i+1]);
+        std::cout << "l2norm of u0 = " << std::setprecision(15) << dg::l2norm(u_[i+1]) << std::endl;
+    }
+    
+}
+
+template< class ContainerType, class RHS>
+void BDF<ContainerType, RHS>::step(value_type t0, value_type& t1, container_type& u, value_type rtol, value_type atol, int solver, value_type damping, value_type beta){
+    dg::blas1::axpby(m_bdf[0],u_[0],0.,sum_prev_x);
+    for (unsigned i = 1; i < m_bdf.size(); i++){
+        dg::blas1::axpby(m_bdf[i],u_[i],1.,sum_prev_x);
+    }
+    tu_ = t1 = t0+dt_;
+    ContainerType upred(u);
+    ContainerType uout(u);
+    ContainerType diff(u);
+    value_type alpha[2] = {2., -1.};
+    dg::blas1::axpby( alpha[0], u_[0], alpha[1],  u_[1], u);
+    std::cout << "AAstart = " << AAstart << std::endl;
+    std::cout << "mMax = " << mMax << std::endl;
+    std::cout << "itmax = " << itmax << std::endl;
+    using namespace std::placeholders;
+    auto fun = std::bind(&BDF<ContainerType, RHS>::func,*this,_1,_2);
+    if (solver == 0) {
+        std::cout << "You have chosen the Anderson Acceleration" << std::endl;
+        andacc<ContainerType>(fun, u, mMax, itmax, atol, rtol, beta, AAstart, damping);
+    } else if (solver == 1){
+        std::cout << "You have chosen the Nonlinear GMRES" << std::endl;
+        ngmres<ContainerType>(fun, u, mMax, itmax, atol, rtol, damping);
+    } else if (solver == 2){
+        std::cout << "You have chosen the de Sterck Nonlinear GMRES" << std::endl;
+        dsngmres<ContainerType>(fun, u, mMax, itmax, atol, rtol, damping);
+    } else if (solver == 3){
+        std::cout << "You have chosen the objective acceleration" << std::endl;
+        o_accel<ContainerType>(fun, u, mMax, itmax, atol, rtol, damping);
+    }else if (solver == 4){
+        std::cout << "You have chosen the Broyden solver" << std::endl;
+        broyden<ContainerType>(fun, u, mMax, itmax, atol, rtol, damping);
+    } else {
+        std::cout << "Solver not specified. Defaulting to Anderson Acceleration" << std::endl;
+        andacc<ContainerType>(fun, u, mMax, itmax, atol, rtol, beta, AAstart,damping);
+    }
+    ////Update u_
+    std::rotate(u_.rbegin(), u_.rbegin() + 1, u_.rend()); //Rotate 1 to the right.
+    dg::blas1::copy(u,u_[0]);
+    std::cout<<"Exited nonlin solver" << std::endl;
+    ////Update sum_x_prev
+    
+}
+}
+*/
 
 
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index a9abf340d..4bfd30107 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1118,7 +1118,7 @@ amplitude  & float &0.01   & - & amplitude $A$ of initial perturbation (blob, tu
 sigma      & float &2      & - & blob variance in units of $\rho_s$ \\
 posX       & float &0.3    & - & blob R-position in units of $a$\\
 posY       & float &0.0    & - & blob Z-position in units of $a$ \\
-sigma\_z    & float &0.25   & - & variance in units of $R_0$ of the fieldline-following initialization \\
+sigma\_z    & float &0.25   & - & toroidal variance in units of $R_0$ of the fieldline-following initialization \\
 k\_psi     & float &0    & - & zonal mode wave number  \\
 nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} \\
 alpha\_mag   & float & 0.05 & - & Width $\alpha$ of the Heaviside in the modified $\psi_p$ function \eqref{eq:modified_psip}\\
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index 32f51abe0..470d9cfb8 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -30,7 +30,6 @@
     "sigma_z"   : 0.5,
     "k_psi"     : 0,
     "nprofileamp" : 4,
-    "bgprofamp"   : 1,
     "bc" :
     {
         "density" : ["NEU", "NEU"],
-- 
GitLab


From ed6b2b7fd733819d5d73054cfaf128df21030e2d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 26 Aug 2019 23:09:50 +0200
Subject: [PATCH 125/540] Write AndersonAcc test file and test BDF

also start documentation
---
 inc/dg/Doxyfile         |   3 +-
 inc/dg/andersonacc.h    |  86 +++++++-------
 inc/dg/andersonacc_t.cu |  66 +++++++++++
 inc/dg/elliptic.h       |  12 ++
 inc/dg/multistep.h      | 246 +++++++++++++++++-----------------------
 inc/dg/multistep_t.cu   |  16 ++-
 6 files changed, 248 insertions(+), 181 deletions(-)
 create mode 100644 inc/dg/andersonacc_t.cu

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 3bcb9934f..e1becbc2d 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -893,7 +893,8 @@ EXCLUDE_SYMBOLS        = hide_*
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           = cg2d_t.cu \
+EXAMPLE_PATH           = andersonacc_t.cu \
+                         cg2d_t.cu \
                          arakawa_t.cu \
                          poisson_t.cu \
                          elliptic_b.cu \
diff --git a/inc/dg/andersonacc.h b/inc/dg/andersonacc.h
index d3944aa13..48729e107 100644
--- a/inc/dg/andersonacc.h
+++ b/inc/dg/andersonacc.h
@@ -46,25 +46,29 @@ void QRdelete1(std::vector<ContainerType>& Q,Mat& R, unsigned m)
  *  f(x) = 0
  *  \f]
  *  described by https://users.wpi.edu/~walker/Papers/Walker-Ni,SINUM,V49,1715-1735.pdf
+ * @snippet andersonacc_t.cu doxygen
  * @copydoc hide_ContainerType
  */
 template<class ContainerType>
 struct AndersonAcceleration
 {
-    using container_type = ContainerType;
-    using value_type = get_value_type<ContainerType>;
+    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
+    using container_type = ContainerType; //!< the type of the vector class in use
     AndersonAcceleration(){}
-//mMax worth trying something between 3 and 10
-//AAstart 0
-//damping Fixed Point damping
-//restart: restart the iteration
     AndersonAcceleration(const ContainerType& copyable, unsigned mMax ):
         m_gval( copyable), m_g_old( m_gval), m_fval( m_gval), m_f_old(m_gval),
         m_df( m_gval), m_DG( mMax, copyable), m_Q( m_DG),
         m_gamma( mMax, 0.), m_Ry( m_gamma),
-        m_R( mMax, m_gamma)
+        m_R( mMax, m_gamma), m_mMax( mMax)
     {}
 
+//mMax worth trying something between 3 and 10
+//AAstart 0
+//damping Fixed Point damping
+//restart: restart the iteration
+    /*!@brief Solve the system \f$ f(x) = b \f$ in the given norm
+     *
+     */
     template<class BinarySubroutine>
     unsigned solve( BinarySubroutine& f, ContainerType& x, const ContainerType& b, const ContainerType& weights,
         value_type rtol, value_type atol, value_type beta, unsigned AAstart, unsigned max_iter,
@@ -76,11 +80,10 @@ struct AndersonAcceleration
     std::vector<value_type> m_gamma, m_Ry;
     std::vector<std::vector<value_type>> m_R;
 
-    value_type m_tol;
-    unsigned m_max_iter, m_mMax;
+    unsigned m_mMax;
 };
+///@cond
 
-/*
 template<class ContainerType>
 template<class BinarySubroutine>
 unsigned AndersonAcceleration<ContainerType>::solve(
@@ -93,32 +96,32 @@ unsigned AndersonAcceleration<ContainerType>::solve(
     }
 
     unsigned mAA = 0;
+    value_type nrmb = sqrt( dg::blas2::dot( b, weights, b));
+    value_type tol = atol+rtol*nrmb;
+    if(verbose)std::cout << "tol = " << tol << std::endl;
 
-    for(unsigned iter=0;iter < max_iter; iter++)
+    for(unsigned iter=0;iter < max_iter-1; iter++)
     {
 
         // Restart if a certain number of iterations are reached. Does not need to be mMax... Just seems to work nicely right now.
-        if (iter % (restart) == 0) {
+        if (iter % restart == 0) {
             if(verbose)std::cout << "Iter = " << iter << std::endl;
             mAA = 0;
         }
 
-        func(x,m_fval);
-        value_type res_norm = sqrt(dg::blas2::dot(m_fval,weights,m_fval));          //l2norm(m_fval)
-
-        dg::blas1::axpby(1.,x,-damping,m_fval,m_gval);                      // m_gval = x - damping*m_fval
+        func( x, m_fval);
+        dg::blas1::axpby( -1., b, 1., m_fval); //f(x) = func - b (residual)
+        value_type res_norm = sqrt(dg::blas2::dot(m_fval,weights,m_fval));  //l2norm(m_fval)
 
-        if(iter == 0){
-            m_tol = std::max(atol*sqrt(numu),rtol*res_norm);
-            if(verbose)std::cout << "tol = " << m_tol << std::endl;
-        }
-        if(verbose)std::cout << "res_norm = " << res_norm << " Against tol = " << m_tol << std::endl;
+        if(verbose)std::cout << "res_norm = " << res_norm << " Against tol = " << tol << std::endl;
         // Test for stopping
-        if (res_norm <= m_tol){
+        if (res_norm <= tol){
             if(verbose)std::cout << "Terminate with residual norm = " << res_norm << std::endl;
-            break;
+            return iter+1;
         }
 
+        dg::blas1::axpby(1.,x,-damping,m_fval,m_gval);                      // m_gval = x - damping*m_fval
+
         if (m_mMax == 0){
             // Without acceleration, update x <- g(x) to obtain the next approximate solution.
 
@@ -129,7 +132,7 @@ unsigned AndersonAcceleration<ContainerType>::solve(
 
             if(iter > AAstart){                                         // Update the m_df vector and the m_DG array.t,
 
-                dg::blas1::axpby(1.,m_fval,-1.,m_f_fold,m_df);                 //m_df = m_fval-m_f_fold;
+                dg::blas1::axpby(1.,m_fval,-1.,m_f_old, m_df);                 //m_df = m_fval-m_f_old;
 
                 if (mAA < m_mMax) {
 
@@ -144,7 +147,7 @@ unsigned AndersonAcceleration<ContainerType>::solve(
                 mAA = mAA + 1;
             }
 
-            dg::blas1::copy(m_fval,m_f_fold);                                //m_f_fold = m_fval;
+            dg::blas1::copy(m_fval,m_f_old);                                //m_f_old = m_fval;
 
             dg::blas1::copy(m_gval,m_g_old);                                //m_g_old = m_gval;
 
@@ -156,8 +159,8 @@ unsigned AndersonAcceleration<ContainerType>::solve(
 
                 if (mAA == 1) {                                         // If mAA == 1, form the initial QR decomposition.
 
-                    R[0][0] = sqrt(dg::blas2::dot(m_df,weights, m_df));
-                    dg::blas1::axpby(1./R[0][0],m_df,0.,Q[0]);
+                    m_R[0][0] = sqrt(dg::blas2::dot(m_df,weights, m_df));
+                    dg::blas1::axpby(1./m_R[0][0],m_df,0.,m_Q[0]);
 
                 } else {                                                // If mAA > 1, update the QR decomposition.
 
@@ -165,17 +168,17 @@ unsigned AndersonAcceleration<ContainerType>::solve(
 
 
                         mAA = mAA - 1;
-                        QRdelete1(Q,R,mAA);
+                        detail::QRdelete1(m_Q,m_R,mAA);
 
                     }
                     // Now update the QR decomposition to incorporate the new column.
                     for (unsigned j = 1; j < mAA; j++) {
-                        R[j-1][mAA-1] = dg::blas2::dot(Q[j-1],weights,m_df);      //Q(:,j)’*m_df; //Changed mAA -> mAA-1
+                        m_R[j-1][mAA-1] = dg::blas2::dot(m_Q[j-1],weights,m_df);      //Q(:,j)’*m_df; //Changed mAA -> mAA-1
 
-                        dg::blas1::axpby(-R[j-1][mAA-1],Q[j-1],1.,m_df);  //m_df = m_df - R(j,mAA)*Q(:,j);
+                        dg::blas1::axpby(-m_R[j-1][mAA-1],m_Q[j-1],1.,m_df);  //m_df = m_df - R(j,mAA)*Q(:,j);
                     }
-                    R[mAA-1][mAA-1] = sqrt(dg::blas2::dot(m_df,weights,m_df));
-                    dg::blas1::axpby(1./R[mAA-1][mAA-1],m_df,0.,Q[mAA-1]);
+                    m_R[mAA-1][mAA-1] = sqrt(dg::blas2::dot(m_df,weights,m_df));
+                    dg::blas1::axpby(1./m_R[mAA-1][mAA-1],m_df,0.,m_Q[mAA-1]);
                 }
 
                 //Calculate condition number of R to figure whether to keep going or call QR delete to reduce Q and R.
@@ -184,11 +187,11 @@ unsigned AndersonAcceleration<ContainerType>::solve(
 
                 //Solve least squares problem.
                 for(int i = (int)mAA-1; i>=0; i--){
-                    m_gamma[i] = dg::blas2::dot(Q[i],weights,m_fval);
-                    for(int j = i + 1; j < mAA; j++){
-                        m_gamma[i] -= R[i][j]*m_gamma[j];
+                    m_gamma[i] = dg::blas2::dot(m_Q[i],weights,m_fval);
+                    for(int j = i + 1; j < (int)mAA; j++){
+                        m_gamma[i] -= m_R[i][j]*m_gamma[j];
                     }
-                    m_gamma[i] /= R[i][i];
+                    m_gamma[i] /= m_R[i][i];
                 }
 
                 //Update new approximate solution x = m_gval - m_DG*gamma
@@ -198,18 +201,20 @@ unsigned AndersonAcceleration<ContainerType>::solve(
                 }
 
                 //In the paper there is a damping terms included.
-                if ((abs(beta) > 0) && (beta != 1)){
+                if ((fabs(beta) > 0) && (beta <= 1)){
                     for(unsigned i = 0; i < mAA; i++) {
                         m_Ry[i] = 0;
                         for(unsigned j = i; j < mAA; j++) {
-                            m_Ry[i] += R[i][j]*m_gamma[j];                  //Check correctness
+                            m_Ry[i] += m_R[i][j]*m_gamma[j];                  //Check correctness
                         }
                     }
                     dg::blas1::axpby(-(1.-beta),m_fval,1,x);              //x = x -(1-beta)*m_fval
                     for(unsigned i = 0; i < mAA; i++) {
-                        dg::blas1::axpby((1.0-beta)*m_Ry[i],Q[i],1.,x);   // x = x - (1-beta)*(-1*Q*R*gamma) = x + (1-beta)*Q*R*gamma = x + (1-beta)*Q*Ry
+                        dg::blas1::axpby((1.0-beta)*m_Ry[i],m_Q[i],1.,x);   // x = x - (1-beta)*(-1*Q*R*gamma) = x + (1-beta)*Q*R*gamma = x + (1-beta)*Q*Ry
                     }
                 }
+                else
+                    throw Error ( Message(_ping_)<< "Insane value for beta: "<<beta<<" should be 0<beta<=1. 1 means no damping!");
             }//Should all mAA
         }
 
@@ -217,5 +222,6 @@ unsigned AndersonAcceleration<ContainerType>::solve(
     return max_iter;
 
 }
-*/
+///@endcond
+
 }//namespace dg
diff --git a/inc/dg/andersonacc_t.cu b/inc/dg/andersonacc_t.cu
new file mode 100644
index 000000000..7f3bb87d2
--- /dev/null
+++ b/inc/dg/andersonacc_t.cu
@@ -0,0 +1,66 @@
+#include <iostream>
+#include <iomanip>
+
+#include "andersonacc.h"
+#include "elliptic.h"
+
+const double lx = 2.*M_PI;
+const double ly = 2.*M_PI;
+
+
+double fct(double x, double y){ return sin(y)*sin(x);}
+double laplace_fct( double x, double y) { return 2*sin(y)*sin(x);}
+double initial( double x, double y) {return sin(0);}
+
+int main()
+{
+    //global relative error in L2 norm is O(h^P)
+    //more N means less iterations for same error
+    unsigned n, Nx, Ny;
+    std::cout << "Type n, Nx and Ny! \n";
+    std::cin >> n >> Nx >> Ny;
+    std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<std::endl;
+    dg::Grid2d grid( 0, lx, 0, ly,n, Nx, Ny, dg::PER, dg::PER);
+    std::cout<<"Evaluate initial condition\n";
+    dg::HVec x = dg::evaluate( initial, grid);
+    unsigned max_iter = n*n*Nx*Ny;
+    const dg::HVec& copyable_vector = x;
+
+//! [doxygen]
+    // create volume and inverse volume on previously defined grid
+    const dg::HVec w2d = dg::create::weights( grid);
+
+    // Create unnormalized Laplacian
+    dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> A( grid, dg::normed);
+
+    // allocate memory in conjugate gradient
+    dg::AndersonAcceleration<dg::HVec > acc( copyable_vector, 10);
+
+    // Evaluate right hand side and solution on the grid
+    dg::HVec b = dg::evaluate ( laplace_fct, grid);
+    const dg::HVec solution = dg::evaluate ( fct, grid);
+
+    const double eps = 1e-6;
+    std::cout << "Number of iterations "<< acc.solve( A, x, b, w2d, eps, eps, 1, 0, max_iter, 1e-6, 10, true)<<std::endl;
+//! [doxygen]
+    std::cout << "For a precision of "<< eps<<std::endl;
+    //compute error
+    dg::HVec error( solution);
+    dg::blas1::axpby( 1.,x,-1.,error);
+
+    dg::HVec Ax(x), resi( b);
+    dg::blas2::symv(  A, x, Ax);
+    dg::blas1::axpby( 1.,Ax,-1.,resi);
+
+    exblas::udouble res;
+    res.d = sqrt(dg::blas2::dot( w2d, x));
+    std::cout << "L2 Norm of x0 is              " << res.d<<"\t"<<res.i << std::endl;
+    res.d = sqrt(dg::blas2::dot(w2d , solution));
+    std::cout << "L2 Norm of Solution is        " << res.d<<"\t"<<res.i << std::endl;
+    res.d = sqrt(dg::blas2::dot(w2d , error));
+    std::cout << "L2 Norm of Error is           " << res.d<<"\t"<<res.i << std::endl;
+    res.d = sqrt(dg::blas2::dot( w2d, resi));
+    std::cout << "L2 Norm of Residuum is        " << res.d<<"\t"<<res.i << std::endl;
+    //Fehler der Integration des Sinus ist vernachlässigbar (vgl. evaluation_t)
+    return 0;
+}
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 7c6c7b26e..52f13f5c2 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -192,6 +192,18 @@ class Elliptic
      * @return  The current scale factor for jump terms
      */
     value_type get_jfactor() const {return m_jfactor;}
+    /**
+     * @brief Compute elliptic term and store in output
+     *
+     * i.e. \c y=M*x
+     * @param x left-hand-side
+     * @param y result
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1>
+    void operator()( const ContainerType0& x, ContainerType1& y){
+        symv( 1, x, 0, y);
+    }
 
     /**
      * @brief Compute elliptic term and store in output
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 81a39ea25..dc31f30ba 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -258,7 +258,7 @@ struct Karniadakis
     }
     SolverType m_solver;
     std::array<ContainerType,3> m_u, m_f;
-    value_type t_, m_dt;
+    value_type m_t, m_dt;
     value_type a[3];
     value_type b[3], g0 = 6./11.;
 };
@@ -269,7 +269,7 @@ template< class RHS, class Diffusion>
 void Karniadakis<ContainerType, SolverType>::init( RHS& f, Diffusion& diff, value_type t0, const ContainerType& u0, value_type dt)
 {
     //operator splitting using explicit Euler for both explicit and implicit part
-    t_ = t0, m_dt = dt;
+    m_t = t0, m_dt = dt;
     blas1::copy(  u0, m_u[0]);
     f( t0, u0, m_f[0]); //f may not destroy u0
     blas1::axpby( 1., m_u[0], -dt, m_f[0], m_f[1]); //Euler step
@@ -300,165 +300,133 @@ void Karniadakis<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, valu
     value_type alpha[2] = {2., -1.};
     //value_type alpha[2] = {1., 0.};
     blas1::axpby( alpha[0], m_u[1], alpha[1],  m_u[2], u); //extrapolate previous solutions
-    t = t_ = t_+ m_dt;
+    t = m_t = m_t + m_dt;
     m_solver.solve( -m_dt*g0, diff, t, u, m_u[0]);
     blas1::copy( u, m_u[0]); //store result
-    f(t_, m_u[0], m_f[0]); //call f on new point
+    f(m_t, m_u[0], m_f[0]); //call f on new point
 }
 ///@endcond
-/*
-template<class ContainerType,class RHS> // , class RHS
-struct BDF{
-    using value_type = dg::get_value_type<ContainerType>;
-    using container_type = ContainerType;
 
-    BDF(){}
-
-    //template<class RHS>
-    BDF( unsigned order, const ContainerType& copyable, RHS& rhs, value_type dt):rhs_(rhs){ //
-        construct( order, copyable, rhs, dt); //rhs, 
-    }
-
-    //template<class RHS>
-    void construct(unsigned order, const ContainerType& copyable, RHS& rhs, value_type dt){ // 
-        //RHS& rhs_ = rhs;
-        //rhs_ = rhs;
-        m_k = order;
-        u_.assign( order, copyable);
-        f_ = copyable;
-        sum_prev_x = copyable;
-        dt_ = dt;
-        //Construct Newton
-        
-        switch (order){
-            case 1: 
-            m_bdf = {1.}; 
-            f_factor = 1.;
-            break;
-            case 2: 
-            m_bdf = {4./3., -1./3.}; 
-            f_factor = 2./3.;
-            break;
-            case 3: 
-            m_bdf = { 18./11., -9./11., 2./11.}; 
-            f_factor = 6./11.;
-            break;
-            case 4: 
-            m_bdf = {48./25., -36./25., 16./25., -3./25.}; 
-            f_factor = 12./25.;
-            break;
-            case 5: 
-            m_bdf = { 300./137., -300./137., 200./137., -75./137., 12./137.}; 
-            f_factor = 60/137.;
-            break;
-            case 6: 
-            m_bdf = { 360./147., -450./147., 400./147., -225./147., 172./147., -10./147.}; 
-            f_factor = 60/147.;
-            break;
-            default: throw dg::Error(dg::Message()<<"Order not implemented in BDF!");
-        }
-        std::cout << "Hello, i'm the BDF constructor :) " << std::endl;
-    }
+/**
+* @brief Struct for Backward differentiation formula implicit multistep time-integration
+* \f[
+* \begin{align}
+    v^{n+1} = \sum_{q=0}^{s-1} \alpha_q v^{n-q} + \Delta t\beta\hat I(t^{n}+\Delta t, v^{n+1})\right]
+    \end{align}
+    \f]
 
-    void func(const ContainerType& uin, ContainerType& uout);           //Function to find root of.
+    which discretizes
+    \f[
+    \frac{\partial v}{\partial t} = \hat I(t,v)
+    \f]
+    where \f$ \hat I \f$ represents the right hand side of the equations.
+    The coefficients for up to order 6 can be found at
+    https://en.wikipedia.org/wiki/Backward_differentiation_formula
+*
+* The necessary Inversion in the imlicit part is provided by the \c SolverType class.
+* Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$). For nonlinear right hand side we recommend the AndersonSolver
+*
+* @note In our experience the implicit treatment of diffusive or hyperdiffusive
+terms can significantly reduce the required number of time steps. This
+outweighs the increased computational cost of the additional inversions.
+* @copydoc hide_note_multistep
+* @copydoc hide_SolverType
+* @copydoc hide_ContainerType
+* @ingroup time
+*/
+template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
+struct BDF
+{
 
-    void init(value_type t0, const ContainerType& u0);                  //Initialise
+    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
+    using container_type = ContainerType; //!< the type of the vector class in use
+    ///@copydoc RungeKutta::RungeKutta()
+    BDF(){}
 
-    void set_mMax(int input_mMax){
-        mMax = input_mMax;
+    ///@copydoc construct()
+    template<class ...SolverParams>
+    BDF( unsigned order, SolverParams&& ...ps):m_solver( std::forward<SolverParams>(ps)...), m_u( order, m_solver.copyable()), m_f(m_solver.copyable()) {
+        init_coeffs(order);
+        m_k = order;
     }
 
-    void set_itmax(int input_itmax){
-        itmax = input_itmax;
+    /*! @brief Reserve memory for integration and construct Solver
+     *
+     * @param order Order of the BDF formula (1 <= order <= 6)
+     * @param ps Parameters that are forwarded to the constructor of \c SolverType
+     * @tparam SolverParams Type of parameters (deduced by the compiler)
+     */
+    template<class ...SolverParams>
+    void construct(unsigned order, SolverParams&& ...ps)
+    {
+        m_solver = SolverType( std::forward<SolverParams>(ps)...);
+        init_coeffs(order);
+        m_k = order;
+        m_u.assign( order, m_solver.copyable());
+        m_f = m_solver.copyable();
     }
 
-    void set_AAstart(int input_AAstart){
-        AAstart = input_AAstart;
-    }    
+    ///@copydoc AdamsBashforth::init()
+    template<class RHS>
+    void init(RHS& rhs, value_type t0, const ContainerType& u0, value_type dt);
 
-    void step(value_type t0, value_type& t1, container_type& u, value_type rtol, value_type atol, int solver, value_type damping, value_type beta);
+    ///@copydoc AdamsBashforth::step()
+    template<class RHS>
+    void step(RHS& rhs, value_type& t0, container_type& u0);
     private:
-        int mMax = 10;
-        int itmax = 100;
-        int AAstart = 0;
-        RHS& rhs_;
-        value_type tu_, dt_, f_factor;
-        std::vector<ContainerType> u_;
-        ContainerType f_;
-        ContainerType sum_prev_x;
-        std::vector<value_type> m_bdf;
-        unsigned m_k;
+    void init_coeffs(unsigned order){
+        switch (order){
+            case 1: m_bdf = {1.}; m_beta = 1.; break;
+            case 2: m_bdf = {4./3., -1./3.}; m_beta = 2./3.; break;
+            case 3: m_bdf = { 18./11., -9./11., 2./11.}; m_beta = 6./11.; break;
+            case 4: m_bdf = {48./25., -36./25., 16./25., -3./25.}; m_beta = 12./25.; break;
+            case 5: m_bdf = { 300./137., -300./137., 200./137., -75./137., 12./137.}; m_beta = 60/137.; break;
+            case 6: m_bdf = { 360./147., -450./147., 400./147., -225./147., 172./147., -10./147.}; m_beta = 60/147.; break;
+            default: throw dg::Error(dg::Message()<<"Order not implemented in BDF!");
+        }
+    }
+    SolverType m_solver;
+    value_type m_tu, m_dt;
+    std::vector<ContainerType> m_u;
+    ContainerType m_f;
+    std::vector<value_type> m_bdf;
+    value_type m_beta;
+    unsigned m_k;
 };
 
-template< class ContainerType, class RHS>
-void BDF<ContainerType, RHS>::func(const ContainerType& x, ContainerType& fval){ //Function to pass to JFNK
-    rhs_(tu_,x,fval);                                                   //fval = rhs(x,t) //tu_ needs to be set
-    dg::blas1::scal(fval,dt_);                                          //fval = fx*dt
-    dg::blas1::axpby(1.,sum_prev_x,f_factor,fval);                      //fval = sum_x_prev + f_factor*fval where sum_x_prev = sum_order x_order
-    dg::blas1::axpby(1.,x,-1.,fval);                                    //fx = x-fx... 
-}
-
-template< class ContainerType, class RHS>
-void BDF<ContainerType, RHS>::init(value_type t0, const ContainerType& u0){
+template< class ContainerType, class SolverType>
+template<class RHS>
+void BDF<ContainerType, SolverType>::init(RHS& rhs, value_type t0,
+    const ContainerType& u0, value_type dt)
+{
+    m_tu = t0, m_dt = dt;
+    dg::blas1::copy(u0, m_u[0]);
     //Perform a number of backward euler steps
-    std::cout << "Initialising method" << std::endl;
-    ContainerType fout(u0);
-    dg::blas1::copy(u0, u_[0]);
-    std::cout << "l2norm of u0 = " << std::setprecision(15) << dg::l2norm(u0) << std::endl;
-    for (unsigned i = 0; i<m_bdf.size()-1; i++){
-        rhs_(t0-i*dt_,u_[i],fout);
-        dg::blas1::axpby(-dt_,fout,1.,u_[i],u_[i+1]);
-        std::cout << "l2norm of u0 = " << std::setprecision(15) << dg::l2norm(u_[i+1]) << std::endl;
+    for (unsigned i = 0; i<m_k-1; i++){
+        rhs(t0-i*dt,m_u[i], m_f);
+        dg::blas1::axpby(-dt,m_f,1.,m_u[i],m_u[i+1]);
     }
-    
+    rhs( t0, u0, m_f); // and set state in f to (t0,u0)
 }
 
-template< class ContainerType, class RHS>
-void BDF<ContainerType, RHS>::step(value_type t0, value_type& t1, container_type& u, value_type rtol, value_type atol, int solver, value_type damping, value_type beta){
-    dg::blas1::axpby(m_bdf[0],u_[0],0.,sum_prev_x);
-    for (unsigned i = 1; i < m_bdf.size(); i++){
-        dg::blas1::axpby(m_bdf[i],u_[i],1.,sum_prev_x);
+template< class ContainerType, class SolverType>
+template<class RHS>
+void BDF<ContainerType, SolverType>::step(RHS& rhs, value_type& t, container_type& u)
+{
+    dg::blas1::axpby( m_bdf[0], m_u[0], 0., m_f);
+    for (unsigned i = 1; i < m_k; i++){
+        dg::blas1::axpby( m_bdf[i], m_u[i], 1., m_f);
     }
-    tu_ = t1 = t0+dt_;
-    ContainerType upred(u);
-    ContainerType uout(u);
-    ContainerType diff(u);
+    t = m_tu = m_tu + m_dt;
+
     value_type alpha[2] = {2., -1.};
-    dg::blas1::axpby( alpha[0], u_[0], alpha[1],  u_[1], u);
-    std::cout << "AAstart = " << AAstart << std::endl;
-    std::cout << "mMax = " << mMax << std::endl;
-    std::cout << "itmax = " << itmax << std::endl;
-    using namespace std::placeholders;
-    auto fun = std::bind(&BDF<ContainerType, RHS>::func,*this,_1,_2);
-    if (solver == 0) {
-        std::cout << "You have chosen the Anderson Acceleration" << std::endl;
-        andacc<ContainerType>(fun, u, mMax, itmax, atol, rtol, beta, AAstart, damping);
-    } else if (solver == 1){
-        std::cout << "You have chosen the Nonlinear GMRES" << std::endl;
-        ngmres<ContainerType>(fun, u, mMax, itmax, atol, rtol, damping);
-    } else if (solver == 2){
-        std::cout << "You have chosen the de Sterck Nonlinear GMRES" << std::endl;
-        dsngmres<ContainerType>(fun, u, mMax, itmax, atol, rtol, damping);
-    } else if (solver == 3){
-        std::cout << "You have chosen the objective acceleration" << std::endl;
-        o_accel<ContainerType>(fun, u, mMax, itmax, atol, rtol, damping);
-    }else if (solver == 4){
-        std::cout << "You have chosen the Broyden solver" << std::endl;
-        broyden<ContainerType>(fun, u, mMax, itmax, atol, rtol, damping);
-    } else {
-        std::cout << "Solver not specified. Defaulting to Anderson Acceleration" << std::endl;
-        andacc<ContainerType>(fun, u, mMax, itmax, atol, rtol, beta, AAstart,damping);
-    }
-    ////Update u_
-    std::rotate(u_.rbegin(), u_.rbegin() + 1, u_.rend()); //Rotate 1 to the right.
-    dg::blas1::copy(u,u_[0]);
-    std::cout<<"Exited nonlin solver" << std::endl;
-    ////Update sum_x_prev
-    
+    if( m_k > 1 ) //everything higher than Euler
+        dg::blas1::axpby( alpha[0], m_u[0], alpha[1],  m_u[1], u);
+    else
+        dg::blas1::copy( m_u[0], u);
+    std::rotate(m_u.rbegin(), m_u.rbegin() + 1, m_u.rend()); //Rotate 1 to the right
+    m_solver.solve( -m_dt*m_beta, rhs, t, u, m_f);
+    dg::blas1::copy( u, m_u[0]);
 }
-}
-*/
-
-
 
 } //namespace dg
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 956c1754e..8885c6bc1 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -82,6 +82,7 @@ struct Full
         m_exp( g, nu), m_imp( g, nu), m_temp( dg::evaluate( dg::one, g))
 
     { }
+    const container& weights(){return m_imp.weights();}
     void operator()( double t, const container& y, container& yp) {
         m_exp( t, y, yp);
         m_imp( t, y, m_temp);
@@ -124,7 +125,7 @@ int main()
     double time = 0.;
     dg::DVec error( sol);
     exblas::udouble res;
-    std::cout << "### Test explicit multistep methods with "<<NT<<"steps\n";
+    std::cout << "### Test explicit multistep methods with "<<NT<<" steps\n";
     for( unsigned s=1; s<6; s++)
     {
         time = 0., y0 = init;
@@ -137,6 +138,19 @@ int main()
         res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
         std::cout << "Relative error AB "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
     }
+    std::cout << "### Test implicit multistep methods with "<<NT<<" steps\n";
+    for( unsigned s=1; s<6; s++)
+    {
+        time = 0., y0 = init;
+        dg::BDF< dg::DVec, dg::FixedPointSolver<dg::DVec> > bdf( s, y0, 100, 1e-10);
+        bdf.init( full, time, y0, dt);
+        //main time loop
+        for( unsigned k=0; k<NT; k++)
+            bdf.step( full, time, y0);
+        dg::blas1::axpby( -1., sol, 1., y0);
+        res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
+        std::cout << "Relative error BDF "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
+    }
     Explicit<dg::DVec> ex( grid, nu);
     Implicit<dg::DMatrix, dg::DVec> im( grid, nu);
     std::cout << "### Test semi-implicit Karniadakis methods with "<<NT<<"steps\n";
-- 
GitLab


From 6d09926c206ffaad5e07d0b6bf7bb3a1b27de0ae Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 27 Aug 2019 16:21:41 +0200
Subject: [PATCH 126/540] Further test and document Anderson Acceleration

---
 inc/dg/andersonacc.h    | 74 +++++++++++++++++++++++------------------
 inc/dg/andersonacc_t.cu | 14 +++++---
 inc/dg/cg.h             |  1 -
 inc/dg/dg_doc.h         |  2 +-
 inc/dg/implicit.h       | 30 ++++++++++-------
 inc/dg/multistep.h      |  8 ++---
 inc/dg/multistep_t.cu   |  5 +--
 inc/dg/simpsons.h       |  2 +-
 8 files changed, 77 insertions(+), 59 deletions(-)

diff --git a/inc/dg/andersonacc.h b/inc/dg/andersonacc.h
index 48729e107..3bb10506d 100644
--- a/inc/dg/andersonacc.h
+++ b/inc/dg/andersonacc.h
@@ -39,13 +39,13 @@ void QRdelete1(std::vector<ContainerType>& Q,Mat& R, unsigned m)
 
 
 
-/*!@brief Anderson Acceleration of Fixed Point Iteration for \f[ f(x) = 0\f]
+/*!@brief Anderson Acceleration of Fixed Point/Richardson Iteration for the nonlinear equation \f[ f(x) = b\f]
  *
- * This class implements the Anderson acceleration of the fixed point iteration algorithm for the problem
- * \f[
- *  f(x) = 0
- *  \f]
+ * This class implements the Anderson acceleration of the fixed point iteration algorithm
  *  described by https://users.wpi.edu/~walker/Papers/Walker-Ni,SINUM,V49,1715-1735.pdf
+ As recommended by  https://arxiv.org/pdf/1803.06673.pdf we periodically restart the acceleration to
+ improve convergence behaviour.
+ *  @ingroup invert
  * @snippet andersonacc_t.cu doxygen
  * @copydoc hide_ContainerType
  */
@@ -54,7 +54,9 @@ struct AndersonAcceleration
 {
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
+    ///@brief Allocate nothing, Call \c construct method before usage
     AndersonAcceleration(){}
+    ///@copydoc construct()
     AndersonAcceleration(const ContainerType& copyable, unsigned mMax ):
         m_gval( copyable), m_g_old( m_gval), m_fval( m_gval), m_f_old(m_gval),
         m_df( m_gval), m_DG( mMax, copyable), m_Q( m_DG),
@@ -62,16 +64,37 @@ struct AndersonAcceleration
         m_R( mMax, m_gamma), m_mMax( mMax)
     {}
 
-//mMax worth trying something between 3 and 10
-//AAstart 0
-//damping Fixed Point damping
-//restart: restart the iteration
+    /*! @brief Allocate memory for the object
+     *
+     * @param copyable A ContainerType must be copy-constructible from this
+     * @param mMax The maximum number of vectors to include in the optimization procedure. \c mMax+1 is the number
+     * of solutions involved in computing the new solution.
+     *  Something between 3 and 10 are good values but higher values mean more storage space that needs to be reserved.
+     *  If \c mMax==0 then the algorithm is equivalent to Fixed Point (or Richardson if the damping parameter is used in the \c solve() method) iteration i.e. no optimization and only 1 solution needed to compute a new solution.
+     */
+    void construct( const ContainerType& copyable, unsigned mMax)
+    {
+        *this = AndersonAcceleration(copyable, mMax);
+    }
+    const ContainerType& copyable() const{ return m_gval;}
+
     /*!@brief Solve the system \f$ f(x) = b \f$ in the given norm
      *
+     * @param f The function \c y=f(x) in the form \c f(x,y). The first argument is the input and the second the output.
+     * @param x Contains an initial guess on input and the solution on output.
+     * @param b The right hand side vector.
+     * @param weights The weights define the norm for the stopping condition of the solver
+     * @param rtol Relative error condition with respect to \c b
+     * @param atol Absolute error condition
+     * @param max_iter Maxmimum number of iterations
+     * @param damping Paramter to prevent too large jumps around the actual solution. Hard to determine in general but values between 0.1 and 1e-3 are good values to begin with. This is the parameter that appears in Richardson iteration.
+     * @param restart Number >= 1 that indicates after how many iterations to restart the acceleration. Periodic restarts are important for this method.  Per default it should be the same value as \c mMax but \c mMax+1 or higher could also be valuable to consider.
+     * @param verbose If true writes intermediate errors to \c std::cout . Avoid in MPI code.
+     * @return Number of iterations used to achieve desired precision
      */
     template<class BinarySubroutine>
     unsigned solve( BinarySubroutine& f, ContainerType& x, const ContainerType& b, const ContainerType& weights,
-        value_type rtol, value_type atol, value_type beta, unsigned AAstart, unsigned max_iter,
+        value_type rtol, value_type atol, unsigned max_iter,
         value_type damping, unsigned restart, bool verbose);
 
     private:
@@ -88,8 +111,8 @@ template<class ContainerType>
 template<class BinarySubroutine>
 unsigned AndersonAcceleration<ContainerType>::solve(
     BinarySubroutine& func, ContainerType& x, const ContainerType& b, const ContainerType& weights,
-    value_type rtol, value_type atol, value_type beta, unsigned AAstart, unsigned max_iter,
-    value_type damping, unsigned restart, bool verbose )
+    value_type rtol, value_type atol, unsigned max_iter,
+    value_type damping, unsigned restart,  bool verbose )
 {
     if (m_mMax == 0){
         if(verbose)std::cout<< "No acceleration will occur" << std::endl;
@@ -100,13 +123,13 @@ unsigned AndersonAcceleration<ContainerType>::solve(
     value_type tol = atol+rtol*nrmb;
     if(verbose)std::cout << "tol = " << tol << std::endl;
 
-    for(unsigned iter=0;iter < max_iter-1; iter++)
+    for(unsigned iter=0;iter < max_iter; iter++)
     {
 
-        // Restart if a certain number of iterations are reached. Does not need to be mMax... Just seems to work nicely right now.
-        if (iter % restart == 0) {
-            if(verbose)std::cout << "Iter = " << iter << std::endl;
+        // Restart from mAA=1 (note that it's incremented further down) if a certain number of iterations are reached.
+        if (iter % (restart) == 0) {
             mAA = 0;
+            if(verbose)std::cout << "Iter = " << iter << std::endl;
         }
 
         func( x, m_fval);
@@ -130,7 +153,7 @@ unsigned AndersonAcceleration<ContainerType>::solve(
         } else {
             // Apply Anderson acceleration.
 
-            if(iter > AAstart){                                         // Update the m_df vector and the m_DG array.t,
+            if(iter > 0){                                         // Update the m_df vector and the m_DG array.t,
 
                 dg::blas1::axpby(1.,m_fval,-1.,m_f_old, m_df);                 //m_df = m_fval-m_f_old;
 
@@ -151,7 +174,7 @@ unsigned AndersonAcceleration<ContainerType>::solve(
 
             dg::blas1::copy(m_gval,m_g_old);                                //m_g_old = m_gval;
 
-            if(mAA==0.){
+            if(mAA==0){ //only the very first iteration
 
                 dg::blas1::copy(m_gval,x);                                // If mAA == 0, update x <- g(x) to obtain the next approximate solution.
 
@@ -200,21 +223,6 @@ unsigned AndersonAcceleration<ContainerType>::solve(
                     dg::blas1::axpby(-m_gamma[i],m_DG[i],1.,x);
                 }
 
-                //In the paper there is a damping terms included.
-                if ((fabs(beta) > 0) && (beta <= 1)){
-                    for(unsigned i = 0; i < mAA; i++) {
-                        m_Ry[i] = 0;
-                        for(unsigned j = i; j < mAA; j++) {
-                            m_Ry[i] += m_R[i][j]*m_gamma[j];                  //Check correctness
-                        }
-                    }
-                    dg::blas1::axpby(-(1.-beta),m_fval,1,x);              //x = x -(1-beta)*m_fval
-                    for(unsigned i = 0; i < mAA; i++) {
-                        dg::blas1::axpby((1.0-beta)*m_Ry[i],m_Q[i],1.,x);   // x = x - (1-beta)*(-1*Q*R*gamma) = x + (1-beta)*Q*R*gamma = x + (1-beta)*Q*Ry
-                    }
-                }
-                else
-                    throw Error ( Message(_ping_)<< "Insane value for beta: "<<beta<<" should be 0<beta<=1. 1 means no damping!");
             }//Should all mAA
         }
 
diff --git a/inc/dg/andersonacc_t.cu b/inc/dg/andersonacc_t.cu
index 7f3bb87d2..f90f7ca28 100644
--- a/inc/dg/andersonacc_t.cu
+++ b/inc/dg/andersonacc_t.cu
@@ -25,23 +25,27 @@ int main()
     dg::HVec x = dg::evaluate( initial, grid);
     unsigned max_iter = n*n*Nx*Ny;
     const dg::HVec& copyable_vector = x;
+    double damping;
+    unsigned restart;
+    std::cout << "Type damping (1e-6) and restart (10) \n";
+    std::cin >> damping >> restart;
 
 //! [doxygen]
-    // create volume and inverse volume on previously defined grid
+    // create volume on previously defined grid
     const dg::HVec w2d = dg::create::weights( grid);
 
-    // Create unnormalized Laplacian
+    // Create normalized Laplacian
     dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> A( grid, dg::normed);
 
-    // allocate memory in conjugate gradient
-    dg::AndersonAcceleration<dg::HVec > acc( copyable_vector, 10);
+    // allocate memory
+    dg::AndersonAcceleration<dg::HVec > acc( copyable_vector, 3);
 
     // Evaluate right hand side and solution on the grid
     dg::HVec b = dg::evaluate ( laplace_fct, grid);
     const dg::HVec solution = dg::evaluate ( fct, grid);
 
     const double eps = 1e-6;
-    std::cout << "Number of iterations "<< acc.solve( A, x, b, w2d, eps, eps, 1, 0, max_iter, 1e-6, 10, true)<<std::endl;
+    std::cout << "Number of iterations "<< acc.solve( A, x, b, w2d, eps, eps, max_iter, damping, restart, true)<<std::endl;
 //! [doxygen]
     std::cout << "For a precision of "<< eps<<std::endl;
     //compute error
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 61801ed78..1539c992f 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -288,7 +288,6 @@ class ChebyshevIteration
      * @brief Allocate memory for the pcg method
      *
      * @param copyable A ContainerType must be copy-constructible from this
-     * @param max_iterations Maximum number of iterations to be used
      */
     void construct( const ContainerType& copyable) {
         m_x1 = m_r = copyable;
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 96f0efe24..6170b1b14 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -48,7 +48,7 @@
  * These algorithms make use only of blas level 1 and 2 functions
  * @{
  *     @defgroup time Time integrators
- *     @defgroup invert Matrix inversion
+ *     @defgroup invert Linear and nonlinear solvers
  *     @defgroup root Root finding
  * @}
  * @defgroup geo Level 3: Topology and Geometry
diff --git a/inc/dg/implicit.h b/inc/dg/implicit.h
index e377df19b..9021753ec 100644
--- a/inc/dg/implicit.h
+++ b/inc/dg/implicit.h
@@ -180,7 +180,7 @@ struct FixedPointSolver
 #ifdef MPI_VERSION
         if(rank==0)
 #endif//MPI
-        std::cout << "# of iterations time solver: "<<number<<"/"<<m_max_iter<<" took "<<ti.diff()<<"s\n";
+        std::cout << "# of iterations Fixed Point time solver: "<<number<<"/"<<m_max_iter<<" took "<<ti.diff()<<"s\n";
 #endif //DG_BENCHMARK
     }
     private:
@@ -207,15 +207,21 @@ struct AndersonSolver
     * @param copyable vector of the size that is later used in \c solve (
      it does not matter what values \c copyable contains, but its size is important;
      the \c solve method can only be called with vectors of the same size)
-    * @param max_iter maimum iteration number in cg
-    * @param eps accuracy parameter for cg
-    */
-    AndersonSolver( const ContainerType& copyable, unsigned mMax, value_type eps):
-        m_acc(copyable, mMax), m_rhs( copyable), m_eps(eps)
+     * @param mMax \c mMax+1 is the maximum number of vectors to include in the optimization procedure.
+     *  Something between 3 and 10 are good values but higher values mean more storage space that needs to be reserved.
+     *  If \c mMax==0 then the algorithm is equivalent to Fixed Point (or Richardson if the damping parameter is used in the \c solver method) iteration
+    * @param eps accuracy parameter for (rtol=atol=eps)
+    * @param max_iter maximum iteration number
+     * @param damping Paramter to prevent too large jumps around the actual solution. Hard to determine in general but values between 0.1 and 1e-3 are good values to begin with. This is the parameter that appears in Richardson iteration.
+     * @param restart Number >= 1 that indicates after how many iterations to restart the acceleration. Periodic restarts are important for this method.  Per default it should be the same value as \c mMax but \c mMax+1 or higher could also be valuable to consider.
+     */
+    AndersonSolver( const ContainerType& copyable, unsigned mMax, value_type eps, unsigned max_iter,
+        value_type damping, unsigned restart):
+        m_acc(copyable, mMax), m_eps(eps), m_damp(damping), m_max(max_iter), m_restart(restart)
         {}
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
-    const ContainerType& copyable()const{ return m_rhs;}
+    const ContainerType& copyable()const{ return m_acc.copyable();}
 
     template< class Implicit> //going to be a reference type
     void solve( value_type alpha, Implicit im, value_type t, ContainerType& y, const ContainerType& rhs)
@@ -228,19 +234,19 @@ struct AndersonSolver
 #endif//MPI
         Timer ti;
         ti.tic();
-        unsigned number = m_acc( implicit, y, m_rhs, im.precond(), im.inv_weights(), m_eps);
+        unsigned number = m_acc.solve( implicit, y, rhs, im.weights(), m_eps, m_eps, m_max, m_damp, m_restart, false);
         ti.toc();
 #ifdef MPI_VERSION
         if(rank==0)
 #endif//MPI
-        std::cout << "# of pcg iterations time solver: "<<number<<"/"<<m_pcg.get_max()<<" took "<<ti.diff()<<"s\n";
+        std::cout << "# of Anderson iterations time solver: "<<number<<"/"<<m_max<<" took "<<ti.diff()<<"s\n";
 #else
-        m_pcg( implicit, y, m_rhs, im.precond(), im.inv_weights(), m_eps);
+        m_acc.solve( implicit, y, rhs, im.weights(), m_eps, m_eps, m_max, m_damp, m_restart, false);
 #endif //DG_BENCHMARK
     }
     private:
     AndersonAcceleration< ContainerType> m_acc;
-    ContainerType m_rhs;
-    value_type m_eps;
+    value_type m_eps, m_damp;
+    unsigned m_max, m_restart;
 };
 }//namespace dg
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index dc31f30ba..99a4b4c29 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -103,13 +103,13 @@ struct AdamsBashforth
     * @brief Advance u0 one timestep
     *
     * @copydoc hide_rhs
-    * @param f right hand side function or functor
+    * @param rhs right hand side function or functor
     * @param t (write-only) contains timestep corresponding to \c u on output
     * @param u (write-only) contains next step of the integration on output
     * @note the implementation is such that on output the last call to the rhs is at the new (t,u). This might be interesting if the call to the rhs changes its state.
     */
     template< class RHS>
-    void step( RHS& f, value_type& t, ContainerType& u);
+    void step( RHS& rhs, value_type& t, ContainerType& u);
   private:
     value_type m_tu, m_dt;
     std::vector<ContainerType> m_f;
@@ -372,7 +372,7 @@ struct BDF
 
     ///@copydoc AdamsBashforth::step()
     template<class RHS>
-    void step(RHS& rhs, value_type& t0, container_type& u0);
+    void step(RHS& rhs, value_type& t, container_type& u);
     private:
     void init_coeffs(unsigned order){
         switch (order){
@@ -381,7 +381,7 @@ struct BDF
             case 3: m_bdf = { 18./11., -9./11., 2./11.}; m_beta = 6./11.; break;
             case 4: m_bdf = {48./25., -36./25., 16./25., -3./25.}; m_beta = 12./25.; break;
             case 5: m_bdf = { 300./137., -300./137., 200./137., -75./137., 12./137.}; m_beta = 60/137.; break;
-            case 6: m_bdf = { 360./147., -450./147., 400./147., -225./147., 172./147., -10./147.}; m_beta = 60/147.; break;
+            case 6: m_bdf = { 360./147., -450./147., 400./147., -225./147., 72./147., -10./147.}; m_beta = 60/147.; break;
             default: throw dg::Error(dg::Message()<<"Order not implemented in BDF!");
         }
     }
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 8885c6bc1..c98e7d6fc 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -139,10 +139,11 @@ int main()
         std::cout << "Relative error AB "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
     }
     std::cout << "### Test implicit multistep methods with "<<NT<<" steps\n";
-    for( unsigned s=1; s<6; s++)
+    for( unsigned s=1; s<7; s++)
     {
         time = 0., y0 = init;
-        dg::BDF< dg::DVec, dg::FixedPointSolver<dg::DVec> > bdf( s, y0, 100, 1e-10);
+        dg::BDF< dg::DVec, dg::AndersonSolver<dg::DVec> > bdf( s, y0, 0, 1e-10, 100, 1, 1);
+        //dg::BDF< dg::DVec, dg::FixedPointSolver<dg::DVec> > bdf( s, y0, 10, 1e-10);
         bdf.init( full, time, y0, dt);
         //main time loop
         for( unsigned k=0; k<NT; k++)
diff --git a/inc/dg/simpsons.h b/inc/dg/simpsons.h
index af81d80c2..2c21af990 100644
--- a/inc/dg/simpsons.h
+++ b/inc/dg/simpsons.h
@@ -41,7 +41,7 @@ struct Simpsons
     {
         set_order(order);
     }
-    ///@copydoc SimpsonsRule(unsigned)
+    ///@copydoc Simpsons(unsigned)
     void set_order( unsigned order){
         m_order=order;
         m_t.resize( order-1);
-- 
GitLab


From 77bc3b7ed0e1359332da0bbf2d8304d95e53eebb Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 10 Sep 2019 18:27:33 +0200
Subject: [PATCH 127/540] Update AndersonAcc using blas1::dot for Minimization

- and making weights a template paramter
- fixing typo in equation
- giving access to solver
---
 README.adoc             |  6 +++---
 inc/dg/andersonacc.h    | 16 ++++++++--------
 inc/dg/andersonacc_t.cu |  2 +-
 inc/dg/multistep.h      |  7 ++++++-
 inc/dg/runge_kutta.h    |  5 +++++
 5 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/README.adoc b/README.adoc
index 650421a28..4bfe5fc22 100644
--- a/README.adoc
+++ b/README.adoc
@@ -436,7 +436,7 @@ compile the tex file(s) of the documentation.
 
 The
 http://feltor-dev.github.io/doc/dg/html/modules.html[documentation]
-of the dG library was generated with
+of the dg library was generated with
 http://www.doxygen.org[Doxygen] and LateX. You can generate a local
 version including informative pdf writeups on implemented numerical
 methods directly from source code. This depends on the `doxygen`,
@@ -455,8 +455,8 @@ For details on how FELTOR's internal Makefiles are configured please see the lin
 
 == 3. Authors, Acknowledgements, Contributions
 
-FELTOR has been developed by Matthias Wiesenberger and Markus Held. Please see the list of https://feltor-dev.github.io/about[contributors]
-and funding.
+FELTOR has been developed by Matthias Wiesenberger and Markus Held. Please see the https://feltor-dev.github.io/about/[Acknowledgements] section on our homepage
+for a list of contributors and funding.
 Also check out our https://feltor-dev.github.io[homepage]
 for general information, wiki pages,
 troubleshooting and guides on how to contribute.
diff --git a/inc/dg/andersonacc.h b/inc/dg/andersonacc.h
index 3bb10506d..0bd3b8d38 100644
--- a/inc/dg/andersonacc.h
+++ b/inc/dg/andersonacc.h
@@ -92,8 +92,8 @@ struct AndersonAcceleration
      * @param verbose If true writes intermediate errors to \c std::cout . Avoid in MPI code.
      * @return Number of iterations used to achieve desired precision
      */
-    template<class BinarySubroutine>
-    unsigned solve( BinarySubroutine& f, ContainerType& x, const ContainerType& b, const ContainerType& weights,
+    template<class BinarySubroutine, class ContainerType2>
+    unsigned solve( BinarySubroutine& f, ContainerType& x, const ContainerType& b, const ContainerType2& weights,
         value_type rtol, value_type atol, unsigned max_iter,
         value_type damping, unsigned restart, bool verbose);
 
@@ -108,9 +108,9 @@ struct AndersonAcceleration
 ///@cond
 
 template<class ContainerType>
-template<class BinarySubroutine>
+template<class BinarySubroutine, class ContainerType2>
 unsigned AndersonAcceleration<ContainerType>::solve(
-    BinarySubroutine& func, ContainerType& x, const ContainerType& b, const ContainerType& weights,
+    BinarySubroutine& func, ContainerType& x, const ContainerType& b, const ContainerType2& weights,
     value_type rtol, value_type atol, unsigned max_iter,
     value_type damping, unsigned restart,  bool verbose )
 {
@@ -182,7 +182,7 @@ unsigned AndersonAcceleration<ContainerType>::solve(
 
                 if (mAA == 1) {                                         // If mAA == 1, form the initial QR decomposition.
 
-                    m_R[0][0] = sqrt(dg::blas2::dot(m_df,weights, m_df));
+                    m_R[0][0] = sqrt(dg::blas1::dot(m_df, m_df));
                     dg::blas1::axpby(1./m_R[0][0],m_df,0.,m_Q[0]);
 
                 } else {                                                // If mAA > 1, update the QR decomposition.
@@ -196,11 +196,11 @@ unsigned AndersonAcceleration<ContainerType>::solve(
                     }
                     // Now update the QR decomposition to incorporate the new column.
                     for (unsigned j = 1; j < mAA; j++) {
-                        m_R[j-1][mAA-1] = dg::blas2::dot(m_Q[j-1],weights,m_df);      //Q(:,j)’*m_df; //Changed mAA -> mAA-1
+                        m_R[j-1][mAA-1] = dg::blas1::dot(m_Q[j-1],m_df);      //Q(:,j)’*m_df; //Changed mAA -> mAA-1
 
                         dg::blas1::axpby(-m_R[j-1][mAA-1],m_Q[j-1],1.,m_df);  //m_df = m_df - R(j,mAA)*Q(:,j);
                     }
-                    m_R[mAA-1][mAA-1] = sqrt(dg::blas2::dot(m_df,weights,m_df));
+                    m_R[mAA-1][mAA-1] = sqrt(dg::blas1::dot(m_df,m_df));
                     dg::blas1::axpby(1./m_R[mAA-1][mAA-1],m_df,0.,m_Q[mAA-1]);
                 }
 
@@ -210,7 +210,7 @@ unsigned AndersonAcceleration<ContainerType>::solve(
 
                 //Solve least squares problem.
                 for(int i = (int)mAA-1; i>=0; i--){
-                    m_gamma[i] = dg::blas2::dot(m_Q[i],weights,m_fval);
+                    m_gamma[i] = dg::blas1::dot(m_Q[i],m_fval);
                     for(int j = i + 1; j < (int)mAA; j++){
                         m_gamma[i] -= m_R[i][j]*m_gamma[j];
                     }
diff --git a/inc/dg/andersonacc_t.cu b/inc/dg/andersonacc_t.cu
index f90f7ca28..238081e5c 100644
--- a/inc/dg/andersonacc_t.cu
+++ b/inc/dg/andersonacc_t.cu
@@ -27,7 +27,7 @@ int main()
     const dg::HVec& copyable_vector = x;
     double damping;
     unsigned restart;
-    std::cout << "Type damping (1e-6) and restart (10) \n";
+    std::cout << "Type damping (1e-3) and restart (10) \n";
     std::cin >> damping >> restart;
 
 //! [doxygen]
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 99a4b4c29..b1d81524b 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -8,6 +8,7 @@
   */
 namespace dg{
 
+//MW: if ever we want to change the SolverType at runtime (with an input parameter e.g.) make it a new parameter in the solve method (either abstract type or template like RHS)
 
 /*! @class hide_explicit_implicit
  * @tparam Explicit The explicit part of the right hand side
@@ -311,7 +312,7 @@ void Karniadakis<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, valu
 * @brief Struct for Backward differentiation formula implicit multistep time-integration
 * \f[
 * \begin{align}
-    v^{n+1} = \sum_{q=0}^{s-1} \alpha_q v^{n-q} + \Delta t\beta\hat I(t^{n}+\Delta t, v^{n+1})\right]
+    v^{n+1} = \sum_{q=0}^{s-1} \alpha_q v^{n-q} + \Delta t\beta\hat I(t^{n}+\Delta t, v^{n+1})
     \end{align}
     \f]
 
@@ -365,6 +366,10 @@ struct BDF
         m_u.assign( order, m_solver.copyable());
         m_f = m_solver.copyable();
     }
+    ///Write access to the internal solver for the implicit part
+    SolverType& solver() { return m_solver;}
+    ///Read access to the internal solver for the implicit part
+    const SolverType& solver() const { return m_solver;}
 
     ///@copydoc AdamsBashforth::init()
     template<class RHS>
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index b9acd2ca1..ddda8b199 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -233,6 +233,7 @@ void ERKStep<ContainerType>::step( RHS& f, value_type t0, const ContainerType& u
 }
 
 
+//MW: if ever we want to change the SolverType at runtime (with an input parameter e.g.) make it a new parameter in the solve method (either abstract type or template like RHS)
 
 
 /*!
@@ -329,6 +330,10 @@ struct ARKStep
         m_kE.assign(m_rkE.num_stages(), m_rhs);
         m_kI.assign(m_rkI.num_stages(), m_rhs);
     }
+    ///Write access to the internal solver for the implicit part
+    SolverType& solver() { return m_solver;}
+    ///Read access to the internal solver for the implicit part
+    const SolverType& solver() const { return m_solver;}
 
     /**
     * @brief Advance one step
-- 
GitLab


From d13f9204513006cd1076679a98927dba655da7e9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 10 Sep 2019 23:12:52 +0200
Subject: [PATCH 128/540] Re-implement feltors serialized netcdf in file
 library

- new file file/easy_output.h
- change names and write documentation
---
 inc/dg/multigrid.h         |   2 +-
 inc/file/Doxyfile          |   2 +-
 inc/file/easy_output.h     | 372 +++++++++++++++++++++++++++++++++++++
 inc/file/nc_utilities.h    |  60 +-----
 src/feltor/feltor.tex      |   3 +-
 src/feltor/feltor_hpc.cu   |  45 +++--
 src/feltor/feltordiag.h    |  18 ++
 src/feltor/serial_netcdf.h | 167 -----------------
 8 files changed, 429 insertions(+), 240 deletions(-)
 create mode 100644 inc/file/easy_output.h
 delete mode 100644 src/feltor/serial_netcdf.h

diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 9c75597c8..c4ea2a0e0 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -264,7 +264,7 @@ struct MultigridCG2d
         //
         // gamma:  typically 1 (V-cycle) or 2 (W-cycle)
         // nu1, nu2: typically in {0,1,2,3}
-        double lmin = 0, lmax = 1; //determine lmin and lmax
+        double lmax = 1; //determine lmin and lmax
 
         // 1. Pre-Smooth nu1 times
         m_cheby[p].solve( op[p], x[p], b[p], 0.1*lmax, 1.1*lmax, nu1);
diff --git a/inc/file/Doxyfile b/inc/file/Doxyfile
index 95bbaf5db..65bbf86dd 100644
--- a/inc/file/Doxyfile
+++ b/inc/file/Doxyfile
@@ -2047,7 +2047,7 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             =
+PREDEFINED             = MPI_VERSION
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The
diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
new file mode 100644
index 000000000..a92cf05a9
--- /dev/null
+++ b/inc/file/easy_output.h
@@ -0,0 +1,372 @@
+#pragma once
+
+#include <exception>
+#include <netcdf.h>
+#include "dg/topology/grid.h"
+#ifdef MPI_VERSION
+#include "dg/backend/mpi_vector.h"
+#include "dg/topology/mpi_grid.h"
+#endif //MPI_VERSION
+
+namespace file
+{
+/**
+ * @brief Class thrown by the NC_ErrorClonePtr
+ */
+struct NC_Error : public std::exception
+{
+
+    /**
+     * @brief Construct from error code
+     *
+     * @param error netcdf error code
+     */
+    NC_Error( int error): error_( error) {}
+    /**
+     * @brief What string
+     *
+     * @return string netcdf error message generated from error code
+     */
+    char const* what() const throw(){
+        return nc_strerror(error_);}
+  private:
+    int error_;
+};
+
+/**
+ * @brief Empty utitlity class that handles return values of netcdf
+ * functions and throws NC_Error if something goes wrong
+ */
+struct NC_Error_Handle
+{
+    /**
+     * @brief Construct from error code
+     *
+     * throws an NC_Error if err is not a success
+     * @param err netcdf error code
+     *
+     * @return Newly instantiated object
+     */
+    NC_Error_Handle operator=( int err)
+    {
+        NC_Error_Handle h;
+        return h(err);
+    }
+    /**
+     * @brief Construct from error code
+     *
+     * throws an NC_Error if err is not a success
+     * @param err netcdf error code
+     *
+     * @return Newly instantiated object
+     */
+    NC_Error_Handle operator()( int err)
+    {
+        if( err)
+            throw NC_Error( err);
+        return *this;
+    }
+};
+
+/**
+* @brief Convenience wrapper around \c nc_put_vara_double()
+*
+* This version is for a time-independent variable,
+* i.e. writes a single variable in one go and is actually equivalent
+* to \c nc_put_var_double. The dimensionality is indicated by the name.
+* @tparam host_vector Type with \c data() member that returns pointer to first element in CPU (host) adress space, meaning it cannot be a GPU vector
+* @param ncid Forwarded to \c nc_put_vara_double
+* @param varid  Forwarded to \c nc_put_vara_double
+* @param toWrite data is forwarded to \c nc_put_vara_double, may be destroyed on output in the mpi version.
+* @param grid The grid from which to construct \c start and \c count variables to forward to \c nc_put_vara_double
+* @param parallel This parameter is ignored in the serial version.
+* In the MPI version this parameter indicates whether each process
+* writes to the file independently in parallel (\c true)
+* or each process funnels its data through the master rank (\c false).
+* @attention In the MPI version, if \c parallel==true a **parallel netcdf** must be
+* linked while if \c parallel==false we need **serial netcdf**.
+* Note that serious performance penalties have been observed on some platforms for parallel netcdf.
+*/
+template<class host_vector>
+void write_static2d(int ncid, int varid, host_vector& toWrite,
+    dg::aTopology2d& grid, bool parallel = false)
+{
+    file::NC_Error_Handle err;
+    size_t start[2] = {0,0}, count[2];
+    count[0] = grid.n()*grid.Ny();
+    count[1] = grid.n()*grid.Nx();
+    err = nc_put_vara_double( ncid, varid, start, count, toWrite.data());
+}
+
+/**
+* @brief Convenience wrapper around \c nc_put_vara_double()
+*
+* This version is for a time-dependent variable,
+* i.e. writes a single time-slice into the file.
+* The dimensionality is indicated by the name.
+* @tparam host_vector Type with \c data() member that returns pointer to first element in CPU (host) adress space, meaning it cannot be a GPU vector
+* @param ncid Forwarded to \c nc_put_vara_double
+* @param varid  Forwarded to \c nc_put_vara_double
+* @param slice The number of the time-slice to write (first element of the \c startp array in \c nc_put_vara_double)
+* @param toWrite data is forwarded to \c nc_put_vara_double, may be destroyed on output in the mpi version.
+* @param grid The grid from which to construct \c start and \c count variables to forward to \c nc_put_vara_double
+* @param parallel This parameter is ignored in the serial version.
+* In the MPI version this parameter indicates whether each process
+* writes to the file independently in parallel (\c true)
+* or each process funnels its data through the master rank (\c false).
+* @attention In the MPI version, if \c parallel==true a **parallel netcdf** must be
+* linked while if \c parallel==false we need **serial netcdf**.
+* Note that serious performance penalties have been observed on some platforms for parallel netcdf.
+*/
+template<class host_vector>
+void write_dynamic2d(int ncid, int varid, unsigned slice, host_vector& toWrite,
+    dg::aTopology2d& grid, bool parallel = false)
+{
+    file::NC_Error_Handle err;
+    size_t start[3] = {slice,0,0}, count[3];
+    count[0] = 1;
+    count[1] = grid.n()*grid.Ny();
+    count[2] = grid.n()*grid.Nx();
+    err = nc_put_vara_double( ncid, varid, start, count,
+        toWrite.data());
+}
+
+///@copydoc write_static2d()
+template<class host_vector>
+void write_static3d(int ncid, int varid, host_vector& toWrite, dg::aTopology3d& grid, bool parallel = false)
+{
+    file::NC_Error_Handle err;
+    size_t start[3] = {0,0,0}, count[3];
+    count[0] = grid.Nz();
+    count[1] = grid.n()*grid.Ny();
+    count[2] = grid.n()*grid.Nx();
+    err = nc_put_vara_double( ncid, varid, start, count,
+        toWrite.data());
+}
+
+///@copydoc write_dynamic2d()
+template<class host_vector>
+void write_dynamic3d(int ncid, int varid, unsigned slice, host_vector& toWrite,
+    dg::aTopology3d& grid, bool parallel = false)
+{
+    file::NC_Error_Handle err;
+    size_t start[3] = {slice, 0,0,0}, count[4];
+    count[0] = 1;
+    count[1] = grid.Nz();
+    count[2] = grid.n()*grid.Ny();
+    count[3] = grid.n()*grid.Nx();
+    err = nc_put_vara_double( ncid, varid, start, count,
+        toWrite.data());
+}
+
+#ifdef MPI_VERSION
+///@copydoc write_static2d()
+template<class host_vector>
+void write_static2d(int ncid, int varid,
+    dg::MPI_Vector<host_vector>& toWrite,
+    dg::aMPITopology3d& grid, bool parallel = false)
+{
+    file::NC_Error_Handle err;
+    size_t start[3] = {0,0}, count[2];
+    count[0] = grid.n()*grid.local().Ny();
+    count[1] = grid.n()*grid.local().Nx();
+    int rank, size;
+    MPI_Comm comm = grid.communicator();
+    MPI_Comm_rank( comm, &rank);
+    MPI_Comm_size( comm, &size);
+    if( !parallel)
+    {
+        MPI_Status status;
+        size_t local_size = grid.local().size();
+        std::vector<int> coords( size*2);
+        for( int rrank=0; rrank<size; rrank++)
+            MPI_Cart_coords( comm, rrank, 2, &coords[2*rrank]);
+        if(rank==0)
+        {
+            for( int rrank=0; rrank<size; rrank++)
+            {
+                if(rrank!=0)
+                    MPI_Recv( toWrite.data().data(), local_size, MPI_DOUBLE,
+                          rrank, rrank, comm, &status);
+                start[0] = coords[2*rrank+1]*count[0],
+                start[1] = coords[2*rrank+0]*count[1],
+                err = nc_put_vara_double( ncid, varid, start, count,
+                    toWrite.data().data());
+            }
+        }
+        else
+            MPI_Send( toWrite.data().data(), local_size, MPI_DOUBLE,
+                      0, rank, comm);
+        MPI_Barrier( comm);
+    }
+    else
+    {
+        int coords[2];
+        MPI_Cart_coords( comm, rank, 2, coords);
+        start[0] = coords[1]*count[0],
+        start[1] = coords[0]*count[1],
+        err = nc_put_vara_double( ncid, varid, start, count,
+            toWrite.data().data());
+    }
+}
+
+///@copydoc write_dynamic2d()
+template<class host_vector>
+void write_dynamic2d(int ncid, int varid, unsigned slice,
+    dg::MPI_Vector<host_vector>& toWrite,
+    dg::aMPITopology2d& grid, bool parallel = false)
+{
+    file::NC_Error_Handle err;
+    size_t start[3] = {slice, 0,0}, count[3];
+    count[0] = 1;
+    count[1] = grid.n()*grid.local().Ny();
+    count[2] = grid.n()*grid.local().Nx();
+    int rank, size;
+    MPI_Comm comm = grid.communicator();
+    MPI_Comm_rank( comm, &rank);
+    MPI_Comm_size( comm, &size);
+    if( parallel)
+    {
+        int coords[2];
+        MPI_Cart_coords( comm, rank, 2, coords);
+        start[1] = coords[1]*count[1],
+        start[2] = coords[0]*count[2],
+        err = nc_put_vara_double( ncid, varid, start, count,
+            toWrite.data().data());
+    }
+    else
+    {
+        MPI_Status status;
+        size_t local_size = grid.local().size();
+        std::vector<int> coords( size*2);
+        for( int rrank=0; rrank<size; rrank++)
+            MPI_Cart_coords( comm, rrank, 2, &coords[2*rrank]);
+        if(rank==0)
+        {
+            for( int rrank=0; rrank<size; rrank++)
+            {
+                if(rrank!=0)
+                    MPI_Recv( toWrite.data().data(), local_size, MPI_DOUBLE,
+                          rrank, rrank, comm, &status);
+                start[1] = coords[2*rrank+1]*count[1],
+                start[2] = coords[2*rrank+0]*count[2],
+                err = nc_put_vara_double( ncid, varid, start, count,
+                    toWrite.data().data());
+            }
+        }
+        else
+            MPI_Send( toWrite.data().data(), local_size, MPI_DOUBLE,
+                      0, rank, comm);
+        MPI_Barrier( comm);
+    }
+}
+
+///@copydoc write_static2d()
+template<class host_vector>
+void write_static3d(int ncid, int varid,
+    dg::MPI_Vector<host_vector>& toWrite,
+    dg::aMPITopology3d& grid, bool parallel = false)
+{
+    file::NC_Error_Handle err;
+    size_t start[3] = {0,0,0}, count[3];
+    count[0] = grid.local().Nz();
+    count[1] = grid.n()*grid.local().Ny();
+    count[2] = grid.n()*grid.local().Nx();
+    int rank, size;
+    MPI_Comm comm = grid.communicator();
+    MPI_Comm_rank( comm, &rank);
+    MPI_Comm_size( comm, &size);
+    if( !parallel)
+    {
+        MPI_Status status;
+        size_t local_size = grid.local().size();
+        std::vector<int> coords( size*3);
+        for( int rrank=0; rrank<size; rrank++)
+            MPI_Cart_coords( comm, rrank, 3, &coords[3*rrank]);
+        if(rank==0)
+        {
+            for( int rrank=0; rrank<size; rrank++)
+            {
+                if(rrank!=0)
+                    MPI_Recv( toWrite.data().data(), local_size, MPI_DOUBLE,
+                          rrank, rrank, comm, &status);
+                start[0] = coords[3*rrank+2]*count[0],
+                start[1] = coords[3*rrank+1]*count[1],
+                start[2] = coords[3*rrank+0]*count[2];
+                err = nc_put_vara_double( ncid, varid, start, count,
+                    toWrite.data().data());
+            }
+        }
+        else
+            MPI_Send( toWrite.data().data(), local_size, MPI_DOUBLE,
+                      0, rank, comm);
+        MPI_Barrier( comm);
+    }
+    else
+    {
+        int coords[3];
+        MPI_Cart_coords( comm, rank, 3, coords);
+        start[0] = coords[2]*count[0],
+        start[1] = coords[1]*count[1],
+        start[2] = coords[0]*count[2];
+        err = nc_put_vara_double( ncid, varid, start, count,
+            toWrite.data().data());
+    }
+}
+
+///@copydoc write_dynamic2d()
+template<class host_vector>
+void write_dynamic3d(int ncid, int varid, unsigned slice,
+    dg::MPI_Vector<host_vector>& toWrite,
+    dg::aMPITopology3d& grid, bool parallel = false)
+{
+    file::NC_Error_Handle err;
+    size_t start[4] = {slice, 0,0,0}, count[4];
+    count[0] = 1;
+    count[1] = grid.local().Nz();
+    count[2] = grid.n()*grid.local().Ny();
+    count[3] = grid.n()*grid.local().Nx();
+    int rank, size;
+    MPI_Comm comm = grid.communicator();
+    MPI_Comm_rank( comm, &rank);
+    MPI_Comm_size( comm, &size);
+    if( parallel)
+    {
+        int coords[3];
+        MPI_Cart_coords( comm, rank, 3, coords);
+        start[1] = coords[2]*count[1],
+        start[2] = coords[1]*count[2],
+        start[3] = coords[0]*count[3];
+        err = nc_put_vara_double( ncid, varid, start, count,
+            toWrite.data().data());
+    }
+    else
+    {
+        MPI_Status status;
+        size_t local_size = grid.local().size();
+        std::vector<int> coords( size*3);
+        for( int rrank=0; rrank<size; rrank++)
+            MPI_Cart_coords( comm, rrank, 3, &coords[3*rrank]);
+        if(rank==0)
+        {
+            for( int rrank=0; rrank<size; rrank++)
+            {
+                if(rrank!=0)
+                    MPI_Recv( toWrite.data().data(), local_size, MPI_DOUBLE,
+                          rrank, rrank, comm, &status);
+                start[1] = coords[3*rrank+2]*count[1],
+                start[2] = coords[3*rrank+1]*count[2],
+                start[3] = coords[3*rrank+0]*count[3];
+                err = nc_put_vara_double( ncid, varid, start, count,
+                    toWrite.data().data());
+            }
+        }
+        else
+            MPI_Send( toWrite.data().data(), local_size, MPI_DOUBLE,
+                      0, rank, comm);
+        MPI_Barrier( comm);
+    }
+}
+#endif //MPI_VERSION
+
+}//namespace file
diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index 4d25a85e1..6ec077d1e 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -3,13 +3,14 @@
 #pragma message( "The inclusion of file/nc_utilities.h is deprecated. Please use dg/file/nc_utilities.h")
 #endif //_INCLUDED_BY_DG_
 
-#include <exception>
 #include <netcdf.h>
 #include "thrust/host_vector.h"
 
 #include "dg/topology/grid.h"
 #include "dg/topology/evaluation.h"
 
+#include "easy_output.h"
+
 /*!@file
  *
  * Contains Error handling class and the define_dimensions functions
@@ -23,63 +24,6 @@
 namespace file
 {
 
-/**
- * @brief Class thrown by the NC_ErrorClonePtr
- */
-struct NC_Error : public std::exception
-{
-
-    /**
-     * @brief Construct from error code
-     *
-     * @param error netcdf error code
-     */
-    NC_Error( int error): error_( error) {}
-    /**
-     * @brief What string
-     *
-     * @return string netcdf error message generated from error code
-     */
-    char const* what() const throw(){
-        return nc_strerror(error_);}
-  private:
-    int error_;
-};
-
-/**
- * @brief Empty utitlity class that handles return values of netcdf
- * functions and throws NC_Error if something goes wrong
- */
-struct NC_Error_Handle
-{
-    /**
-     * @brief Construct from error code
-     *
-     * throws an NC_Error if err is not a success
-     * @param err netcdf error code
-     *
-     * @return Newly instantiated object
-     */
-    NC_Error_Handle operator=( int err)
-    {
-        NC_Error_Handle h;
-        return h(err);
-    }
-    /**
-     * @brief Construct from error code
-     *
-     * throws an NC_Error if err is not a success
-     * @param err netcdf error code
-     *
-     * @return Newly instantiated object
-     */
-    NC_Error_Handle operator()( int err)
-    {
-        if( err)
-            throw NC_Error( err);
-        return *this;
-    }
-};
 /**
  * @brief Define an unlimited time dimension and variable following
   <a href="http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html">CF-conventions</a>
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 4bfd30107..e15c8f9e6 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1031,9 +1031,10 @@ discontinuous Galerkin on structured grid
 \toprule
 \rowcolor{gray!50}\textbf{Term} &  \textbf{Method} & \textbf{Description}  \\ \midrule
     coordinate system & Cylindrical & equidistant discretization of $[R_{\min},R_{\max}] \times [Z_{\min},Z_{\max}] \times [0,2\pi]$ (Eq.~\eqref{eq:box}, equal number of Gaussian nodes in $R$ and $Z$, equidistant planes in $\varphi$ with one Gaussian node \\
+Advection terms & direct DG & DG approximation with centered flux of derivatives \\
+Elliptic terms & local DG & The local DG approximation with centered flux \\
 Helmholtz and Elliptic matrix inversions & multigrid/ conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
 Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017} \\
-Curvature terms & direct DG & regular DG approximation of derivatives \\
 time & Multistep "Karniadakis" & \\
 \qquad explicit & Multistep "Karniadakis" & $3$rd order explicit\\
 \qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion and Resistive terms. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index cc9c60e77..2bf125006 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -36,7 +36,6 @@ using Geometry = dg::CylindricalGrid3d;
 
 #include "init.h"
 #include "feltordiag.h"
-#include "serial_netcdf.h"
 
 #ifdef FELTOR_MPI
 //ATTENTION: in slurm should be used with --signal=SIGINT@30 (<signal>@<time in seconds>)
@@ -170,8 +169,6 @@ int main( int argc, char* argv[])
     HVec resultH = dg::evaluate( dg::zero, grid);
     DVec resultD = dg::evaluate( dg::zero, grid);
 
-    feltor::ManageOutput output(g3d_out);
-
     std::array<DVec, 3> gradPsip;
     gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
@@ -243,7 +240,7 @@ int main( int argc, char* argv[])
         MPI_OUT std::cout << "Computing "<<record.name<<"\n";
         record.function( resultH, var, grid, gp, mag);
         dg::blas2::symv( projectH, resultH, transferH);
-        output.output_static3d( ncid, vecID, transferH);
+        file::write_static3d( ncid, vecID, transferH, g3d_out);
         MPI_OUT err = nc_redef(ncid);
     }
 
@@ -299,7 +296,7 @@ int main( int argc, char* argv[])
         record.function( resultD, var);
         dg::blas2::symv( projectD, resultD, transferD);
         dg::assign( transferD, transferH);
-        output.output_dynamic3d( ncid, id4d.at(record.name), start, transferH);
+        file::write_dynamic3d( ncid, id4d.at(record.name), start, transferH, g3d_out);
     }
     for( auto& record : feltor::diagnostics2d_list)
     {
@@ -317,7 +314,11 @@ int main( int argc, char* argv[])
         tti.toc();
         MPI_OUT std::cout<< name << " Computing average took "<<tti.diff()<<"\n";
         tti.tic();
-        output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+#ifdef FELTOR_MPI
+        //only the globally first slice should write
+        if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
+#endif //FELTOR_MPI
+            file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
         tti.tic();
@@ -327,7 +328,11 @@ int main( int argc, char* argv[])
         feltor::slice_vector3d( transferD, transferD2d, local_size2d);
         dg::assign( transferD2d, transferH2d);
         if( record.integral) time_integrals[name].init( time, transferH2d);
-        output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+#ifdef FELTOR_MPI
+        //only the globally first slice should write
+        if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
+#endif //FELTOR_MPI
+            file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
     }
@@ -433,7 +438,7 @@ int main( int argc, char* argv[])
             record.function( resultD, var);
             dg::blas2::symv( projectD, resultD, transferD);
             dg::assign( transferD, transferH);
-            output.output_dynamic3d( ncid, id4d.at(record.name), start, transferH);
+            file::write_dynamic3d( ncid, id4d.at(record.name), start, transferH, g3d_out);
         }
         for( auto& record : feltor::diagnostics2d_list)
         {
@@ -442,12 +447,20 @@ int main( int argc, char* argv[])
                 std::string name = record.name+"_ta2d";
                 transferH2d = time_integrals.at(name).get_integral();
                 time_integrals.at(name).flush();
-                output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+#ifdef FELTOR_MPI
+                //only the globally first slice should write
+                if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
+#endif //FELTOR_MPI
+                    file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
 
                 name = record.name+"_2d";
                 transferH2d = time_integrals.at(name).get_integral( );
                 time_integrals.at(name).flush( );
-                output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+#ifdef FELTOR_MPI
+                //only the globally first slice should write
+                if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
+#endif //FELTOR_MPI
+                    file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
             }
             else //manage the time integrators
             {
@@ -457,13 +470,21 @@ int main( int argc, char* argv[])
                 std::string name = record.name+"_ta2d";
                 dg::assign( transferD, transferH);
                 toroidal_average( transferH, transferH2d, false);
-                output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+#ifdef FELTOR_MPI
+                //only the globally first slice should write
+                if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
+#endif //FELTOR_MPI
+                    file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
 
                 // 2d data of plane varphi = 0
                 name = record.name+"_2d";
                 feltor::slice_vector3d( transferD, transferD2d, local_size2d);
                 dg::assign( transferD2d, transferH2d);
-                output.output_dynamic2d_slice( ncid, id3d.at(name), start, transferH2d);
+#ifdef FELTOR_MPI
+                //only the globally first slice should write
+                if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
+#endif //FELTOR_MPI
+                    file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
             }
         }
         MPI_OUT err = nc_close(ncid);
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index ed16263e1..df9b583ba 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -719,4 +719,22 @@ std::vector<Record> diagnostics2d_list = {
 // These two lists signify the quantities for accuracy computation
 std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
 std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt"};
+
+template<class Container>
+void slice_vector3d( const Container& transfer, Container& transfer2d, size_t local_size2d)
+{
+#ifdef FELTOR_MPI
+    thrust::copy(
+        transfer.data().begin(),
+        transfer.data().begin() + local_size2d,
+        transfer2d.data().begin()
+    );
+#else
+    thrust::copy(
+        transfer.begin(),
+        transfer.begin() + local_size2d,
+        transfer2d.begin()
+    );
+#endif
+}
 }//namespace feltor
diff --git a/src/feltor/serial_netcdf.h b/src/feltor/serial_netcdf.h
deleted file mode 100644
index cce0c25e3..000000000
--- a/src/feltor/serial_netcdf.h
+++ /dev/null
@@ -1,167 +0,0 @@
-#pragma once
-
-namespace feltor
-{
-template<class Container>
-void slice_vector3d( const Container& transfer, Container& transfer2d, size_t local_size2d)
-{
-#ifdef FELTOR_MPI
-    thrust::copy(
-        transfer.data().begin(),
-        transfer.data().begin() + local_size2d,
-        transfer2d.data().begin()
-    );
-#else
-    thrust::copy(
-        transfer.begin(),
-        transfer.begin() + local_size2d,
-        transfer2d.begin()
-    );
-#endif
-}
-//Manages the communication for serial netcdf output
-struct ManageOutput
-{
-    ManageOutput( const Geometry& g3d_out)
-    {
-        std::unique_ptr<typename Geometry::perpendicular_grid> g2d_out_ptr  ( dynamic_cast<typename Geometry::perpendicular_grid*>( g3d_out.perp_grid()));
-#ifdef FELTOR_MPI
-        m_local_size3d = g3d_out.local().size();
-        m_local_size2d = g2d_out_ptr->local().size();
-        m_comm3d = g3d_out.communicator();
-        m_comm2d = g2d_out_ptr->communicator();
-        MPI_Comm_rank( m_comm2d, &m_rank2d);
-        MPI_Comm_size( m_comm2d, &m_size2d);
-        MPI_Comm_rank( m_comm3d, &m_rank3d);
-        MPI_Comm_size( m_comm3d, &m_size3d);
-
-        m_coords2d.resize(m_size2d*2);
-        m_coords.resize(m_size3d*3);
-        for( int rrank=0; rrank<m_size2d; rrank++)
-            MPI_Cart_coords( m_comm2d, rrank, 2, &m_coords2d[2*rrank]);
-        for( int rrank=0; rrank<m_size3d; rrank++)
-            MPI_Cart_coords( m_comm3d, rrank, 3, &m_coords[3*rrank]);
-        m_count4d[0] = 1;
-        m_count4d[1] = g3d_out.local().Nz();
-        m_count4d[2] = g3d_out.n()*(g3d_out.local().Ny());
-        m_count4d[3] = g3d_out.n()*(g3d_out.local().Nx());
-        m_count3d[0] = 1;
-        m_count3d[1] = g3d_out.n()*(g3d_out.local().Ny()),
-        m_count3d[2] = g3d_out.n()*(g3d_out.local().Nx());
-#else //FELTOR_MPI
-        m_count4d[0] = 1;
-        m_count4d[1] = g3d_out.Nz();
-        m_count4d[2] = g3d_out.n()*g3d_out.Ny();
-        m_count4d[3] = g3d_out.n()*g3d_out.Nx();
-        m_count3d[0] = 1;
-        m_count3d[1] = g3d_out.n()*g3d_out.Ny();
-        m_count3d[2] = g3d_out.n()*g3d_out.Nx();
-        m_local_size3d = g3d_out.size();
-        m_local_size2d = g2d_out_ptr->size();
-#endif //FELTOR_MPI
-    }
-    //must enddef first
-    void output_static3d(int ncid, int vecID, HVec& transferH) const
-    {
-        file::NC_Error_Handle err;
-        size_t start4d[4] = {0,0,0,0};
-#ifdef FELTOR_MPI
-        MPI_Status status;
-        if(m_rank3d==0)
-        {
-            for( int rrank=0; rrank<m_size3d; rrank++)
-            {
-                if(rrank!=0)
-                    MPI_Recv( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
-                          rrank, rrank, m_comm3d, &status);
-                start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
-                start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
-                start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
-                err = nc_put_vara_double( ncid, vecID, &start4d[1], &m_count4d[1],
-                    transferH.data().data());
-            }
-        }
-        else
-            MPI_Send( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
-                      0, m_rank3d, m_comm3d);
-        MPI_Barrier( m_comm3d);
-#else
-        err = nc_put_vara_double( ncid, vecID, &start4d[1], &m_count4d[1],
-            transferH.data());
-#endif // FELTOR_MPI
-    }
-    void output_dynamic3d(int ncid, int vecID, unsigned start, HVec& transferH) const
-    {
-        size_t start4d[4] = {start, 0, 0, 0};
-        file::NC_Error_Handle err;
-#ifdef FELTOR_MPI
-        MPI_Status status;
-        if(m_rank3d==0)
-        {
-            for( int rrank=0; rrank<m_size3d; rrank++)
-            {
-                if(rrank!=0)
-                    MPI_Recv( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
-                          rrank, rrank, m_comm3d, &status);
-                start4d[1] = m_coords[3*rrank+2]*m_count4d[1],
-                start4d[2] = m_coords[3*rrank+1]*m_count4d[2],
-                start4d[3] = m_coords[3*rrank+0]*m_count4d[3];
-                err = nc_put_vara_double( ncid, vecID, start4d, m_count4d,
-                    transferH.data().data());
-            }
-        }
-        else
-            MPI_Send( transferH.data().data(), m_local_size3d, MPI_DOUBLE,
-                      0, m_rank3d, m_comm3d);
-        MPI_Barrier( m_comm3d);
-#else
-        err = nc_put_vara_double( ncid, vecID, start4d, m_count4d,
-            transferH.data());
-#endif // FELTOR_MPI
-    }
-
-//all send to their rank2d 0 but only rank3d 0 writes into file
-    void output_dynamic2d_slice(int ncid, int vecID, unsigned start, HVec& transferH2d) const
-    {
-        file::NC_Error_Handle err;
-        size_t start3d[3] = {start, 0, 0};
-#ifdef FELTOR_MPI
-        MPI_Status status;
-        // 2d data of plane varphi = 0
-        if(m_rank2d==0)
-        {
-            for( int rrank=0; rrank<m_size2d; rrank++)
-            {
-                if(rrank!=0)
-                    MPI_Recv( transferH2d.data().data(), m_local_size2d, MPI_DOUBLE,
-                          rrank, rrank, m_comm2d, &status);
-                start3d[1] = m_coords2d[2*rrank+1]*m_count3d[1],
-                start3d[2] = m_coords2d[2*rrank+0]*m_count3d[2];
-                if(m_rank3d==0)//only global rank 0 writes out
-                    err = nc_put_vara_double( ncid, vecID, start3d, m_count3d,
-                        transferH2d.data().data());
-            }
-        }
-        else
-            MPI_Send( transferH2d.data().data(), m_local_size2d, MPI_DOUBLE,
-                      0, m_rank2d, m_comm2d);
-        MPI_Barrier( m_comm2d);
-        MPI_Barrier( m_comm3d); //all processes synchronize
-#else
-        err = nc_put_vara_double( ncid, vecID, start3d, m_count3d,
-            transferH2d.data());
-#endif // FELTOR_MPI
-    }
-    private:
-    unsigned m_local_size2d;
-    unsigned m_local_size3d;
-    size_t m_count3d[3];
-    size_t m_count4d[4];
-    int m_rank3d, m_size3d, m_rank2d, m_size2d;
-#ifdef FELTOR_MPI
-    MPI_Comm m_comm2d, m_comm3d;
-    std::vector<int> m_coords, m_coords2d;
-#endif //FELTOR_MPI
-};
-
-}//namespace feltor
-- 
GitLab


From ddb7871bf3daf76b6c149e369726ec8fb0293fbc Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 11 Sep 2019 14:01:22 +0200
Subject: [PATCH 129/540] Refactor toefl_mpi.cu into toefl_hpc.cu

- make one out of two programs
- change interface for easy_output functions
---
 inc/file/easy_output.h   | 108 ++++++++++---------
 src/feltor/feltor_hpc.cu |  18 ++--
 src/toefl/Makefile       |   4 +-
 src/toefl/toefl.tex      |  11 +-
 src/toefl/toefl_hpc.cu   | 175 +++++++++++++++++++++---------
 src/toefl/toefl_mpi.cu   | 224 ---------------------------------------
 6 files changed, 196 insertions(+), 344 deletions(-)
 delete mode 100644 src/toefl/toefl_mpi.cu

diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
index a92cf05a9..be93f9e63 100644
--- a/inc/file/easy_output.h
+++ b/inc/file/easy_output.h
@@ -71,56 +71,62 @@ struct NC_Error_Handle
 /**
 * @brief Convenience wrapper around \c nc_put_vara_double()
 *
+* The purpose of this function is mainly to simplify output in an MPI environment and to provide
+* the same interface also in a shared memory system for uniform programming.
 * This version is for a time-independent variable,
 * i.e. writes a single variable in one go and is actually equivalent
-* to \c nc_put_var_double. The dimensionality is indicated by the name.
+* to \c nc_put_var_double. The dimensionality is given by the grid.
 * @tparam host_vector Type with \c data() member that returns pointer to first element in CPU (host) adress space, meaning it cannot be a GPU vector
 * @param ncid Forwarded to \c nc_put_vara_double
 * @param varid  Forwarded to \c nc_put_vara_double
-* @param toWrite data is forwarded to \c nc_put_vara_double, may be destroyed on output in the mpi version.
 * @param grid The grid from which to construct \c start and \c count variables to forward to \c nc_put_vara_double
+* @param data data is forwarded to \c nc_put_vara_double, may be destroyed on output in the mpi version.
 * @param parallel This parameter is ignored in the serial version.
 * In the MPI version this parameter indicates whether each process
 * writes to the file independently in parallel (\c true)
-* or each process funnels its data through the master rank (\c false).
+* or each process funnels its data through the master rank (\c false),
+* which involves communication but may be faster than the former method.
 * @attention In the MPI version, if \c parallel==true a **parallel netcdf** must be
 * linked while if \c parallel==false we need **serial netcdf**.
 * Note that serious performance penalties have been observed on some platforms for parallel netcdf.
 */
 template<class host_vector>
-void write_static2d(int ncid, int varid, host_vector& toWrite,
-    dg::aTopology2d& grid, bool parallel = false)
+void put_var_double(int ncid, int varid, dg::aTopology2d& grid,
+    host_vector& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[2] = {0,0}, count[2];
     count[0] = grid.n()*grid.Ny();
     count[1] = grid.n()*grid.Nx();
-    err = nc_put_vara_double( ncid, varid, start, count, toWrite.data());
+    err = nc_put_vara_double( ncid, varid, start, count, data.data());
 }
 
 /**
 * @brief Convenience wrapper around \c nc_put_vara_double()
 *
+* The purpose of this function is mainly to simplify output in an MPI environment and to provide
+* the same interface also in a shared memory system for uniform programming.
 * This version is for a time-dependent variable,
 * i.e. writes a single time-slice into the file.
-* The dimensionality is indicated by the name.
+* The dimensionality is given by the grid.
 * @tparam host_vector Type with \c data() member that returns pointer to first element in CPU (host) adress space, meaning it cannot be a GPU vector
 * @param ncid Forwarded to \c nc_put_vara_double
 * @param varid  Forwarded to \c nc_put_vara_double
 * @param slice The number of the time-slice to write (first element of the \c startp array in \c nc_put_vara_double)
-* @param toWrite data is forwarded to \c nc_put_vara_double, may be destroyed on output in the mpi version.
 * @param grid The grid from which to construct \c start and \c count variables to forward to \c nc_put_vara_double
+* @param data data is forwarded to \c nc_put_vara_double, may be destroyed on output in the mpi version.
 * @param parallel This parameter is ignored in the serial version.
 * In the MPI version this parameter indicates whether each process
 * writes to the file independently in parallel (\c true)
-* or each process funnels its data through the master rank (\c false).
+* or each process funnels its data through the master rank (\c false),
+* which involves communication but may be faster than the former method.
 * @attention In the MPI version, if \c parallel==true a **parallel netcdf** must be
 * linked while if \c parallel==false we need **serial netcdf**.
 * Note that serious performance penalties have been observed on some platforms for parallel netcdf.
 */
 template<class host_vector>
-void write_dynamic2d(int ncid, int varid, unsigned slice, host_vector& toWrite,
-    dg::aTopology2d& grid, bool parallel = false)
+void put_vara_double(int ncid, int varid, unsigned slice,
+    dg::aTopology2d& grid, host_vector& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[3] = {slice,0,0}, count[3];
@@ -128,12 +134,13 @@ void write_dynamic2d(int ncid, int varid, unsigned slice, host_vector& toWrite,
     count[1] = grid.n()*grid.Ny();
     count[2] = grid.n()*grid.Nx();
     err = nc_put_vara_double( ncid, varid, start, count,
-        toWrite.data());
+        data.data());
 }
 
-///@copydoc write_static2d()
+///@copydoc put_var_double()
 template<class host_vector>
-void write_static3d(int ncid, int varid, host_vector& toWrite, dg::aTopology3d& grid, bool parallel = false)
+void put_var_double(int ncid, int varid, dg::aTopology3d& grid,
+    host_vector& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[3] = {0,0,0}, count[3];
@@ -141,13 +148,13 @@ void write_static3d(int ncid, int varid, host_vector& toWrite, dg::aTopology3d&
     count[1] = grid.n()*grid.Ny();
     count[2] = grid.n()*grid.Nx();
     err = nc_put_vara_double( ncid, varid, start, count,
-        toWrite.data());
+        data.data());
 }
 
-///@copydoc write_dynamic2d()
+///@copydoc put_vara_double()
 template<class host_vector>
-void write_dynamic3d(int ncid, int varid, unsigned slice, host_vector& toWrite,
-    dg::aTopology3d& grid, bool parallel = false)
+void put_vara_double(int ncid, int varid, unsigned slice,
+    dg::aTopology3d& grid, host_vector& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[3] = {slice, 0,0,0}, count[4];
@@ -156,15 +163,14 @@ void write_dynamic3d(int ncid, int varid, unsigned slice, host_vector& toWrite,
     count[2] = grid.n()*grid.Ny();
     count[3] = grid.n()*grid.Nx();
     err = nc_put_vara_double( ncid, varid, start, count,
-        toWrite.data());
+        data.data());
 }
 
 #ifdef MPI_VERSION
-///@copydoc write_static2d()
+///@copydoc put_var_double()
 template<class host_vector>
-void write_static2d(int ncid, int varid,
-    dg::MPI_Vector<host_vector>& toWrite,
-    dg::aMPITopology3d& grid, bool parallel = false)
+void put_var_double(int ncid, int varid, dg::aMPITopology2d& grid,
+    dg::MPI_Vector<host_vector>& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[3] = {0,0}, count[2];
@@ -186,16 +192,16 @@ void write_static2d(int ncid, int varid,
             for( int rrank=0; rrank<size; rrank++)
             {
                 if(rrank!=0)
-                    MPI_Recv( toWrite.data().data(), local_size, MPI_DOUBLE,
+                    MPI_Recv( data.data().data(), local_size, MPI_DOUBLE,
                           rrank, rrank, comm, &status);
                 start[0] = coords[2*rrank+1]*count[0],
                 start[1] = coords[2*rrank+0]*count[1],
                 err = nc_put_vara_double( ncid, varid, start, count,
-                    toWrite.data().data());
+                    data.data().data());
             }
         }
         else
-            MPI_Send( toWrite.data().data(), local_size, MPI_DOUBLE,
+            MPI_Send( data.data().data(), local_size, MPI_DOUBLE,
                       0, rank, comm);
         MPI_Barrier( comm);
     }
@@ -206,15 +212,15 @@ void write_static2d(int ncid, int varid,
         start[0] = coords[1]*count[0],
         start[1] = coords[0]*count[1],
         err = nc_put_vara_double( ncid, varid, start, count,
-            toWrite.data().data());
+            data.data().data());
     }
 }
 
-///@copydoc write_dynamic2d()
+///@copydoc put_vara_double()
 template<class host_vector>
-void write_dynamic2d(int ncid, int varid, unsigned slice,
-    dg::MPI_Vector<host_vector>& toWrite,
-    dg::aMPITopology2d& grid, bool parallel = false)
+void put_vara_double(int ncid, int varid, unsigned slice,
+    dg::aMPITopology2d& grid, dg::MPI_Vector<host_vector>& data,
+    bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[3] = {slice, 0,0}, count[3];
@@ -232,7 +238,7 @@ void write_dynamic2d(int ncid, int varid, unsigned slice,
         start[1] = coords[1]*count[1],
         start[2] = coords[0]*count[2],
         err = nc_put_vara_double( ncid, varid, start, count,
-            toWrite.data().data());
+            data.data().data());
     }
     else
     {
@@ -246,26 +252,26 @@ void write_dynamic2d(int ncid, int varid, unsigned slice,
             for( int rrank=0; rrank<size; rrank++)
             {
                 if(rrank!=0)
-                    MPI_Recv( toWrite.data().data(), local_size, MPI_DOUBLE,
+                    MPI_Recv( data.data().data(), local_size, MPI_DOUBLE,
                           rrank, rrank, comm, &status);
                 start[1] = coords[2*rrank+1]*count[1],
                 start[2] = coords[2*rrank+0]*count[2],
                 err = nc_put_vara_double( ncid, varid, start, count,
-                    toWrite.data().data());
+                    data.data().data());
             }
         }
         else
-            MPI_Send( toWrite.data().data(), local_size, MPI_DOUBLE,
+            MPI_Send( data.data().data(), local_size, MPI_DOUBLE,
                       0, rank, comm);
         MPI_Barrier( comm);
     }
 }
 
-///@copydoc write_static2d()
+///@copydoc put_var_double()
 template<class host_vector>
-void write_static3d(int ncid, int varid,
-    dg::MPI_Vector<host_vector>& toWrite,
-    dg::aMPITopology3d& grid, bool parallel = false)
+void put_var_double(int ncid, int varid,
+    dg::aMPITopology3d& grid, dg::MPI_Vector<host_vector>& data,
+    bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[3] = {0,0,0}, count[3];
@@ -288,17 +294,17 @@ void write_static3d(int ncid, int varid,
             for( int rrank=0; rrank<size; rrank++)
             {
                 if(rrank!=0)
-                    MPI_Recv( toWrite.data().data(), local_size, MPI_DOUBLE,
+                    MPI_Recv( data.data().data(), local_size, MPI_DOUBLE,
                           rrank, rrank, comm, &status);
                 start[0] = coords[3*rrank+2]*count[0],
                 start[1] = coords[3*rrank+1]*count[1],
                 start[2] = coords[3*rrank+0]*count[2];
                 err = nc_put_vara_double( ncid, varid, start, count,
-                    toWrite.data().data());
+                    data.data().data());
             }
         }
         else
-            MPI_Send( toWrite.data().data(), local_size, MPI_DOUBLE,
+            MPI_Send( data.data().data(), local_size, MPI_DOUBLE,
                       0, rank, comm);
         MPI_Barrier( comm);
     }
@@ -310,15 +316,15 @@ void write_static3d(int ncid, int varid,
         start[1] = coords[1]*count[1],
         start[2] = coords[0]*count[2];
         err = nc_put_vara_double( ncid, varid, start, count,
-            toWrite.data().data());
+            data.data().data());
     }
 }
 
-///@copydoc write_dynamic2d()
+///@copydoc put_vara_double()
 template<class host_vector>
-void write_dynamic3d(int ncid, int varid, unsigned slice,
-    dg::MPI_Vector<host_vector>& toWrite,
-    dg::aMPITopology3d& grid, bool parallel = false)
+void put_vara_double(int ncid, int varid, unsigned slice,
+    dg::aMPITopology3d& grid, dg::MPI_Vector<host_vector>& data,
+    bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[4] = {slice, 0,0,0}, count[4];
@@ -338,7 +344,7 @@ void write_dynamic3d(int ncid, int varid, unsigned slice,
         start[2] = coords[1]*count[2],
         start[3] = coords[0]*count[3];
         err = nc_put_vara_double( ncid, varid, start, count,
-            toWrite.data().data());
+            data.data().data());
     }
     else
     {
@@ -352,17 +358,17 @@ void write_dynamic3d(int ncid, int varid, unsigned slice,
             for( int rrank=0; rrank<size; rrank++)
             {
                 if(rrank!=0)
-                    MPI_Recv( toWrite.data().data(), local_size, MPI_DOUBLE,
+                    MPI_Recv( data.data().data(), local_size, MPI_DOUBLE,
                           rrank, rrank, comm, &status);
                 start[1] = coords[3*rrank+2]*count[1],
                 start[2] = coords[3*rrank+1]*count[2],
                 start[3] = coords[3*rrank+0]*count[3];
                 err = nc_put_vara_double( ncid, varid, start, count,
-                    toWrite.data().data());
+                    data.data().data());
             }
         }
         else
-            MPI_Send( toWrite.data().data(), local_size, MPI_DOUBLE,
+            MPI_Send( data.data().data(), local_size, MPI_DOUBLE,
                       0, rank, comm);
         MPI_Barrier( comm);
     }
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 2bf125006..d0f603615 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -240,7 +240,7 @@ int main( int argc, char* argv[])
         MPI_OUT std::cout << "Computing "<<record.name<<"\n";
         record.function( resultH, var, grid, gp, mag);
         dg::blas2::symv( projectH, resultH, transferH);
-        file::write_static3d( ncid, vecID, transferH, g3d_out);
+        file::put_var_double( ncid, vecID, g3d_out, transferH);
         MPI_OUT err = nc_redef(ncid);
     }
 
@@ -296,7 +296,7 @@ int main( int argc, char* argv[])
         record.function( resultD, var);
         dg::blas2::symv( projectD, resultD, transferD);
         dg::assign( transferD, transferH);
-        file::write_dynamic3d( ncid, id4d.at(record.name), start, transferH, g3d_out);
+        file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
     }
     for( auto& record : feltor::diagnostics2d_list)
     {
@@ -318,7 +318,7 @@ int main( int argc, char* argv[])
         //only the globally first slice should write
         if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
 #endif //FELTOR_MPI
-            file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
+            file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
         tti.tic();
@@ -332,7 +332,7 @@ int main( int argc, char* argv[])
         //only the globally first slice should write
         if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
 #endif //FELTOR_MPI
-            file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
+            file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
     }
@@ -438,7 +438,7 @@ int main( int argc, char* argv[])
             record.function( resultD, var);
             dg::blas2::symv( projectD, resultD, transferD);
             dg::assign( transferD, transferH);
-            file::write_dynamic3d( ncid, id4d.at(record.name), start, transferH, g3d_out);
+            file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
         }
         for( auto& record : feltor::diagnostics2d_list)
         {
@@ -451,7 +451,7 @@ int main( int argc, char* argv[])
                 //only the globally first slice should write
                 if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
 #endif //FELTOR_MPI
-                    file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
+                    file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
 
                 name = record.name+"_2d";
                 transferH2d = time_integrals.at(name).get_integral( );
@@ -460,7 +460,7 @@ int main( int argc, char* argv[])
                 //only the globally first slice should write
                 if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
 #endif //FELTOR_MPI
-                    file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
+                    file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
             }
             else //manage the time integrators
             {
@@ -474,7 +474,7 @@ int main( int argc, char* argv[])
                 //only the globally first slice should write
                 if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
 #endif //FELTOR_MPI
-                    file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
+                    file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
 
                 // 2d data of plane varphi = 0
                 name = record.name+"_2d";
@@ -484,7 +484,7 @@ int main( int argc, char* argv[])
                 //only the globally first slice should write
                 if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
 #endif //FELTOR_MPI
-                    file::write_dynamic2d( ncid, id3d.at(name), start, transferH2d, *g2d_out_ptr);
+                    file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
             }
         }
         MPI_OUT err = nc_close(ncid);
diff --git a/src/toefl/Makefile b/src/toefl/Makefile
index c6e2b3085..8940f4bca 100644
--- a/src/toefl/Makefile
+++ b/src/toefl/Makefile
@@ -16,8 +16,8 @@ toeflR: toeflR.cu toeflR.cuh
 toefl_hpc: toefl_hpc.cu toeflR.cuh
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK  -g
 
-toefl_mpi: toefl_mpi.cu toeflR.cuh
-	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK -g
+toefl_mpi: toefl_hpc.cu toeflR.cuh
+	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DTOEFL_MPI -DDG_BENCHMARK -g
 
 doc:
 	mkdir -p doc;
diff --git a/src/toefl/toefl.tex b/src/toefl/toefl.tex
index 608637beb..90ac7dbdc 100644
--- a/src/toefl/toefl.tex
+++ b/src/toefl/toefl.tex
@@ -132,9 +132,9 @@ time &  Karniadakis multistep & $3rd$ order explicit, diffusion $2nd$ order impl
 \end{longtable}
 
 \section{Compilation and useage}
-There are three programs toeflR.cu, toefl\_hpc.cu and toefl\_mpi.cu . Compilation with 
+There are two programs toeflR.cu and toefl\_hpc.cu . Compilation with
 \begin{verbatim}
-make device = <omp or gpu>
+make <toeflR toefl_hpc toefl_mpi> device = <omp gpu>
 \end{verbatim}
 Run with
 \begin{verbatim}
@@ -146,9 +146,10 @@ echo np_x np_y | mpirun -n np_x*np_y path/to/feltor/src/toefl/toefl_mpi\
 All programs write performance informations to std::cout.
 The first is for shared memory systems (OpenMP/GPU) and opens a terminal window with life simulation results.
  The
-second is for shared memory systems and uses serial netcdf.
-The third is for distributed
-memory systems (MPI+OpenMP/GPU) using parallel netcdf and expects the distribution of processes in the
+second can be compiled for both shared and distributed memory systems and uses serial netcdf in both cases
+to write results to a file.
+For distributed
+memory systems (MPI+OpenMP/GPU) the program expects the distribution of processes in the
 x and y directions as command line input parameters.
 
 \subsection{Input file structure}
diff --git a/src/toefl/toefl_hpc.cu b/src/toefl/toefl_hpc.cu
index 5653c0dca..650e4f38d 100644
--- a/src/toefl/toefl_hpc.cu
+++ b/src/toefl/toefl_hpc.cu
@@ -2,22 +2,77 @@
 #include <iomanip>
 #include <vector>
 
+#ifdef TOEFL_MPI
+#include <mpi.h>
+#endif //FELTOR_MPI
+
 #include "dg/algorithm.h"
 #include "dg/file/nc_utilities.h"
 
 #include "toeflR.cuh"
 #include "parameters.h"
 
+#ifdef TOEFL_MPI
+using HVec = dg::MHVec;
+using DVec = dg::MDVec;
+using HMatrix = dg::MHMatrix;
+using DMatrix = dg::MDMatrix;
+using IDMatrix = dg::MIDMatrix;
+using IHMatrix = dg::MIHMatrix;
+using Geometry = dg::CartesianMPIGrid2d;
+#define MPI_OUT if(rank==0)
+#else //TOEFL_MPI
+using HVec = dg::HVec;
+using DVec = dg::DVec;
+using HMatrix = dg::HMatrix;
+using DMatrix = dg::DMatrix;
+using IDMatrix = dg::IDMatrix;
+using IHMatrix = dg::IHMatrix;
+using Geometry = dg::CartesianGrid2d;
+#define MPI_OUT
+#endif //TOEFL_MPI
+
 int main( int argc, char* argv[])
 {
-    //Parameter initialisation
+#ifdef TOEFL_MPI
+    ////////////////////////////////setup MPI///////////////////////////////
+    int provided;
+    MPI_Init_thread( &argc, &argv, MPI_THREAD_FUNNELED, &provided);
+    if( provided != MPI_THREAD_FUNNELED)
+    {
+        std::cerr << "wrong mpi-thread environment provided!\n";
+        return -1;
+    }
+    int periods[2] = {false, true}; //non-, periodic
+    int rank, size;
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    MPI_Comm_size( MPI_COMM_WORLD, &size);
+#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+    int num_devices=0;
+    cudaGetDeviceCount(&num_devices);
+    if(num_devices==0){std::cerr << "No CUDA capable devices found"<<std::endl; return -1;}
+    int device = rank % num_devices; //assume # of gpus/node is fixed
+    cudaSetDevice( device);
+#endif//cuda
+    int np[2];
+    if(rank==0)
+    {
+        std::cin>> np[0] >> np[1];
+        std::cout << "Computing with "<<np[0]<<" x "<<np[1]<<" = "<<size<<std::endl;
+        assert( size == np[0]*np[1]);
+    }
+    MPI_Bcast( np, 2, MPI_INT, 0, MPI_COMM_WORLD);
+    MPI_Comm comm;
+    MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
+#endif//TOEFL_MPI
+    ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     Json::CharReaderBuilder parser;
     parser["collectComments"] = false;
     std::string errs;
     if( argc != 3)
     {
-        std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
+        MPI_OUT std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
         return -1;
     }
     else
@@ -25,22 +80,30 @@ int main( int argc, char* argv[])
         std::ifstream is(argv[1]);
         parseFromStream( parser, is, &js, &errs); //read input without comments
     }
-    std::cout << js<<std::endl;
+    MPI_OUT std::cout << js<<std::endl;
     const Parameters p( js);
-    p.display( std::cout);
+    MPI_OUT p.display( std::cout);
 
     ////////////////////////////////set up computations///////////////////////////
-    dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
-    dg::Grid2d grid_out( 0, p.lx, 0, p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
+    Geometry grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y
+        #ifdef TOEFL_MPI
+        , comm
+        #endif //TOEFL_MPI
+    );
+    Geometry grid_out( 0, p.lx, 0, p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y
+        #ifdef TOEFL_MPI
+        , comm
+        #endif //TOEFL_MPI
+    );
     //create RHS
-    toefl::Explicit< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > exp( grid, p);
-    toefl::Implicit< dg::CartesianGrid2d, dg::DMatrix, dg::DVec > imp( grid, p.nu);
+    toefl::Explicit< Geometry, DMatrix, DVec > exp( grid, p);
+    toefl::Implicit< Geometry, DMatrix, DVec > imp( grid, p.nu);
     /////////////////////create initial vector////////////////////////////////////
     dg::Gaussian g( p.posX*p.lx, p.posY*p.ly, p.sigma, p.sigma, p.amp);
-    std::vector<dg::DVec> y0(2, dg::evaluate( g, grid)), y1(y0); // n_e' = gaussian
+    std::vector<DVec> y0(2, dg::evaluate( g, grid)), y1(y0); // n_e' = gaussian
     dg::blas2::symv( exp.gamma(), y0[0], y0[1]); // n_e = \Gamma_i n_i -> n_i = ( 1+alphaDelta) n_e' + 1
     {
-        dg::DVec v2d = dg::create::inv_weights(grid);
+        DVec v2d = dg::create::inv_weights(grid);
         dg::blas2::symv( v2d, y0[1], y0[1]);
     }
     if( p.equations == "gravity_local" || p.equations == "gravity_global" || p.equations == "drift_global"){
@@ -48,41 +111,44 @@ int main( int argc, char* argv[])
     }
     //////////////////initialisation of timekarniadakis and first step///////////////////
     double time = 0;
-    dg::Karniadakis< std::vector<dg::DVec> > karniadakis( y0, y0[0].size(), p.eps_time);
+    dg::Karniadakis< std::vector<DVec> > karniadakis( y0, y0[0].size(), p.eps_time);
     karniadakis.init( exp, imp, time, y0, p.dt);
     y1 = y0;
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
     int ncid;
-    err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
+    MPI_OUT err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     std::string input = js.toStyledString();
-    err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
+    MPI_OUT err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
+#ifdef TOEFL_MPI
+    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
+#else //TOEFL_MPI
     err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
+#endif //TOEFL_MPI
     //field IDs
     std::string names[4] = {"electrons", "ions", "potential", "vorticity"};
     int dataIDs[4];
     for( unsigned i=0; i<4; i++){
-        err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);}
+        MPI_OUT err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);}
 
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    MPI_OUT err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, dissID, dEdtID;
-    err = nc_def_var( ncid, "energy",      NC_DOUBLE, 1, &EtimeID, &energyID);
-    err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
-    err = nc_def_var( ncid, "dissipation", NC_DOUBLE, 1, &EtimeID, &dissID);
-    err = nc_def_var( ncid, "dEdt",        NC_DOUBLE, 1, &EtimeID, &dEdtID);
-    err = nc_enddef(ncid);
-    dg::DVec transfer( dg::evaluate( dg::zero, grid));
+    MPI_OUT err = nc_def_var( ncid, "energy",      NC_DOUBLE, 1, &EtimeID, &energyID);
+    MPI_OUT err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
+    MPI_OUT err = nc_def_var( ncid, "dissipation", NC_DOUBLE, 1, &EtimeID, &dissID);
+    MPI_OUT err = nc_def_var( ncid, "dEdt",        NC_DOUBLE, 1, &EtimeID, &dEdtID);
+    MPI_OUT err = nc_enddef(ncid);
+    DVec transfer( dg::evaluate( dg::zero, grid));
     ///////////////////////////////////first output/////////////////////////
-    size_t count[3] = {1, grid_out.n()*grid_out.Ny(), grid_out.n()*grid_out.Nx()};
-    size_t start[3] = {0, 0, 0};
+    size_t start = 0, count = 1;
     size_t Ecount[] = {1};
     size_t Estart[] = {0};
-    std::vector<dg::DVec> transferD(4, dg::evaluate(dg::zero, grid_out));
-    dg::HVec transferH(dg::evaluate(dg::zero, grid_out));
-    dg::IDMatrix interpolate = dg::create::interpolation( grid_out, grid);
+    std::vector<DVec> transferD(4, dg::evaluate(dg::zero, grid_out));
+    HVec transferH(dg::evaluate(dg::zero, grid_out));
+    IDMatrix interpolate = dg::create::interpolation( grid_out, grid);
     dg::blas2::symv( interpolate, y1[0], transferD[0]);
     dg::blas2::symv( interpolate, y1[1], transferD[1]);
     dg::blas2::symv( interpolate, exp.potential()[0], transferD[2]);
@@ -91,10 +157,10 @@ int main( int argc, char* argv[])
     for( int k=0;k<4; k++)
     {
         dg::blas1::transfer( transferD[k], transferH);
-        err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH.data() );
+        file::put_vara_double( ncid, dataIDs[k], start, grid_out, transferH);
     }
-    err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-    err = nc_close(ncid);
+    MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
+    MPI_OUT err = nc_close(ncid);
     ///////////////////////////////////////Timeloop/////////////////////////////////
     const double mass0 = exp.mass(), mass_blob0 = mass0 - grid.lx()*grid.ly();
     double E0 = exp.energy(), E1 = 0, diff = 0;
@@ -117,61 +183,64 @@ int main( int argc, char* argv[])
             karniadakis.step( exp, imp, time, y1);
             //store accuracy details
             {
-                std::cout << "(m_tot-m_0)/m_0: "<< (exp.mass()-mass0)/mass_blob0<<"\t";
+                MPI_OUT std::cout << "(m_tot-m_0)/m_0: "<< (exp.mass()-mass0)/mass_blob0<<"\t";
                 E0 = E1;
                 E1 = exp.energy();
                 diff = (E1 - E0)/p.dt;
                 double diss = exp.energy_diffusion( );
-                std::cout << "diff: "<< diff<<" diss: "<<diss<<"\t";
-                std::cout << "Accuracy: "<< 2.*(diff-diss)/(diff+diss)<<"\n";
+                MPI_OUT std::cout << "diff: "<< diff<<" diss: "<<diss<<"\t";
+                MPI_OUT std::cout << "Accuracy: "<< 2.*(diff-diss)/(diff+diss)<<"\n";
             }
             Estart[0] += 1;
             {
-                err = nc_open(argv[2], NC_WRITE, &ncid);
+                MPI_OUT err = nc_open(argv[2], NC_WRITE, &ncid);
                 double ener=exp.energy(), mass=exp.mass(), diff=exp.mass_diffusion(), dEdt=exp.energy_diffusion();
-                err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
-                err = nc_put_vara_double( ncid, energyID,   Estart, Ecount, &ener);
-                err = nc_put_vara_double( ncid, massID,     Estart, Ecount, &mass);
-                err = nc_put_vara_double( ncid, dissID,     Estart, Ecount, &diff);
-                err = nc_put_vara_double( ncid, dEdtID,     Estart, Ecount, &dEdt);
-                err = nc_close(ncid);
+                MPI_OUT err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
+                MPI_OUT err = nc_put_vara_double( ncid, energyID,   Estart, Ecount, &ener);
+                MPI_OUT err = nc_put_vara_double( ncid, massID,     Estart, Ecount, &mass);
+                MPI_OUT err = nc_put_vara_double( ncid, dissID,     Estart, Ecount, &diff);
+                MPI_OUT err = nc_put_vara_double( ncid, dEdtID,     Estart, Ecount, &dEdt);
+                MPI_OUT err = nc_close(ncid);
             }
         }
         //////////////////////////write fields////////////////////////
-        start[0] = i;
+        start = i;
         dg::blas2::symv( interpolate, y1[0], transferD[0]);
         dg::blas2::symv( interpolate, y1[1], transferD[1]);
         dg::blas2::symv( interpolate, exp.potential()[0], transferD[2]);
         dg::blas2::symv( imp.laplacianM(), exp.potential()[0], transfer);
         dg::blas2::symv( interpolate, transfer, transferD[3]);
-        err = nc_open(argv[2], NC_WRITE, &ncid);
+        MPI_OUT err = nc_open(argv[2], NC_WRITE, &ncid);
         for( int k=0;k<4; k++)
         {
             dg::blas1::transfer( transferD[k], transferH);
-            err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH.data() );
+            file::put_vara_double( ncid, dataIDs[k], start, grid_out, transferH);
         }
-        err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-        err = nc_close(ncid);
+        MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
+        MPI_OUT err = nc_close(ncid);
 
 #ifdef DG_BENCHMARK
         ti.toc();
         step+=p.itstp;
-        std::cout << "\n\t Step "<<step <<" of "<<p.itstp*p.maxout <<" at time "<<time;
-        std::cout << "\n\t Average time for one step: "<<ti.diff()/(double)p.itstp<<"s\n\n"<<std::flush;
+        MPI_OUT std::cout << "\n\t Step "<<step <<" of "<<p.itstp*p.maxout <<" at time "<<time;
+        MPI_OUT std::cout << "\n\t Average time for one step: "<<ti.diff()/(double)p.itstp<<"s\n\n"<<std::flush;
 #endif//DG_BENCHMARK
     }
     }
-    catch( dg::Fail& fail) { 
-        std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-        std::cerr << "Does Simulation respect CFL condition?\n";
+    catch( dg::Fail& fail) {
+        MPI_OUT std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
+        MPI_OUT std::cerr << "Does Simulation respect CFL condition?\n";
     }
-    t.toc(); 
+    t.toc();
     unsigned hour = (unsigned)floor(t.diff()/3600);
     unsigned minute = (unsigned)floor( (t.diff() - hour*3600)/60);
     double second = t.diff() - hour*3600 - minute*60;
-    std::cout << std::fixed << std::setprecision(2) <<std::setfill('0');
-    std::cout <<"Computation Time \t"<<hour<<":"<<std::setw(2)<<minute<<":"<<second<<"\n";
-    std::cout <<"which is         \t"<<t.diff()/p.itstp/p.maxout<<"s/step\n";
+    MPI_OUT std::cout << std::fixed << std::setprecision(2) <<std::setfill('0');
+    MPI_OUT std::cout <<"Computation Time \t"<<hour<<":"<<std::setw(2)<<minute<<":"<<second<<"\n";
+    MPI_OUT std::cout <<"which is         \t"<<t.diff()/p.itstp/p.maxout<<"s/step\n";
+#ifdef TOEFL_MPI
+    MPI_Finalize();
+#endif //TOEFL_MPI
 
     return 0;
 
diff --git a/src/toefl/toefl_mpi.cu b/src/toefl/toefl_mpi.cu
deleted file mode 100644
index 57d405e73..000000000
--- a/src/toefl/toefl_mpi.cu
+++ /dev/null
@@ -1,224 +0,0 @@
-#include <iostream>
-#include <iomanip>
-#include <vector>
-
-#include <mpi.h> //activate mpi
-
-#include "netcdf_par.h"
-#include "file/nc_utilities.h"
-
-#include "toeflR.cuh"
-#include "dg/algorithm.h"
-#include "parameters.h"
-
-
-int main( int argc, char* argv[])
-{
-    ////////////////////////////////setup MPI///////////////////////////////
-    int provided;
-    MPI_Init_thread( &argc, &argv, MPI_THREAD_FUNNELED, &provided);
-    if( provided != MPI_THREAD_FUNNELED)
-    {
-        std::cerr << "wrong mpi-thread environment provided!\n";
-        return -1;
-    }
-    int periods[2] = {false, true}; //non-, periodic
-    int rank, size;
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-    MPI_Comm_size( MPI_COMM_WORLD, &size);
-#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
-    int num_devices=0;
-    cudaGetDeviceCount(&num_devices);
-    if(num_devices==0){std::cerr << "No CUDA capable devices found"<<std::endl; return -1;}
-    int device = rank % num_devices; //assume # of gpus/node is fixed
-    cudaSetDevice( device);
-#endif//cuda
-    int np[2];
-    if(rank==0)
-    {
-        std::cin>> np[0] >> np[1];
-        std::cout << "Computing with "<<np[0]<<" x "<<np[1]<<" = "<<size<<std::endl;
-        assert( size == np[0]*np[1]);
-    }
-    MPI_Bcast( np, 2, MPI_INT, 0, MPI_COMM_WORLD);
-    MPI_Comm comm;
-    MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
-    ////////////////////////Parameter initialisation//////////////////////////
-    Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false; //important since we want to write to netcdf
-    std::string errs;
-    if( argc != 3)
-    {
-        if(rank==0)std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
-        return -1;
-    }
-    else
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs);
-    }
-    const Parameters p( js);
-    if(rank==0)p.display( std::cout);
-
-    ////////////////////////////////set up computations///////////////////////////
-    dg::MPIGrid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y, comm);
-    dg::MPIGrid2d grid_out( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y, comm);
-    //create RHS
-    toefl::Explicit< dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec > test( grid, p);
-    toefl::Implicit< dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec > diffusion( grid, p.nu);
-    //////////////////create initial vector///////////////////////////////////////
-    dg::Gaussian g( p.posX*p.lx, p.posY*p.ly, p.sigma, p.sigma, p.amp);
-    std::vector<dg::MDVec> y0(2, dg::evaluate( g, grid)), y1(y0); // n_e' = gaussian
-    dg::blas2::symv( test.gamma(), y0[0], y0[1]); // n_e = \Gamma_i n_i -> n_i = ( 1+alphaDelta) n_e' + 1
-    {
-        dg::MDVec v2d = dg::create::inv_weights(grid);
-        dg::blas2::symv( v2d, y0[1], y0[1]);
-    }
-    if( p.equations == "gravity_local" || p.equations == "gravity_global" || p.equations == "drift_global" ){
-        y0[1] = dg::evaluate( dg::zero, grid);
-    }
-    //////////////////initialisation of timekarniadakis and first step///////////////////
-    double time = 0;
-    dg::Karniadakis< std::vector<dg::MDVec> > karniadakis( y0, y0[0].size(), p.eps_time);
-    karniadakis.init( test, diffusion, time, y0, p.dt);
-    y1 = y0;
-    /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid; MPI_Info info = MPI_INFO_NULL;
-    err = nc_create_par( argv[2],NC_NETCDF4|NC_MPIIO|NC_CLOBBER,comm,info, &ncid);
-    std::string input = js.toStyledString();
-    err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
-    //field IDs
-    std::string names[4] = {"electrons", "ions", "potential", "vorticity"};
-    int dataIDs[4];
-    for( unsigned i=0; i<4; i++){
-        err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);}
-
-    //energy IDs
-    int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
-    int energyID, massID, dissID, dEdtID;
-    err = nc_def_var( ncid, "energy",      NC_DOUBLE, 1, &EtimeID, &energyID);
-    err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
-    err = nc_def_var( ncid, "dissipation", NC_DOUBLE, 1, &EtimeID, &dissID);
-    err = nc_def_var( ncid, "dEdt",        NC_DOUBLE, 1, &EtimeID, &dEdtID);
-      //mpi specific part
-      for(unsigned i=0; i<4; i++)
-          err = nc_var_par_access( ncid, dataIDs[i], NC_COLLECTIVE);
-      err = nc_var_par_access( ncid, tvarID, NC_COLLECTIVE);
-      err = nc_var_par_access( ncid, EtimevarID, NC_COLLECTIVE);
-      err = nc_var_par_access( ncid, energyID, NC_COLLECTIVE);
-      err = nc_var_par_access( ncid, massID, NC_COLLECTIVE);
-      err = nc_var_par_access( ncid, dissID, NC_COLLECTIVE);
-      err = nc_var_par_access( ncid, dEdtID, NC_COLLECTIVE);
-      err = nc_enddef(ncid);
-      int dims[2],  coords[2];
-      MPI_Cart_get( comm, 2, dims, periods, coords);
-      dg::MDVec transfer( dg::evaluate( dg::zero, grid));
-    ///////////////////////////////////first output/////////////////////////
-    size_t count[3] = {1, grid_out.n()*grid_out.local().Ny(), grid_out.n()*grid_out.local().Nx()};
-    size_t start[3] = {0, coords[1]*count[1], coords[0]*count[2]};
-    size_t Ecount[] = {1};
-    size_t Estart[] = {0};
-    std::vector<dg::DVec> transferD(4, dg::evaluate(dg::zero, grid_out.local()));
-    dg::HVec transferH(dg::evaluate(dg::zero, grid_out.local()));
-    dg::IDMatrix interpolate = dg::create::interpolation( grid_out.local(), grid.local());
-    dg::blas2::symv( interpolate, y1[0].data(), transferD[0]);
-    dg::blas2::symv( interpolate, y1[1].data(), transferD[1]);
-    dg::blas2::symv( interpolate, test.potential()[0].data(), transferD[2]);
-    dg::blas2::symv( diffusion.laplacianM(), test.potential()[0], transfer);
-    dg::blas2::symv( interpolate, transfer.data(), transferD[3]);
-    for( unsigned k=0; k<4; k++)
-    {
-        dg::blas1::transfer( transferD[k], transferH);
-        err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH.data() );
-    }
-    err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-    //err = nc_close(ncid);
-    ///////////////////////////////////////Timeloop/////////////////////////////////
-    const double mass0 = test.mass(), mass_blob0 = mass0 - grid.global().lx()*grid.global().ly();
-    double E0 = test.energy(), E1 = 0, diff = 0;
-    dg::Timer t;
-    t.tic();
-    try
-    {
-#ifdef DG_BENCHMARK
-    unsigned step = 0;
-#endif //DG_BENCHMARK
-    for( unsigned i=1; i<=p.maxout; i++)
-    {
-
-#ifdef DG_BENCHMARK
-        dg::Timer ti;
-        ti.tic();
-#endif//DG_BENCHMARK
-        for( unsigned j=0; j<p.itstp; j++)
-        {
-            karniadakis.step( test, diffusion, time, y1);
-            //store accuracy details
-            {
-                if(rank==0)std::cout << "(m_tot-m_0)/m_0: "<< (test.mass()-mass0)/mass_blob0<<"\t";
-                E0 = E1;
-                E1 = test.energy();
-                diff = (E1 - E0)/p.dt;
-                double diss = test.energy_diffusion( );
-                if(rank==0)std::cout << "diff: "<< diff<<" diss: "<<diss<<"\t";
-                if(rank==0)std::cout << "Accuracy: "<< 2.*(diff-diss)/(diff+diss)<<"\n";
-            }
-            Estart[0] += 1;
-            {
-                //err = nc_open(argv[2], NC_WRITE, &ncid);
-                double ener=test.energy(), mass=test.mass(), diff=test.mass_diffusion(), dEdt=test.energy_diffusion();
-                err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
-                err = nc_put_vara_double( ncid, energyID,   Estart, Ecount, &ener);
-                err = nc_put_vara_double( ncid, massID,     Estart, Ecount, &mass);
-                err = nc_put_vara_double( ncid, dissID,     Estart, Ecount, &diff);
-                err = nc_put_vara_double( ncid, dEdtID,     Estart, Ecount, &dEdt);
-                //err = nc_close(ncid);
-            }
-        }
-        //////////////////////////write fields////////////////////////
-        start[0] = i;
-        dg::blas2::symv( interpolate, y1[0].data(), transferD[0]);
-        dg::blas2::symv( interpolate, y1[1].data(), transferD[1]);
-        dg::blas2::symv( interpolate, test.potential()[0].data(), transferD[2]);
-        dg::blas2::symv( diffusion.laplacianM(), test.potential()[0], transfer);
-        dg::blas2::symv( interpolate, transfer.data(), transferD[3]);
-        //err = nc_open(argv[2], NC_WRITE, &ncid);
-        for( int k=0;k<4; k++)
-        {
-            dg::blas1::transfer( transferD[k], transferH);
-            err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH.data() );
-        }
-        err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-        //err = nc_close(ncid);
-
-#ifdef DG_BENCHMARK
-        ti.toc();
-        step+=p.itstp;
-        if(rank==0)std::cout << "\n\t Step "<<step <<" of "<<p.itstp*p.maxout <<" at time "<<time;
-        if(rank==0)std::cout << "\n\t Average time for one step: "<<ti.diff()/(double)p.itstp<<"s\n\n"<<std::flush;
-#endif//DG_BENCHMARK
-    }
-    }
-    catch( dg::Fail& fail) {
-        if(rank==0)std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-        if(rank==0)std::cerr << "Does Simulation respect CFL condition?\n";
-    }
-    t.toc();
-    unsigned hour = (unsigned)floor(t.diff()/3600);
-    unsigned minute = (unsigned)floor( (t.diff() - hour*3600)/60);
-    double second = t.diff() - hour*3600 - minute*60;
-    if(rank==0)std::cout << std::fixed << std::setprecision(2) <<std::setfill('0');
-    if(rank==0)std::cout <<"Computation Time \t"<<hour<<":"<<std::setw(2)<<minute<<":"<<second<<"\n";
-    if(rank==0)std::cout <<"which is         \t"<<t.diff()/p.itstp/p.maxout<<"s/step\n";
-    nc_close(ncid);
-    MPI_Finalize();
-
-    return 0;
-
-}
-
-- 
GitLab


From 74241df8d8887659db2fecb0702ebc8e26ed1699 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 11 Sep 2019 17:14:13 +0200
Subject: [PATCH 130/540] Intermediate commit to restructure init.h

- feltor does not compile
---
 inc/dg/topology/grid.h     |   4 +-
 inc/dg/topology/mpi_grid.h |   2 +-
 inc/file/easy_output.h     |   2 +-
 src/feltor/feltor_hpc.cu   |  43 +---
 src/feltor/feltordiag.h    |  19 +-
 src/feltor/init.h          | 441 +++++++++++++++++++++++--------------
 src/feltor/parameters.h    |   1 +
 7 files changed, 308 insertions(+), 204 deletions(-)

diff --git a/inc/dg/topology/grid.h b/inc/dg/topology/grid.h
index 5034a1735..7a3f25bfb 100644
--- a/inc/dg/topology/grid.h
+++ b/inc/dg/topology/grid.h
@@ -249,7 +249,7 @@ struct RealGrid1d
 
 /**
  * @brief An abstract base class for two-dimensional grids
- * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
+ * @note although it is abstract, objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  * @ingroup basictopology
  */
 template<class real_type>
@@ -473,7 +473,7 @@ struct aRealTopology2d
 
 /**
  * @brief An abstract base class for three-dimensional grids
- * @note although it is abstract objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
+ * @note although it is abstract, objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  * @ingroup basictopology
  */
 template<class real_type>
diff --git a/inc/dg/topology/mpi_grid.h b/inc/dg/topology/mpi_grid.h
index bb62d20ed..26056f0fa 100644
--- a/inc/dg/topology/mpi_grid.h
+++ b/inc/dg/topology/mpi_grid.h
@@ -30,7 +30,7 @@ namespace dg
  * Represents the global grid coordinates and the process topology.
  * It just divides the given (global) box into nonoverlapping (local) subboxes that are attributed to each process
  * @note a single cell is never divided across processes.
- * @note although it is abstract objects, are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
+ * @note although it is abstract, objects are not meant to be hold on the heap via a base class pointer ( we protected the destructor)
  * @attention
  * The access functions \c n() \c Nx() ,... all return the global parameters. If you want to have the local ones call the \c local() function.
  * @ingroup basictopology
diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
index be93f9e63..6b7413561 100644
--- a/inc/file/easy_output.h
+++ b/inc/file/easy_output.h
@@ -157,7 +157,7 @@ void put_vara_double(int ncid, int varid, unsigned slice,
     dg::aTopology3d& grid, host_vector& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
-    size_t start[3] = {slice, 0,0,0}, count[4];
+    size_t start[4] = {slice, 0,0,0}, count[4];
     count[0] = 1;
     count[1] = grid.Nz();
     count[2] = grid.n()*grid.Ny();
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index d0f603615..a760a76d8 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -298,6 +298,11 @@ int main( int argc, char* argv[])
         dg::assign( transferD, transferH);
         file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
     }
+    bool write2d = true;
+#ifdef FELTOR_MPI
+    //only the globally first slice should write
+    if( !(g3d_out.local().z0() - g3d_out.global().z0() < 1e-14) ) write2d = false;
+#endif //FELTOR_MPI
     for( auto& record : feltor::diagnostics2d_list)
     {
         dg::Timer tti;
@@ -314,11 +319,7 @@ int main( int argc, char* argv[])
         tti.toc();
         MPI_OUT std::cout<< name << " Computing average took "<<tti.diff()<<"\n";
         tti.tic();
-#ifdef FELTOR_MPI
-        //only the globally first slice should write
-        if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
-#endif //FELTOR_MPI
-            file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+        if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
         tti.tic();
@@ -328,11 +329,7 @@ int main( int argc, char* argv[])
         feltor::slice_vector3d( transferD, transferD2d, local_size2d);
         dg::assign( transferD2d, transferH2d);
         if( record.integral) time_integrals[name].init( time, transferH2d);
-#ifdef FELTOR_MPI
-        //only the globally first slice should write
-        if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
-#endif //FELTOR_MPI
-            file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+        if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
     }
@@ -447,22 +444,14 @@ int main( int argc, char* argv[])
                 std::string name = record.name+"_ta2d";
                 transferH2d = time_integrals.at(name).get_integral();
                 time_integrals.at(name).flush();
-#ifdef FELTOR_MPI
-                //only the globally first slice should write
-                if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
-#endif //FELTOR_MPI
-                    file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+                if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
 
                 name = record.name+"_2d";
                 transferH2d = time_integrals.at(name).get_integral( );
                 time_integrals.at(name).flush( );
-#ifdef FELTOR_MPI
-                //only the globally first slice should write
-                if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
-#endif //FELTOR_MPI
-                    file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+                if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
             }
-            else //manage the time integrators
+            else // compute from scratch
             {
                 record.function( resultD, var);
                 dg::blas2::symv( projectD, resultD, transferD);
@@ -470,21 +459,13 @@ int main( int argc, char* argv[])
                 std::string name = record.name+"_ta2d";
                 dg::assign( transferD, transferH);
                 toroidal_average( transferH, transferH2d, false);
-#ifdef FELTOR_MPI
-                //only the globally first slice should write
-                if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
-#endif //FELTOR_MPI
-                    file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+                if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
 
                 // 2d data of plane varphi = 0
                 name = record.name+"_2d";
                 feltor::slice_vector3d( transferD, transferD2d, local_size2d);
                 dg::assign( transferD2d, transferH2d);
-#ifdef FELTOR_MPI
-                //only the globally first slice should write
-                if( g3d_out.local().z0() - g3d_out.global().z0() < 1e-14)
-#endif //FELTOR_MPI
-                    file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+                if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
             }
         }
         MPI_OUT err = nc_close(ncid);
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index df9b583ba..49b0bc2c1 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -12,8 +12,10 @@
 
 namespace feltor{
 
-// This file constitutes the diagnostics module of feltor
-// You can register you own diagnostics in one of the diagnostics lists further down
+// This file constitutes the diagnostics module for feltor
+// The way it works is that it allocates global lists of Records that describe what goes into the file
+// You can register you own diagnostics in one of three diagnostics lists (static 3d, dynamic 3d and
+// dynamic 2d) further down
 // which will then be applied during a simulation
 
 namespace routines{
@@ -138,7 +140,8 @@ void jacobian(
 }
 }//namespace routines
 
-//Here, we neeed the typedefs
+//From here on, we use the typedefs to ease the notation
+/
 struct Variables{
     feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
     feltor::Parameters p;
@@ -149,7 +152,7 @@ struct Variables{
 struct Record{
     std::string name;
     std::string long_name;
-    bool integral;
+    bool integral; //indicates whether the function should be time-integrated
     std::function<void( DVec&, Variables&)> function;
 };
 
@@ -159,6 +162,9 @@ struct Record_static{
     std::function<void( HVec&, Variables&, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag)> function;
 };
 
+///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 //Here is a list of static (time-independent) 3d variables that go into the output
 std::vector<Record_static> diagnostics3d_static_list = {
     { "BR", "R-component of magnetic field in cylindrical coordinates",
@@ -716,7 +722,10 @@ std::vector<Record> diagnostics2d_list = {
 
 };
 
-// These two lists signify the quantities for accuracy computation
+///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+// These two lists signify the quantities involved in accuracy computation
 std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
 std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt"};
 
diff --git a/src/feltor/init.h b/src/feltor/init.h
index c8884f3b7..ad75d8cc5 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -18,66 +18,93 @@ struct TorpexSource
     double m_R0, m_Z0, m_a, m_b, m_c;
 };
 
-//We use the typedefs and MPI_OUT
-struct Initialize
+HVec xpoint_damping(const Geometry& grid
+    const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
-    Initialize( feltor::Parameters p, dg::geo::solovev::Parameters gp,
-        dg::geo::TokamakMagneticField mag) : p(p), gp(gp), mag(mag)
+    HVec xpoint_damping = dg::evaluate( dg::one, grid);
+    if( gp.hasXpoint() )
     {
+        double RX = gp.R_0 - 1.1*gp.triangularity*gp.a;
+        double ZX = -1.1*gp.elongation*gp.a;
+        dg::geo::findXpoint( mag.get_psip(), RX, ZX);
+        xpoint_damping = dg::pullback(
+            dg::geo::ZCutter(-1.1*gp.elongation*gp.a), grid);
     }
-    HVec profile(const Geometry& grid)const{
-        //First the profile and the source (on the host since we want to output those)
-        HVec profile = dg::pullback( dg::geo::Compose<dg::LinearX>( mag.psip(),
-            p.nprofamp/mag.psip()(mag.R0(), 0.), 0.), grid);
-        dg::blas1::pointwiseDot( profile_damping(grid), profile, profile);
-        return profile;
-    }
-    HVec xpoint_damping(const Geometry& grid)const{
-        HVec xpoint_damping = dg::evaluate( dg::one, grid);
-        if( gp.hasXpoint() )
-            xpoint_damping = dg::pullback(
-                dg::geo::ZCutter(-1.1*gp.elongation*gp.a), grid);
-        return xpoint_damping;
-    }
-    HVec source_damping(const Geometry& grid)const{
-        HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-            //first change coordinate from psi to (psi_0 - psip)/psi_0
-            dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-            //then shift
-            p.rho_source, p.alpha, -1), grid);
-        dg::blas1::pointwiseDot( xpoint_damping(grid), source_damping, source_damping);
-        return source_damping;
-    }
-    HVec damping_damping(const Geometry& grid)const{
-        HVec damping_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-            //first change coordinate from psi to (psi_0 - psip)/psi_0
-            dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-            //then shift
-            p.rho_damping, p.alpha, +1), grid);
-        return damping_damping;
-    }
-    HVec profile_damping(const Geometry& grid)const{
-        HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
-            mag.psip(), -p.alpha, p.alpha, -1), grid);
-        dg::blas1::pointwiseDot( xpoint_damping(grid), profile_damping, profile_damping);
-        return profile_damping;
-    }
-    template<class Feltor>
-    std::array<std::array<DVec,2>,2> init_from_parameters(Feltor& feltor, const Geometry& grid){
+    return xpoint_damping;
+}
+HVec damping_damping(const Geometry& grid
+    const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+{
+    HVec damping_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
+        //first change coordinate from psi to (psi_0 - psip)/psi_0
+        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
+        //then shift
+        p.rho_damping, p.alpha, +1), grid);
+    return damping_damping;
+}
+HVec profile_damping(const Geometry& grid
+    const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+{
+    HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
+        mag.psip(), -p.alpha, p.alpha, -1), grid);
+    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), profile_damping, profile_damping);
+    return profile_damping;
+}
+HVec profile(const Geometry& grid,
+    const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag ){
+    //First the profile and the source (on the host since we want to output those)
+    HVec profile = dg::pullback( dg::geo::Compose<dg::LinearX>( mag.psip(),
+        p.nprofamp/mag.psip()(mag.R0(), 0.), 0.), grid);
+    dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), profile, profile);
+    return profile;
+}
+
+
+void init_ni(
+    std::array<std::array<DVec,2>,2>& y0,
+    Explicit<Geometry, IDMatrix, DMatrix, DVec>& feltor,
+    Geometry& grid, const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+{
 #ifdef FELTOR_MPI
-        int rank;
-        MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    int rank;
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 #endif
-        std::array<std::array<DVec,2>,2> y0;
-        //Now perturbation
-        HVec ntilde = dg::evaluate(dg::zero,grid);
-        if( p.initne == "blob" || p.initne == "straight blob")
+    MPI_OUT std::cout << "initialize ni" << std::endl;
+    feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
+    double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
+    MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
+    if( minimalni <= -1)
+    {
+        throw dg::Error(dg::Message()<< "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n"
+            << "Minimum Ni value "<<minimalni+1);
+    }
+};
+
+
+std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
+    Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+    Geometry& grid, const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+> > initial_conditions =
+{
+    { "blob",
+        []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
+            Geometry& grid, const feltor::Parameters& p,
+            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
+            std::array<std::array<DVec,2>,2> y0;
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid,p,gp,mag));
+            HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
             dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
             if( p.symmetric)
                 ntilde = dg::pullback( init0, grid);
-            else if( p.initne == "blob")//rounds =3 ->2*3-1
+            else
             {
                 dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
                     fieldaligned( mag, grid, p.bcxN, p.bcyN,
@@ -85,7 +112,27 @@ struct Initialize
                 //evaluate should always be used with mx,my > 1
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 3);
             }
-            else if( p.initne == "straight blob")//rounds =1 ->2*1-1
+            dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
+            init_ni( y0, f,p,gp,mag);
+
+            dg::blas1::copy( 0., y0[1][0]); //set we = 0
+            dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
+            return y0;
+        }
+    },
+    { "straight_blob",
+        []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
+            Geometry& grid, const feltor::Parameters& p,
+            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        {
+            std::array<std::array<DVec,2>,2> y0;
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid,p,gp,mag));
+            HVec ntilde = dg::evaluate(dg::zero,grid);
+            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
+            dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
+            if( p.symmetric)
+                ntilde = dg::pullback( init0, grid);
+            else
             {
                 dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
                     fieldaligned( mag, grid, p.bcxN, p.bcyN,
@@ -93,9 +140,22 @@ struct Initialize
                 //evaluate should always be used with mx,my > 1
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
+            dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
+            init_ni( y0, f,p,gp,mag);
+
+            dg::blas1::copy( 0., y0[1][0]); //set we = 0
+            dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
+            return y0;
         }
-        else if( p.initne == "turbulence")
+    },
+    { "turbulence",
+        []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
+            Geometry& grid, const feltor::Parameters& p,
+            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
+            std::array<std::array<DVec,2>,2> y0;
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid,p,gp,mag));
+            HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
             dg::BathRZ init0(16,16,grid.x0(),grid.y0(), 30.,2.,p.amp);
             if( p.symmetric)
@@ -108,131 +168,184 @@ struct Initialize
                 //evaluate should always be used with mx,my > 1
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
-            dg::blas1::pointwiseDot( profile_damping(grid), ntilde, ntilde);
+            dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), ntilde, ntilde);
+            dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
+            init_ni( y0, f,p,gp,mag);
+
+            dg::blas1::copy( 0., y0[1][0]); //set we = 0
+            dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
+            return y0;
         }
-        else if( p.initne == "zonal")
+    },
+    { "zonal",
+        []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
+            Geometry& grid, const feltor::Parameters& p,
+            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
+            std::array<std::array<DVec,2>,2> y0;
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid,p,gp,mag));
+            HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::geo::ZonalFlow init0(mag.psip(), p.amp, 0., p.k_psi);
             ntilde = dg::pullback( init0, grid);
-            dg::blas1::pointwiseDot( profile_damping(grid), ntilde, ntilde);
-        }
-        else
-            MPI_OUT std::cerr <<"WARNING: Unknown initial condition!\n";
-        y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid));
-        dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-        MPI_OUT std::cout << "initialize ni" << std::endl;
-        feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
-        double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
-        MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
-        if( minimalni <= -1)
-        {
-            throw dg::Error(dg::Message()<< "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n"
-                << "Minimum Ni value "<<minimalni+1);
-        }
+            dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), ntilde, ntilde);
+            dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
+            init_ni( y0, f,p,gp,mag);
 
-        dg::blas1::copy( 0., y0[1][0]); //set we = 0
-        dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
-        return y0;
+            dg::blas1::copy( 0., y0[1][0]); //set we = 0
+            dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
+            return y0;
+        }
     }
+};
 
-    //everyone reads their portion of the input data
-    std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Geometry& grid, double& time){
+HVec source_damping(const Geometry& grid
+    const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+{
+    HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
+        //first change coordinate from psi to (psi_0 - psip)/psi_0
+        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
+        //then shift
+        p.rho_source, p.alpha, -1), grid);
+    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), source_damping, source_damping);
+    return source_damping;
+}
+std::map<std::string, std::function< DVec(
+    bool& fixed_profile, DVec& ne_profile,
+    Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+    Geometry& grid, const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+> > source_profiles =
+{
+    {"profile",
+        []( bool& fixed_profile, DVec& ne_profile,
+        Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+        Geometry& grid, const feltor::Parameters& p,
+        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        {
+            fixed_profile = true;
+            ne_profile = dg::construct<DVec>( profile(grid, p,gp,mag));
+            DVec source_profile = dg::construct<DVec> ( source_damping( grid, p,gp,mag));
+            return source_profile;
+        }
+    },
+    {"influx",
+        []( bool& fixed_profile, DVec& ne_profile,
+        Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+        Geometry& grid, const feltor::Parameters& p,
+        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        {
+            fixed_profile = false;
+            DVec source_profile = dg::construct<DVec> ( source_damping( grid, p,gp,mag));
+            return source_profile;
+        }
+    },
+    {"torpex",
+        []( bool& fixed_profile, DVec& ne_profile,
+        Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+        Geometry& grid, const feltor::Parameters& p,
+        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        {
+            fixed_profile = false;
+            DVec source_profile = dg::construct<DVec> ( dg::pullback( TorpexSource(0,0, 1,1,1 ), grid));
+            return source_profile;
+        }
+    },
+};
+//We use the typedefs and MPI_OUT
+//
+//everyone reads their portion of the input data
+//don't forget to also read source profiles
+std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Geometry& grid, double& time){
 #ifdef FELTOR_MPI
-        int rank;
-        MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    int rank;
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 #endif
-        std::array<std::array<DVec,2>,2> y0;
-        ///////////////////read in and show inputfile
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
-        size_t lengthIN;
-        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
-        std::string inputIN( lengthIN, 'x');
-        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
-
-        Json::Value jsIN;
-        std::stringstream is(inputIN);
-        Json::CharReaderBuilder parser;
-        parser["collectComments"] = false;
-        std::string errs;
-        parseFromStream( parser, is, &jsIN, &errs); //read input without comments
-        const feltor::Parameters pIN(  jsIN);
-        MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
-        MPI_OUT std::cout << " file parameters:" << std::endl;
-        MPI_OUT pIN.display( std::cout);
-
-        // Now read in last timestep
-        Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
-            pIN.n_out, pIN.Nx_out, pIN.Ny_out, pIN.symmetric ? 1 : pIN.Nz_out, pIN.bcxN, pIN.bcyN, dg::PER
-            #ifdef FELTOR_MPI
-            , grid.communicator()
-            #endif //FELTOR_MPI
-            );
-        IHMatrix interpolateIN = dg::create::interpolation( grid, grid_IN);
+    std::array<std::array<DVec,2>,2> y0;
+    ///////////////////read in and show inputfile
+    file::NC_Error_Handle errIN;
+    int ncidIN;
+    errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
+    size_t lengthIN;
+    errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
+    std::string inputIN( lengthIN, 'x');
+    errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
+
+    Json::Value jsIN;
+    std::stringstream is(inputIN);
+    Json::CharReaderBuilder parser;
+    parser["collectComments"] = false;
+    std::string errs;
+    parseFromStream( parser, is, &jsIN, &errs); //read input without comments
+    const feltor::Parameters pIN(  jsIN);
+    MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
+    MPI_OUT std::cout << " file parameters:" << std::endl;
+    MPI_OUT pIN.display( std::cout);
 
+    // Now read in last timestep
+    Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
+        pIN.n_out, pIN.Nx_out, pIN.Ny_out, pIN.symmetric ? 1 : pIN.Nz_out, pIN.bcxN, pIN.bcyN, dg::PER
         #ifdef FELTOR_MPI
-        int dimsIN[3],  coordsIN[3];
-        int periods[3] = {false, false, true}; //non-, non-, periodic
-        MPI_Cart_get( grid.communicator(), 3, dimsIN, periods, coordsIN);
-        size_t countIN[4] = {1, grid_IN.local().Nz(),
-            grid_IN.n()*(grid_IN.local().Ny()),
-            grid_IN.n()*(grid_IN.local().Nx())};
-        size_t startIN[4] = {0, coordsIN[2]*countIN[1],
-                                coordsIN[1]*countIN[2],
-                                coordsIN[0]*countIN[3]};
-        #else //FELTOR_MPI
-        size_t startIN[4] = {0, 0, 0, 0};
-        size_t countIN[4] = {1, grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
-            grid_IN.n()*grid_IN.Nx()};
+        , grid.communicator()
         #endif //FELTOR_MPI
-        std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
-        HVec transferINH( dg::evaluate(dg::zero, grid_IN));
-
-        std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
-
-        int timeIDIN;
-        /////////////////////Get time length and initial data///////////////////////////
-        errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
-        errIN = nc_inq_dimlen(ncidIN, timeIDIN, &startIN[0]);
-        startIN[0] -= 1;
-        errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
-        errIN = nc_get_vara_double( ncidIN, timeIDIN, startIN, countIN, &time);
-        MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
-        for( unsigned i=0; i<5; i++)
-        {
-            int dataID;
-            errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataID);
-            errIN = nc_get_vara_double( ncidIN, dataID, startIN, countIN,
-                #ifdef FELTOR_MPI
-                    transferINH.data().data()
-                #else //FELTOR_MPI
-                    transferINH.data()
-                #endif //FELTOR_MPI
-                );
-            dg::blas2::gemv( interpolateIN, transferINH, transferINHvec[i]);
-        }
-        errIN = nc_close(ncidIN);
-        /// ///////////////Now Construct initial fields
-        //
-        //Convert to N-1 and W
-        dg::blas1::plus( transferINHvec[0], -1.);
-        dg::blas1::plus( transferINHvec[1], -1.);
-        dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
-        dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
-
-        dg::assign( transferINHvec[0], y0[0][0]); //ne-1
-        dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
-        dg::assign( transferINHvec[2], y0[1][0]); //We
-        dg::assign( transferINHvec[3], y0[1][1]); //Wi
-        return y0;
-    }
+        );
+    IHMatrix interpolateIN = dg::create::interpolation( grid, grid_IN);
 
-    private:
-    feltor::Parameters p;
-    dg::geo::solovev::Parameters gp;
-    dg::geo::TokamakMagneticField mag;
-};
+    #ifdef FELTOR_MPI
+    int dimsIN[3],  coordsIN[3];
+    int periods[3] = {false, false, true}; //non-, non-, periodic
+    MPI_Cart_get( grid.communicator(), 3, dimsIN, periods, coordsIN);
+    size_t countIN[4] = {1, grid_IN.local().Nz(),
+        grid_IN.n()*(grid_IN.local().Ny()),
+        grid_IN.n()*(grid_IN.local().Nx())};
+    size_t startIN[4] = {0, coordsIN[2]*countIN[1],
+                            coordsIN[1]*countIN[2],
+                            coordsIN[0]*countIN[3]};
+    #else //FELTOR_MPI
+    size_t startIN[4] = {0, 0, 0, 0};
+    size_t countIN[4] = {1, grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
+        grid_IN.n()*grid_IN.Nx()};
+    #endif //FELTOR_MPI
+    std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
+    HVec transferINH( dg::evaluate(dg::zero, grid_IN));
+
+    std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
+
+    int timeIDIN;
+    /////////////////////Get time length and initial data///////////////////////////
+    errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
+    errIN = nc_inq_dimlen(ncidIN, timeIDIN, &startIN[0]);
+    startIN[0] -= 1;
+    errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
+    errIN = nc_get_vara_double( ncidIN, timeIDIN, startIN, countIN, &time);
+    MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
+    for( unsigned i=0; i<5; i++)
+    {
+        int dataID;
+        errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataID);
+        errIN = nc_get_vara_double( ncidIN, dataID, startIN, countIN,
+            #ifdef FELTOR_MPI
+                transferINH.data().data()
+            #else //FELTOR_MPI
+                transferINH.data()
+            #endif //FELTOR_MPI
+            );
+        dg::blas2::gemv( interpolateIN, transferINH, transferINHvec[i]);
+    }
+    errIN = nc_close(ncidIN);
+    /// ///////////////Now Construct initial fields
+    //
+    //Convert to N-1 and W
+    dg::blas1::plus( transferINHvec[0], -1.);
+    dg::blas1::plus( transferINHvec[1], -1.);
+    dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
+    dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
 
+    dg::assign( transferINHvec[0], y0[0][0]); //ne-1
+    dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
+    dg::assign( transferINHvec[2], y0[1][0]); //We
+    dg::assign( transferINHvec[3], y0[1][1]); //Wi
+    return y0;
+}
 
 } //namespace feltor
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 8a44d070a..76e5ffd1b 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -6,6 +6,7 @@
 #include "json/json.h"
 
 namespace feltor{
+/// If you need more parameters, just go ahead and extend the list
 struct Parameters
 {
     unsigned n, Nx, Ny, Nz;
-- 
GitLab


From 8075694215bb18b95bc64d62968e6fe40f3b135d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 11 Sep 2019 23:51:03 +0200
Subject: [PATCH 131/540] Further work on initialization

- what should we write into file?
---
 src/feltor/feltor.h      |  6 ++-
 src/feltor/feltor_hpc.cu | 13 +++++--
 src/feltor/feltordiag.h  | 15 +++----
 src/feltor/init.h        | 84 +++++++++++++++++++++++++++++++++-------
 4 files changed, 91 insertions(+), 27 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 481425312..a68e7ec68 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -294,8 +294,9 @@ struct Explicit
     }
 
     //source strength, profile - 1
-    void set_source( Container profile, double omega_source, Container source)
+    void set_source( bool fixed_profile, Container profile, double omega_source, Container source)
     {
+        m_fixed_profile = fixed_profile;
         m_profne = profile;
         m_omega_source = omega_source;
         m_source = source;
@@ -355,6 +356,7 @@ struct Explicit
 
     const feltor::Parameters m_p;
     double m_omega_source = 0.;
+    bool m_fixed_profile = true;
 
 };
 
@@ -835,7 +837,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     //Add source terms
     if( m_omega_source != 0)
     {
-        if( m_p.source_type == "profile")
+        if( m_fixed_profile )
             dg::blas1::subroutine( routines::ComputeSource(), m_s[0][0], y[0][0],
                 m_profne, m_source, m_omega_source);
         else
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index a760a76d8..2e92748e8 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -185,12 +185,17 @@ int main( int argc, char* argv[])
     /// //////////////////The initial field///////////////////////////////////////////
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
-    feltor::Initialize init( p, gp, mag);
     if( argc == 4)
-        y0 = init.init_from_parameters(feltor, grid);
+        y0 = feltor::initial_conditions.at[p.initne]( feltor, grid, p,gp,mag );
     if( argc == 5)
-        y0 = init.init_from_file(argv[4], grid, time);
-    feltor.set_source( init.profile(grid), p.omega_source, init.source_damping(grid));
+        y0 = feltor::init_from_file(argv[4], grid, time);
+
+    bool fixed_profile;
+    DVec profile;
+    DVec source_profile = feltor::source_profile.at[p.source_type](
+        fixed_profile, profile, feltor, grid, p, gp, mag);
+
+    feltor.set_source( fixed_profile, profile, p.omega_source, source_profile);
 
     /// //////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 49b0bc2c1..4d36af7ef 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -190,22 +190,23 @@ std::vector<Record_static> diagnostics3d_static_list = {
              result = dg::pullback( mag.psip(), grid);
         }
     },
-    { "Nprof", "Density profile",
+    { "Nprof", "Density profile (that the source may force)",
         []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            Initialize init(v.p, gp, mag);
-            result = init.profile(grid);
+            result = profile(grid, p, gp, mag);
         }
     },
     { "Source", "Source region",
         []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            Initialize init(v.p, gp, mag);
-            result = init.source_damping(grid);
+            result = source_damping(grid, p, gp, mag);
+            bool fixed_profile;
+            DVec profile;
+            result = feltor::source_profile.at[p.source_type](
+                fixed_profile, profile, v.f, grid, v.p, gp, mag);
         }
     },
     { "Damping", "Damping region for initial profile",
         []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            Initialize init(v.p, gp, mag);
-            result = init.profile_damping(grid);
+            result = profile_damping(grid, v.p, gp, mag);
         }
     },
     { "xc", "x-coordinate in Cartesian coordinate system",
diff --git a/src/feltor/init.h b/src/feltor/init.h
index ad75d8cc5..2d9055301 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -18,6 +18,27 @@ struct TorpexSource
     double m_R0, m_Z0, m_a, m_b, m_c;
 };
 
+struct Radius
+{
+    Radius ( double R0, double Z0): m_R0(R0), m_Z0(Z0) {}
+    DG_DEVICE
+    double operator()( double R, double Z) const{
+        return sqrt( (R-m_R0)*(R-m_R0) - (Z-m_Z0)*(Z-m_Z0));
+    }
+    private:
+    double m_R0, m_Z0;
+};
+HVec circular_damping( const Geometry& grid
+    const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+{
+    HVec circular = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
+        Radius( mag.R0(), 0.),
+        gp.a*p.rho_damping, gp.a*p.alpha, -1), grid);
+    return circular;
+}
+
+
 HVec xpoint_damping(const Geometry& grid
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
@@ -29,7 +50,7 @@ HVec xpoint_damping(const Geometry& grid
         double ZX = -1.1*gp.elongation*gp.a;
         dg::geo::findXpoint( mag.get_psip(), RX, ZX);
         xpoint_damping = dg::pullback(
-            dg::geo::ZCutter(-1.1*gp.elongation*gp.a), grid);
+            dg::geo::ZCutter(ZX), grid);
     }
     return xpoint_damping;
 }
@@ -62,6 +83,18 @@ HVec profile(const Geometry& grid,
     dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), profile, profile);
     return profile;
 }
+HVec source_damping(const Geometry& grid
+    const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+{
+    HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
+        //first change coordinate from psi to (psi_0 - psip)/psi_0
+        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
+        //then shift
+        p.rho_source, p.alpha, -1), grid);
+    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), source_damping, source_damping);
+    return source_damping;
+}
 
 
 void init_ni(
@@ -191,6 +224,40 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
             init_ni( y0, f,p,gp,mag);
 
+            dg::blas1::copy( 0., y0[1][0]); //set we = 0
+            dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
+            return y0;
+        }
+    },
+    { "turbulence_on_gaussian",
+        []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
+            Geometry& grid, const feltor::Parameters& p,
+            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        {
+            dg::Gaussian profile( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
+                p.sigma, p.nprofamp);
+            std::array<std::array<DVec,2>,2> y0;
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(
+                dg::pullback( profile, grid) );
+
+            HVec ntilde = dg::evaluate(dg::zero,grid);
+            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
+            dg::BathRZ init0(16,16,grid.x0(),grid.y0(), 30.,2.,p.amp);
+            if( p.symmetric)
+                ntilde = dg::pullback( init0, grid);
+            else
+            {
+                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
+                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
+                //evaluate should always be used with mx,my > 1
+                ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
+            }
+            dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0] );
+            dg::blas1::pointwiseDot( circular_damping(grid,p,gp,mag),
+                y0[0][0], y0[0][0] );
+            init_ni( y0, f,p,gp,mag);
+
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
             return y0;
@@ -198,18 +265,6 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     }
 };
 
-HVec source_damping(const Geometry& grid
-    const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
-{
-    HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-        //first change coordinate from psi to (psi_0 - psip)/psi_0
-        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift
-        p.rho_source, p.alpha, -1), grid);
-    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), source_damping, source_damping);
-    return source_damping;
-}
 std::map<std::string, std::function< DVec(
     bool& fixed_profile, DVec& ne_profile,
     Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
@@ -247,7 +302,8 @@ std::map<std::string, std::function< DVec(
         const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = false;
-            DVec source_profile = dg::construct<DVec> ( dg::pullback( TorpexSource(0,0, 1,1,1 ), grid));
+            DVec source_profile = dg::construct<DVec> ( dg::pullback(
+                TorpexSource(0.98, -0.02, 0.0335, 0.05, 565 ), grid) );
             return source_profile;
         }
     },
-- 
GitLab


From fa6cc93be5b4af905ed920761b88fa82f0916a77 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 12 Sep 2019 15:21:25 +0200
Subject: [PATCH 132/540] Further work on initial and source conditions

- and some documentation in feltor.tex
- programs compile again but untested
---
 inc/dg/functors.h           |   2 +-
 src/feltor/feltor.cu        |  14 ++-
 src/feltor/feltor.tex       |  93 ++++++++++--------
 src/feltor/feltor_hpc.cu    |  15 +--
 src/feltor/feltordiag.h     |  19 ++--
 src/feltor/init.h           | 188 ++++++++++--------------------------
 src/feltor/init_from_file.h | 102 +++++++++++++++++++
 7 files changed, 236 insertions(+), 197 deletions(-)
 create mode 100644 src/feltor/init_from_file.h

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index f4e1f4fbc..0763cde27 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -92,7 +92,7 @@ struct Gaussian
      * @brief Return the value of the gaussian
      *
      * \f[
-       f(x,y) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
+       f(x,y) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}\right)}
        \f]
      * @param x x - coordinate
      * @param y y - coordinate
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 9c3dfae4f..cde2c5aca 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -81,9 +81,15 @@ int main( int argc, char* argv[])
     /////////////////////The initial field///////////////////////////////////////////
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
-    feltor::Initialize init( p, gp, mag);
-    y0 = init.init_from_parameters(feltor, grid);
-    feltor.set_source( init.profile(grid), p.omega_source, init.source_damping(grid));
+    y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+    bool fixed_profile;
+
+    HVec profile;
+    HVec source_profile = feltor::source_profiles.at(p.source_type)(
+        fixed_profile, profile, grid, p, gp, mag);
+
+    feltor.set_source( fixed_profile, dg::construct<DVec>(profile), p.omega_source, dg::construct<DVec>(source_profile));
+
 
     ////////////////////////create timer and timestepper
     //
@@ -142,7 +148,7 @@ int main( int argc, char* argv[])
             else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
             {
                 dg::assign( *pair.second, hvisual);
-                dg::blas1::axpby( 1., hvisual, -1., init.profile(grid), hvisual);
+                dg::blas1::axpby( 1., hvisual, -1., profile, hvisual);
             }
             else
                 dg::assign( *pair.second, hvisual);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index e15c8f9e6..d4611bfb2 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -670,10 +670,10 @@ We define the simulation box as
 $[ R_{\min}, R_{\max}]\times [Z_{\min}, Z_{\max}] \times [0,2\pi]$,
 where we define
 \begin{align} \label{eq:box}
-    R_{\min}&=R_0-\varepsilon^{R-}a\quad
-    &&R_{\max}=R_0+\varepsilon^{R+}a\nonumber\\
-    Z_{\min}&=-\varepsilon^{Z-}ae\quad
-    &&Z_{\max}=\varepsilon^{Z+}ae
+    R_{\min}&=R_0-\varepsilon_{R-}a\quad
+    &&R_{\max}=R_0+\varepsilon_{R+}a\nonumber\\
+    Z_{\min}&=-\varepsilon_{Z-}ae\quad
+    &&Z_{\max}=\varepsilon_{Z+}ae
 \end{align}
 where $a$ is the minor radius, $e$ is the elongation of the flux surfaces and
 the $\varepsilon$ are free parameters to be specified by the user.
@@ -760,29 +760,41 @@ wavelength $k_\psi$ aligned with the magnetic flux surfaces.
     \tilde n_{\text{zonal}}(R,Z) &= A \sin (2\pi k_\psi \psi_p(R,Z)) \nonumber\\
 \tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta_{\alpha}(\psi_p(R, Z)+\alpha) H(Z-Z_X)
 \end{align}
+\subsubsection{Turbulence on Gaussian profile}
+Instead of the flux-aligned profile we can also choose a toroidally symmetric Gaussian profile
+\begin{align} \label{eq:profile_blob}
+  n_{prof}(R,Z) = n_0 + \triangle n_{peak} \exp\left( -\frac{(R - R_0 - p_x a)^2 + (Z-p_ya)^2}{\sigma^2} \right)
+\end{align}
+on top of which we can add the turbulent bath $\tilde n_{\text{bath}}$ and finally dampen it by
+\begin{align}\label{eq:turbulence_on_gaussian}
+n_e(R,Z,\varphi,0) = (n_{prof}(R,Z) + \tilde n_{\text{bath}})\Theta_\alpha( 1- \sqrt{(R-R_0)^2 + Z^2}/a)
+\end{align}
 
 \subsection{Sinks and sources} \label{sec:sources}
-The idea for the source terms $S_N$ is to fix the profile $n_{\text{prof}}$ in the
+We can choose the source terms $S_N$ to either force a profile
+$n_{\text{prof}}$ or provide a constant influx of particles in the
 core of our domain, where our model does not apply.
 We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
-  S_{n_e}(R,Z,\varphi, t) &= \omega_s
-    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta_\alpha( \rho_{s} -\rho(R,Z)) H(Z-Z_X)\\
+  S_{n_e}(R,Z,\varphi, t) &= \omega_s \begin{cases}
+    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta_\alpha( \rho_{s} -\rho(R,Z)) H(Z-Z_X) \quad \text{ forced}\\
+    S_{prof}(R,Z)\quad \text{ influx}
+    \end{cases} \\
     \rho(R,Z) &:= \frac{\psi_{p,\min}- \psi_p(R,Z) }{\psi_{p,\min}}, \quad
     \psi_p(R,Z):= (1-\rho(R,Z))\psi_{p,\min},\\ \text{In general }\psi_{p,\min} &= \psi_p(R_O, Z_O) \neq\psi_{p}(R_0,0)
 \end{align}
 with $0 < \rho_{s}<1$
 where $\omega_s$ is the source strength parameter and $R_O$, $Z_O$ are the coordinates of the O-point.
-This will result in exponential adaption of the core
+The forced source will result in exponential adaption of the core
 density profile of the form $n_e \propto n_{prof}+(n_{prof}-n_{e,0})e^{-\omega_st}$.
 
-As an alternative we can also choose a constant influx
+We can choose the constant influx
 \begin{align} \label{eq:electron_source_influx}
-  S_{n_e}(R,Z,\varphi, t) &= \omega_s\Theta_\alpha( \rho_{s} -\rho(R,Z)) H(Z-Z_X)
+  S_{prof}(R,Z) &= \Theta_\alpha( \rho_{s} -\rho(R,Z)) H(Z-Z_X)
 \end{align}
 or a Torpex inspired source profile
 \begin{align} \label{eq:electron_source_torpex}
-  S_{n_e}(R,Z,\varphi, t) &= \omega_s
+  S_{prof}(R,Z) &= 
   \begin{cases}
     \exp\left( - \frac{(R-R_0)^2}{a^2 }- \frac{(Z-Z_0)^2}{b^2}\right) \text{ if} R > R_0 \\
     \frac{1}{2}\exp\left( - \frac{(R-R_0)^2}{a^2} -2c(R-R_0)(Z-Z_0)- \frac{(Z-Z_0)^2}{b^2} \right) \\
@@ -1075,22 +1087,24 @@ n      & integer & 3 & - &Number of Gaussian nodes in R and Z \\
 Nx     & integer &52& - &Number of grid points in R \\
 Ny     & integer &52& - &Number of grid points in Z \\
 Nz     & integer &16& - &Number of grid points in $\varphi$ (determines dt since parallel velocity dominates timestep) \\
-dt     & integer &1e-2& - &initial time stepsize in units of $c_s/\rho_s$ \\
+dt     & integer &1e-2& - & time stepsize in units of $c_s/\rho_s$ \\
 compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing points in x and y: output contains n*Nx/c[0] points in x,
     (has to divde Nx evenly), and n*Ny/c[1] points in y,
     (has to divde Ny evenly)\\
-inner\_loop & integer & 2  & 1 & Number of time steps between diagnostics updates \\
-itstp       & integer & 2  & - & Number of updates for one output \\
-maxout      & integer & 10 & - & Total Number of fields outputs excluding first (The total number of time steps is maxout$\cdot$itstp$\cdot$inner\_loop) \\
-eps\_time   & float & 1e-7  & - & Accuracy of solver for implicit part in time-stepper (if too low, you'll see oscillations in $u_e$ and/or $\phi$) \\
-rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper \\
-eps\_pol    & float & 1e-6  & - &  Accuracy of residual of the inversion of polarisation and induction Eq. (should not be more than a factor 10 from eps\_time for $\beta\neq  0$ ) \\
-jumpfactor  & float & 1 & 1 & Jumpfactor $\in \left[0.01,1\right]$ in Elliptic\\
-eps\_gamma  & float & 1e-6  & - & Accuracy of $\Gamma_1$  \\
+inner\_loop & integer & 2  & 1 & Number of time steps between updates to the time integrated quantities. Note that integrating selected
+quantities in time during the simulation is how we maintain the time-resolution in the file output (cf. \ref{sec:output_file}).\\
+itstp       & integer & 2  & - &{ \tt inner\_loop*itstp} is the number of timesteps between file outputs (2d and 3d quantities);
+Note that 1d and 0d quantities can only be computed in diagnostics since we can't compute flux-integrals in parallel in MPI. \\
+maxout      & integer & 10 & - & Total Number of fields outputs excluding first (The total number of time steps is {\tt maxout$\cdot$itstp$\cdot$inner\_loop}) \\
+eps\_time   & float & 1e-7  & - & Tolerance for solver for implicit part in time-stepper (if too low, you'll see oscillations in $u_e$ and/or $\phi$) \\
+rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper. (Ignored in Multistep) \\
+eps\_pol    & float & 1e-6  & - &  Tolerance for residual of the inversion of polarisation and induction Eq. (should not be more than a factor 10 from eps\_time for $\beta\neq  0$ ) \\
+jumpfactor  & float & 1 & 1 & Jumpfactor $\in \left[0.01,1\right]$ in the local DG method for the elliptic terms\\
+eps\_gamma  & float & 1e-6  & - & Tolerance for $\Gamma_1$  \\
 stages      & integer & 3 & 3 & number of stages in multigrid, $2^{\text{stages-1}}$
 has to evenly divide both $N_x$ and $N_y$\\
-refineDS     & integer[2] & [10,10] & [10,10] & refinement factor in DS in R- and Z-direction\\
-rk4eps     & float & 1e-5 & 1e-5 & Accuracy of fieldline integrator in DS\\
+refineDS     & integer[2] & [10,10] & [10,10] & refinement factor in FCI approach in R- and Z-direction\\
+rk4eps     & float & 1e-5 & 1e-5 & Accuracy of fieldline integrator in FCI\\
 mu         & float & -0.000272121& - & $\mu_e =-m_e/m_i$.
     One of $\left\{ -0.000544617, -0.000272121, -0.000181372 \right\}$\\
 tau        & float &1      & - & $\tau = T_i/T_e$  \\
@@ -1105,29 +1119,33 @@ bc & dict & & & Dictionary of boundary conditions (note that $A_\parallel$ has t
 \qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y for $n_e$ and $N_i$\\
 \qquad velocity  & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $u_e$ and $U_i$ and $A_\parallel$\\
 \qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $\phi$ and $\psi$\\
-    boxscaleR  & float[2] & [1.1,1.1]     & [1.05,1.05] & $[\varepsilon^{R-}, \varepsilon^{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
-    boxscaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon^{Z-}, \varepsilon^{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box} \\
+    boxscaleR  & float[2] & [1.1,1.1]     & [1.05,1.05] & $[\varepsilon_{R-}, \varepsilon_{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
+    boxscaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon_{Z-}, \varepsilon_{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box} \\
 initne    & string & "turbulence"     & "blob"  & initial condition for the
 perturbation $\tilde n$ in \eqref{eq:initial_ne}. "zonal" (Eq.~\eqref{eq:initial_zonal_flow}),
     "blob" = blob simulations (several rounds fieldaligned),
     "straight blob" = straight blob simulation( 1 round fieldaligned),
-    "turbulence" = turbulence simulations ( 1 round fieldaligned, Eq.~\eqref{eq:initial_turbulent})\\
+    "turbulence" = turbulence simulations ( 1 round fieldaligned, Eq.~\eqref{eq:initial_turbulent})
+    "turbulence\_on\_gaussian" = Gaussian bg. profile with turbulence perturbation Eq.~\eqref{eq:turbulence_on_gaussian}
+    See the file {\tt init.h} to add your own custom condition.
+    \\
 initphi   & string & "zero"  & "balance" & initial condition for $\phi$ and thus $N_i$ (Eq.~\eqref{eq:initphi}: "zero" : $\phi = 0$, vanishing
 electric potential, "balance": ExB vorticity equals ion diamagnetic vorticity (For $\tau_i =0 $ both are the same)
 \\
 amplitude  & float &0.01   & - & amplitude $A$ of initial perturbation (blob, turbulent bath or zonal flow)  \\
-sigma      & float &2      & - & blob variance in units of $\rho_s$ \\
-posX       & float &0.3    & - & blob R-position in units of $a$\\
-posY       & float &0.0    & - & blob Z-position in units of $a$ \\
+sigma      & float &2      & - & Gaussian variance in units of $\rho_s$ \\
+posX       & float &0.3    & - & Gaussian R-position in units of $a$\\
+posY       & float &0.0    & - & Gaussian Z-position in units of $a$ \\
 sigma\_z    & float &0.25   & - & toroidal variance in units of $R_0$ of the fieldline-following initialization \\
-k\_psi     & float &0    & - & zonal mode wave number  \\
-nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} \\
+k\_psi     & float &0    & - & zonal mode wave number (only for "zonal" initial condition)  \\
+nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}\\
 alpha\_mag   & float & 0.05 & - & Width $\alpha$ of the Heaviside in the modified $\psi_p$ function \eqref{eq:modified_psip}\\
 alpha       & float & 0.2 & - & Width $\alpha$ of the Heaviside Eq.~\eqref{eq:approx_heaviside} in the density and source profiles (should be small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes) \\
 source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
 source\_type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
-"influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx}, "torpex": Torpex inspired
-source profile Eq.~\eqref{eq:electron_source_torpex}\\
+"influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx},
+"torpex": Torpex inspired source profile Eq.~\eqref{eq:electron_source_torpex},
+    See the file {\tt init.h} to add your own custom source. \\
 rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
 %damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
 rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0$ in Eq.~\eqref{eq:modified_psip}  \\
@@ -1147,8 +1165,8 @@ File format: json
     B      & float & 1 &  1 & Solovev parameter in Eq.~\eqref{eq:solovev} \\
     c      & float[12] &  - & - & Solovev coefficients in Eq.~\eqref{eq:solovev} \\
     R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ in Eq.~\eqref{eq:solovev} (This is the only geometry quantity to change if $\rho_s$ changes)\\
-    elongation    & float & 1 & - & Elongation $e$ \\
-    triangularity & float & 0 & - & Triangularity $\delta$ \\
+    elongation    & float & 1 & - & Elongation $e$, used in determining the box size Eq.~\eqref{eq:box} and the initial guess for the location of the X-point $Z_X = -1.1 ea$ \\
+    triangularity & float & 0 & - & Triangularity $\delta$, used in the initial guess for the location of the X-point $R_X = R_0-1.1\delta a$ \\
     inverseaspectratio & float & 0.16667 & - & minor to major radius $a/R_0$ \\
 \bottomrule
 \end{longtable}
@@ -1180,9 +1198,8 @@ xc           & Dataset & 3 (z,y,x) & Cartesian x-coordinate $x=R\sin(\varphi)$ \
 yc           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
 zc           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 Psip             & Dataset & 3 (z,y,x) & Flux function $\psi_p(R,Z)$ \\
-Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ \\
+Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ used in the forcing source \\
 Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta_\alpha(\rho_{s} - \rho(R,Z)) H(Z-Z_X)$\\
-Damping          & Dataset & 3 (z,y,x) & Damping profile for initial profile \\
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
@@ -1197,10 +1214,10 @@ X\_ta2d          & Dataset & 3 (time,y,x) & Toroidal average $\langle X
     \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
 Y\_tt\_2d        & Dataset & 3 (time,y,x) & Time integrated (between two outputs) selected plane
 $\int_{t_0}^{t_1}\d t Y(\varphi=0) $
-where $t_1 - t_0 = $dt*inner\_loop*itstp\\
+where $t_1 - t_0 = ${\tt dt*inner\_loop*itstp} and {\tt itstp} is the number of discretization points\\
 Y\_tt\_ta2d      & Dataset & 3 (time,y,x) & Time integrated (between two outputs) toroidal average (Eq.~\eqref{eq:phi_average})
 $\int_{t_0}^{t_1}\d t \langle Y \rangle_\varphi$
-where $t_1 - t_0 = $dt*inner\_loop*itstp\\
+where $t_1 - t_0 = ${\tt dt*inner\_loop*itstp} and {\tt itstp} is the number of discretization points\\
 \bottomrule
 \end{longtable}
 where the time integrals are computed with the help of Simpson's rule
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 2e92748e8..2498533e6 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -35,6 +35,7 @@ using Geometry = dg::CylindricalGrid3d;
 #endif //FELTOR_MPI
 
 #include "init.h"
+#include "init_from_file.h"
 #include "feltordiag.h"
 
 #ifdef FELTOR_MPI
@@ -186,16 +187,18 @@ int main( int argc, char* argv[])
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
     if( argc == 4)
-        y0 = feltor::initial_conditions.at[p.initne]( feltor, grid, p,gp,mag );
+        y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
     if( argc == 5)
-        y0 = feltor::init_from_file(argv[4], grid, time);
+        y0 = feltor::init_from_file(argv[4], grid, p,time);
 
     bool fixed_profile;
-    DVec profile;
-    DVec source_profile = feltor::source_profile.at[p.source_type](
-        fixed_profile, profile, feltor, grid, p, gp, mag);
+    {
+    HVec profile;
+    HVec source_profile = feltor::source_profiles.at(p.source_type)(
+        fixed_profile, profile, grid, p, gp, mag);
 
-    feltor.set_source( fixed_profile, profile, p.omega_source, source_profile);
+    feltor.set_source( fixed_profile, dg::construct<DVec>(profile), p.omega_source, dg::construct<DVec>(source_profile));
+    }
 
     /// //////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 4d36af7ef..ab37135e1 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -141,7 +141,7 @@ void jacobian(
 }//namespace routines
 
 //From here on, we use the typedefs to ease the notation
-/
+
 struct Variables{
     feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
     feltor::Parameters p;
@@ -192,21 +192,18 @@ std::vector<Record_static> diagnostics3d_static_list = {
     },
     { "Nprof", "Density profile (that the source may force)",
         []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            result = profile(grid, p, gp, mag);
+            result = dg::evaluate( dg::zero, grid);
+            bool fixed_profile;
+            HVec source = feltor::source_profiles.at(v.p.source_type)(
+                fixed_profile, result, grid, v.p, gp, mag);
         }
     },
     { "Source", "Source region",
         []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            result = source_damping(grid, p, gp, mag);
             bool fixed_profile;
-            DVec profile;
-            result = feltor::source_profile.at[p.source_type](
-                fixed_profile, profile, v.f, grid, v.p, gp, mag);
-        }
-    },
-    { "Damping", "Damping region for initial profile",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            result = profile_damping(grid, v.p, gp, mag);
+            HVec profile;
+            result = feltor::source_profiles.at(v.p.source_type)(
+                fixed_profile, profile, grid, v.p, gp, mag);
         }
     },
     { "xc", "x-coordinate in Cartesian coordinate system",
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 2d9055301..375105627 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -3,6 +3,8 @@
 
 namespace feltor
 {
+namespace detail
+{
 
 struct TorpexSource
 {
@@ -14,6 +16,10 @@ struct TorpexSource
         return 0.5*exp( - (R-m_R0)*(R-m_R0)/m_a/m_a -2.*m_c*(R-m_R0)*(Z-m_Z0)- (Z-m_Z0)*(Z-m_Z0)/m_b/m_b )
               +0.5*exp( - (R-m_R0)*(R-m_R0)/m_a/m_a +2.*m_c*(R-m_R0)*(Z-m_Z0)- (Z-m_Z0)*(Z-m_Z0)/m_b/m_b );
     }
+    DG_DEVICE
+    double operator()( double R, double Z, double p) const{
+        return this->operator()(R,Z);
+    }
     private:
     double m_R0, m_Z0, m_a, m_b, m_c;
 };
@@ -28,18 +34,18 @@ struct Radius
     private:
     double m_R0, m_Z0;
 };
-HVec circular_damping( const Geometry& grid
+HVec circular_damping( const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     HVec circular = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
         Radius( mag.R0(), 0.),
-        gp.a*p.rho_damping, gp.a*p.alpha, -1), grid);
+        gp.a, gp.a*p.alpha, -1), grid);
     return circular;
 }
 
 
-HVec xpoint_damping(const Geometry& grid
+HVec xpoint_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
@@ -54,7 +60,7 @@ HVec xpoint_damping(const Geometry& grid
     }
     return xpoint_damping;
 }
-HVec damping_damping(const Geometry& grid
+HVec damping_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
@@ -65,7 +71,7 @@ HVec damping_damping(const Geometry& grid
         p.rho_damping, p.alpha, +1), grid);
     return damping_damping;
 }
-HVec profile_damping(const Geometry& grid
+HVec profile_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
@@ -83,7 +89,7 @@ HVec profile(const Geometry& grid,
     dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), profile, profile);
     return profile;
 }
-HVec source_damping(const Geometry& grid
+HVec source_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
@@ -100,7 +106,7 @@ HVec source_damping(const Geometry& grid
 void init_ni(
     std::array<std::array<DVec,2>,2>& y0,
     Explicit<Geometry, IDMatrix, DMatrix, DVec>& feltor,
-    Geometry& grid, const feltor::Parameters& p,
+    const Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
 {
 #ifdef FELTOR_MPI
@@ -117,21 +123,25 @@ void init_ni(
             << "Minimum Ni value "<<minimalni+1);
     }
 };
+}//namespace detail
 
+/* The purpose of this file is to provide an interface for custom initial conditions and
+ * source profiles.  Just add your own to the relevant map below.
+ */
 
 std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
-    Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
-    Geometry& grid, const feltor::Parameters& p,
+    Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
+    const Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
 > > initial_conditions =
 {
     { "blob",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
-            Geometry& grid, const feltor::Parameters& p,
+            const Geometry& grid, const feltor::Parameters& p,
             const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             std::array<std::array<DVec,2>,2> y0;
-            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid,p,gp,mag));
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
             dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
@@ -146,7 +156,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 3);
             }
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-            init_ni( y0, f,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,gp,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -155,11 +165,11 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     },
     { "straight_blob",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
-            Geometry& grid, const feltor::Parameters& p,
+            const Geometry& grid, const feltor::Parameters& p,
             const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             std::array<std::array<DVec,2>,2> y0;
-            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid,p,gp,mag));
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
             dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
@@ -174,7 +184,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-            init_ni( y0, f,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,gp,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -183,11 +193,11 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     },
     { "turbulence",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
-            Geometry& grid, const feltor::Parameters& p,
+            const Geometry& grid, const feltor::Parameters& p,
             const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             std::array<std::array<DVec,2>,2> y0;
-            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid,p,gp,mag));
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
             dg::BathRZ init0(16,16,grid.x0(),grid.y0(), 30.,2.,p.amp);
@@ -201,9 +211,9 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 //evaluate should always be used with mx,my > 1
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
-            dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), ntilde, ntilde);
+            dg::blas1::pointwiseDot( detail::profile_damping(grid,p,gp,mag), ntilde, ntilde);
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-            init_ni( y0, f,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,gp,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -212,17 +222,17 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     },
     { "zonal",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
-            Geometry& grid, const feltor::Parameters& p,
+            const Geometry& grid, const feltor::Parameters& p,
             const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             std::array<std::array<DVec,2>,2> y0;
-            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(profile(grid,p,gp,mag));
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::geo::ZonalFlow init0(mag.psip(), p.amp, 0., p.k_psi);
             ntilde = dg::pullback( init0, grid);
-            dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), ntilde, ntilde);
+            dg::blas1::pointwiseDot( detail::profile_damping(grid,p,gp,mag), ntilde, ntilde);
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-            init_ni( y0, f,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,gp,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -231,14 +241,14 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     },
     { "turbulence_on_gaussian",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
-            Geometry& grid, const feltor::Parameters& p,
+            const Geometry& grid, const feltor::Parameters& p,
             const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
-            dg::Gaussian profile( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
+            dg::Gaussian prof( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
                 p.sigma, p.nprofamp);
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(
-                dg::pullback( profile, grid) );
+                dg::pullback( prof, grid) );
 
             HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
@@ -254,9 +264,9 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0] );
-            dg::blas1::pointwiseDot( circular_damping(grid,p,gp,mag),
+            dg::blas1::pointwiseDot( dg::construct<DVec>(detail::circular_damping(grid,p,gp,mag)),
                 y0[0][0], y0[0][0] );
-            init_ni( y0, f,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,gp,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -265,143 +275,47 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     }
 };
 
-std::map<std::string, std::function< DVec(
-    bool& fixed_profile, DVec& ne_profile,
-    Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+std::map<std::string, std::function< HVec(
+    bool& fixed_profile, //indicate whether a profile should be forced (yes or no)
+    HVec& ne_profile, //construct profile if yes, do nothing if no
     Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
 > > source_profiles =
 {
     {"profile",
-        []( bool& fixed_profile, DVec& ne_profile,
-        Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+        []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
         const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = true;
-            ne_profile = dg::construct<DVec>( profile(grid, p,gp,mag));
-            DVec source_profile = dg::construct<DVec> ( source_damping( grid, p,gp,mag));
+            ne_profile = dg::construct<HVec>( detail::profile(grid, p,gp,mag));
+            HVec source_profile = dg::construct<HVec> ( detail::source_damping( grid, p,gp,mag));
             return source_profile;
         }
     },
     {"influx",
-        []( bool& fixed_profile, DVec& ne_profile,
-        Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+        []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
         const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = false;
-            DVec source_profile = dg::construct<DVec> ( source_damping( grid, p,gp,mag));
+            HVec source_profile = dg::construct<HVec> ( detail::source_damping( grid, p,gp,mag));
             return source_profile;
         }
     },
     {"torpex",
-        []( bool& fixed_profile, DVec& ne_profile,
-        Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+        []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
         const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = false;
-            DVec source_profile = dg::construct<DVec> ( dg::pullback(
-                TorpexSource(0.98, -0.02, 0.0335, 0.05, 565 ), grid) );
+            double rhosinm = 0.98 / gp.R_0;
+            double rhosinm2 = rhosinm*rhosinm;
+            HVec source_profile = dg::construct<HVec> ( dg::pullback(
+                detail::TorpexSource(0.98/rhosinm, -0.02/rhosinm, 0.0335/rhosinm, 0.05/rhosinm, 565*rhosinm2 ), grid) );
             return source_profile;
         }
     },
 };
-//We use the typedefs and MPI_OUT
-//
-//everyone reads their portion of the input data
-//don't forget to also read source profiles
-std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Geometry& grid, double& time){
-#ifdef FELTOR_MPI
-    int rank;
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-#endif
-    std::array<std::array<DVec,2>,2> y0;
-    ///////////////////read in and show inputfile
-    file::NC_Error_Handle errIN;
-    int ncidIN;
-    errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
-    size_t lengthIN;
-    errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
-    std::string inputIN( lengthIN, 'x');
-    errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
-
-    Json::Value jsIN;
-    std::stringstream is(inputIN);
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    parseFromStream( parser, is, &jsIN, &errs); //read input without comments
-    const feltor::Parameters pIN(  jsIN);
-    MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
-    MPI_OUT std::cout << " file parameters:" << std::endl;
-    MPI_OUT pIN.display( std::cout);
-
-    // Now read in last timestep
-    Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
-        pIN.n_out, pIN.Nx_out, pIN.Ny_out, pIN.symmetric ? 1 : pIN.Nz_out, pIN.bcxN, pIN.bcyN, dg::PER
-        #ifdef FELTOR_MPI
-        , grid.communicator()
-        #endif //FELTOR_MPI
-        );
-    IHMatrix interpolateIN = dg::create::interpolation( grid, grid_IN);
-
-    #ifdef FELTOR_MPI
-    int dimsIN[3],  coordsIN[3];
-    int periods[3] = {false, false, true}; //non-, non-, periodic
-    MPI_Cart_get( grid.communicator(), 3, dimsIN, periods, coordsIN);
-    size_t countIN[4] = {1, grid_IN.local().Nz(),
-        grid_IN.n()*(grid_IN.local().Ny()),
-        grid_IN.n()*(grid_IN.local().Nx())};
-    size_t startIN[4] = {0, coordsIN[2]*countIN[1],
-                            coordsIN[1]*countIN[2],
-                            coordsIN[0]*countIN[3]};
-    #else //FELTOR_MPI
-    size_t startIN[4] = {0, 0, 0, 0};
-    size_t countIN[4] = {1, grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
-        grid_IN.n()*grid_IN.Nx()};
-    #endif //FELTOR_MPI
-    std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
-    HVec transferINH( dg::evaluate(dg::zero, grid_IN));
-
-    std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
-
-    int timeIDIN;
-    /////////////////////Get time length and initial data///////////////////////////
-    errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
-    errIN = nc_inq_dimlen(ncidIN, timeIDIN, &startIN[0]);
-    startIN[0] -= 1;
-    errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
-    errIN = nc_get_vara_double( ncidIN, timeIDIN, startIN, countIN, &time);
-    MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
-    for( unsigned i=0; i<5; i++)
-    {
-        int dataID;
-        errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataID);
-        errIN = nc_get_vara_double( ncidIN, dataID, startIN, countIN,
-            #ifdef FELTOR_MPI
-                transferINH.data().data()
-            #else //FELTOR_MPI
-                transferINH.data()
-            #endif //FELTOR_MPI
-            );
-        dg::blas2::gemv( interpolateIN, transferINH, transferINHvec[i]);
-    }
-    errIN = nc_close(ncidIN);
-    /// ///////////////Now Construct initial fields
-    //
-    //Convert to N-1 and W
-    dg::blas1::plus( transferINHvec[0], -1.);
-    dg::blas1::plus( transferINHvec[1], -1.);
-    dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
-    dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
-
-    dg::assign( transferINHvec[0], y0[0][0]); //ne-1
-    dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
-    dg::assign( transferINHvec[2], y0[1][0]); //We
-    dg::assign( transferINHvec[3], y0[1][1]); //Wi
-    return y0;
-}
 
 } //namespace feltor
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
new file mode 100644
index 000000000..8f524265e
--- /dev/null
+++ b/src/feltor/init_from_file.h
@@ -0,0 +1,102 @@
+#pragma once
+
+
+#include "dg/file/nc_utilities.h"
+
+namespace feltor
+{//We use the typedefs and MPI_OUT
+//
+//everyone reads their portion of the input data
+//don't forget to also read source profiles
+std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Geometry& grid, const Parameters& p, double& time){
+#ifdef FELTOR_MPI
+    int rank;
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+#endif
+    std::array<std::array<DVec,2>,2> y0;
+    ///////////////////read in and show inputfile
+    file::NC_Error_Handle errIN;
+    int ncidIN;
+    errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
+    size_t lengthIN;
+    errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
+    std::string inputIN( lengthIN, 'x');
+    errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
+
+    Json::Value jsIN;
+    std::stringstream is(inputIN);
+    Json::CharReaderBuilder parser;
+    parser["collectComments"] = false;
+    std::string errs;
+    parseFromStream( parser, is, &jsIN, &errs); //read input without comments
+    const feltor::Parameters pIN(  jsIN);
+    MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
+    MPI_OUT std::cout << " file parameters:" << std::endl;
+    MPI_OUT pIN.display( std::cout);
+
+    // Now read in last timestep
+    Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
+        pIN.n_out, pIN.Nx_out, pIN.Ny_out, pIN.symmetric ? 1 : pIN.Nz_out, pIN.bcxN, pIN.bcyN, dg::PER
+        #ifdef FELTOR_MPI
+        , grid.communicator()
+        #endif //FELTOR_MPI
+        );
+    IHMatrix interpolateIN = dg::create::interpolation( grid, grid_IN);
+
+    #ifdef FELTOR_MPI
+    int dimsIN[3],  coordsIN[3];
+    int periods[3] = {false, false, true}; //non-, non-, periodic
+    MPI_Cart_get( grid.communicator(), 3, dimsIN, periods, coordsIN);
+    size_t countIN[4] = {1, grid_IN.local().Nz(),
+        grid_IN.n()*(grid_IN.local().Ny()),
+        grid_IN.n()*(grid_IN.local().Nx())};
+    size_t startIN[4] = {0, coordsIN[2]*countIN[1],
+                            coordsIN[1]*countIN[2],
+                            coordsIN[0]*countIN[3]};
+    #else //FELTOR_MPI
+    size_t startIN[4] = {0, 0, 0, 0};
+    size_t countIN[4] = {1, grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
+        grid_IN.n()*grid_IN.Nx()};
+    #endif //FELTOR_MPI
+    std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
+    HVec transferINH( dg::evaluate(dg::zero, grid_IN));
+
+    std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
+
+    int timeIDIN;
+    /////////////////////Get time length and initial data///////////////////////////
+    errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
+    errIN = nc_inq_dimlen(ncidIN, timeIDIN, &startIN[0]);
+    startIN[0] -= 1;
+    errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
+    errIN = nc_get_vara_double( ncidIN, timeIDIN, startIN, countIN, &time);
+    MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
+    for( unsigned i=0; i<5; i++)
+    {
+        int dataID;
+        errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataID);
+        errIN = nc_get_vara_double( ncidIN, dataID, startIN, countIN,
+            #ifdef FELTOR_MPI
+                transferINH.data().data()
+            #else //FELTOR_MPI
+                transferINH.data()
+            #endif //FELTOR_MPI
+            );
+        dg::blas2::gemv( interpolateIN, transferINH, transferINHvec[i]);
+    }
+    errIN = nc_close(ncidIN);
+    /// ///////////////Now Construct initial fields
+    //
+    //Convert to N-1 and W
+    dg::blas1::plus( transferINHvec[0], -1.);
+    dg::blas1::plus( transferINHvec[1], -1.);
+    dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
+    dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
+
+    dg::assign( transferINHvec[0], y0[0][0]); //ne-1
+    dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
+    dg::assign( transferINHvec[2], y0[1][0]); //We
+    dg::assign( transferINHvec[3], y0[1][1]); //Wi
+    return y0;
+}
+}//namespace feltor
-- 
GitLab


From ff9aaba135a24783e25f6db73fa01b9a9db47595 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 13 Sep 2019 13:16:22 +0200
Subject: [PATCH 133/540] Debug init and test some feltor runs

- now ready to test on cluster
---
 config/devices/devices.mk                     |   2 +-
 inc/dg/functors.h                             | 109 ++++++++----------
 inc/geometries/geometry_diag.cu               |   5 +-
 src/feltor/feltor.cu                          |   5 +-
 src/feltor/feltor.tex                         |  34 +++---
 src/feltor/feltor_hpc.cu                      |   3 +-
 src/feltor/geometry/compass.json              |  23 ++++
 ...eometry_params.js => geometry_params.json} |   0
 ...msXpoint.js => geometry_paramsXpoint.json} |   0
 src/feltor/geometry/torpex.json               |  23 ++++
 src/feltor/init.h                             |   8 +-
 11 files changed, 128 insertions(+), 84 deletions(-)
 create mode 100644 src/feltor/geometry/compass.json
 rename src/feltor/geometry/{geometry_params.js => geometry_params.json} (100%)
 rename src/feltor/geometry/{geometry_paramsXpoint.js => geometry_paramsXpoint.json} (100%)
 create mode 100644 src/feltor/geometry/torpex.json

diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 4de178dc8..1a57c9fe4 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -21,6 +21,6 @@ OPT=-O3 -xMIC-AVX512
 #OPT=-O3 -mavx512er
 endif #device=mic
 ifeq ($(strip $(device)),skl)
-OPT=-xCORE-AVX512 -mtune=skylake -O3 
+OPT=-xCORE-AVX512 -O3 # -mtune=skylake # do not use mtune according to cineca
 #OPT=-O3 -mtune=skylake
 endif #device=mic
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 0763cde27..525e18bf9 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -69,7 +69,7 @@ DG_DEVICE
 };
 
 /**
- * @brief Functor returning a gaussian
+ * @brief Functor returning a 2d Gaussian
  * \f[
    f(x,y) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}\right)}
    \f]
@@ -77,19 +77,18 @@ DG_DEVICE
 struct Gaussian
 {
     /**
-     * @brief Functor returning a gaussian
+     * @brief Functor returning a Gaussian
      *
      * @param x0 x-center-coordinate
      * @param y0 y-center-coordinate
      * @param sigma_x x - variance
      * @param sigma_y y - variance
      * @param amp Amplitude
-     * @param kz wavenumber in z direction
      */
-    Gaussian( double x0, double y0, double sigma_x, double sigma_y, double amp, double kz = 1.)
-        : x00(x0), y00(y0), sigma_x(sigma_x), sigma_y(sigma_y), amplitude(amp), kz_(kz){}
+    Gaussian( double x0, double y0, double sigma_x, double sigma_y, double amp)
+        : x00(x0), y00(y0), sigma_x(sigma_x), sigma_y(sigma_y), amplitude(amp){}
     /**
-     * @brief Return the value of the gaussian
+     * @brief Return the value of the Gaussian
      *
      * \f[
        f(x,y) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}\right)}
@@ -107,9 +106,9 @@ struct Gaussian
                           (y-y00)*(y-y00)/2./sigma_y/sigma_y) );
     }
     /**
-     * @brief Return the value of the gaussian modulated by a cosine
+     * @brief Return the value of the Gaussian
      * \f[
-       f(x,y,z) = A\cos(kz)e^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
+       f(x,y,z) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
        \f]
      * @param x x - coordinate
      * @param y y - coordinate
@@ -120,12 +119,10 @@ struct Gaussian
     DG_DEVICE
     double operator()(double x, double y, double z) const
     {
-        return  amplitude*cos(kz_*z)*
-                   exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x +
-                          (y-y00)*(y-y00)/2./sigma_y/sigma_y) );
+        return  this->operator()(x,y);
     }
   private:
-    double  x00, y00, sigma_x, sigma_y, amplitude, kz_;
+    double  x00, y00, sigma_x, sigma_y, amplitude;
 
 };
 
@@ -206,7 +203,7 @@ struct Cauchy
 };
 
 /**
-* @brief The 3d gaussian
+* @brief The 3d Gaussian
 * \f[
 f(x,y,z) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2} + \frac{(z-z_0)^2}{2\sigma_z^2}\right)}
 \f]
@@ -214,7 +211,7 @@ f(x,y,z) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y
 struct Gaussian3d
 {
     /**
-     * @brief Functor returning a gaussian
+     * @brief Functor returning a Gaussian
      *
      * @param x0 x-center-coordinate
      * @param y0 y-center-coordinate
@@ -227,7 +224,7 @@ struct Gaussian3d
     Gaussian3d( double x0, double y0, double z0, double sigma_x, double sigma_y, double sigma_z, double amp)
         : x00(x0), y00(y0), z00(z0), sigma_x(sigma_x), sigma_y(sigma_y), sigma_z(sigma_z), amplitude(amp){}
     /**
-     * @brief Return a 2d gaussian
+     * @brief Return a 2d Gaussian
      *
      * \f[
        f(x,y) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
@@ -245,7 +242,7 @@ struct Gaussian3d
                           (y-y00)*(y-y00)/2./sigma_y/sigma_y) );
     }
     /**
-     * @brief Return the value of the gaussian
+     * @brief Return the value of the Gaussian
      *
      * \f[
        f(x,y) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}+\frac{(z-z_0)^2}{2\sigma_z^2})}
@@ -259,16 +256,10 @@ struct Gaussian3d
     DG_DEVICE
     double operator()(double x, double y, double z) const
     {
-//         if (z== z00)
-//         {
-            return  amplitude*
-                    exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x +
-                           (z-z00)*(z-z00)/2./sigma_z/sigma_z +
-                           (y-y00)*(y-y00)/2./sigma_y/sigma_y) );
-//         }
-//         else {
-//         return 0.;
-//         }
+        return  amplitude*
+                exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x +
+                       (z-z00)*(z-z00)/2./sigma_z/sigma_z +
+                       (y-y00)*(y-y00)/2./sigma_y/sigma_y) );
     }
   private:
     double  x00, y00, z00, sigma_x, sigma_y, sigma_z, amplitude;
@@ -852,33 +843,33 @@ struct TanhProfX {
 /**
  * @brief An approximation to Heaviside using polynomials
      \f[ \begin{cases}
-     0 \text{ if } x < x_0-a \\
-        ((16 a^3 - 29 a^2 (x - x_0) + 20 a (x - x_0)^2 - 5 (x - x_0)^3) (a + x - 
-   x_0)^4)/(32 a^7) \text{ if } |x-x_0| < a \\
-        1  \text{ if } x > x_0 + a
+     0 \text{ if } x < x_b-a \\
+        ((16 a^3 - 29 a^2 (x - x_b) + 20 a (x - x_b)^2 - 5 (x - x_b)^3) (a + x -
+   x_b)^4)/(32 a^7) \text{ if } |x-x_b| < a \\
+        1  \text{ if } x > x_b + a
      \end{cases}\f]
 
-     This function is 3 times continuously differentiable, takes the value 0.5 at x0 and
-     has a transition width a on both sides of x0.
+     This function is 3 times continuously differentiable, takes the value 0.5 at xb and
+     has a transition width a on both sides of xb.
  */
 struct PolynomialHeaviside {
     /**
-     * @brief Construct with x0, width and sign
+     * @brief Construct with xb, width and sign
      *
-     * @param x0 boundary value
+     * @param xb boundary value
      * @param a transition width
-     * @param sign either +1 (original) or -1 (the function is mirrored at the \c x=x0 axis: f(2x0-x))
+     * @param sign either +1 (original Heaviside) or -1 (the function is mirrored at the \c x=xb axis: f(2xb-x))
      */
-    PolynomialHeaviside(double x0, double a, int sign = +1) :
-        x0(x0), a(a), m_s(sign){}
+    PolynomialHeaviside(double xb, double a, int sign = +1) :
+        x0(xb), a(a), m_s(sign){}
     DG_DEVICE
     double operator() (double x)const
     {
-        if( m_s == -1) x = 2*x0-x; //mirror 
+        if( m_s == -1) x = 2*x0-x; //mirror
         if ( x < x0-a) return 0;
         if ( x > x0+a) return 1;
         return ((16.*a*a*a - 29.*a*a*(x - x0)
-               + 20.*a*(x - x0)*(x - x0) 
+               + 20.*a*(x - x0)*(x - x0)
                - 5.*(x - x0)*(x-x0)*(x-x0))
                *(a + x - x0)*(a + x - x0)
                *(a + x - x0)*(a + x - x0))/(32.*a*a*a * a*a*a*a);
@@ -891,30 +882,30 @@ struct PolynomialHeaviside {
 /**
  * @brief The integral of PolynomialHeaviside approximates xH(x)
      \f[ \begin{cases}
-     x_0 \text{ if } x < x_0-a \\
-     x_0 + ((35 a^3 - 47 a^2 (x - x0) + 25 a (x - x0)^2 - 5 (x - x0)^3) (a + x - x0)^5)/(256 a^7)
-        \text{ if } |x-x_0| < a \\
-        x  \text{ if } x > x_0 + a
+     x_b \text{ if } x < x_b-a \\
+     x_b + ((35 a^3 - 47 a^2 (x - x_b) + 25 a (x - x_b)^2 - 5 (x - x_b)^3) (a + x - x_b)^5)/(256 a^7)
+        \text{ if } |x-x_b| < a \\
+        x  \text{ if } x > x_b + a
      \end{cases}\f]
 
      This function is 4 times continuously differentiable,
-     has a transition width \c a on both sides of \c x0, where it transitions from the
-     constant \c x0 to the linear function \c x.
+     has a transition width \c a on both sides of \c xb, where it transitions from the
+     constant \c xb to the linear function \c x.
  */
 struct IPolynomialHeaviside {
     /**
-     * @brief Construct with x0, width and sign
+     * @brief Construct with xb, width and sign
      *
-     * @param x0 boundary value
+     * @param xb boundary value
      * @param a transition width
-     * @param sign either +1 (original) or -1 (the function is point mirrored at \c x=x0: 2*x0-f(2x0-x))
+     * @param sign either +1 (original) or -1 (the function is point mirrored at \c x=xb: 2*xb-f(2xb-x))
      */
-    IPolynomialHeaviside(double x0, double a, int sign = +1) :
-        x0(x0), a(a), m_s(sign){}
+    IPolynomialHeaviside(double xb, double a, int sign = +1) :
+        x0(xb), a(a), m_s(sign){}
     DG_DEVICE
     double operator() (double x)const
     {
-        if( m_s == -1) x = 2*x0-x; //mirror 
+        if( m_s == -1) x = 2*x0-x; //mirror
         double result;
         if ( x < x0-a) result =  x0;
         else if ( x > x0+a) result =  x;
@@ -935,27 +926,27 @@ struct IPolynomialHeaviside {
 /**
  * @brief The derivative of PolynomialHeaviside approximates delta(x)
      \f[ \begin{cases}
-     0 \text{ if } x < x_0-a || x > x_0+a \\
-     (35 (a + x - x0)^3 (a - x + x0)^3)/(32 a^7)
-        \text{ if } |x-x_0| < a 
+     0 \text{ if } x < x_b-a || x > x_b+a \\
+     (35 (a + x - x_b)^3 (a - x + x_b)^3)/(32 a^7)
+        \text{ if } |x-x_b| < a
      \end{cases}\f]
 
-     This function is 2 times continuously differentiable, is symmetric around \c x0
+     This function is 2 times continuously differentiable, is symmetric around \c xb
      and has a width \c a on both sides of \c x0.
      The integral over this function yields 1.
  */
 struct DPolynomialHeaviside {
     /**
-     * @brief Construct with x0, width and sign
+     * @brief Construct with xb, width and sign
      *
-     * @param x0 boundary value
+     * @param xb boundary value
      * @param a transition width
      * @param sign either +1 (original) or -1 (the function is mirrored at \c x=x0)
      * (since this function is symmetric this parameter is ignored, it's there to be
      * consistent with PolynomialHeaviside)
      */
-    DPolynomialHeaviside(double x0, double a, int sign = +1) :
-        x0(x0), a(a){}
+    DPolynomialHeaviside(double xb, double a, int sign = +1) :
+        x0(xb), a(a){}
     DG_DEVICE
     double operator() (double x)const
     {
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 353077c06..1f8748621 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -88,7 +88,7 @@ int main( int argc, char* argv[])
     if( !(argc == 4 || argc == 3))
     {
         std::cerr << "ERROR: Wrong number of arguments!\n";
-        std::cerr << " Usage: "<< argv[0]<<" [input.js] [geom.js] [output.nc]\n";
+        std::cerr << " Usage: "<< argv[0]<<" [input.json] [geom.json] [output.nc]\n";
         std::cerr << " ( Minimum input json file is { \"n\" : 3, \"Nx\": 100, \"Ny\":100 })\n";
         std::cerr << "Or \n Usage: "<< argv[0]<<" [file.nc] [output.nc]\n";
         std::cerr << " ( Program searches for string variables 'inputfile' and 'geomfile' in file.nc and tries a json parser)\n";
@@ -146,7 +146,8 @@ int main( int argc, char* argv[])
 
     //Test coefficients
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    //mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(), 0.), p.alpha_mag);
+    if(p.alpha_mag > 0 )
+        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(), 0.), p.alpha_mag);
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
     if( gp.hasXpoint())
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index cde2c5aca..396cece59 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -57,7 +57,8 @@ int main( int argc, char* argv[])
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
+    if( p.alpha_mag > 0.)
+        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
     //create RHS
     std::cout << "Constructing Explicit...\n";
@@ -84,7 +85,7 @@ int main( int argc, char* argv[])
     y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
     bool fixed_profile;
 
-    HVec profile;
+    HVec profile = dg::evaluate( dg::zero, grid);
     HVec source_profile = feltor::source_profiles.at(p.source_type)(
         fixed_profile, profile, grid, p, gp, mag);
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index d4611bfb2..9364faae0 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -376,29 +376,29 @@ of introducing a strong shear layer limiting the scrape-off layer width.
 We define an approximation to the step function with width $\alpha$
 \begin{align}
 \Theta_\alpha(\psi) := \begin{cases}
-    1 & \text{ for } \psi < - \alpha  \\
-    \frac{1}{32 \alpha^7}  \left(16 \alpha^3+29 \alpha^2 \psi+20 \alpha \psi^2+5 \psi^3\right) (\alpha-\psi)^4
+    0 & \text{ for } \psi < - \alpha  \\
+    \frac{1}{32 \alpha^7}  \left(16 \alpha^3-29 \alpha^2 \psi+20 \alpha \psi^2-5 \psi^3\right) (\alpha+\psi)^4
     &\text{ for } -\alpha<\psi<+\alpha \\
-    0 & \text{ for } \psi > \alpha 
+    1 & \text{ for } \psi > \alpha 
 \end{cases}
-    \approx H(-\psi)
+    \approx H(\psi)
 \label{eq:approx_heaviside}
 \end{align}
 if $H(\psi)$ is the Heaviside step function.
 An integral of this function is
 \begin{align}
 \theta_\alpha(\psi) := \begin{cases}
-\psi &\text{ for } \psi < -\alpha \\
-    - \frac{1}{256 \alpha^7} \left(35 \alpha^3+47 \alpha^2 \psi+25 \alpha \psi^2+5 \psi^3\right) (\alpha-\psi)^5
+    0 &\text{ for } \psi < -\alpha \\
+    \frac{1}{256 \alpha^7} \left(35 \alpha^3-47 \alpha^2 \psi+25 \alpha \psi^2-5 \psi^3\right) (\alpha+\psi)^5
      &\text{ for } -\alpha<\psi<+\alpha \\
-    0 &\text{ for } \psi > \alpha
+\psi &\text{ for } \psi > \alpha
 \end{cases}
-    \approx \psi H(-\psi)
+    \approx \psi H(\psi)
 \end{align}
 
 We now use 
 \begin{align}
-\theta_\alpha(\psi-\psi_0)+\psi_0
+-\theta_\alpha(\psi_0 - \psi)+\psi_0 \approx (\psi- \psi_0)H(\psi_0-\psi) + \psi_0
 \label{eq:modified_psip}
 \end{align}
 instead of $\psi$ for the computation of the
@@ -715,7 +715,7 @@ Note that we should take care to intitialize a smooth profile with ideally well-
 Let us define a flux-aligned density profile as
 \begin{align} \label{eq:density_profile}
   n_{\text{prof}}(R,Z)=
-      n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_p(R_0,0)}\Theta_{\alpha}(\psi_p(R, Z)+\alpha) H(Z-Z_X)
+      n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_p(R_0,0)}\Theta_{\alpha}(-\psi_p(R, Z)-\alpha) H(Z-Z_X)
 \end{align}
 The second Heaviside is multiplied only if the equilibrium $\psi_p$ has an
 X-point and avoids a profile in the private flux region. The factor $\alpha$ provides a smooth transition
@@ -751,14 +751,14 @@ last closed flux surface. Notice that the core region is rather stable
 and quickly damps away fluctuations.
 Again, we transform this to all poloidal planes along the magnetic field lines and multiply the bath with
 \begin{align} \label{eq:initial_turbulent}
-\tilde n_e(R,Z,\varphi) = \tilde n_{\text{bath}}(R,Z,\varphi)\Theta_{\alpha}(\psi_p(R, Z)+\alpha) H(Z-Z_X)
+\tilde n_e(R,Z,\varphi) = \tilde n_{\text{bath}}(R,Z,\varphi)\Theta_{\alpha}(-\psi_p(R, Z)-\alpha) H(Z-Z_X)
 \end{align}
 \subsubsection{Zonal flows}
 We can initialize the R-Z plane with zonal flows of amplitude $A$ and
 wavelength $k_\psi$ aligned with the magnetic flux surfaces.
 \begin{align} \label{eq:initial_zonal_flow}
     \tilde n_{\text{zonal}}(R,Z) &= A \sin (2\pi k_\psi \psi_p(R,Z)) \nonumber\\
-\tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta_{\alpha}(\psi_p(R, Z)+\alpha) H(Z-Z_X)
+\tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta_{\alpha}(-\psi_p(R, Z)-\alpha) H(Z-Z_X)
 \end{align}
 \subsubsection{Turbulence on Gaussian profile}
 Instead of the flux-aligned profile we can also choose a toroidally symmetric Gaussian profile
@@ -777,7 +777,7 @@ core of our domain, where our model does not apply.
 We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
   S_{n_e}(R,Z,\varphi, t) &= \omega_s \begin{cases}
-    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta_\alpha( \rho_{s} -\rho(R,Z)) H(Z-Z_X) \quad \text{ forced}\\
+    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta_\alpha( \rho(R,Z) - \rho_s) H(Z-Z_X) \quad \text{ forced}\\
     S_{prof}(R,Z)\quad \text{ influx}
     \end{cases} \\
     \rho(R,Z) &:= \frac{\psi_{p,\min}- \psi_p(R,Z) }{\psi_{p,\min}}, \quad
@@ -790,7 +790,7 @@ density profile of the form $n_e \propto n_{prof}+(n_{prof}-n_{e,0})e^{-\omega_s
 
 We can choose the constant influx
 \begin{align} \label{eq:electron_source_influx}
-  S_{prof}(R,Z) &= \Theta_\alpha( \rho_{s} -\rho(R,Z)) H(Z-Z_X)
+  S_{prof}(R,Z) &= \Theta_\alpha( \rho(R,Z) - \rho_s) H(Z-Z_X)
 \end{align}
 or a Torpex inspired source profile
 \begin{align} \label{eq:electron_source_torpex}
@@ -1139,7 +1139,6 @@ posY       & float &0.0    & - & Gaussian Z-position in units of $a$ \\
 sigma\_z    & float &0.25   & - & toroidal variance in units of $R_0$ of the fieldline-following initialization \\
 k\_psi     & float &0    & - & zonal mode wave number (only for "zonal" initial condition)  \\
 nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}\\
-alpha\_mag   & float & 0.05 & - & Width $\alpha$ of the Heaviside in the modified $\psi_p$ function \eqref{eq:modified_psip}\\
 alpha       & float & 0.2 & - & Width $\alpha$ of the Heaviside Eq.~\eqref{eq:approx_heaviside} in the density and source profiles (should be small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes) \\
 source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
 source\_type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
@@ -1148,7 +1147,8 @@ source\_type & string & "profile" & "profile" & The type of source to use: "prof
     See the file {\tt init.h} to add your own custom source. \\
 rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
 %damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
-rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0$ in Eq.~\eqref{eq:modified_psip}  \\
+alpha\_mag   & float & 0.05 & - & Width $\alpha$ of the Heaviside in the modified $\psi_p$ function \eqref{eq:modified_psip}. If zero, then we do not modify the magnetic field and rho\_damping is ignored.\\
+rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0 = (1-\rho_d)\psi_{p,\min}$ in Eq.~\eqref{eq:modified_psip}. \\
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
@@ -1199,7 +1199,7 @@ yc           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
 zc           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 Psip             & Dataset & 3 (z,y,x) & Flux function $\psi_p(R,Z)$ \\
 Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ used in the forcing source \\
-Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta_\alpha(\rho_{s} - \rho(R,Z)) H(Z-Z_X)$\\
+Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta_\alpha(\rho(R,Z) - \rho_s) H(Z-Z_X)$\\
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 2498533e6..539774519 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -149,7 +149,8 @@ int main( int argc, char* argv[])
 #endif
 
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
+    if( p.alpha_mag > 0.)
+        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
     //create RHS
     MPI_OUT std::cout << "Constructing Explicit...\n";
diff --git a/src/feltor/geometry/compass.json b/src/feltor/geometry/compass.json
new file mode 100644
index 000000000..ed5f7e807
--- /dev/null
+++ b/src/feltor/geometry/compass.json
@@ -0,0 +1,23 @@
+{
+	"A" : 1,
+	"R_0" : 178.941,
+	"c" :
+	[
+		0.1837793492930699307570501845612999759808,
+        0.003053802605442159220296950705726992590288,
+        -0.1882242965994715344669754151680388020196,
+        -0.08356260086850207612663444582876793886111,
+        0.1400828823828941556097064593061472639108,
+        -0.1464795892953832541951287559065793362435,
+        -0.006527941916323078493138471704332722876669,
+        0.05608898864420130592521710379149509407494,
+        0.3846091415179281423168465616340378123959,
+        -0.1950822491569537061735371149653311041446,
+        -0.04578101700693340619317565018736939517299,
+        0.006936460124168526675792408803022789327478
+	],
+	"equilibrium" : "solovev",
+	"elongation" : 1.6594,
+	"inverseaspectratio" : 0.35714285714,
+	"triangularity" : 0.4
+}
diff --git a/src/feltor/geometry/geometry_params.js b/src/feltor/geometry/geometry_params.json
similarity index 100%
rename from src/feltor/geometry/geometry_params.js
rename to src/feltor/geometry/geometry_params.json
diff --git a/src/feltor/geometry/geometry_paramsXpoint.js b/src/feltor/geometry/geometry_paramsXpoint.json
similarity index 100%
rename from src/feltor/geometry/geometry_paramsXpoint.js
rename to src/feltor/geometry/geometry_paramsXpoint.json
diff --git a/src/feltor/geometry/torpex.json b/src/feltor/geometry/torpex.json
new file mode 100644
index 000000000..c9467fcde
--- /dev/null
+++ b/src/feltor/geometry/torpex.json
@@ -0,0 +1,23 @@
+{
+    "A": 0,
+    "B": 4.752951718389757e-06,
+    "c": [
+        0.047717395667799645,
+        0.9559689342704147,
+        -0.9100332687345335,
+        -0.5820663196518737,
+        0.5651064668675106,
+        -0.4018415371030592,
+        -0.014888980816708646,
+        -4.933621033399979e-13,
+        -1.9628885576793177e-12,
+        1.3157047951347136e-12,
+        3.2965297296431085e-13,
+        -3.2602277468028835e-14
+    ],
+    "R_0": 500,
+    "elongation": 1,
+    "equilibrium": "fit",
+    "triangularity": 0,
+    "inverseaspectratio": 0.2
+}
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 375105627..317022e33 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -29,7 +29,7 @@ struct Radius
     Radius ( double R0, double Z0): m_R0(R0), m_Z0(Z0) {}
     DG_DEVICE
     double operator()( double R, double Z) const{
-        return sqrt( (R-m_R0)*(R-m_R0) - (Z-m_Z0)*(Z-m_Z0));
+        return sqrt( (R-m_R0)*(R-m_R0) + (Z-m_Z0)*(Z-m_Z0));
     }
     private:
     double m_R0, m_Z0;
@@ -277,7 +277,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
 
 std::map<std::string, std::function< HVec(
     bool& fixed_profile, //indicate whether a profile should be forced (yes or no)
-    HVec& ne_profile, //construct profile if yes, do nothing if no
+    HVec& ne_profile, //construct profile if yes, do nothing or construct (determines what is written in output fiele) if no
     Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
 > > source_profiles =
@@ -299,6 +299,7 @@ std::map<std::string, std::function< HVec(
         const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = false;
+            ne_profile = dg::construct<HVec>( detail::profile(grid, p,gp,mag));
             HVec source_profile = dg::construct<HVec> ( detail::source_damping( grid, p,gp,mag));
             return source_profile;
         }
@@ -308,6 +309,9 @@ std::map<std::string, std::function< HVec(
         Geometry& grid, const feltor::Parameters& p,
         const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
+            dg::Gaussian prof( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
+                p.sigma, p.nprofamp);
+            ne_profile = dg::pullback( prof, grid);
             fixed_profile = false;
             double rhosinm = 0.98 / gp.R_0;
             double rhosinm2 = rhosinm*rhosinm;
-- 
GitLab


From 5a77c3f99e8f19961b8da6a61395542f23e2f0f9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 16 Sep 2019 18:22:20 +0200
Subject: [PATCH 134/540] Forward - Backward discretization parallel current

- in feltor
---
 src/feltor/feltor.h   | 6 +++---
 src/feltor/feltor.tex | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index a68e7ec68..885c6abd9 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -764,7 +764,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         //---------------------density--------------------------//
         //density: -Div ( NUb)
         m_ds_N.centered( y[0][i], m_dsN[i]);
-        m_ds_U.centered( fields[1][i], m_dsU[i]);
+        m_ds_U.backward( fields[1][i], m_dsU[i]);
         dg::blas1::pointwiseDot(-1., m_dsN[i], fields[1][i],
             -1., fields[0][i], m_dsU[i], 1., yp[0][i] );
         dg::blas1::pointwiseDot( -1., fields[0][i],fields[1][i],m_divb,
@@ -780,8 +780,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         m_ds_U.centered(-0.5, m_temp1, 1., yp[1][i]);
         // force terms: -tau/mu * ds lnN -1/mu * ds Phi
         // (These two terms converge slowly and require high z resolution)
-        m_ds_N.centered(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
-        m_ds_P.centered(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
+        m_ds_N.forward(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
+        m_ds_P.forward(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
         // diffusion: + nu_par Delta_par U
         dg::blas1::pointwiseDot(m_p.nu_parallel, m_divb, m_dsU[i],
                                 1., yp[1][i]);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 9364faae0..fc33f8af3 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1046,7 +1046,8 @@ discontinuous Galerkin on structured grid
 Advection terms & direct DG & DG approximation with centered flux of derivatives \\
 Elliptic terms & local DG & The local DG approximation with centered flux \\
 Helmholtz and Elliptic matrix inversions & multigrid/ conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
-Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017} \\
+Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017}. The terms $\nabla_\parallel N$ and $\nabla_\parallel \phi$ in the velocity equation use a forward difference, while the term $\nabla_\parallel U$ in the
+density equation uses backward difference. This is to avoid a too wide stencil for the diverence of the current and increases stability for low resistivity. \\
 time & Multistep "Karniadakis" & \\
 \qquad explicit & Multistep "Karniadakis" & $3$rd order explicit\\
 \qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion and Resistive terms. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
-- 
GitLab


From 2b5d6f6ccf3785aeda50d30cc31fc42d3a0df3e6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 18 Sep 2019 18:52:26 +0200
Subject: [PATCH 135/540] Redesign parameters for Psi again

- to be able to swap the magnetic field directions
- Recompute Torpex parameters
---
 inc/dg/functors.h                   |   3 +
 inc/geometries/Doxyfile             |   2 +-
 inc/geometries/geometry_diag.cu     | 130 ++++++++++++-----------
 inc/geometries/solovev.h            | 159 ++++++++++++++++------------
 inc/geometries/solovev_parameters.h |  42 +++++---
 src/feltor/feltor.tex               |  37 ++++---
 src/feltor/geometry/torpex.json     |  23 ++--
 7 files changed, 224 insertions(+), 172 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 525e18bf9..662f82fb0 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -768,6 +768,7 @@ struct Heaviside
      * @param xb boundary value
      * @param sign either +1 or -1, If -1, we mirror the Heaviside at
      *  the \c x=x_b axis, i.e. we swap the < sign in the definition to >
+     * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
      */
     Heaviside( double xb, int sign = +1):
         m_xb(xb), m_s(sign){ }
@@ -819,6 +820,7 @@ struct TanhProfX {
      * @param sign sign of the Tanh, defines the damping direction
      * @param bgamp background amplitude \c B
      * @param profamp profile amplitude \c A
+     * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
      */
     TanhProfX(double xb, double width, int sign =1,double bgamp = 0.,
         double profamp = 1.) :
@@ -859,6 +861,7 @@ struct PolynomialHeaviside {
      * @param xb boundary value
      * @param a transition width
      * @param sign either +1 (original Heaviside) or -1 (the function is mirrored at the \c x=xb axis: f(2xb-x))
+     * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
      */
     PolynomialHeaviside(double xb, double a, int sign = +1) :
         x0(xb), a(a), m_s(sign){}
diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index fe893c156..dbe9526e4 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -2049,7 +2049,7 @@ INCLUDE_FILE_PATTERNS  =
 # recursively expanded use the := operator instead of the = operator.
 # This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
 
-PREDEFINED             =
+PREDEFINED             = JSONCPP_VERSION_STRING
 
 # If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then this
 # tag can be used to specify a list of macro names that should be expanded. The
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 1f8748621..2ca48b2b3 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -165,7 +165,8 @@ int main( int argc, char* argv[])
     std::cout << "psip( 1, 0) "<<psip0<<"\n";
     //Find O-point
     double R_O = gp.R_0, Z_O = 0.;
-    dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
+    if( !gp.isToroidal() )
+        dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
     const double psipmin = mag.psip()(R_O, Z_O);
 
     std::cout << "O-point "<<R_O<<" "<<Z_O<<" with Psip = "<<psipmin<<std::endl;
@@ -274,8 +275,9 @@ int main( int argc, char* argv[])
         std::cout << "Generate X-point flux-aligned grid!\n";
         double RX = gp.R_0-1.1*gp.triangularity*gp.a;
         double ZX = -1.1*gp.elongation*gp.a;
+        dg::geo::findXpoint( mag.get_psip(), RX, ZX);
         dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), RX, ZX) ;
-        dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, R_X, Z_X, mag.R0(), 0, 0, true);
+        dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, RX, ZX, mag.R0(), 0, 0, true);
         double fx_0 = 1./8.;
         psipmax = -fx_0/(1.-fx_0)*psipmin;
         std::cout << "psi 1 is          "<<psipmax<<"\n";
@@ -339,72 +341,76 @@ int main( int argc, char* argv[])
     }
 
     ///////////////////Compute flux average////////////////////
-    std::cout << "Compute flux averages\n";
-    dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
-    if( gp.hasXpoint() )
-        dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
-    dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, mag, psipog2d, xpoint_weights);
-    dg::Grid1d grid1d(psipmin, psipmax, npsi ,Npsi,dg::NEU);
-    map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
-        "Flux surface average of psi with delta function");
-    if( gp.equilibrium == "solovev")
+    dg::Grid1d grid1d( 0,1,npsi,Npsi, dg::NEU);
+    if( !gp.isToroidal())
     {
-        dg::geo::SafetyFactor     qprof( mag);
-        map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
-            "q-profile (Safety factor) using direct integration");
-    }
-    else
-    {
-        dg::geo::SafetyFactorAverage     qprof( grid2d, mag);
-        map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
-            "q-profile (Safety factor) using average integration");
-    }
-    map1d.emplace_back("psip1d",    dg::evaluate( dg::cooX1d, grid1d),
-        "Flux label psi evaluated on grid1d");
-    dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
-    dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
-    map1d.emplace_back("rho", rho,
-        "Alternative flux label rho = -psi/psimin + 1");
-    fsa.set_container( (dg::DVec)curvNablaBGradPsip);
-    map1d.emplace_back("curvNablaB_fsa",   dg::evaluate( fsa,      grid1d),
-        "Flux surface average of true Nabla B curvature Dot Grad Psip with delta function");
-    fsa.set_container( (dg::DVec)curvKappaKGradPsip);
-    map1d.emplace_back("curvKappaK_fsa",   dg::evaluate( fsa,      grid1d),
-        "Flux surface average of true Kappa curvature Dot Grad Psip with delta function");
-    fsa.set_container( (dg::DVec)gradPsip);
-    map1d.emplace_back("gradPsip_fsa",   dg::evaluate( fsa,      grid1d),
-        "Flux surface average of |Grad Psip| with delta function");
+        std::cout << "Compute flux averages\n";
+        dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
+        if( gp.hasXpoint() )
+            dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
+        dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, mag, psipog2d, xpoint_weights);
+        grid1d = dg::Grid1d (psipmin, psipmax, npsi ,Npsi,dg::NEU);
+        map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
+            "Flux surface average of psi with delta function");
+        if( gp.equilibrium == "solovev")
+        {
+            dg::geo::SafetyFactor     qprof( mag);
+            map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
+                "q-profile (Safety factor) using direct integration");
+        }
+        else
+        {
+            dg::geo::SafetyFactorAverage     qprof( grid2d, mag);
+            map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
+                "q-profile (Safety factor) using average integration");
+        }
+        map1d.emplace_back("psip1d",    dg::evaluate( dg::cooX1d, grid1d),
+            "Flux label psi evaluated on grid1d");
+        dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
+        dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
+        map1d.emplace_back("rho", rho,
+            "Alternative flux label rho = -psi/psimin + 1");
+        fsa.set_container( (dg::DVec)curvNablaBGradPsip);
+        map1d.emplace_back("curvNablaB_fsa",   dg::evaluate( fsa,      grid1d),
+            "Flux surface average of true Nabla B curvature Dot Grad Psip with delta function");
+        fsa.set_container( (dg::DVec)curvKappaKGradPsip);
+        map1d.emplace_back("curvKappaK_fsa",   dg::evaluate( fsa,      grid1d),
+            "Flux surface average of true Kappa curvature Dot Grad Psip with delta function");
+        fsa.set_container( (dg::DVec)gradPsip);
+        map1d.emplace_back("gradPsip_fsa",   dg::evaluate( fsa,      grid1d),
+            "Flux surface average of |Grad Psip| with delta function");
 
-    //other flux labels
-    dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, mag);
-    fsi.set_right( xpoint_weights);
+        //other flux labels
+        dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, mag);
+        fsi.set_right( xpoint_weights);
 
-    dg::HVec areaT_psip = dg::evaluate( fsi, grid1d);
-    dg::HVec w1d = dg::create::weights( grid1d);
-    double volumeCoarea = 2.*M_PI*dg::blas1::dot( areaT_psip, w1d);
+        dg::HVec areaT_psip = dg::evaluate( fsi, grid1d);
+        dg::HVec w1d = dg::create::weights( grid1d);
+        double volumeCoarea = 2.*M_PI*dg::blas1::dot( areaT_psip, w1d);
 
-    //area
-    fsi.set_left( dg::evaluate( dg::geo::GradPsip(mag), grid2d));
-    dg::HVec psi_area = dg::evaluate( fsi, grid1d);
-    dg::blas1::scal(psi_area, 2.*M_PI);
-    map1d.emplace_back( "psi_area", psi_area,
-        "Flux area with delta function");
+        //area
+        fsi.set_left( dg::evaluate( dg::geo::GradPsip(mag), grid2d));
+        dg::HVec psi_area = dg::evaluate( fsi, grid1d);
+        dg::blas1::scal(psi_area, 2.*M_PI);
+        map1d.emplace_back( "psi_area", psi_area,
+            "Flux area with delta function");
 
-    dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, mag);
-    std::cout << "Delta Rho for Flux surface integrals = "<<-fsi.get_deltapsi()/psipmin<<"\n";
+        dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, mag);
+        std::cout << "Delta Rho for Flux surface integrals = "<<-fsi.get_deltapsi()/psipmin<<"\n";
 
-    fvi.set_right( xpoint_weights);
-    dg::HVec psi_vol = dg::evaluate( fvi, grid1d);
-    dg::blas1::scal(psi_vol, 2.*M_PI);
-    map1d.emplace_back( "psi_vol", psi_vol,
-        "Flux volume with delta function");
-    double volumeFVI = 2.*M_PI*fvi(psipmax);
-    std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeCoarea<<" "<<volumeFVI
-              <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
-    if(gp.hasXpoint()){
-        std::cout << "VOLUME TEST WITH X Grid FORMULA: "<<volumeXGrid<<" "<<volumeFVI
-                  <<" rel error = "<<fabs(volumeXGrid-volumeFVI)/volumeFVI<<"\n";
-    };
+        fvi.set_right( xpoint_weights);
+        dg::HVec psi_vol = dg::evaluate( fvi, grid1d);
+        dg::blas1::scal(psi_vol, 2.*M_PI);
+        map1d.emplace_back( "psi_vol", psi_vol,
+            "Flux volume with delta function");
+        double volumeFVI = 2.*M_PI*fvi(psipmax);
+        std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeCoarea<<" "<<volumeFVI
+                  <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
+        if(gp.hasXpoint()){
+            std::cout << "VOLUME TEST WITH X Grid FORMULA: "<<volumeXGrid<<" "<<volumeFVI
+                      <<" rel error = "<<fabs(volumeXGrid-volumeFVI)/volumeFVI<<"\n";
+        };
+    }
 
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index dcc77a991..2be586da8 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -34,10 +34,10 @@ namespace solovev
  * @brief \f[ \hat{\psi}_p  \f]
  *
  * \f[ \hat{\psi}_p(R,Z) =
-      \hat{R}_0\Bigg\{B\bar{R}^4/8 + A \left[ 1/2 \bar{R}^2  \ln{(\bar{R}   )}-(\bar{R}^4 )/8\right]
+      \hat{R}_0P_{\psi}\Bigg\{\bar{R}^4/8 + A \left[ 1/2 \bar{R}^2  \ln{(\bar{R}   )}-(\bar{R}^4 )/8\right]
       + \sum_{i=1}^{12} c_i\bar \psi_{pi}(\bar R, \bar Z) \Bigg\}
       =
-      \hat{R}_0\Bigg\{B\bar{R}^4/8 + A \left[ 1/2 \bar{R}^2  \ln{(\bar{R}   )}-(\bar{R}^4 )/8\right]
+      \hat{R}_0P_{\psi}\Bigg\{\bar{R}^4/8 + A \left[ 1/2 \bar{R}^2  \ln{(\bar{R}   )}-(\bar{R}^4 )/8\right]
       +c_1+
       c_2 \bar{R}^2 +
       c_3 \left[  \bar{Z}^2-\bar{R}^2  \ln{(\bar{R}   )} \right] +
@@ -64,14 +64,14 @@ struct Psip: public aCylindricalFunctor<Psip>
      *
      * @param gp geometric parameters
      */
-    Psip( Parameters gp ): R_0_(gp.R_0), A_(gp.A), B_(gp.B), c_(gp.c) {}
+    Psip( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) {}
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,Zn5,Zn6,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn4 = Rn2*Rn2;
         Zn = Z/R_0_; Zn2 = Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2; Zn5 = Zn3*Zn2; Zn6 = Zn3*Zn3;
         lgRn= log(Rn);
-        return   R_0_*( B_*Rn4/8.+ A_ * ( 1./2.* Rn2* lgRn-(Rn4)/8.)
+        return   R_0_*pp_*( Rn4/8.+ A_ * ( 1./2.* Rn2* lgRn-(Rn4)/8.)
                       + c_[0]  //c_[0] entspricht c_1
               + c_[1]  *Rn2
               + c_[2]  *(Zn2 - Rn2 * lgRn )
@@ -88,7 +88,7 @@ struct Psip: public aCylindricalFunctor<Psip>
                       );
     }
   private:
-    double R_0_, A_, B_;
+    double R_0_, A_, pp_;
     std::vector<double> c_;
 };
 
@@ -96,9 +96,9 @@ struct Psip: public aCylindricalFunctor<Psip>
  * @brief \f[ \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}} \f]
  *
  * \f[ \frac{\partial  \hat{\psi}_p }{ \partial \hat{R}} =
-      \Bigg\{ 2 c_2 \bar{R} +(\bar{R}^3 )/2+2 c_9 \bar{R}  \bar{Z}
+      P_\psi \Bigg\{ 2 c_2 \bar{R} +(\bar{R}^3 )/2+2 c_9 \bar{R}  \bar{Z}
       +c_4 (4 \bar{R}^3 -8 \bar{R}  \bar{Z}^2)+c_{11}
-      (12 B \bar{R}^3  \bar{Z}-8 \bar{R}  \bar{Z}^3
+      (12  \bar{R}^3  \bar{Z}-8 \bar{R}  \bar{Z}^3
       +c_6 (6 \bar{R}^5 -48 \bar{R}^3  \bar{Z}^2+16 \bar{R}  \bar{Z}^4)+c_3 (-\bar{R} -2 \bar{R}  \ln{(\bar{R}   )})+
       A ((\bar{R} )/2-(\bar{R}^3 )/2+\bar{R}  \ln{(\bar{R}   )})
       +c_{10} (-3 \bar{R}  \bar{Z}-6 \bar{R}  \bar{Z} \ln{(\bar{R}   )})+c_5 (3 \bar{R}^3 -30 \bar{R}
@@ -113,14 +113,14 @@ struct Psip: public aCylindricalFunctor<Psip>
 struct PsipR: public aCylindricalFunctor<PsipR>
 {
     ///@copydoc Psip::Psip()
-    PsipR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), B_(gp.B), c_(gp.c) {}
+    PsipR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) {}
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn3,Rn5,Zn,Zn2,Zn3,Zn4,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn3 = Rn2*Rn;  Rn5 = Rn3*Rn2;
         Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
         lgRn= log(Rn);
-        return   (Rn3/2.*B_ + (Rn/2. - Rn3/2. + Rn*lgRn)* A_ +
+        return   pp_*(Rn3/2. + (Rn/2. - Rn3/2. + Rn*lgRn)* A_ +
         2.* Rn* c_[1] + (-Rn - 2.* Rn*lgRn)* c_[2] + (4.*Rn3 - 8.* Rn *Zn2)* c_[3] +
         (3. *Rn3 - 30.* Rn *Zn2 + 12. *Rn3*lgRn -  24.* Rn *Zn2*lgRn)* c_[4]
         + (6 *Rn5 - 48 *Rn3 *Zn2 + 16.* Rn *Zn4)*c_[5]
@@ -131,14 +131,14 @@ struct PsipR: public aCylindricalFunctor<PsipR>
           );
     }
   private:
-    double R_0_, A_, B_;
+    double R_0_, A_, pp_;
     std::vector<double> c_;
 };
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}\f]
  *
  * \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}=
-     \hat{R}_0^{-1} \Bigg\{ 2 c_2 +B(3 \hat{\bar{R}}^2 )/2+2 c_9  \bar{Z}+c_4 (12 \bar{R}^2 -8  \bar{Z}^2)+c_{11}
+     \hat{R}_0^{-1} P_\psi \Bigg\{ 2 c_2 +(3 \hat{\bar{R}}^2 )/2+2 c_9  \bar{Z}+c_4 (12 \bar{R}^2 -8  \bar{Z}^2)+c_{11}
       (36 \bar{R}^2  \bar{Z}-8  \bar{Z}^3)
       +c_6 (30 \bar{R}^4 -144 \bar{R}^2  \bar{Z}^2+16  \bar{Z}^4)+c_3 (-3 -2  \ln{(\bar{R}
       )})+
@@ -154,14 +154,14 @@ struct PsipR: public aCylindricalFunctor<PsipR>
 struct PsipRR: public aCylindricalFunctor<PsipRR>
 {
     ///@copydoc Psip::Psip()
-    PsipRR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), B_(gp.B), c_(gp.c) {}
+    PsipRR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) {}
     double do_compute(double R, double Z) const
     {
        double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,lgRn;
        Rn = R/R_0_; Rn2 = Rn*Rn;  Rn4 = Rn2*Rn2;
        Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
        lgRn= log(Rn);
-       return   1./R_0_*( (3.* Rn2)/2.*B_ + (3./2. - (3. *Rn2)/2. +lgRn) *A_ +  2.* c_[1] + (-3. - 2.*lgRn)* c_[2] + (12. *Rn2 - 8. *Zn2) *c_[3] +
+       return   pp_/R_0_*( (3.* Rn2)/2. + (3./2. - (3. *Rn2)/2. +lgRn) *A_ +  2.* c_[1] + (-3. - 2.*lgRn)* c_[2] + (12. *Rn2 - 8. *Zn2) *c_[3] +
          (21. *Rn2 - 54. *Zn2 + 36. *Rn2*lgRn - 24. *Zn2*lgRn)* c_[4]
          + (30. *Rn4 - 144. *Rn2 *Zn2 + 16.*Zn4)*c_[5] + (-165. *Rn4 + 2160. *Rn2 *Zn2 - 640. *Zn4 - 450. *Rn4*lgRn +
       2160. *Rn2 *Zn2*lgRn - 240. *Zn4*lgRn)* c_[6] +
@@ -170,14 +170,14 @@ struct PsipRR: public aCylindricalFunctor<PsipRR>
  +   (-120. *Rn2* Zn - 240. *Zn3 + 720. *Rn2* Zn*lgRn - 160. *Zn3*lgRn)* c_[11]);
     }
   private:
-    double R_0_, A_, B_;
+    double R_0_, A_, pp_;
     std::vector<double> c_;
 };
 /**
  * @brief \f[\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\f]
  *
  * \f[\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}=
-      \Bigg\{c_8 +c_9 \bar{R}^2 +2 c_3  \bar{Z}-8 c_4 \bar{R}^2  \bar{Z}+c_{11}
+      P_\psi \Bigg\{c_8 +c_9 \bar{R}^2 +2 c_3  \bar{Z}-8 c_4 \bar{R}^2  \bar{Z}+c_{11}
       (3 \bar{R}^4 -12 \bar{R}^2  \bar{Z}^2)+c_6 (-24 \bar{R}^4  \bar{Z}+32 \bar{R}^2  \bar{Z}^3)
       +c_{10} (3  \bar{Z}^2-3 \bar{R}^2
       \ln{(\bar{R}   )})+c_5 (-18 \bar{R}^2  \bar{Z}+8  \bar{Z}^3-24 \bar{R}^2  \bar{Z}
@@ -190,7 +190,7 @@ struct PsipRR: public aCylindricalFunctor<PsipRR>
 struct PsipZ: public aCylindricalFunctor<PsipZ>
 {
     ///@copydoc Psip::Psip()
-    PsipZ( Parameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
+    PsipZ( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) { }
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,Zn5,lgRn;
@@ -198,7 +198,7 @@ struct PsipZ: public aCylindricalFunctor<PsipZ>
         Zn = Z/R_0_; Zn2 = Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2; Zn5 = Zn3*Zn2;
         lgRn= log(Rn);
 
-        return   (2.* Zn* c_[2]
+        return   pp_*(2.* Zn* c_[2]
             -  8. *Rn2* Zn* c_[3] +
               ((-18.)*Rn2 *Zn + 8. *Zn3 - 24. *Rn2* Zn*lgRn) *c_[4]
             + ((-24.) *Rn4* Zn + 32. *Rn2 *Zn3)* c_[5]
@@ -211,14 +211,14 @@ struct PsipZ: public aCylindricalFunctor<PsipZ>
 
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, pp_;
     std::vector<double> c_;
 };
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}\f]
 
    \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}=
-      \hat{R}_0^{-1} \Bigg\{2 c_3 -8 c_4 \bar{R}^2 +6 c_{10}  \bar{Z}-24 c_{11}
+      \hat{R}_0^{-1} P_\psi \Bigg\{2 c_3 -8 c_4 \bar{R}^2 +6 c_{10}  \bar{Z}-24 c_{11}
       \bar{R}^2  \bar{Z}+c_6 (-24 \bar{R}^4 +96 \bar{R}^2  \bar{Z}^2)
       +c_5 (-18 \bar{R}^2 +24  \bar{Z}^2-24 \bar{R}^2  \ln{(\bar{R}   )})+
       c_{12} (160  \bar{Z}^3-480 \bar{R}^2  \bar{Z} \ln{(\bar{R}   )})
@@ -228,25 +228,25 @@ struct PsipZ: public aCylindricalFunctor<PsipZ>
 struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 {
     ///@copydoc Psip::Psip()
-    PsipZZ( Parameters gp): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
+    PsipZZ( Parameters gp): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) { }
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn4 = Rn2*Rn2;
         Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
         lgRn= log(Rn);
-        return   1./R_0_*( 2.* c_[2] - 8. *Rn2* c_[3] + (-18. *Rn2 + 24. *Zn2 - 24. *Rn2*lgRn) *c_[4] + (-24.*Rn4 + 96. *Rn2 *Zn2) *c_[5]
+        return   pp_/R_0_*( 2.* c_[2] - 8. *Rn2* c_[3] + (-18. *Rn2 + 24. *Zn2 - 24. *Rn2*lgRn) *c_[4] + (-24.*Rn4 + 96. *Rn2 *Zn2) *c_[5]
         + (150. *Rn4 - 1680. *Rn2 *Zn2 + 240. *Zn4 + 360. *Rn4*lgRn - 1440. *Rn2 *Zn2*lgRn)* c_[6] + 6.* Zn* c_[9] -  24. *Rn2 *Zn *c_[10] + (160. *Zn3 - 480. *Rn2* Zn*lgRn) *c_[11]);
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, pp_;
     std::vector<double> c_;
 };
 /**
  * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}\f]
 
   \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}=
-        \hat{R}_0^{-1} \Bigg\{2 c_9 \bar{R} -16 c_4 \bar{R}  \bar{Z}+c_{11}
+        \hat{R}_0^{-1} P_\psi \Bigg\{2 c_9 \bar{R} -16 c_4 \bar{R}  \bar{Z}+c_{11}
       (12 \bar{R}^3 -24 \bar{R}  \bar{Z}^2)+c_6 (-96 \bar{R}^3  \bar{Z}+64 \bar{R}  \bar{Z}^3)
       + c_{10} (-3 \bar{R} -6 \bar{R}  \ln{(\bar{R}   )})
       +c_5 (-60 \bar{R}  \bar{Z}-48 \bar{R}  \bar{Z} \ln{(\bar{R}   )})
@@ -258,40 +258,42 @@ struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 struct PsipRZ: public aCylindricalFunctor<PsipRZ>
 {
     ///@copydoc Psip::Psip()
-    PsipRZ( Parameters gp ): R_0_(gp.R_0), A_(gp.A), c_(gp.c) { }
+    PsipRZ( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) { }
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn3,Zn,Zn2,Zn3,lgRn;
         Rn = R/R_0_; Rn2 = Rn*Rn; Rn3 = Rn2*Rn;
         Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn;
         lgRn= log(Rn);
-        return   1./R_0_*(
+        return   pp_/R_0_*(
               -16.* Rn* Zn* c_[3] + (-60.* Rn* Zn - 48.* Rn* Zn*lgRn)* c_[4] + (-96. *Rn3* Zn + 64.*Rn *Zn3)* c_[5]
             + (960. *Rn3 *Zn - 1600.* Rn *Zn3 + 1440. *Rn3* Zn*lgRn - 960. *Rn *Zn3*lgRn) *c_[6] +  2.* Rn* c_[8] + (-3.* Rn - 6.* Rn*lgRn)* c_[9]
             + (12. *Rn3 - 24.* Rn *Zn2) *c_[10] + (-120. *Rn3 - 240. *Rn *Zn2 + 240. *Rn3*lgRn -   480.* Rn *Zn2*lgRn)* c_[11]
                  );
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, pp_;
     std::vector<double> c_;
 };
 
 /**
  * @brief \f[\hat{I}\f]
 
-    \f[\hat{I}= \sqrt{-2 A \hat{\psi}_p / \hat{R}_0 +1}\f]
+    \f[\hat{I}= P_I \sqrt{-2 A \hat{\psi}_p / \hat{R}_0/P_\psi +1}\f]
  */
 struct Ipol: public aCylindricalFunctor<Ipol>
 {
     ///@copydoc Psip::Psip()
-    Ipol(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), psip_(gp) { }
+    Ipol(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp) {
+        if( gp.pp == 0.)
+            pp_ = 1.; //safety measure to avoid divide by zero errors
+    }
     double do_compute(double R, double Z) const
     {
-        //sign before A changed to -
-        return sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.);
+        return pi_*sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.);
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, pp_, pi_;
     Psip psip_;
 };
 /**
@@ -300,13 +302,16 @@ struct Ipol: public aCylindricalFunctor<Ipol>
 struct IpolR: public aCylindricalFunctor<IpolR>
 {
     ///@copydoc Psip::Psip()
-    IpolR(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), psip_(gp), psipR_(gp) { }
+    IpolR(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp), psipR_(gp) {
+        if( gp.pp == 0.)
+            pp_ = 1.; //safety measure to avoid divide by zero errors
+    }
     double do_compute(double R, double Z) const
     {
-        return -1./sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipR_(R,Z)/R_0_);
+        return -pi_/sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.)*(A_*psipR_(R,Z)/R_0_/pp_);
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, pp_, pi_;
     Psip psip_;
     PsipR psipR_;
 };
@@ -316,13 +321,16 @@ struct IpolR: public aCylindricalFunctor<IpolR>
 struct IpolZ: public aCylindricalFunctor<IpolZ>
 {
     ///@copydoc Psip::Psip()
-    IpolZ(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), psip_(gp), psipZ_(gp) { }
+    IpolZ(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp), psipZ_(gp) {
+        if( gp.pp == 0.)
+            pp_ = 1.; //safety measure to avoid divide by zero errors
+    }
     double do_compute(double R, double Z) const
     {
-        return -1./sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipZ_(R,Z)/R_0_);
+        return -pi_/sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.)*(A_*psipZ_(R,Z)/R_0_/pp_);
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, pp_, pi_;
     Psip psip_;
     PsipZ psipZ_;
 };
@@ -349,8 +357,8 @@ namespace mod
 
 struct Psip: public aCylindricalFunctor<Psip>
 {
-    Psip( Parameters gp, double psi0, double alpha) :
-        m_ipoly( psi0, alpha, -1), m_psip(gp)
+    Psip( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_ipoly( psi0, alpha, sign), m_psip(gp)
     { }
     double do_compute(double R, double Z) const
     {
@@ -363,8 +371,8 @@ struct Psip: public aCylindricalFunctor<Psip>
 };
 struct PsipR: public aCylindricalFunctor<PsipR>
 {
-    PsipR( Parameters gp, double psi0, double alpha) :
-        m_poly( psi0, alpha, -1), m_psip(gp), m_psipR(gp)
+    PsipR( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_psip(gp), m_psipR(gp)
     { }
     double do_compute(double R, double Z) const
     {
@@ -379,8 +387,8 @@ struct PsipR: public aCylindricalFunctor<PsipR>
 };
 struct PsipZ: public aCylindricalFunctor<PsipZ>
 {
-    PsipZ( Parameters gp, double psi0, double alpha) :
-        m_poly( psi0, alpha, -1), m_psip(gp), m_psipZ(gp)
+    PsipZ( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_psip(gp), m_psipZ(gp)
     { }
     double do_compute(double R, double Z) const
     {
@@ -396,8 +404,8 @@ struct PsipZ: public aCylindricalFunctor<PsipZ>
 
 struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 {
-    PsipZZ( Parameters gp, double psi0, double alpha) :
-        m_poly( psi0, alpha, -1), m_dpoly( psi0, alpha, -1), m_psip(gp), m_psipZ(gp), m_psipZZ(gp)
+    PsipZZ( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipZ(gp), m_psipZZ(gp)
     { }
     double do_compute(double R, double Z) const
     {
@@ -415,8 +423,8 @@ struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 };
 struct PsipRR: public aCylindricalFunctor<PsipRR>
 {
-    PsipRR( Parameters gp, double psi0, double alpha) :
-        m_poly( psi0, alpha, -1), m_dpoly( psi0, alpha, -1), m_psip(gp), m_psipR(gp), m_psipRR(gp)
+    PsipRR( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipR(gp), m_psipRR(gp)
     { }
     double do_compute(double R, double Z) const
     {
@@ -434,8 +442,8 @@ struct PsipRR: public aCylindricalFunctor<PsipRR>
 };
 struct PsipRZ: public aCylindricalFunctor<PsipRZ>
 {
-    PsipRZ( Parameters gp, double psi0, double alpha) :
-        m_poly( psi0, alpha, -1), m_dpoly( psi0, alpha, -1), m_psip(gp), m_psipR(gp), m_psipZ(gp), m_psipRZ(gp)
+    PsipRZ( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipR(gp), m_psipZ(gp), m_psipRZ(gp)
     { }
     double do_compute(double R, double Z) const
     {
@@ -456,52 +464,66 @@ struct PsipRZ: public aCylindricalFunctor<PsipRZ>
 
 struct Ipol: public aCylindricalFunctor<Ipol>
 {
-    Ipol(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), psip_(gp, psi0, alpha) { }
+    Ipol(  Parameters gp, double psi0, double alpha, double sign = -1):
+        R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp, psi0, alpha, sign) {
+            if( gp.pp == 0.)
+                pp_ = 1.; //safety measure to avoid divide by zero errors
+    }
     double do_compute(double R, double Z) const
     {
-        return sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.);
+        return pi_*sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.);
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, pp_, pi_;
     mod::Psip psip_;
 };
 struct IpolR: public aCylindricalFunctor<IpolR>
 {
-    IpolR(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), psip_(gp, psi0, alpha), psipR_(gp, psi0, alpha) { }
+    IpolR(  Parameters gp, double psi0, double alpha, double sign = -1 ):
+        R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp, psi0, alpha,
+        sign), psipR_(gp, psi0, alpha, sign) {
+            if( gp.pp == 0.)
+                pp_ = 1.; //safety measure to avoid divide by zero errors
+    }
     double do_compute(double R, double Z) const
     {
-        return -1./sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipR_(R,Z)/R_0_);
+        return -pi_/sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.)*(A_*psipR_(R,Z)/R_0_/pp_);
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, pp_, pi_;
     mod::Psip psip_;
     mod::PsipR psipR_;
 };
 struct IpolZ: public aCylindricalFunctor<IpolZ>
 {
-    IpolZ(  Parameters gp, double psi0, double alpha ):  R_0_(gp.R_0), A_(gp.A), psip_(gp, psi0, alpha), psipZ_(gp, psi0, alpha) { }
+    IpolZ(  Parameters gp, double psi0, double alpha, double sign = -1 ):
+        R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi),
+        psip_(gp, psi0, alpha, sign), psipZ_(gp, psi0, alpha, sign) {
+            if( gp.pp == 0.)
+                pp_ = 1.; //safety measure to avoid divide by zero errors
+    }
     double do_compute(double R, double Z) const
     {
-        return -1./sqrt(-2.*A_* psip_(R,Z) /R_0_ + 1.)*(A_*psipZ_(R,Z)/R_0_);
+        return -pi_/sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.)*(A_*psipZ_(R,Z)/R_0_/pp_);
     }
   private:
-    double R_0_, A_;
+    double R_0_, A_, pp_, pi_;
     mod::Psip psip_;
     mod::PsipZ psipZ_;
 };
 
 static inline dg::geo::CylindricalFunctorsLvl2 createPsip( Parameters gp,
-    double psi0, double alpha)
+    double psi0, double alpha, double sign = -1)
 {
-    return CylindricalFunctorsLvl2( Psip(gp, psi0, alpha), PsipR(gp, psi0,
-    alpha), PsipZ(gp, psi0, alpha), PsipRR(gp, psi0, alpha), PsipRZ(gp,
-    psi0, alpha), PsipZZ(gp, psi0, alpha));
+    return CylindricalFunctorsLvl2( Psip(gp, psi0, alpha, sign), PsipR(gp,
+    psi0, alpha, sign), PsipZ(gp, psi0, alpha, sign), PsipRR(gp, psi0, alpha,
+    sign), PsipRZ(gp, psi0, alpha, sign), PsipZZ(gp, psi0, alpha, sign));
 }
 static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp,
-    double psi0, double alpha)
+    double psi0, double alpha, double sign = -1)
 {
-    return CylindricalFunctorsLvl1( Ipol(gp, psi0, alpha), IpolR(gp, psi0,
-    alpha), IpolZ(gp, psi0, alpha));
+    return CylindricalFunctorsLvl1( Ipol(gp, psi0, alpha, sign), IpolR(gp,
+    psi0, alpha, sign), IpolZ(gp, psi0, alpha, sign));
 }
 
 } //namespace mod
@@ -539,14 +561,15 @@ static inline dg::geo::TokamakMagneticField createSolovevField(
  * @param gp Solovev parameters
  * @param psi0 above this value psi is modified to a constant psi0
  * @param alpha determines how quickly the modification acts (smaller is quicker)
+ * @param sign determines which side of Psi to dampen (negative or positive)
  * @return A magnetic field object
  * @ingroup geom
  */
 static inline dg::geo::TokamakMagneticField createModifiedSolovevField(
-    dg::geo::solovev::Parameters gp, double psi0, double alpha)
+    dg::geo::solovev::Parameters gp, double psi0, double alpha, double sign = -1)
 {
-    return TokamakMagneticField( gp.R_0, solovev::mod::createPsip(gp,
-        psi0, alpha), solovev::mod::createIpol(gp, psi0, alpha));
+    return TokamakMagneticField( gp.R_0, solovev::mod::createPsip(gp, psi0,
+    alpha, sign), solovev::mod::createIpol(gp, psi0, alpha, sign));
 }
 
 } //namespace geo
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index 98ff9c751..5cdb05139 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -18,8 +18,10 @@ namespace solovev
  */
 struct Parameters
 {
-    double A,B, //!< A and B coefficients, B is not there originally but is added for more flexibility
-           R_0, //!< central tokamak radius
+    double A, //!< A coefficient
+           R_0, //!< major tokamak radius
+           pp, //!< prefactor for Psi_p
+           pi, //!< prefactor for current I
            a,  //!<  little tokamak radius
            elongation, //!< elongation of the magnetic surfaces
            triangularity, //!< triangularity of the magnetic surfaces
@@ -34,22 +36,26 @@ struct Parameters
 #ifdef JSONCPP_VERSION_STRING
     /**
      * @brief Construct from Json dataset
-     * @param js Must contain the variables "A", "c", "R_0", "inverseaspectratio", "elongation", "triangularity", "alpha", "rk4eps" (1e-5), "psip_min" (0), "psip_max" (0), "psip_max_cut" (0), "psip_max_lim" (1e10), "equilibrium" ("solovev")
-     * @note the default values in brackets are taken if the variables are not found in the input
+     * @param js Can contain the variables "A" (0), "c" (0), "PP" (1.), "PI"
+     * (1.), "R_0" , "inverseaspectratio" , "elongation" (1), "triangularity"
+     * (0), "alpha"  (0.), "rk4eps" (1e-5), "psip_min" (0), "psip_max" (0),
+     * "psip_max_cut" (0), "psip_max_lim" (1e10), "equilibrium" ("solovev")
+     * @note the default values in brackets are taken if the variables are not found in the input file
      * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
      */
     Parameters( const Json::Value& js) {
         A  = js.get("A", 0).asDouble();
-        B  = js.get("B", 1).asDouble();
+        pp  = js.get("PP", 1).asDouble();
+        pi  = js.get("PI", 1).asDouble();
         c.resize(12);
         for (unsigned i=0;i<12;i++)
-            c[i] = js["c"][i].asDouble();
+            c[i] = js["c"].get(i,0).asDouble();
 
         R_0  = js["R_0"].asDouble();
         a  = R_0*js["inverseaspectratio"].asDouble();
-        elongation=js["elongation"].asDouble();
-        triangularity=js["triangularity"].asDouble();
-        alpha=js["alpha"].asDouble();
+        elongation=js.get("elongation",1).asDouble();
+        triangularity=js.get("triangularity",0).asDouble();
+        alpha=js.get("alpha",0.).asDouble();
         rk4eps=js.get("rk4eps",1e-5).asDouble();
         psipmin= js.get("psip_min",0).asDouble();
         psipmax= js.get("psip_max",0).asDouble();
@@ -67,7 +73,8 @@ struct Parameters
     {
         Json::Value js;
         js["A"] = A;
-        js["B"] = B;
+        js["PP"] = pp;
+        js["PI"] = pi;
         for (unsigned i=0;i<12;i++) js["c"][i] = c[i];
         js["R_0"] = R_0;
         js["inverseaspectratio"] = a/R_0;
@@ -84,7 +91,7 @@ struct Parameters
     }
 #endif // JSONCPP_VERSION_STRING
     /**
-    * @brief True if all coefficients \c c_i==0 with \c 7<=i<12
+    * @brief True if any coefficient \c c_i!=0 with \c 7<=i<12
     *
     * The Xpoint is situated close to
      <tt> R_X = R_0-1.1*triangularity*a</tt>
@@ -99,12 +106,23 @@ struct Parameters
                 Xpoint = true;
         return Xpoint;
     }
+    /**
+    * @brief True if \c pp==0
+    *
+    * @return \c true if the flux function is a constant
+    */
+    bool isToroidal() const{
+        if( pp == 0)
+            return true;
+        return false;
+    }
     ///Write variables as a formatted string
     void display( std::ostream& os = std::cout ) const
     {
         os << "Geometrical parameters are: \n"
             <<" A               = "<<A<<"\n"
-            <<" B [optional]    = "<<B<<"\n";
+            <<" Prefactor Psi   = "<<pp<<"\n"
+            <<" Prefactor I     = "<<pi<<"\n";
         for( unsigned i=0; i<12; i++)
             os<<" c"<<i+1<<"\t\t = "<<c[i]<<"\n";
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index fc33f8af3..32d4b2850 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -174,19 +174,24 @@ from where we recover the Grad-Shafranov equation
 with $\Delta^*_\perp \psi_p = R\partial_R (R^{-1}\psi_R) + \psi_{ZZ}$.
 The Solov'ev assumptions consist of \(A/R_0 = -I \frac{d I}{d  \psi_p }\) and \((1-A)/R_0 = -\frac{d p}{d  \psi_p }\), where \(A\) is a constant~\cite{Cerfon2010,Cerfon2014}.
 By integration over \(\psi_p\) we find
-\begin{align}\label{eq:solovevassumption}
- p(\psi_p) &= (A-1)\psi_p/R_0,  &
- I(\psi_p) &= \sqrt{-2 A \psi_p/R_0 + 1}, &
-    j_{\hat\varphi} &= \left[(A-1)R^2/R_0^2 - A \right]/R.
-\end{align}
-Now, we introduce \(\bar{R} \equiv \frac{R}{R_0}\) and \(\bar{Z} \equiv\frac{Z}{R_0}\)
-and solve Equations~\eqref{eq:GSEdimless} and~\eqref{eq:solovevassumption} to obtain
+$
+ p(\psi_p) = (A-1)\psi_p/R_0$,
+ $I(\psi_p) = \sqrt{-2 A \psi_p/R_0 + 1}$,
+ and
+    $j_{\hat\varphi} = \left[(A-1)R^2/R_0^2 - A \right]/R $.
+Note that if $\psi_p$, $I(\psi)$ and $p(\psi)$ are a solution to Eq.~\eqref{eq:GSEdimless}
+then so are $\mathcal P_\psi \psi_p$ , $\mathcal P_\psi I(\psi_p)$ and $\mathcal P_\psi^2 p(\psi_p)$.
+Also note that for $A=0$ the constant current $I$ is arbitrary $\mathcal P_I$.
+
+We introduce \(\bar{R} \equiv \frac{R}{R_0}\) and \(\bar{Z} \equiv\frac{Z}{R_0}\)
+and represent a solution to Equation~\eqref{eq:GSEdimless} as~\cite{Cerfon2010}
 \begin{align}\label{eq:solovev}
- \psi_p (R,Z) &= R_0 \left[ A\left( \frac{1}{2} \bar{R}^2 \ln{\bar{R}}
-   - \frac{1}{8}\bar{R}^4\right)+ B\frac{1}{8}\bar{R}^4 
-   + \sum_{i=1}^{12} c_{i}  \bar{\psi}_{pi}\right],
+ \psi_p (R,Z) &= \mathcal P_{\psi} R_0 \left[ A\left( \frac{1}{2} \bar{R}^2 \ln{\bar{R}}
+   - \frac{1}{8}\bar{R}^4\right)+ \frac{1}{8}\bar{R}^4 
+   + \sum_{i=1}^{12} c_{i}  \bar{\psi}_{pi}\right],\\
+   I(\psi_p) &= \mathcal P_I\sqrt{ - 2A\psi_p/(R_0\mathcal P_{\psi}) +1},
 \end{align}
-with
+with $\mathcal P_I = \pm \mathcal P_\psi$ for $A\neq 0$, $p(\psi_p) = \mathcal P_\psi (A-1)\psi_p/R_0$ and
 \rowcolors{2}{gray!25}{white}
 \begin{longtable}{>{\RaggedRight}p{7cm}>{\RaggedRight}p{7cm}}
 \toprule
@@ -219,9 +224,11 @@ $\bar{\psi}_{p11}=3 \bar{Z}\bar{R}^4 - 4\bar{Z}^3\bar{R}^2$\\
 \bottomrule
 \end{longtable}
 The choice of the coefficients \(c_{i}\) and \(A\) determines the actual form of the magnetic field, while $R_0$ appears as an artificial scaling factor (note here that a change in $\rho_s$ changes $R_0$ but not the form or size of the dimensional equilibrium magnetic field).
-Note that $B$ is not part of the original work ($B=1$ per default) and has been added for additional flexibility in fitting the coefficients to an existing experimental equilibrium.
+The scaling factors $\mathcal P_\psi$ and $\mathcal P_I$ are mainly introduced to maximize the flexibility e.g. to adapt the solution to experimental equilibria or to reverse the sign of the magnetic field.
+Remember that $\mathcal  P_I \neq \mathcal P_\psi$ only yields a solution for $A=0$.
+
 Eq.~\eqref{eq:solovev} allows axisymmetric equilibria with e.g. single and asymmetric double X-point configurations, force-free states,
-field reversed configurations and low and high beta tokamak equilbria. This casts this simple analytical equilibrium to the ideal choice in order to study geometric effects (e.g. inverse aspect ratio, elongation and triangularity) in magnetised plasmas.
+field reversed configurations and low and high beta tokamak equilibria. This casts this simple analytical equilibrium to the ideal choice in order to study geometric effects (e.g. inverse aspect ratio, elongation and triangularity) in magnetised plasmas.
 
 Note that
 \begin{align}
@@ -273,9 +280,9 @@ This simplifies the curvature operators to:
 \end{align}
 and
 \begin{align}
- \vec{\nabla} \cdot \vec{\mathcal{K}}_{{\nabla\times\bhat}} &\approx \frac{1}{R B^2} \frac{\partial B}{\partial Z}.
+ \vec{\nabla} \cdot \vec{\mathcal{K}}_{{\nabla\times\bhat}} &\approx \frac{1}{R B^2} \frac{\partial B}{\partial Z},
 \end{align}
-which, results in a vanishing divergence of the curvature operators \( \vec{\nabla} \cdot \vec{ \mathcal{K} } = 0\).
+which results in a vanishing divergence of the curvature operators \( \vec{\nabla} \cdot \vec{ \mathcal{K} } = 0\).
 
 Note that in an actual toroidal field we have
 \begin{align}
diff --git a/src/feltor/geometry/torpex.json b/src/feltor/geometry/torpex.json
index c9467fcde..8c2338da4 100644
--- a/src/feltor/geometry/torpex.json
+++ b/src/feltor/geometry/torpex.json
@@ -1,23 +1,18 @@
 {
     "A": 0,
-    "B": 4.752951718389757e-06,
     "c": [
-        0.047717395667799645,
-        0.9559689342704147,
-        -0.9100332687345335,
-        -0.5820663196518737,
-        0.5651064668675106,
-        -0.4018415371030592,
-        -0.014888980816708646,
-        -4.933621033399979e-13,
-        -1.9628885576793177e-12,
-        1.3157047951347136e-12,
-        3.2965297296431085e-13,
-        -3.2602277468028835e-14
+        10039.52880126578,
+        201131.63164937127,
+        -191466.97098005487,
+        -122464.17681875188,
+        118895.89887501781,
+        -84545.68043438873,
+        -3132.5756495908304
     ],
+    "PP": -4.752951718389757e-06,
+    "PI": -1,
     "R_0": 500,
     "elongation": 1,
-    "equilibrium": "fit",
     "triangularity": 0,
     "inverseaspectratio": 0.2
 }
-- 
GitLab


From 112afa83ed4cc9bcd6c5c4deba0a8eebbad7688b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 20 Sep 2019 15:37:30 +0200
Subject: [PATCH 136/540] Add source terms for mass and energy to feltordiag

---
 src/feltor/feltor.h     |  1 +
 src/feltor/feltor.tex   | 41 ++++++++++++++++++++++++++++-------------
 src/feltor/feltordiag.h | 30 +++++++++++++++++++++++++++++-
 3 files changed, 58 insertions(+), 14 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 885c6abd9..5bd6b8b6c 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -848,6 +848,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
 
         dg::blas1::axpby( 1., m_s[0][0], 1.0, yp[0][0]);
         dg::blas1::axpby( 1., m_s[0][1], 1.0, yp[0][1]);
+        //currently we ignore m_s[1]
     }
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( yp[0][0], dg::plus_equals(), manufactured::SNe{
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 32d4b2850..7f5cf4bb0 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -185,12 +185,15 @@ Also note that for $A=0$ the constant current $I$ is arbitrary $\mathcal P_I$.
 
 We introduce \(\bar{R} \equiv \frac{R}{R_0}\) and \(\bar{Z} \equiv\frac{Z}{R_0}\)
 and represent a solution to Equation~\eqref{eq:GSEdimless} as~\cite{Cerfon2010}
-\begin{align}\label{eq:solovev}
+\begin{subequations}
+\label{eq:solovev}
+\begin{align}
  \psi_p (R,Z) &= \mathcal P_{\psi} R_0 \left[ A\left( \frac{1}{2} \bar{R}^2 \ln{\bar{R}}
    - \frac{1}{8}\bar{R}^4\right)+ \frac{1}{8}\bar{R}^4 
    + \sum_{i=1}^{12} c_{i}  \bar{\psi}_{pi}\right],\\
    I(\psi_p) &= \mathcal P_I\sqrt{ - 2A\psi_p/(R_0\mathcal P_{\psi}) +1},
 \end{align}
+\end{subequations}
 with $\mathcal P_I = \pm \mathcal P_\psi$ for $A\neq 0$, $p(\psi_p) = \mathcal P_\psi (A-1)\psi_p/R_0$ and
 \rowcolors{2}{gray!25}{white}
 \begin{longtable}{>{\RaggedRight}p{7cm}>{\RaggedRight}p{7cm}}
@@ -616,7 +619,7 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
     + \mu NU\mathcal K_{\nabla\times\bhat}(\psi) \nonumber\\
     &= -\tau \left(\bhat + \tilde{\vec b}_\perp\right)\cdot \nabla N 
     -N \left( \left(\bhat+\tilde{\vec b}_\perp\right)\cdot \nabla \psi + \frac{\partial A_\parallel}{\partial t}\right) 
-    - \eta n_e^2(U_i-u_e) + \mu N(\Lambda_U + S_U)
+    - \eta n_e^2(U_i-u_e) + \mu N\Lambda_U
 \label{}
 \end{align}
 with
@@ -828,6 +831,7 @@ so that $\nabla_\perp^2 S_{n_e}$ is well defined.
 %  S_{U}(R,Z,\varphi, t) := -\omega_d U \Theta( \rho(R,Z) - \rho_d)
 %\end{align}
 %with $\rho_d > 1$.
+Currently we do not implement any source terms for the velocity $U$.
 
 \subsection{Implemented form}
 The form that we implement avoids derivatives on the product of
@@ -855,7 +859,7 @@ two functions for which we have no boundary conditions
         -2\tau U\mathcal K_{\nabla\times\bhat}(\ln N)
         - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
         + \nu_\perp\Delta_\perp U
-        + \nu_\parallel \Delta_\parallel U + S_U,
+        + \nu_\parallel \Delta_\parallel U,
         \label{eq:EgyrofluidU} \\
         W&:= \left( U + \frac{A_\parallel}{\mu}\right)
     \end{align}
@@ -942,7 +946,7 @@ with ( $z_e=-1$ and $z_i=+1$)
   \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U^2 \right)
   \left(\nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N\right)  +  \mu NU\left(\nu_\perp\Delta_\perp U + \nu_\parallel\Delta_\parallel U\right) \right]
 \nonumber \\
-  S_{\mathcal E} =&  \sum_s  z\left[ \left(\tau\left( 1+\ln{N}\right) +\psi + \frac{1}{2} \mu U^2 \right)S_{N}  + \mu NU S_U\right]
+  S_{\mathcal E} =&  \sum_s  z\left[ \left(\tau\left( 1+\ln{N}\right) +\psi + \frac{1}{2} \mu U^2 \right)S_{N}\right]
 \nonumber \\
   R_{\mathcal E} =&  -\eta_\parallel  \left[ n_e(U_i-u_e)\right]^2.
 \end{align}
@@ -1091,9 +1095,9 @@ Input file format: json
 \begin{longtable}{llll>{\RaggedRight}p{6cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
-n      & integer & 3 & - &Number of Gaussian nodes in R and Z \\
-Nx     & integer &52& - &Number of grid points in R \\
-Ny     & integer &52& - &Number of grid points in Z \\
+n      & integer & 3 & - &Number of Gaussian nodes in R and Z (we always take 3) \\
+Nx     & integer &52& - &Number of grid points in R (increase if your simulations crash) \\
+Ny     & integer &52& - &Number of grid points in Z (increase if your simulations crash) \\
 Nz     & integer &16& - &Number of grid points in $\varphi$ (determines dt since parallel velocity dominates timestep) \\
 dt     & integer &1e-2& - & time stepsize in units of $c_s/\rho_s$ \\
 compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing points in x and y: output contains n*Nx/c[0] points in x,
@@ -1119,10 +1123,18 @@ tau        & float &1      & - & $\tau = T_i/T_e$  \\
 beta       & float & 5e-6  & 0 & Plasma beta $5\cdot 10^{-6}$ (TJK), $4\cdot 10^{-3}$ (Compass), If $0$, then the model is electrostatic \\
 nu\_perp   & float &1e-3   & - & perpendicular viscosity $\nu_\perp$ \\
 perp\_diff & string & "viscous" & "viscous" & "viscous": $\Lambda_\perp\propto \nu_\perp\Delta_\perp$ , "hyperviscous": $\Lambda_\perp \propto -\nu_\perp\Delta_\perp^2$\\
-nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$ \\
+nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$
+(dimensional analysis reveals there can be a factor $(R_0/\rho_s)^2$ between
+$\nu_\perp $n and $\nu_\parallel$ for $\nu_\parallel$ to become relevant for
+the dynamics)\\
 resistivity & float &1e-4  & - & parallel resistivity parameter Eq.~\eqref{eq:resistivity}\\
-curvmode  & string & "low beta"  & "toroidal"& curvature mode ("low beta", "true": no approximation - requires significantly more resolution in Nz, "toroidal": toroidal field approx - elliptic equation does not need communication in z)  \\
-symmetric & bool & false & false & If true, initialize all quantities symmetric in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used to construct the parallel derivatives and then overwritten to $N_z\equiv 1$. \\
+curvmode  & string & "low beta"  & "toroidal"& curvature mode ("low beta",
+"true": no approximation - requires significantly more resolution in Nz,
+"toroidal": toroidal field approx - elliptic equation does not need
+communication in z)  \\
+symmetric & bool & false & false & If true, initialize all quantities symmetric
+in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used
+to construct the parallel derivatives and then overwritten to $N_z\equiv 1$. \\
 bc & dict & & & Dictionary of boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
 \qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y for $n_e$ and $N_i$\\
 \qquad velocity  & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $u_e$ and $U_i$ and $A_\parallel$\\
@@ -1170,8 +1182,9 @@ File format: json
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
     A      & float & 0 &  0 & Solovev parameter in Eq.~\eqref{eq:solovev} \\
-    B      & float & 1 &  1 & Solovev parameter in Eq.~\eqref{eq:solovev} \\
     c      & float[12] &  - & - & Solovev coefficients in Eq.~\eqref{eq:solovev} \\
+    PP     & float & 1 &  1 & Prefactor $\mathcal P_\psi$ for $\psi_p$ in Eq.~\eqref{eq:solovev} \\
+    PI     & float & 1 &  1 & Prefactor $\mathcal P_I$ for $I$ in Eq.~\eqref{eq:solovev} \\
     R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ in Eq.~\eqref{eq:solovev} (This is the only geometry quantity to change if $\rho_s$ changes)\\
     elongation    & float & 1 & - & Elongation $e$, used in determining the box size Eq.~\eqref{eq:box} and the initial guess for the location of the X-point $Z_X = -1.1 ea$ \\
     triangularity & float & 0 & - & Triangularity $\delta$, used in the initial guess for the location of the X-point $R_X = R_0-1.1\delta a$ \\
@@ -1257,7 +1270,9 @@ and X $\in$
     ne2 & $n_e^2$ &
     phi2 & $\phi^2$ \\
     nephi & $n_e\phi$ &
-     & \\
+    sne & $S_{n_e}$ \\
+    see & $z_e(\tau_e (1+\ln n_e) + \phi + \frac{1}{2}\mu_e u_e^2) S_{n_e} $ &
+    sei & $z_i(\tau_i (1+\ln N_i) + \psi + \frac{1}{2}\mu_i U_i^2) S_{N_i} $ \\
     nelnne &$ z_e\tau_e n_e \ln n_e$ &
     nilnni &$ z_i\tau_i N_i \ln N_i$ \\
     aperp2 &$ (\nabla_\perp A_\parallel)^2/2/\beta$ &
@@ -1270,7 +1285,7 @@ and X $\in$
     odiai &$\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i$ \\
 \bottomrule
 \end{longtable}
-and the sources and currents Y $\in$
+and the time-averaged quantities Y $\in$
 \begin{longtable}{ll}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}\\
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index ab37135e1..6d1447565 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -105,6 +105,11 @@ struct RadialEnergyFlux{
         return m_z*(m_tau*(1+log(ne))+P+0.5*m_mu*ue*ue)*lapMperpN
                 + m_z*m_mu*ne*ue*lapMperpU;
     }
+    //energy source
+    DG_DEVICE double operator()( double ne, double ue, double P,
+        double source){
+        return m_z*(m_tau*(1+log(ne))+P+0.5*m_mu*ue*ue)*source;
+    }
     private:
     double m_tau, m_mu, m_z;
 };
@@ -447,6 +452,11 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., result);
         }
     },
+    {"sne", "Source term for electron density", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::copy( v.f.sources()[0][0], result);
+        }
+    },
     /// ------------------- Energy terms ------------------------//
     {"nelnne", "Entropy electrons", false,
         []( DVec& result, Variables& v ) {
@@ -506,6 +516,24 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( -v.p.eta, result, result, 0., result);
         }
     },
+    {"see", "Energy sink/source for electrons", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
+                v.f.density(0), v.f.velocity(0), v.f.potential(0),
+                v.f.sources()[0][0]
+            );
+        }
+    },
+    {"sei", "Energy sink/source for ions", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
+                v.f.density(1), v.f.velocity(1), v.f.potential(1),
+                v.f.sources()[0][1]
+            );
+        }
+    },
     /// ------------------ Energy flux terms ------------------------//
     {"jsee_tt", "Radial electron energy flux without induction contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
@@ -725,7 +753,7 @@ std::vector<Record> diagnostics2d_list = {
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 // These two lists signify the quantities involved in accuracy computation
 std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
-std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt"};
+std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt", "see", "sei"};
 
 template<class Container>
 void slice_vector3d( const Container& transfer, Container& transfer2d, size_t local_size2d)
-- 
GitLab


From 1f23a489a75c026e4e98b4dccbc2c5dd53451a6d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 23 Sep 2019 16:28:23 +0200
Subject: [PATCH 137/540] Add Davide Makefile

---
 config/davide.mk | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 config/davide.mk

diff --git a/config/davide.mk b/config/davide.mk
new file mode 100644
index 000000000..6480f6870
--- /dev/null
+++ b/config/davide.mk
@@ -0,0 +1,10 @@
+ifeq ($(strip $(HPC_SYSTEM)),davide)
+CFLAGS=-Wall -std=c++11 -DWITHOUT_VCL -mcpu=power8 # -mavx -mfma #flags for CC
+OPT=-O3 # optimization flags for host code
+NVCC=nvcc #CUDA compiler
+NVCCARCH=-arch sm_60 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
+NVCCFLAGS= -std=c++11 -Xcompiler "-mcpu=power8 -Wall"# -mavx -mfma" #flags for NVCC
+INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
+LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
+LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
+endif
-- 
GitLab


From f6e838c5546dab87e735b6bc56f388d73d47b39d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 23 Sep 2019 16:16:11 +0200
Subject: [PATCH 138/540] Change sign of toroidal approx operators

- missed that the first time
---
 inc/geometries/geometry_diag.cu | 12 +++---
 inc/geometries/magnetic_field.h | 72 ++++++++++++++++++++++-----------
 src/feltor/feltor.h             | 28 ++++++++++---
 src/feltor/feltor.tex           | 11 +++--
 src/feltor/implicit.h           |  8 +++-
 src/feltor/init.h               |  4 +-
 6 files changed, 93 insertions(+), 42 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 2ca48b2b3..ef55390e9 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -210,12 +210,12 @@ int main( int argc, char* argv[])
         {"Divb", "The divergence of the magnetic unit vector", dg::geo::Divb(mag)},
         {"B_R", "Derivative of Bmodule in R", dg::geo::BR(mag)},
         {"B_Z", "Derivative of Bmodule in Z", dg::geo::BZ(mag)},
-        {"CurvatureNablaBR",  "R-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBR(mag)},
-        {"CurvatureNablaBZ",  "Z-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBZ(mag)},
-        {"CurvatureKappaR",   "R-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaR(mag)},
-        {"CurvatureKappaZ",   "Z-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaZ(mag)},
-        {"DivCurvatureKappa", "Divergence of the (toroidal) Kappa B curvature vector", dg::geo::DivCurvatureKappa(mag)},
-        {"DivCurvatureNablaB","Divergence of the (toroidal) Nabla B curvature vector", dg::geo::DivCurvatureNablaB(mag)},
+        {"CurvatureNablaBR",  "R-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBR(mag,+1)},
+        {"CurvatureNablaBZ",  "Z-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBZ(mag,+1)},
+        {"CurvatureKappaR",   "R-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaR(mag,+1)},
+        {"CurvatureKappaZ",   "Z-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaZ(mag,+1)},
+        {"DivCurvatureKappa", "Divergence of the (toroidal) Kappa B curvature vector", dg::geo::DivCurvatureKappa(mag,+1)},
+        {"DivCurvatureNablaB","Divergence of the (toroidal) Nabla B curvature vector", dg::geo::DivCurvatureNablaB(mag,+1)},
         {"TrueCurvatureNablaBR", "R-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBR(mag)},
         {"TrueCurvatureNablaBZ", "Z-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBZ(mag)},
         {"TrueCurvatureNablaBP", "Contravariant Phi-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBP(mag)},
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 146aca54d..2cabc92e4 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -141,8 +141,6 @@ struct BR: public aCylindricalFunctor<BR>
     {
         double Rn;
         Rn = R/mag_.R0();
-        //sign before A changed to +
-        //return -( Rn*Rn/invB_(R,Z)/invB_(R,Z)+ qampl_*qampl_*Rn *A_*psipR_(R,Z) - R  *(psipZ_(R,Z)*psipRZ_(R,Z)+psipR_(R,Z)*psipRR_(R,Z)))/(R*Rn*Rn/invB_(R,Z));
         return -1./R/invB_(R,Z) + invB_(R,Z)/Rn/Rn*(mag_.ipol()(R,Z)*mag_.ipolR()(R,Z) + mag_.psipR()(R,Z)*mag_.psipRR()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipRZ()(R,Z));
     }
   private:
@@ -165,8 +163,6 @@ struct BZ: public aCylindricalFunctor<BZ>
     {
         double Rn;
         Rn = R/mag_.R0();
-        //sign before A changed to -
-        //return (-qampl_*qampl_*A_/R_0_*psipZ_(R,Z) + psipR_(R,Z)*psipRZ_(R,Z)+psipZ_(R,Z)*psipZZ_(R,Z))/(Rn*Rn/invB_(R,Z));
         return (invB_(R,Z)/Rn/Rn)*(mag_.ipol()(R,Z)*mag_.ipolZ()(R,Z) + mag_.psipR()(R,Z)*mag_.psipRZ()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipZZ()(R,Z));
     }
   private:
@@ -180,12 +176,18 @@ struct BZ: public aCylindricalFunctor<BZ>
 ///@copydoc hide_toroidal_approximation_note
 struct CurvatureNablaBR: public aCylindricalFunctor<CurvatureNablaBR>
 {
-    CurvatureNablaBR(const TokamakMagneticField& mag): invB_(mag), bZ_(mag) { }
+    CurvatureNablaBR(const TokamakMagneticField& mag, int sign): invB_(mag), bZ_(mag) {
+        if( sign >0)
+            m_sign = +1.;
+        else
+            m_sign = -1;
+    }
     double do_compute( double R, double Z) const
     {
-        return -invB_(R,Z)*invB_(R,Z)*bZ_(R,Z);
+        return -m_sign*invB_(R,Z)*invB_(R,Z)*bZ_(R,Z);
     }
     private:
+    double m_sign;
     InvB invB_;
     BZ bZ_;
 };
@@ -196,12 +198,18 @@ struct CurvatureNablaBR: public aCylindricalFunctor<CurvatureNablaBR>
 ///@copydoc hide_toroidal_approximation_note
 struct CurvatureNablaBZ: public aCylindricalFunctor<CurvatureNablaBZ>
 {
-    CurvatureNablaBZ( const TokamakMagneticField& mag): invB_(mag), bR_(mag) { }
+    CurvatureNablaBZ( const TokamakMagneticField& mag, int sign): invB_(mag), bR_(mag) {
+        if( sign >0)
+            m_sign = +1.;
+        else
+            m_sign = -1;
+    }
     double do_compute( double R, double Z) const
     {
-        return invB_(R,Z)*invB_(R,Z)*bR_(R,Z);
+        return m_sign*invB_(R,Z)*invB_(R,Z)*bR_(R,Z);
     }
     private:
+    double m_sign;
     InvB invB_;
     BR bR_;
 };
@@ -213,7 +221,7 @@ struct CurvatureNablaBZ: public aCylindricalFunctor<CurvatureNablaBZ>
 struct CurvatureKappaR: public aCylindricalFunctor<CurvatureKappaR>
 {
     CurvatureKappaR( ){ }
-    CurvatureKappaR( const TokamakMagneticField& mag){ }
+    CurvatureKappaR( const TokamakMagneticField& mag, int sign = +1){ }
     double do_compute( double R, double Z) const
     {
         return  0.;
@@ -227,12 +235,18 @@ struct CurvatureKappaR: public aCylindricalFunctor<CurvatureKappaR>
 ///@copydoc hide_toroidal_approximation_note
 struct CurvatureKappaZ: public aCylindricalFunctor<CurvatureKappaZ>
 {
-    CurvatureKappaZ( const TokamakMagneticField& mag): invB_(mag) { }
+    CurvatureKappaZ( const TokamakMagneticField& mag, int sign): invB_(mag) {
+        if( sign >0)
+            m_sign = +1.;
+        else
+            m_sign = -1;
+    }
     double do_compute( double R, double Z) const
     {
-        return -invB_(R,Z)/R;
+        return -m_sign*invB_(R,Z)/R;
     }
     private:
+    double m_sign;
     InvB invB_;
 };
 
@@ -242,22 +256,29 @@ struct CurvatureKappaZ: public aCylindricalFunctor<CurvatureKappaZ>
 ///@copydoc hide_toroidal_approximation_note
 struct DivCurvatureKappa: public aCylindricalFunctor<DivCurvatureKappa>
 {
-    DivCurvatureKappa( const TokamakMagneticField& mag): invB_(mag), bZ_(mag){ }
+    DivCurvatureKappa( const TokamakMagneticField& mag, int sign): invB_(mag), bZ_(mag){
+        if( sign >0)
+            m_sign = +1.;
+        else
+            m_sign = -1;
+    }
     double do_compute( double R, double Z) const
     {
-        return bZ_(R,Z)*invB_(R,Z)*invB_(R,Z)/R;
+        return m_sign*bZ_(R,Z)*invB_(R,Z)*invB_(R,Z)/R;
     }
     private:
+    double m_sign;
     InvB invB_;
     BZ bZ_;
 };
+
 ///@brief Approximate \f$  \vec{\nabla}\cdot \mathcal{K}_{\nabla B}  \f$
 ///
 ///  \f$  \vec{\hat{\nabla}}\cdot \mathcal{\hat{K}}_{\nabla B}  = -\frac{1}{\hat{R}  \hat{B}^2 } \partial_{\hat{Z}} \hat{B}\f$
 ///@copydoc hide_toroidal_approximation_note
 struct DivCurvatureNablaB: public aCylindricalFunctor<DivCurvatureNablaB>
 {
-    DivCurvatureNablaB( const TokamakMagneticField& mag): div_(mag){ }
+    DivCurvatureNablaB( const TokamakMagneticField& mag, int sign): div_(mag, sign){ }
     double do_compute( double R, double Z) const
     {
         return -div_(R,Z);
@@ -533,33 +554,38 @@ inline CylindricalVectorLvl0 createBHat( const TokamakMagneticField& mag){
 }
 
 /**
- * @brief Contravariant components of the unit vector field (0, 0, 1/R)
+ * @brief Contravariant components of the unit vector field (0, 0, +/- 1/R)
  * in cylindrical coordinates.
+ * @param sign indicate positive or negative unit vector
  * @return the tuple dg::geo::Constant(0), dg::geo::Constant(0), \f$ 1/R \f$
  * @note This is equivalent to inserting a toroidal magnetic field into the \c dg::geo::createBHat function.
  */
-inline CylindricalVectorLvl0 createEPhi( ){
-    return CylindricalVectorLvl0( Constant(0), Constant(0), [](double x, double y){ return 1./x;});
+inline CylindricalVectorLvl0 createEPhi( int sign ){
+    if( sign > 0)
+        return CylindricalVectorLvl0( Constant(0), Constant(0), [](double x, double y){ return 1./x;});
+    return CylindricalVectorLvl0( Constant(0), Constant(0), [](double x, double y){ return -1./x;});
 }
 /**
  * @brief Approximate curvature vector field (CurvatureNablaBR, CurvatureNablaBZ, Constant(0))
  *
  * @param mag the tokamak magnetic field
- * @return the tuple CurvatureNablaBR, CurvatureNablaBZ, dg::geo::Constant(0) constructed from mag
+ * @param sign indicate positive or negative unit vector in approximation
+ * @return the tuple \c CurvatureNablaBR, \c CurvatureNablaBZ, \c dg::geo::Constant(0) constructed from \c mag
  * @note The contravariant components in cylindrical coordinates
  */
-inline CylindricalVectorLvl0 createCurvatureNablaB( const TokamakMagneticField& mag){
-    return CylindricalVectorLvl0( CurvatureNablaBR(mag), CurvatureNablaBZ(mag), Constant(0));
+inline CylindricalVectorLvl0 createCurvatureNablaB( const TokamakMagneticField& mag, int sign){
+    return CylindricalVectorLvl0( CurvatureNablaBR(mag, sign), CurvatureNablaBZ(mag, sign), Constant(0));
 }
 /**
  * @brief Approximate curvature vector field (CurvatureKappaR, CurvatureKappaZ, Constant(0))
  *
  * @param mag the tokamak magnetic field
- * @return the tuple CurvatureKappaR, CurvatureKappaZ, dg::geo::Constant(0) constructed from mag
+ * @param sign indicate positive or negative unit vector in approximation
+ * @return the tuple \c CurvatureKappaR, \c CurvatureKappaZ, \c dg::geo::Constant(0) constructed from \c mag
  * @note The contravariant components in cylindrical coordinates
  */
-inline CylindricalVectorLvl0 createCurvatureKappa( const TokamakMagneticField& mag){
-    return CylindricalVectorLvl0( CurvatureKappaR(mag), CurvatureKappaZ(mag), Constant(0));
+inline CylindricalVectorLvl0 createCurvatureKappa( const TokamakMagneticField& mag, int sign){
+    return CylindricalVectorLvl0( CurvatureKappaR(mag, sign), CurvatureKappaZ(mag, sign), Constant(0));
 }
 /**
  * @brief True curvature vector field (TrueCurvatureKappaR, TrueCurvatureKappaZ, TrueCurvatureKappaP)
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 5bd6b8b6c..ca0ba92d9 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -375,14 +375,26 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_mag(
     }
     else if( p.curvmode == "low beta")
     {
-        curvNabla = curvKappa = dg::geo::createCurvatureNablaB(mag);
+        curvNabla = curvKappa = dg::geo::createCurvatureNablaB(mag, +1);
         dg::assign( dg::evaluate(dg::zero, g), m_divCurvKappa);
     }
     else if( p.curvmode == "toroidal")
     {
-        curvNabla = dg::geo::createCurvatureNablaB(mag);
-        curvKappa = dg::geo::createCurvatureKappa(mag);
-        dg::assign(  dg::pullback(dg::geo::DivCurvatureKappa(mag), g),
+        curvNabla = dg::geo::createCurvatureNablaB(mag, +1);
+        curvKappa = dg::geo::createCurvatureKappa(mag, +1);
+        dg::assign(  dg::pullback(dg::geo::DivCurvatureKappa(mag, +1), g),
+            m_divCurvKappa);
+    }
+    else if( p.curvmode == "low beta negative")
+    {
+        curvNabla = curvKappa = dg::geo::createCurvatureNablaB(mag, -1);
+        dg::assign( dg::evaluate(dg::zero, g), m_divCurvKappa);
+    }
+    else if( p.curvmode == "toroidal negative")
+    {
+        curvNabla = dg::geo::createCurvatureNablaB(mag, -1);
+        curvKappa = dg::geo::createCurvatureKappa(mag, -1);
+        dg::assign(  dg::pullback(dg::geo::DivCurvatureKappa(mag, -1), g),
             m_divCurvKappa);
     }
     dg::pushForward(curvNabla.x(), curvNabla.y(), curvNabla.z(),
@@ -416,9 +428,11 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
             dg::forward, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
 
     // in Poisson we take EPhi except for the true curvmode
-    bhat = dg::geo::createEPhi();
+    bhat = dg::geo::createEPhi(+1);
     if( p.curvmode == "true")
         bhat = dg::geo::createBHat(mag);
+    else if ( p.curvmode == "toroidal negative" || p.curvmode == "low beta negative")
+        bhat = dg::geo::createEPhi(-1);
     dg::pushForward(bhat.x(), bhat.y(), bhat.z(), m_b[0], m_b[1], m_b[2], g);
     dg::SparseTensor<Container> metric = g.metric();
     dg::tensor::inv_multiply3d( metric, m_b[0], m_b[1], m_b[2],
@@ -447,9 +461,11 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
     const Grid& g, feltor::Parameters p, dg::geo::TokamakMagneticField mag)
 {
     /////////////////////////init elliptic and helmholtz operators/////////
-    auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
+    auto bhat = dg::geo::createEPhi(+1); //bhat = ephi except when "true"
     if( p.curvmode == "true")
         bhat = dg::geo::createBHat( mag);
+    else if ( p.curvmode == "toroidal negative" || p.curvmode == "low beta negative")
+        bhat = dg::geo::createEPhi(-1);
     m_multi_chi = m_multigrid.project( m_temp0);
     m_multi_pol.resize(p.stages);
     m_multi_invgammaP.resize(p.stages);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 7f5cf4bb0..e1e536174 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1128,10 +1128,15 @@ nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$
 $\nu_\perp $n and $\nu_\parallel$ for $\nu_\parallel$ to become relevant for
 the dynamics)\\
 resistivity & float &1e-4  & - & parallel resistivity parameter Eq.~\eqref{eq:resistivity}\\
-curvmode  & string & "low beta"  & "toroidal"& curvature mode ("low beta",
+curvmode  & string & "low beta"  & "toroidal" &
+curvature mode (
+"low beta",
 "true": no approximation - requires significantly more resolution in Nz,
 "toroidal": toroidal field approx - elliptic equation does not need
-communication in z)  \\
+communication in z,
+"low beta negative": reversed sign of low beta,
+"toroidal negative": reversed sign of toroidal
+) \\
 symmetric & bool & false & false & If true, initialize all quantities symmetric
 in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used
 to construct the parallel derivatives and then overwritten to $N_z\equiv 1$. \\
@@ -1146,7 +1151,7 @@ perturbation $\tilde n$ in \eqref{eq:initial_ne}. "zonal" (Eq.~\eqref{eq:initial
     "blob" = blob simulations (several rounds fieldaligned),
     "straight blob" = straight blob simulation( 1 round fieldaligned),
     "turbulence" = turbulence simulations ( 1 round fieldaligned, Eq.~\eqref{eq:initial_turbulent})
-    "turbulence\_on\_gaussian" = Gaussian bg. profile with turbulence perturbation Eq.~\eqref{eq:turbulence_on_gaussian}
+    "turbulence on gaussian" = Gaussian bg. profile with turbulence perturbation Eq.~\eqref{eq:turbulence_on_gaussian}
     See the file {\tt init.h} to add your own custom condition.
     \\
 initphi   & string & "zero"  & "balance" & initial condition for $\phi$ and thus $N_i$ (Eq.~\eqref{eq:initphi}: "zero" : $\phi = 0$, vanishing
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 754511e6a..b1ebd3ea3 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -46,9 +46,11 @@ struct ImplicitDensity
         m_p = p;
         m_lapM_perpN.construct( g, p.bcxN, p.bcyN,dg::PER, dg::normed, dg::centered);
         dg::assign( dg::evaluate( dg::zero, g), m_temp);
-        auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
+        auto bhat = dg::geo::createEPhi(+1); //bhat = ephi except when "true"
         if( p.curvmode == "true")
             bhat = dg::geo::createBHat(mag);
+        else if ( p.curvmode == "toroidal negative" || p.curvmode == "low beta negative")
+            bhat = dg::geo::createEPhi(-1);
         dg::SparseTensor<Container> hh
             = dg::geo::createProjectionTensor( bhat, g);
         //set perpendicular projection tensor h
@@ -114,9 +116,11 @@ struct ImplicitVelocity
         m_apar = m_temp;
         m_fields[0][0] = m_fields[0][1] = m_temp;
         m_fields[1][0] = m_fields[1][1] = m_temp;
-        auto bhat = dg::geo::createEPhi(); //bhat = ephi except when "true"
+        auto bhat = dg::geo::createEPhi(+1); //bhat = ephi except when "true"
         if( p.curvmode == "true")
             bhat = dg::geo::createBHat(mag);
+        else if ( p.curvmode == "toroidal negative" || p.curvmode == "low beta negative")
+            bhat = dg::geo::createEPhi(-1);
         dg::SparseTensor<Container> hh
             = dg::geo::createProjectionTensor( bhat, g);
         //set perpendicular projection tensor h
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 317022e33..955409ba2 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -163,7 +163,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             return y0;
         }
     },
-    { "straight_blob",
+    { "straight blob",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
             const Geometry& grid, const feltor::Parameters& p,
             const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
@@ -239,7 +239,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             return y0;
         }
     },
-    { "turbulence_on_gaussian",
+    { "turbulence on gaussian",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
             const Geometry& grid, const feltor::Parameters& p,
             const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
-- 
GitLab


From 8d4a387c32a70960468c3a24898e545e42931c66 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 24 Sep 2019 19:54:37 +0200
Subject: [PATCH 139/540] Restructure nc_utilities to include dimension names

---
 inc/file/nc_utilities.h | 172 +++++++++++++++-------------------------
 1 file changed, 66 insertions(+), 106 deletions(-)

diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index 6ec077d1e..d2145e458 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -72,204 +72,164 @@ static inline int define_limited_time( int ncid, const char* name, int size, int
 /**
  * @brief Define a 1d dimension and create a coordinate variable together with its data points in a netcdf file
  *
- * By netcdf conventions a variable with the same name as a dimension is called a coordinate variable.
+ * @note By netcdf conventions a variable with the same name as a dimension is called a coordinate variable.
  * @param ncid file ID
- * @param name Name of dimension (input)
  * @param dimID dimension ID (output)
- * @param points pointer to data (input)
- * @param size size of data points (input)
+ * @param g The 1d DG grid from which data points are generated (input)
+ * @param name_dim Name of dimension (input)
+ * @param axis The axis attribute (input), ("X", "Y" or "Z")
  *
  * @return netcdf error code if any
  */
-static inline int define_dimension( int ncid, const char* name, int* dimID, const double * points, int size)
+static inline int define_dimension( int ncid, int* dimID, const dg::Grid1d& g, std::string name_dim = "x", std::string axis = "X")
 {
     int retval;
-    if( (retval = nc_def_dim( ncid, name, size, dimID)) ) { return retval;}
+    std::string long_name = name_dim+"-coordinate in Computational coordinate system";
+    thrust::host_vector<double> points = dg::create::abscissas( g);
+    if( (retval = nc_def_dim( ncid, name_dim.data(), points.size(), dimID)) ) { return retval;}
     int varID;
-    if( (retval = nc_def_var( ncid, name, NC_DOUBLE, 1, dimID, &varID))){return retval;}
+    if( (retval = nc_def_var( ncid, name_dim.data(), NC_DOUBLE, 1, dimID, &varID))){return retval;}
     if( (retval = nc_enddef(ncid)) ) {return retval;} //not necessary for NetCDF4 files
-    if( (retval = nc_put_var_double( ncid, varID, points)) ){ return retval;}
+    if( (retval = nc_put_var_double( ncid, varID, points.data())) ){ return retval;}
     if( (retval = nc_redef(ncid))) {return retval;} //not necessary for NetCDF4 files
-    return retval;
-}
-/**
- * @brief Define a 1d dimension and create a coordinate variable together with its data points in a netcdf file
- *
- * By netcdf conventions a variable with the same name as a dimension is called a coordinate variable.
- * @param ncid file ID
- * @param name Name of dimension (input)
- * @param dimID dimension ID (output)
- * @param g The 1d DG grid from which data points are generated (input)
- *
- * @return netcdf error code if any
- */
-static inline int define_dimension( int ncid, const char* name, int* dimID, const dg::Grid1d& g)
-{
-    thrust::host_vector<double> points = dg::create::abscissas( g);
-    return define_dimension( ncid, name, dimID, points.data(), points.size());
-}
-
-///@cond
-namespace detail{
-static inline int define_x_dimension( int ncid, int* dimID, const dg::Grid1d& g)
-{
-    int retval;
-    std::string long_name = "x-coordinate in Computational coordinate system";
-    if( (retval = define_dimension( ncid, "x", dimID, g))){ return retval;}
-    retval = nc_put_att_text( ncid, *dimID, "axis", 1, "X");
+    retval = nc_put_att_text( ncid, *dimID, "axis", axis.size(), axis.data());
     retval = nc_put_att_text( ncid, *dimID, "long_name", long_name.size(), long_name.data());
     return retval;
 }
-static inline int define_y_dimension( int ncid, int* dimID, const dg::Grid1d& g)
-{
-    int retval;
-    std::string long_name = "y-coordinate in Computational coordinate system";
-    if( (retval = define_dimension( ncid, "y", dimID, g))){ return retval;}
-    retval = nc_put_att_text( ncid, *dimID, "axis", 1, "Y");
-    retval = nc_put_att_text( ncid, *dimID, "long_name", long_name.size(), long_name.data());
-    return retval;
-}
-static inline int define_z_dimension( int ncid, int* dimID, const dg::Grid1d& g)
-{
-    int retval;
-    std::string long_name = "z-coordinate in Computational coordinate system";
-    if( (retval = define_dimension( ncid, "z", dimID, g))){ return retval;}
-    retval = nc_put_att_text( ncid, *dimID, "axis", 1, "Z");
-    retval = nc_put_att_text( ncid, *dimID, "long_name", long_name.size(), long_name.data());
-    return retval;
-}
-}//namespace detail
-///@endcond
 
 /**
  * @brief Define a 1d time-dependent dimension variable together with its data points
  *
- * Dimensions are named x, and time
+ * Dimensions have attribute of (time, X)
  * @param ncid file ID
- * @param dimsIDs dimension IDs (time, x)
- * @param tvarID time variable ID
+ * @param dimsIDs dimension IDs (time, X)
+ * @param tvarID time variable ID (unlimited)
  * @param g The 1d DG grid from which data points are generated
+ * @param name_dims Names for the dimension variables
  *
  * @return netcdf error code if any
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::Grid1d& g)
+static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::Grid1d& g, std::array<std::string,2> name_dims = {"time","x"})
 {
     int retval;
-    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[1], g))){ return retval;}
-    if( (retval = define_time( ncid, "time", &dimsIDs[0], tvarID)) ){ return retval;}
-
-    return retval;
+    retval = define_time( ncid, name_dims[0].data(), &dimsIDs[0], tvarID);
+    if(retval)
+        return retval;
+    return define_dimension( ncid, &dimsIDs[1], g, name_dims[1], "X");
 }
 /**
  * @brief Define 2d dimensions and associate values in NetCDF-file
  *
- * Dimensions are named y, x
+ * Dimensions have attributes of (Y, X)
  * @param ncid file ID
- * @param dimsIDs (write - only) 2D array of dimension IDs (y,x)
+ * @param dimsIDs (write - only) 2D array of dimension IDs (Y,X)
  * @param g The 2d grid from which to derive the dimensions
+ * @param name_dims Names for the dimension variables
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology2d& g)
+static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology2d& g, std::array<std::string,2> name_dims = {"y", "x"})
 {
     dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
     int retval;
-    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[1], gx))){ return retval;}
-    if( (retval = detail::define_y_dimension( ncid, &dimsIDs[0], gy))){ return retval;}
-
-    return retval;
+    retval = define_dimension( ncid, &dimsIDs[0], gy, name_dims[0], "Y");
+    if(retval)
+        return retval;
+    return define_dimension( ncid, &dimsIDs[1], gx, name_dims[1], "X");
 }
 /**
  * @brief Define 2d time-dependent dimensions and associate values in NetCDF-file
  *
- * Dimensions are named time, y and x
+ * Dimensions have attributes of (time, Y, X)
  * @param ncid file ID
- * @param dimsIDs (write - only) 3D array of dimension IDs (time, y,x)
- * @param tvarID (write - only) The ID of the time variable
+ * @param dimsIDs (write - only) 3D array of dimension IDs (time, Y,X)
+ * @param tvarID (write - only) The ID of the time variable ( unlimited)
  * @param g The 2d grid from which to derive the dimensions
+ * @param name_dims Names for the dimension variables ( time, Y, X)
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aTopology2d& g)
+static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aTopology2d& g, std::array<std::string,3> name_dims = {"time", "y", "x"})
 {
-    dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
-    dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
     int retval;
-    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[2], gx))){ return retval;}
-    if( (retval = detail::define_y_dimension( ncid, &dimsIDs[1], gy))){ return retval;}
-    if( (retval = define_time( ncid, "time", &dimsIDs[0], tvarID)) ){ return retval;}
-
-    return retval;
+    retval = define_time( ncid, name_dims[0].data(), &dimsIDs[0], tvarID);
+    if(retval)
+        return retval;
+    return define_dimensions( ncid, &dimsIDs[1], g, {name_dims[1], name_dims[2]});
 }
 
 /**
  * @brief Define 2d time-dependent (limited) dimensions and associate values in NetCDF-file
  *
- * Dimensions are named x, y, and time (limited)
+ * Dimensions have attributes of (time, Y, X)
  * @param ncid file ID
- * @param dimsIDs (write - only) 3D array of dimension IDs (time, y,x)
+ * @param dimsIDs (write - only) 3D array of dimension IDs (time, Y,X)
  * @param size The size of the time variable
- * @param tvarID (write - only) The ID of the time variable
+ * @param tvarID (write - only) The ID of the time variable (limited)
  * @param g The 2d grid from which to derive the dimensions
+ * @param name_dims Names for the dimension variables (time, Y, X)
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_limtime_xy( int ncid, int* dimsIDs, int size, int* tvarID, const dg::aTopology2d& g)
+static inline int define_limtime_xy( int ncid, int* dimsIDs, int size, int* tvarID, const dg::aTopology2d& g, std::array<std::string, 3> name_dims = {"time", "y", "x"})
 {
-    dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
-    dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
     int retval;
-    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[2], gx)));
-    if( (retval = detail::define_y_dimension( ncid, &dimsIDs[1], gy)));
-    if( (retval = define_limited_time( ncid, "time", size, &dimsIDs[0], tvarID)) ){ return retval;}
-
-    return retval;
+    retval = define_limited_time( ncid, name_dims[0].data(), size, &dimsIDs[0], tvarID);
+    if(retval)
+        return retval;
+    return define_dimensions( ncid, &dimsIDs[1], g, {name_dims[1], name_dims[2]});
 }
 /**
  * @brief Define 3d dimensions and associate values in NetCDF-file
  *
- * Dimensions are named x, y, z
+ * Dimensions have attributes ( Z, Y, X)
  * @param ncid file ID
- * @param dimsIDs (write - only) 3D array of dimension IDs (z,y,x)
+ * @param dimsIDs (write - only) 3D array of dimension IDs (Z,Y,X)
  * @param g The grid from which to derive the dimensions
+ * @param name_dims Names for the dimension variables ( Z, Y, X)
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology3d& g)
+static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology3d& g, std::array<std::string, 3> name_dims = {"z", "y", "x"})
 {
     dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
     dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
     dg::Grid1d gz( g.z0(), g.z1(), 1, g.Nz());
     int retval;
-    if( (retval = detail::define_x_dimension( ncid, &dimsIDs[2], gx)));
-    if( (retval = detail::define_y_dimension( ncid, &dimsIDs[1], gy)));
-    if( (retval = detail::define_z_dimension( ncid, &dimsIDs[0], gz)));
-    return retval;
+    retval = define_dimension( ncid, &dimsIDs[0], gz, name_dims[0], "Z");
+    if(retval)
+        return retval;
+    retval = define_dimension( ncid, &dimsIDs[1], gy, name_dims[1], "Y");
+    if(retval)
+        return retval;
+    return define_dimension( ncid, &dimsIDs[2], gx, name_dims[2], "X");
 }
 
 /**
  * @brief Define 3d time-dependent dimensions and associate values in NetCDF-file
  *
- * Dimensions are named x, y, z, and time
+ * Dimensions have attributes ( time, Z, Y, X)
  * @param ncid file ID
- * @param dimsIDs (write - only) 4D array of dimension IDs (time, z,y,x)
- * @param tvarID (write - only) The ID of the time variable
+ * @param dimsIDs (write - only) 4D array of dimension IDs (time, Z,Y,X)
+ * @param tvarID (write - only) The ID of the time variable ( unlimited)
  * @param g The grid from which to derive the dimensions
+ * @param name_dims Names for the dimension variables ( time, Z, Y, X)
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aTopology3d& g)
+static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aTopology3d& g, std::array<std::string, 4> name_dims = {"time", "z", "y", "x"})
 {
     int retval;
-    if( (retval = define_dimensions( ncid, &dimsIDs[1], g)) ){ return retval;}
-    if( (retval = define_time( ncid, "time", &dimsIDs[0], tvarID)) ){ return retval;}
-    return retval;
+    retval = define_time( ncid, "time", &dimsIDs[0], tvarID);
+    if(retval)
+        return retval;
+    return define_dimensions( ncid, &dimsIDs[1], g, {name_dims[1], name_dims[2], name_dims[3]});
 }
 
 
-- 
GitLab


From fb56aadbdf05032e94464d6f0f70aed0bb42f10a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 24 Sep 2019 21:27:44 +0200
Subject: [PATCH 140/540] Write out static 2d and restart 3d fields in feltor

---
 inc/file/nc_utilities.h  | 26 ++++++++++++
 src/feltor/feltor.h      |  3 ++
 src/feltor/feltor_hpc.cu | 56 ++++++++++++++++++++-----
 src/feltor/feltordiag.h  | 91 ++++++++++++++++++++++++++++++++++++++++
 src/toefl/toefl_hpc.cu   |  6 +--
 5 files changed, 166 insertions(+), 16 deletions(-)

diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index d2145e458..ecd444559 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -8,6 +8,9 @@
 
 #include "dg/topology/grid.h"
 #include "dg/topology/evaluation.h"
+#ifdef MPI_VERSION
+#include "dg/topology/mpi_grid.h"
+#endif //MPI_VERSION
 
 #include "easy_output.h"
 
@@ -233,5 +236,28 @@ static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const
 }
 
 
+#ifdef MPI_VERSION
+
+/// Convenience function that just calls the corresponding serial version with the global grid
+static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aMPITopology2d& g, std::array<std::string,2> name_dims = {"y", "x"})
+{
+    return define_dimensions( ncid, dimsIDs, g.global(), name_dims);
+}
+/// Convenience function that just calls the corresponding serial version with the global grid
+static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aMPITopology2d& g, std::array<std::string,3> name_dims = {"time", "y", "x"})
+{
+    return define_dimensions( ncid, dimsIDs, tvarID, g.global(), name_dims);
+}
+/// Convenience function that just calls the corresponding serial version with the global grid
+static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aMPITopology3d& g, std::array<std::string, 3> name_dims = {"z", "y", "x"})
+{
+    return define_dimensions( ncid, dimsIDs, g.global(), name_dims);
+}
+/// Convenience function that just calls the corresponding serial version with the global grid
+static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aMPITopology3d& g, std::array<std::string, 4> name_dims = {"time", "z", "y", "x"})
+{
+    return define_dimensions( ncid, dimsIDs, tvarID, g.global(), name_dims);
+}
+#endif //MPI_VERSION
 
 } //namespace file
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index ca0ba92d9..1cbb754fe 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -245,6 +245,9 @@ struct Explicit
     const std::array<Container, 3> & curvKappa () const {
         return m_curvKappa;
     }
+    const Container& divCurvKappa() const {
+        return m_divCurvKappa;
+    }
     const Container& bphi( ) const { return m_bphi; }
     const Container& binv( ) const { return m_binv; }
     const Container& divb( ) const { return m_divb; }
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 539774519..33e7176dc 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -179,7 +179,7 @@ int main( int argc, char* argv[])
         feltor, p, gradPsip, gradPsip
     };
     // the vector ids
-    std::map<std::string, int> id3d, id4d;
+    std::map<std::string, int> id3d, id4d, restart_ids;
 
     double dEdt = 0, accuracy = 0;
     double E0 = 0.;
@@ -229,13 +229,15 @@ int main( int argc, char* argv[])
             pair.first.data(), pair.second.size(), pair.second.data());
 
     // Define dimensions (t,z,y,x)
-    int dim_ids[4], tvarID;
+    int dim_ids[4], restart_dim_ids[3], tvarID;
+    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out, {"time", "z", "y", "x"});
+    MPI_OUT err = file::define_dimensions( ncid, restart_dim_ids, grid, {"zr", "yr", "xr"});
+    int dim_ids3d[3] = {dim_ids[0], dim_ids[2], dim_ids[3]};
+    bool write2d = true;
 #ifdef FELTOR_MPI
-    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out.global());
-#else //FELTOR_MPI
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out);
+    //only the globally first slice should write
+    if( !(g3d_out.local().z0() - g3d_out.global().z0() < 1e-14) ) write2d = false;
 #endif //FELTOR_MPI
-    int dim_ids3d[3] = {dim_ids[0], dim_ids[2], dim_ids[3]};
 
     //create & output static 3d variables into file
     for ( auto& record : feltor::diagnostics3d_static_list)
@@ -252,6 +254,21 @@ int main( int argc, char* argv[])
         file::put_var_double( ncid, vecID, g3d_out, transferH);
         MPI_OUT err = nc_redef(ncid);
     }
+    //create & output static 2d variables into file
+    for ( auto& record : feltor::diagnostics2d_static_list)
+    {
+        int vecID;
+        MPI_OUT err = nc_def_var( ncid, record.name.data(), NC_DOUBLE, 2,
+            &dim_ids[2], &vecID);
+        MPI_OUT err = nc_put_att_text( ncid, vecID,
+            "long_name", record.long_name.size(), record.long_name.data());
+        MPI_OUT err = nc_enddef( ncid);
+        MPI_OUT std::cout << "Computing2d "<<record.name<<"\n";
+        record.function( resultH, var, grid, gp, mag);
+        dg::blas2::symv( projectH, resultH, transferH);
+        if(write2d)file::put_var_double( ncid, vecID, *g2d_out_ptr, transferH);
+        MPI_OUT err = nc_redef(ncid);
+    }
 
     //Create field IDs
     for( auto& record : feltor::diagnostics3d_list)
@@ -264,6 +281,16 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_put_att_text( ncid, id4d.at(name), "long_name", long_name.size(),
             long_name.data());
     }
+    for( auto& record : feltor::restart3d_list)
+    {
+        std::string name = record.name;
+        std::string long_name = record.long_name;
+        restart_ids[name] = 0;//creates a new entry for all processes
+        MPI_OUT err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, restart_dim_ids,
+            &restart_ids.at(name));
+        MPI_OUT err = nc_put_att_text( ncid, restart_ids.at(name), "long_name", long_name.size(),
+            long_name.data());
+    }
     for( auto& record : feltor::diagnostics2d_list)
     {
         std::string name = record.name + "_ta2d";
@@ -307,11 +334,12 @@ int main( int argc, char* argv[])
         dg::assign( transferD, transferH);
         file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
     }
-    bool write2d = true;
-#ifdef FELTOR_MPI
-    //only the globally first slice should write
-    if( !(g3d_out.local().z0() - g3d_out.global().z0() < 1e-14) ) write2d = false;
-#endif //FELTOR_MPI
+    for( auto& record : feltor::restart3d_list)
+    {
+        record.function( resultD, var);
+        dg::assign( transferD, transferH);
+        file::put_var_double( ncid, restart_ids.at(record.name), grid, transferH);
+    }
     for( auto& record : feltor::diagnostics2d_list)
     {
         dg::Timer tti;
@@ -446,6 +474,12 @@ int main( int argc, char* argv[])
             dg::assign( transferD, transferH);
             file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
         }
+        for( auto& record : feltor::restart3d_list)
+        {
+            record.function( resultD, var);
+            dg::assign( transferD, transferH);
+            file::put_var_double( ncid, restart_ids.at(record.name), grid, transferH);
+        }
         for( auto& record : feltor::diagnostics2d_list)
         {
             if(record.integral) // we already computed the output...
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 6d1447565..df4a3d86e 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -274,6 +274,75 @@ std::vector<Record> diagnostics3d_list = {
     }
 };
 
+//Here is a list of static (time-independent) 2d variables that go into the output
+//( we make 3d variables here that but only the first 2d slice is output)
+std::vector<Record_static> diagnostics2d_static_list = {
+    { "Psip2d", "Flux-function psi",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = dg::pullback( mag.psip(), grid);
+        }
+    },
+    { "Ipol", "Poloidal current",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = dg::pullback( mag.ipol(), grid);
+        }
+    },
+    { "Bmodule", "Magnetic field strength",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = dg::pullback( dg::geo::Bmodule(mag), grid);
+        }
+    },
+    { "Divb", "The divergence of the magnetic unit vector",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.divb();
+        }
+    },
+    { "InvB", "Inverse of Bmodule",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.binv();
+        }
+    },
+    { "CurvatureKappaR", "R-component of the Kappa B curvature vector",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.curvKappa()[0];
+        }
+    },
+    { "CurvatureKappaZ", "Z-component of the Kappa B curvature vector",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.curvKappa()[1];
+        }
+    },
+    { "CurvatureKappaP", "Contravariant Phi-component of the Kappa B curvature vector",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.curvKappa()[2];
+        }
+    },
+    { "DivCurvatureKappa", "Divergence of the Kappa B curvature vector",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.divCurvKappa();
+        }
+    },
+    { "CurvatureR", "R-component of the curvature vector",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.curv()[0];
+        }
+    },
+    { "CurvatureZ", "Z-component of the full curvature vector",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.curv()[1];
+        }
+    },
+    { "CurvatureP", "Contravariant Phi-component of the full curvature vector",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.curv()[2];
+        }
+    },
+    { "bphi", "Contravariant Phi-component of the magnetic unit vector",
+        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+            result = v.f.bphi();
+        }
+    }
+};
 // and here are all the 2d outputs we want to produce
 std::vector<Record> diagnostics2d_list = {
     {"electrons", "Electron density", false,
@@ -751,6 +820,28 @@ std::vector<Record> diagnostics2d_list = {
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+std::vector<Record> restart3d_list = {
+    {"restart_electrons", "electron density", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.density(0), result);
+        }
+    },
+    {"restart_ions", "ion density", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.density(1), result);
+        }
+    },
+    {"restart_Ue", "parallel electron velocity", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.velocity(0), result);
+        }
+    },
+    {"restart_Ui", "parallel ion velocity", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.velocity(1), result);
+        }
+    }
+};
 // These two lists signify the quantities involved in accuracy computation
 std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
 std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt", "see", "sei"};
diff --git a/src/toefl/toefl_hpc.cu b/src/toefl/toefl_hpc.cu
index 650e4f38d..a27541c9a 100644
--- a/src/toefl/toefl_hpc.cu
+++ b/src/toefl/toefl_hpc.cu
@@ -121,11 +121,7 @@ int main( int argc, char* argv[])
     std::string input = js.toStyledString();
     MPI_OUT err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
-#ifdef TOEFL_MPI
-    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
-#else //TOEFL_MPI
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
-#endif //TOEFL_MPI
+    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     //field IDs
     std::string names[4] = {"electrons", "ions", "potential", "vorticity"};
     int dataIDs[4];
-- 
GitLab


From f0e652e232f25153090b68585d294776a8e8c726 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 24 Sep 2019 21:44:32 +0200
Subject: [PATCH 141/540] Adapt init_from_file routine to read restart fields

- also correct missing induction in restart fields
---
 src/feltor/feltordiag.h     |  5 +++++
 src/feltor/init_from_file.h | 29 +++++++++++++++--------------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index df4a3d86e..34d4dafab 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -840,6 +840,11 @@ std::vector<Record> restart3d_list = {
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.velocity(1), result);
         }
+    },
+    {"restart_induction", "parallel magnetic induction", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.induction(), result);
+        }
     }
 };
 // These two lists signify the quantities involved in accuracy computation
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index 8f524265e..333591ef0 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -36,7 +36,7 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
 
     // Now read in last timestep
     Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
-        pIN.n_out, pIN.Nx_out, pIN.Ny_out, pIN.symmetric ? 1 : pIN.Nz_out, pIN.bcxN, pIN.bcyN, dg::PER
+        pIN.n, pIN.Nx, pIN.Ny, pIN.symmetric ? 1 : pIN.Nz, pIN.bcxN, pIN.bcyN, dg::PER
         #ifdef FELTOR_MPI
         , grid.communicator()
         #endif //FELTOR_MPI
@@ -47,29 +47,30 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     int dimsIN[3],  coordsIN[3];
     int periods[3] = {false, false, true}; //non-, non-, periodic
     MPI_Cart_get( grid.communicator(), 3, dimsIN, periods, coordsIN);
-    size_t countIN[4] = {1, grid_IN.local().Nz(),
-        grid_IN.n()*(grid_IN.local().Ny()),
-        grid_IN.n()*(grid_IN.local().Nx())};
-    size_t startIN[4] = {0, coordsIN[2]*countIN[1],
-                            coordsIN[1]*countIN[2],
-                            coordsIN[0]*countIN[3]};
+    size_t countIN[3] = {grid_IN.local().Nz(),
+            grid_IN.n()*(grid_IN.local().Ny()),
+            grid_IN.n()*(grid_IN.local().Nx())};
+    size_t startIN[3] = {coordsIN[2]*countIN[1],
+                         coordsIN[1]*countIN[2],
+                         coordsIN[0]*countIN[3]};
     #else //FELTOR_MPI
-    size_t startIN[4] = {0, 0, 0, 0};
-    size_t countIN[4] = {1, grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
+    size_t startIN[3] = {0, 0, 0};
+    size_t countIN[3] = {grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
         grid_IN.n()*grid_IN.Nx()};
     #endif //FELTOR_MPI
     std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
     HVec transferINH( dg::evaluate(dg::zero, grid_IN));
 
-    std::string namesIN[5] = {"electrons", "ions", "Ue", "Ui", "induction"};
+    std::string namesIN[5] = {"restart_electrons", "restart_ions", "restart_Ue", "restart_Ui", "restart_induction"};
 
     int timeIDIN;
+    size_t size_time, count_time = 1;
     /////////////////////Get time length and initial data///////////////////////////
     errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
-    errIN = nc_inq_dimlen(ncidIN, timeIDIN, &startIN[0]);
-    startIN[0] -= 1;
+    errIN = nc_inq_dimlen(ncidIN, timeIDIN, &size_time);
     errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
-    errIN = nc_get_vara_double( ncidIN, timeIDIN, startIN, countIN, &time);
+    size_time -= 1;
+    errIN = nc_get_vara_double( ncidIN, timeIDIN, &size_time, &count_time, &time);
     MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
     for( unsigned i=0; i<5; i++)
     {
@@ -85,7 +86,7 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
         dg::blas2::gemv( interpolateIN, transferINH, transferINHvec[i]);
     }
     errIN = nc_close(ncidIN);
-    /// ///////////////Now Construct initial fields
+    /// ///////////////Now Construct initial fields ////////////////////////
     //
     //Convert to N-1 and W
     dg::blas1::plus( transferINHvec[0], -1.);
-- 
GitLab


From bc9774050a30d65f90725e29c21976a8fc6bcca4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 25 Sep 2019 15:13:13 +0200
Subject: [PATCH 142/540] Fix bug in restart fields in feltor_hpc

---
 src/feltor/feltor.tex       | 4 +++-
 src/feltor/feltor_hpc.cu    | 8 ++++----
 src/feltor/init_from_file.h | 6 +++---
 3 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index e1e536174..ba0707848 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1326,7 +1326,9 @@ The program \texttt{feltor\_hpc.cu} has the possibility to initialize time and t
 the results of a previous simulation. This behaviour is enabled by giving an additional file \texttt{initial.nc}
 to the command line. In this case the \texttt{initne} and \texttt{initphi} parameters of the input
 file are ignored. Instead, the fields \texttt{electrons, ions, Ue, Ui, induction} at the latest timestep
-are read from the given file, interpolated to the current grid and used to initialize the simulation.
+are read from the given file to initialize the simulation.
+Note that to enable a loss-less continuation of the simulation we output special restart fields into the output file that in contrast to the other fields
+are not compressed.
 Apart from that the behaviour of the program is unchanged i.e. the magnetic field, profiles, resolutions, etc.
 are all taken from the regular input files. This means that the user must take care that these are consistent
 with the paramters in the existing \texttt{initial.nc} file. Also note that we try to discourage
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 33e7176dc..cd2bba77d 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -337,8 +337,8 @@ int main( int argc, char* argv[])
     for( auto& record : feltor::restart3d_list)
     {
         record.function( resultD, var);
-        dg::assign( transferD, transferH);
-        file::put_var_double( ncid, restart_ids.at(record.name), grid, transferH);
+        dg::assign( resultD, resultH);
+        file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
     }
     for( auto& record : feltor::diagnostics2d_list)
     {
@@ -477,8 +477,8 @@ int main( int argc, char* argv[])
         for( auto& record : feltor::restart3d_list)
         {
             record.function( resultD, var);
-            dg::assign( transferD, transferH);
-            file::put_var_double( ncid, restart_ids.at(record.name), grid, transferH);
+            dg::assign( resultD, resultH);
+            file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
         }
         for( auto& record : feltor::diagnostics2d_list)
         {
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index 333591ef0..d71f53e94 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -50,9 +50,9 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     size_t countIN[3] = {grid_IN.local().Nz(),
             grid_IN.n()*(grid_IN.local().Ny()),
             grid_IN.n()*(grid_IN.local().Nx())};
-    size_t startIN[3] = {coordsIN[2]*countIN[1],
-                         coordsIN[1]*countIN[2],
-                         coordsIN[0]*countIN[3]};
+    size_t startIN[3] = {coordsIN[2]*countIN[0],
+                         coordsIN[1]*countIN[1],
+                         coordsIN[0]*countIN[2]};
     #else //FELTOR_MPI
     size_t startIN[3] = {0, 0, 0};
     size_t countIN[3] = {grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
-- 
GitLab


From bf332dafec0cb9e0fdf1d32e1964b769adfbee24 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 26 Sep 2019 13:50:06 +0200
Subject: [PATCH 143/540] Implement simple and fast average backend

---
 inc/dg/topology/average.h       | 73 +++++++++++++++++++++++++----
 inc/dg/topology/average_b.cu    | 20 ++++++--
 inc/dg/topology/average_mpi.h   | 83 +++++++++++++++++++++++++++++----
 inc/dg/topology/average_mpib.cu | 16 +++++--
 inc/dg/topology/average_mpit.cu |  4 +-
 inc/dg/topology/average_t.cu    |  6 +--
 inc/geometries/average.h        |  4 +-
 src/feltor/feltor.tex           | 12 +++--
 src/feltor/feltor_hpc.cu        |  2 +-
 9 files changed, 180 insertions(+), 40 deletions(-)

diff --git a/inc/dg/topology/average.h b/inc/dg/topology/average.h
index 4f587552d..a6d9743f1 100644
--- a/inc/dg/topology/average.h
+++ b/inc/dg/topology/average.h
@@ -4,11 +4,30 @@
 #include "weights.h"
 #include "dg/blas1.h"
 #include "dg/backend/average_dispatch.h"
+#include "dg/backend/view.h"
 
 /*! @file
   @brief Classes for poloidal and toroidal average computations.
   */
 namespace dg{
+///@cond
+template<class container>
+void simple_average( unsigned nx, unsigned ny, const container& in0, const container& in1, container& out)
+{
+    const double* in0_ptr = thrust::raw_pointer_cast( in0.data());
+    const double* in1_ptr = thrust::raw_pointer_cast( in1.data());
+          double* out_ptr = thrust::raw_pointer_cast( out.data());
+    dg::View<const container> in0_view( in0_ptr, nx), in1_view( in1_ptr, nx);
+    dg::View<container> out_view( out_ptr, nx);
+    dg::blas1::pointwiseDot( 1., in0_view, in1_view, 0, out_view);
+    for( unsigned i=1; i<ny; i++)
+    {
+        in0_view.construct( in0_ptr+i*nx, nx);
+        in1_view.construct( in1_ptr+i*nx, nx);
+        dg::blas1::pointwiseDot( 1., in0_view, in1_view, 1, out_view);
+    }
+}
+///@endcond
 
 /**
  * @brief Topological average computations in a Cartesian topology
@@ -38,8 +57,13 @@ struct Average
      *
      * @param g the grid from which to take the dimensionality and sizes
      * @param direction the direction or plane over which to average when calling \c operator() (at the moment cannot be \c coo3d::xz or \c coo3d::y)
+     * @param mode either "exact" ( uses the exact and reproducible dot product for the summation) or "simple" (uses inexact but much faster direct summation) use simple if you do not need the reproducibility
+     * @note computing in "exact" mode is especially difficult if the averaged
+     * direction is small compared to the remaining dimensions and for GPUs in
+     * general, expect to gain a factor 10-1000 (no joke) from going to
+     * "simple" mode in these cases
      */
-    Average( const aTopology2d& g, enum coo2d direction)
+    Average( const aTopology2d& g, enum coo2d direction, std::string mode = "exact") : m_mode(mode)
     {
         m_nx = g.Nx()*g.n(), m_ny = g.Ny()*g.n();
         m_w=dg::construct<ContainerType>(dg::create::weights(g, direction));
@@ -48,46 +72,64 @@ struct Average
         unsigned size1d = 0;
         if( direction == coo2d::x)
         {
+            dg::blas1::scal( m_temp, 1./g.lx());
             dg::blas1::scal( m_w, 1./g.lx());
             size1d = m_ny;
+            if( "simple" == mode)
+                dg::transpose( m_nx, m_ny, m_temp, m_w);
         }
         else
         {
             m_transpose = true;
             dg::blas1::scal( m_temp, 1./g.ly());
-            dg::transpose( m_nx, m_ny, m_temp, m_w);
+            dg::blas1::scal( m_w, 1./g.ly());
+            if( "exact" == mode)
+                dg::transpose( m_nx, m_ny, m_temp, m_w);
             size1d = m_nx;
         }
         thrust::host_vector<double> t1d( size1d);
         m_temp1d = dg::construct<ContainerType>( t1d);
+        if( !("exact"==mode || "simple" == mode))
+            throw dg::Error( dg::Message( _ping_) << "Mode must either be exact or simple!");
+
     }
 
     ///@copydoc Average()
-    Average( const aTopology3d& g, enum coo3d direction)
+    Average( const aTopology3d& g, enum coo3d direction, std::string mode = "exact"): m_mode(mode)
     {
         m_w = dg::construct<ContainerType>(dg::create::weights(g, direction));
         m_temp = m_w;
         m_transpose = false;
         unsigned nx = g.n()*g.Nx(), ny = g.n()*g.Ny(), nz = g.Nz();
         if( direction == coo3d::x) {
+            dg::blas1::scal( m_temp, 1./g.lx());
             dg::blas1::scal( m_w, 1./g.lx());
             m_nx = nx, m_ny = ny*nz;
+            if( "simple" == mode)
+                dg::transpose( m_nx, m_ny, m_temp, m_w);
         }
         else if( direction == coo3d::z) {
             m_transpose = true;
             dg::blas1::scal( m_temp, 1./g.lz());
+            dg::blas1::scal( m_w, 1./g.lz());
             m_nx = nx*ny, m_ny = nz;
-            dg::transpose( m_nx, m_ny, m_temp, m_w);
+            if( "exact" == mode)
+                dg::transpose( m_nx, m_ny, m_temp, m_w);
         }
         else if( direction == coo3d::xy) {
+            dg::blas1::scal( m_temp, 1./g.lx()/g.ly());
             dg::blas1::scal( m_w, 1./g.lx()/g.ly());
             m_nx = nx*ny, m_ny = nz;
+            if( "simple" == mode)
+                dg::transpose( m_nx, m_ny, m_temp, m_w);
         }
         else if( direction == coo3d::yz) {
             m_transpose = true;
             dg::blas1::scal( m_temp, 1./g.ly()/g.lz());
+            dg::blas1::scal( m_w, 1./g.ly()/g.lz());
             m_nx = nx, m_ny = ny*nz;
-            dg::transpose( m_nx, m_ny, m_temp, m_w);
+            if( "exact" == mode)
+                dg::transpose( m_nx, m_ny, m_temp, m_w);
         }
         else
             std::cerr << "Warning: this direction is not implemented\n";
@@ -97,6 +139,8 @@ struct Average
         else
             m_temp1d = dg::construct<ContainerType>(
                 thrust::host_vector<double>( m_nx,0.));
+        if( !("exact"==mode || "simple" == mode))
+            throw dg::Error( dg::Message( _ping_) << "Mode must either be exact or simple!");
     }
     /**
      * @brief Compute the average as configured in the constructor
@@ -114,7 +158,13 @@ struct Average
         if( !m_transpose)
         {
             //temp1d has size m_ny
-            dg::average( m_nx, m_ny, src, m_w, m_temp1d);
+            if( "exact" == m_mode)
+                dg::average( m_nx, m_ny, src, m_w, m_temp1d);
+            else
+            {
+                dg::transpose( m_nx, m_ny, src, m_temp);
+                dg::simple_average( m_ny, m_nx, m_temp, m_w, m_temp1d);
+            }
             if( extend )
                 dg::extend_column( m_nx, m_ny, m_temp1d, res);
             else
@@ -123,8 +173,13 @@ struct Average
         else
         {
             //temp1d has size m_nx
-            dg::transpose( m_nx, m_ny, src, m_temp);
-            dg::average( m_ny, m_nx, m_temp, m_w, m_temp1d);
+            if( "exact" == m_mode)
+            {
+                dg::transpose( m_nx, m_ny, src, m_temp);
+                dg::average( m_ny, m_nx, m_temp, m_w, m_temp1d);
+            }
+            else
+                dg::simple_average( m_nx, m_ny, src, m_w, m_temp1d);
             if( extend )
                 dg::extend_line( m_nx, m_ny, m_temp1d, res);
             else
@@ -136,7 +191,7 @@ struct Average
     unsigned m_nx, m_ny;
     ContainerType m_w, m_temp, m_temp1d;
     bool m_transpose;
-
+    std::string m_mode;
 };
 
 
diff --git a/inc/dg/topology/average_b.cu b/inc/dg/topology/average_b.cu
index a4332f581..13b4b7273 100644
--- a/inc/dg/topology/average_b.cu
+++ b/inc/dg/topology/average_b.cu
@@ -17,8 +17,10 @@ int main()
     std::cin >> n >> Nx >> Ny;
     const dg::Grid2d g( 0, lx, 0, ly, n, Nx, Ny);
 
-    dg::Average<dg::HVec> pol(g, dg::coo2d::y);
-    dg::Average<dg::DVec> pol_device(g, dg::coo2d::y);
+    dg::Average<dg::HVec> pol(g, dg::coo2d::y, "simple");
+    dg::Average<dg::DVec> pol_device(g, dg::coo2d::y, "simple");
+    dg::Average<dg::HVec> pol_ex(g, dg::coo2d::y, "exact");
+    dg::Average<dg::DVec> pol_device_ex(g, dg::coo2d::y, "exact");
     dg::Timer t;
 
     dg::HVec vector = dg::evaluate( function ,g), vector_y( vector);
@@ -28,15 +30,25 @@ int main()
     dg::HVec w2d = dg::create::weights( g);
     dg::DVec w2d_device( w2d);
     t.tic();
+    for( unsigned i=0; i<100; i++)
+        pol_ex( vector, vector_y);
+    t.toc();
+    std::cout << "Assembly of average (exact)  vector on host took:      "<<t.diff()/100.<<"s\n";
+    t.tic();
+    for( unsigned i=0; i<100; i++)
+        pol_device_ex( dvector, dvector_y);
+    t.toc();
+    std::cout << "Assembly of average (exact)  vector on device took:    "<<t.diff()/100.<<"s\n";
+    t.tic();
     for( unsigned i=0; i<100; i++)
         pol( vector, vector_y);
     t.toc();
-    std::cout << "Assembly of average vector on host took:      "<<t.diff()/100.<<"s\n";
+    std::cout << "Assembly of average (simple) vector on host took:      "<<t.diff()/100.<<"s\n";
     t.tic();
     for( unsigned i=0; i<100; i++)
         pol_device( dvector, dvector_y);
     t.toc();
-    std::cout << "Assembly of average vector on device took:    "<<t.diff()/100.<<"s\n";
+    std::cout << "Assembly of average (simple) vector on device took:    "<<t.diff()/100.<<"s\n";
     dg::blas1::axpby( 1., solution, -1., vector_y, vector);
     std::cout << "Result of integration on host is:     "<<dg::blas1::dot( vector, w2d)<<std::endl; //should be zero
     dg::blas1::axpby( 1., dsolution, -1., dvector_y, dvector);
diff --git a/inc/dg/topology/average_mpi.h b/inc/dg/topology/average_mpi.h
index 4453f1260..fb581fa14 100644
--- a/inc/dg/topology/average_mpi.h
+++ b/inc/dg/topology/average_mpi.h
@@ -10,6 +10,30 @@
   */
 namespace dg{
 
+///@cond
+template<class container>
+void simple_mpi_average( unsigned nx, unsigned ny, const container& in0, const container& in1, container& out, MPI_Comm comm)
+{
+    const double* in0_ptr = thrust::raw_pointer_cast( in0.data());
+    const double* in1_ptr = thrust::raw_pointer_cast( in1.data());
+          double* out_ptr = thrust::raw_pointer_cast( out.data());
+    dg::View<const container> in0_view( in0_ptr, nx), in1_view( in1_ptr, nx);
+    dg::View<container> out_view( out_ptr, nx);
+    dg::blas1::pointwiseDot( 1., in0_view, in1_view, 0, out_view);
+    for( unsigned i=1; i<ny; i++)
+    {
+        in0_view.construct( in0_ptr+i*nx, nx);
+        in1_view.construct( in1_ptr+i*nx, nx);
+        dg::blas1::pointwiseDot( 1., in0_view, in1_view, 1, out_view);
+    }
+    static thrust::host_vector<double> send_buf;
+    send_buf.resize( nx);
+    dg::assign( out_view, send_buf);
+    MPI_Allreduce(MPI_IN_PLACE, send_buf.data(), nx, MPI_DOUBLE, MPI_SUM, comm);
+    dg::assign( send_buf, out);
+}
+///@endcond
+
 /**
  * @brief MPI specialized class for average computations
  *
@@ -25,8 +49,13 @@ struct Average<MPI_Vector<container> >
      *
      * @param g the grid from which to take the dimensionality and sizes
      * @param direction the direction or plane over which to average when calling \c operator() (at the moment cannot be \c coo3d::xz or \c coo3d::y)
+     * @param mode either "exact" ( uses the exact and reproducible dot product for the summation) or "simple" (uses inexact but much faster direct summation) use simple if you do not need the reproducibility
+     * @note computing in "exact" mode is especially difficult if the averaged
+     * direction is small compared to the remaining dimensions and for GPUs in
+     * general, expect to gain a factor 10-1000 (no joke) from going to
+     * "simple" mode in these cases
      */
-    Average( const aMPITopology2d& g, enum coo2d direction)
+    Average( const aMPITopology2d& g, enum coo2d direction, std::string mode = "exact") : m_mode( mode)
     {
         m_nx = g.local().Nx()*g.n(), m_ny = g.local().Ny()*g.n();
         m_w=dg::construct<MPI_Vector<container>>(dg::create::weights(g, direction));
@@ -37,15 +66,20 @@ struct Average<MPI_Vector<container> >
         if( direction == dg::coo2d::x)
         {
             dg::blas1::scal( m_w, 1./g.lx());
+            dg::blas1::scal( m_temp, 1./g.lx());
             size1d = m_ny;
             remain_dims[0] = true;
+            if( "simple" == mode)
+                dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
         }
         else
         {
             m_transpose = true;
             remain_dims[1] = true;
+            dg::blas1::scal( m_w, 1./g.ly());
             dg::blas1::scal( m_temp, 1./g.ly());
-            dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
+            if( "exact" == mode)
+                dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
             size1d = m_nx;
         }
 
@@ -60,10 +94,12 @@ struct Average<MPI_Vector<container> >
         // with that construct the reduce mpi vec
         thrust::host_vector<double> t1d( size1d);
         m_temp1d = MPI_Vector<container>( dg::construct<container>( t1d), comm2);
+        if( !("exact"==mode || "simple" == mode))
+            throw dg::Error( dg::Message( _ping_) << "Mode must either be exact or simple!");
     }
 
     ///@copydoc Average()
-    Average( const aMPITopology3d& g, enum coo3d direction)
+    Average( const aMPITopology3d& g, enum coo3d direction, std::string mode = "exact") : m_mode( mode)
     {
         m_w = dg::construct<MPI_Vector<container>>(dg::create::weights(g, direction));
         m_temp = m_w;
@@ -73,27 +109,37 @@ struct Average<MPI_Vector<container> >
         m_transpose = false;
         if( direction == dg::coo3d::x) {
             dg::blas1::scal( m_w, 1./g.lx());
+            dg::blas1::scal( m_temp, 1./g.lx());
             m_nx = nx, m_ny = ny*nz;
             remain_dims[0] = true;
+            if( "simple" == mode)
+                dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
         }
         else if( direction == dg::coo3d::z) {
             m_transpose = true;
             remain_dims[2] = true;
             m_nx = nx*ny, m_ny = nz;
+            dg::blas1::scal( m_w, 1./g.lz());
             dg::blas1::scal( m_temp, 1./g.lz());
-            dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
+            if( "exact" == mode)
+                dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
         }
         else if( direction == dg::coo3d::xy) {
             dg::blas1::scal( m_w, 1./g.lx()/g.ly());
+            dg::blas1::scal( m_temp, 1./g.lx()/g.ly());
             m_nx = nx*ny, m_ny = nz;
             remain_dims[0] = remain_dims[1] = true;
+            if( "simple" == mode)
+                dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
         }
         else if( direction == dg::coo3d::yz) {
             m_transpose = true;
             m_nx = nx, m_ny = ny*nz;
             remain_dims[1] = remain_dims[2] = true;
+            dg::blas1::scal( m_w, 1./g.ly()/g.lz());
             dg::blas1::scal( m_temp, 1./g.ly()/g.lz());
-            dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
+            if( "exact" == mode)
+                dg::transpose( m_nx, m_ny, m_temp.data(), m_w.data());
         }
         else
             std::cerr << "Warning: this direction is not implemented\n";
@@ -112,6 +158,8 @@ struct Average<MPI_Vector<container> >
         else
             t1d = thrust::host_vector<double>( m_nx,0.);
         m_temp1d = MPI_Vector<container>( dg::construct<container>( t1d), comm2);
+        if( !("exact"==mode || "simple" == mode))
+            throw dg::Error( dg::Message( _ping_) << "Mode must either be exact or simple!");
     }
     /**
      * @brief Compute the average as configured in the constructor
@@ -131,8 +179,16 @@ struct Average<MPI_Vector<container> >
         if( !m_transpose)
         {
             //temp1d has size m_ny
-            dg::mpi_average( m_nx, m_ny, src.data(), m_w.data(),
-                m_temp1d.data(), m_comm, m_comm_mod, m_comm_mod_reduce);
+            if( "exact" == m_mode)
+                dg::mpi_average( m_nx, m_ny, src.data(), m_w.data(),
+                    m_temp1d.data(), m_comm, m_comm_mod, m_comm_mod_reduce);
+            else
+            {
+                dg::transpose( m_nx, m_ny, src.data(), m_temp.data());
+                dg::simple_mpi_average( m_ny, m_nx, m_temp.data(), m_w.data(),
+                    m_temp1d.data(), m_comm);
+            }
+
             if( extend )
                 dg::extend_column( m_nx, m_ny, m_temp1d.data(), res.data());
             else
@@ -141,8 +197,16 @@ struct Average<MPI_Vector<container> >
         else
         {
             //temp1d has size m_nx
-            dg::transpose( m_nx, m_ny, src.data(), m_temp.data());
-            dg::mpi_average( m_ny, m_nx, m_temp.data(), m_w.data(), m_temp1d.data(), m_comm, m_comm_mod, m_comm_mod_reduce);
+            if( "exact" == m_mode)
+            {
+                dg::transpose( m_nx, m_ny, src.data(), m_temp.data());
+                dg::mpi_average( m_ny, m_nx, m_temp.data(), m_w.data(),
+                    m_temp1d.data(), m_comm, m_comm_mod, m_comm_mod_reduce);
+            }
+            else
+                dg::simple_mpi_average( m_nx, m_ny, src.data(), m_w.data(),
+                    m_temp1d.data(), m_comm);
+
             if( extend )
                 dg::extend_line( m_nx, m_ny, m_temp1d.data(), res.data());
             else
@@ -154,6 +218,7 @@ struct Average<MPI_Vector<container> >
     MPI_Vector<container> m_w, m_temp, m_temp1d;
     bool m_transpose;
     MPI_Comm m_comm, m_comm_mod, m_comm_mod_reduce;
+    std::string m_mode;
 };
 
 
diff --git a/inc/dg/topology/average_mpib.cu b/inc/dg/topology/average_mpib.cu
index 9a16cda18..6dfbcd0d1 100644
--- a/inc/dg/topology/average_mpib.cu
+++ b/inc/dg/topology/average_mpib.cu
@@ -29,17 +29,23 @@ int main(int argc, char* argv[])
     dg::Timer t;
 
 
-    dg::Average<dg::MHVec > pol(g, dg::coo2d::y);
-    dg::MHVec vector = dg::evaluate( function ,g), average_y( vector);
-    const dg::MHVec solution = dg::evaluate( pol_average, g);
+    dg::Average<dg::MDVec > pol(g, dg::coo2d::y, "simple");
+    dg::Average<dg::MDVec > pol_ex(g, dg::coo2d::y, "exact");
+    dg::MDVec vector = dg::evaluate( function ,g), average_y( vector);
+    const dg::MDVec solution = dg::evaluate( pol_average, g);
     t.tic();
     for( unsigned i=0; i<100; i++)
         pol( vector, average_y);
     t.toc();
-    if(rank==0)std::cout << "Assembly of average vector took:      "<<t.diff()/100.<<"s\n";
+    if(rank==0)std::cout << "Assembly of average (simple) vector took:      "<<t.diff()/100.<<"s\n";
+    t.tic();
+    for( unsigned i=0; i<100; i++)
+        pol_ex( vector, average_y);
+    t.toc();
+    if(rank==0)std::cout << "Assembly of average (exact)  vector took:      "<<t.diff()/100.<<"s\n";
 
     dg::blas1::axpby( 1., solution, -1., average_y, vector);
-    dg::MHVec w2d = dg::create::weights(g);
+    dg::MDVec w2d = dg::create::weights(g);
     double norm = dg::blas2::dot(vector, w2d, vector);
     if(rank==0)std::cout << "Distance to solution is: "<<        sqrt(norm)<<std::endl;
 
diff --git a/inc/dg/topology/average_mpit.cu b/inc/dg/topology/average_mpit.cu
index 360d53cbc..add08c1bd 100644
--- a/inc/dg/topology/average_mpit.cu
+++ b/inc/dg/topology/average_mpit.cu
@@ -27,7 +27,7 @@ int main(int argc, char* argv[])
     //![doxygen]
     dg::MPIGrid3d g( 0, lx, 0, ly, 0, lz, n, Nx, Ny, Nz, comm);
 
-    dg::Average<dg::MDVec > avg(g, dg::coo3d::z);
+    dg::Average<dg::MDVec > avg(g, dg::coo3d::z, "exact");
 
     const dg::MDVec vector = dg::evaluate( function ,g);
     dg::MDVec average_z;
@@ -44,7 +44,7 @@ int main(int argc, char* argv[])
     if(rank==0)std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i<<std::endl;
     if(rank==0)std::cout << "(Converges with 2nd order).\n";
     if(rank==0)std::cout << "Averaging x ... \n";
-    dg::Average< dg::MDVec> tor( g, dg::coo3d::x);
+    dg::Average< dg::MDVec> tor( g, dg::coo3d::x, "exact");
     average_z = vector;
     tor( vector, average_z);
     solution = dg::evaluate( x_average, g);
diff --git a/inc/dg/topology/average_t.cu b/inc/dg/topology/average_t.cu
index 5b1419c0f..f2eab250b 100644
--- a/inc/dg/topology/average_t.cu
+++ b/inc/dg/topology/average_t.cu
@@ -17,7 +17,7 @@ int main()
     //![doxygen]
     const dg::Grid2d g( 0, lx, 0, ly, n, Nx, Ny);
 
-    dg::Average< dg::DVec > pol(g, dg::coo2d::y);
+    dg::Average< dg::DVec > pol(g, dg::coo2d::y, "simple");
 
     const dg::DVec vector = dg::evaluate( function ,g);
     dg::DVec average_y;
@@ -33,9 +33,9 @@ int main()
     res.d = sqrt( dg::blas2::dot( average_y, w1d, average_y));
     std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i<<std::endl;
     std::cout << "Averaging x ... \n";
-    dg::Average< dg::DVec> tor( g, dg::coo2d::x);
+    dg::Average< dg::DVec> tor( g, dg::coo2d::x, "simple" );
     average_y = vector;
-    tor( vector, average_y);
+    tor( vector, average_y, true);
     solution = dg::evaluate( tor_average, g);
     dg::blas1::axpby( 1., solution, -1., average_y);
     const dg::DVec w2d = dg::create::weights( g);
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index c39753f42..554d9474b 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -100,7 +100,7 @@ struct FluxSurfaceIntegral
     /**
      * @brief Set the right function to integrate
      *
-     * @param f the container containing the discretized function
+     * @param g the container containing the discretized function
      */
     void set_right( const container& g){
         dg::blas1::copy( g, m_g);
@@ -160,7 +160,7 @@ struct FluxVolumeIntegral
     /**
      * @brief Set the right function to integrate
      *
-     * @param f the container containing the discretized function
+     * @param g the container containing the discretized function
      */
     void set_right( const container& g){
         dg::blas1::copy( g, m_g);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index ba0707848..40292739b 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1095,12 +1095,12 @@ Input file format: json
 \begin{longtable}{llll>{\RaggedRight}p{6cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
-n      & integer & 3 & - &Number of Gaussian nodes in R and Z (we always take 3) \\
+n      & integer & 3 & - &Number of Gaussian nodes in R and Z (we practically always take 3) \\
 Nx     & integer &52& - &Number of grid points in R (increase if your simulations crash) \\
 Ny     & integer &52& - &Number of grid points in Z (increase if your simulations crash) \\
 Nz     & integer &16& - &Number of grid points in $\varphi$ (determines dt since parallel velocity dominates timestep) \\
 dt     & integer &1e-2& - & time stepsize in units of $c_s/\rho_s$ \\
-compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing points in x and y: output contains n*Nx/c[0] points in x,
+compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing points in x and y (pojecting the polynomials onto a coarser grid): output contains n*Nx/c[0] points in x,
     (has to divde Nx evenly), and n*Ny/c[1] points in y,
     (has to divde Ny evenly)\\
 inner\_loop & integer & 2  & 1 & Number of time steps between updates to the time integrated quantities. Note that integrating selected
@@ -1141,9 +1141,11 @@ symmetric & bool & false & false & If true, initialize all quantities symmetric
 in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used
 to construct the parallel derivatives and then overwritten to $N_z\equiv 1$. \\
 bc & dict & & & Dictionary of boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
-\qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y for $n_e$ and $N_i$\\
-\qquad velocity  & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $u_e$ and $U_i$ and $A_\parallel$\\
-\qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $\phi$ and $\psi$\\
+\qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y for $n_e$ and $N_i$, DIR means
+no outflow by convection while NEU means no outflow by diffusion\\
+\qquad velocity  & char[2] & [NEU,NEU] & - & boundary conditions in x and y for $u_e$ and $U_i$ and $A_\parallel$, DIR is in general not very stable, NEU works better\\
+\qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $\phi$ and $\psi$, DIR means that
+the $v_{E,\perp}=0$ on the boundary (i.e. no outflow by \ExB drift), NEU can mean lowering the timestep \\
     boxscaleR  & float[2] & [1.1,1.1]     & [1.05,1.05] & $[\varepsilon_{R-}, \varepsilon_{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
     boxscaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon_{Z-}, \varepsilon_{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box} \\
 initne    & string & "turbulence"     & "blob"  & initial condition for the
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index cd2bba77d..b5315ce31 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -161,7 +161,7 @@ int main( int argc, char* argv[])
 
     // helper variables for output computations
     std::map<std::string, dg::Simpsons<HVec>> time_integrals;
-    dg::Average<HVec> toroidal_average( g3d_out, dg::coo3d::z);
+    dg::Average<HVec> toroidal_average( g3d_out, dg::coo3d::z, "simple");
     dg::MultiMatrix<HMatrix,HVec> projectH = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
     dg::MultiMatrix<DMatrix,DVec> projectD = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
     HVec transferH( dg::evaluate(dg::zero, g3d_out));
-- 
GitLab


From b25ca6ea7755ec1286828e19035d1828bbe79eb3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 1 Oct 2019 13:36:59 +0200
Subject: [PATCH 144/540] Update version checking code

- and some docu in feltor
---
 inc/dg/backend/version.cu |  17 ++---
 src/feltor/feltor.tex     | 145 +++++++++++++++++++++++++++-----------
 src/feltor/parameters.h   |   2 +-
 3 files changed, 111 insertions(+), 53 deletions(-)

diff --git a/inc/dg/backend/version.cu b/inc/dg/backend/version.cu
index 1d7fbc37c..828dd1416 100644
--- a/inc/dg/backend/version.cu
+++ b/inc/dg/backend/version.cu
@@ -2,14 +2,8 @@
 #include <cusp/version.h>
 #include <iostream>
 
-#ifndef CUDA_VERSION
-#define CUDA_VERSION 0
-#endif
-
 int main(void)
 {
-    int cuda_major =  CUDA_VERSION / 1000;
-    int cuda_minor = (CUDA_VERSION % 1000) / 10;
 
     int thrust_major = THRUST_MAJOR_VERSION;
     int thrust_minor = THRUST_MINOR_VERSION;
@@ -19,10 +13,13 @@ int main(void)
     int cusp_minor = CUSP_MINOR_VERSION;
     int cusp_subminor = CUSP_SUBMINOR_VERSION;
 
-    if( CUDA_VERSION==0)
-        std::cout << "Cuda is not needed for host-only compilation!\n";
-    else
-        std::cout << "CUDA   v" << cuda_major   << "." << cuda_minor   << std::endl;
+#ifdef __NVCC__
+    int cuda_major = __CUDACC_VER_MAJOR__;
+    int cuda_minor = __CUDACC_VER_MINOR__;
+    std::cout << "CUDA   v" << cuda_major   << "." << cuda_minor   << std::endl;
+#else
+    std::cout << "Cuda is not needed for host-only compilation!\n";
+#endif
     std::cout << "Thrust v" << thrust_major << "." << thrust_minor << "."<<thrust_subminor << std::endl;
     std::cout << "Cusp   v" << cusp_major   << "." << cusp_minor   << "."<<cusp_subminor<< std::endl;
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 40292739b..a7649cff5 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1095,39 +1095,81 @@ Input file format: json
 \begin{longtable}{llll>{\RaggedRight}p{6cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
-n      & integer & 3 & - &Number of Gaussian nodes in R and Z (we practically always take 3) \\
-Nx     & integer &52& - &Number of grid points in R (increase if your simulations crash) \\
-Ny     & integer &52& - &Number of grid points in Z (increase if your simulations crash) \\
-Nz     & integer &16& - &Number of grid points in $\varphi$ (determines dt since parallel velocity dominates timestep) \\
+n      & integer & 3 & - &Number of Gaussian nodes in R and Z (we practically always take 3)
+\\
+Nx     & integer &52& - &Number of grid points in R (increase if your simulations crash)
+\\
+Ny     & integer &52& - &Number of grid points in Z (increase if your simulations crash)
+\\
+Nz     & integer &16& - &Number of grid points in $\varphi$ (determines dt
+since parallel velocity dominates timestep)
+\\
 dt     & integer &1e-2& - & time stepsize in units of $c_s/\rho_s$ \\
-compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing points in x and y (pojecting the polynomials onto a coarser grid): output contains n*Nx/c[0] points in x,
-    (has to divde Nx evenly), and n*Ny/c[1] points in y,
-    (has to divde Ny evenly)\\
-inner\_loop & integer & 2  & 1 & Number of time steps between updates to the time integrated quantities. Note that integrating selected
-quantities in time during the simulation is how we maintain the time-resolution in the file output (cf. \ref{sec:output_file}).\\
-itstp       & integer & 2  & - &{ \tt inner\_loop*itstp} is the number of timesteps between file outputs (2d and 3d quantities);
-Note that 1d and 0d quantities can only be computed in diagnostics since we can't compute flux-integrals in parallel in MPI. \\
-maxout      & integer & 10 & - & Total Number of fields outputs excluding first (The total number of time steps is {\tt maxout$\cdot$itstp$\cdot$inner\_loop}) \\
-eps\_time   & float & 1e-7  & - & Tolerance for solver for implicit part in time-stepper (if too low, you'll see oscillations in $u_e$ and/or $\phi$) \\
-rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper. (Ignored in Multistep) \\
-eps\_pol    & float & 1e-6  & - &  Tolerance for residual of the inversion of polarisation and induction Eq. (should not be more than a factor 10 from eps\_time for $\beta\neq  0$ ) \\
-jumpfactor  & float & 1 & 1 & Jumpfactor $\in \left[0.01,1\right]$ in the local DG method for the elliptic terms\\
-eps\_gamma  & float & 1e-6  & - & Tolerance for $\Gamma_1$  \\
+compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing
+points in x and y (pojecting the polynomials onto a coarser grid): output
+contains n*Nx/c[0] points in x, (has to divde Nx evenly), and n*Ny/c[1] points
+in y, (has to divde Ny evenly). 2 or 3 are reasonable values.
+\\
+inner\_loop & integer & 2  & 1 & Number of time steps between updates to the
+time integrated quantities. (Although the diagnostics is quite fast sometimes
+you need to amortize the time spent on it). Note that integrating selected
+quantities in time during the simulation is how we maintain the time-resolution
+in the file output (cf. \ref{sec:output_file}). Choose as low as you can get
+away with (between 1 and 10).
+\\
+itstp       & integer & 2  & - &{ \tt inner\_loop*itstp} is the number of
+timesteps between file outputs (2d and 3d quantities); Note that 1d and 0d
+quantities can only be computed post-simulation since we can't compute
+flux-integrals in parallel in MPI.
+\\
+maxout      & integer & 10 & - & Total Number of fields outputs excluding first
+(The total number of time steps is {\tt maxout$\cdot$itstp$\cdot$inner\_loop})
+If you want to let the simulation run for a certain time instead just choose
+this parameter very large and let the simulation hit the time-limit.
+\\
+eps\_time   & float & 1e-7  & - & Tolerance for solver for implicit part in
+time-stepper (if too low, you'll see oscillations in $u_e$ and/or $\phi$)
+\\
+rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper. (Ignored in Multistep)
+\\
+eps\_pol    & float & 1e-6  & - &  Tolerance for residual of the inversion of polarisation and induction Eq. (should not be more than a factor 10 from eps\_time for $\beta\neq  0$ )
+\\
+jumpfactor  & float & 1 & 1 & Jumpfactor $\in \left[0.01,1\right]$ in the local DG method for the elliptic terms. (Don't touch unless you know what you're doing.
+\\
+eps\_gamma  & float & 1e-6  & - & Tolerance for $\Gamma_1$
+\\
 stages      & integer & 3 & 3 & number of stages in multigrid, $2^{\text{stages-1}}$
-has to evenly divide both $N_x$ and $N_y$\\
-refineDS     & integer[2] & [10,10] & [10,10] & refinement factor in FCI approach in R- and Z-direction\\
-rk4eps     & float & 1e-5 & 1e-5 & Accuracy of fieldline integrator in FCI\\
+has to evenly divide both $N_x$ and $N_y$
+\\
+refineDS     & integer[2] & [10,10] & [10,10] & refinement factor in FCI approach in R- and Z-direction.
+We use [1,1], higher values take more time.
+\\
+rk4eps     & float & 1e-6 & 1e-6 & Accuracy of fieldline integrator in FCI. The default is reasonable.
+\\
 mu         & float & -0.000272121& - & $\mu_e =-m_e/m_i$.
-    One of $\left\{ -0.000544617, -0.000272121, -0.000181372 \right\}$\\
-tau        & float &1      & - & $\tau = T_i/T_e$  \\
-beta       & float & 5e-6  & 0 & Plasma beta $5\cdot 10^{-6}$ (TJK), $4\cdot 10^{-3}$ (Compass), If $0$, then the model is electrostatic \\
-nu\_perp   & float &1e-3   & - & perpendicular viscosity $\nu_\perp$ \\
-perp\_diff & string & "viscous" & "viscous" & "viscous": $\Lambda_\perp\propto \nu_\perp\Delta_\perp$ , "hyperviscous": $\Lambda_\perp \propto -\nu_\perp\Delta_\perp^2$\\
+    One of $\left\{ -0.000544617, -0.000272121, -0.000181372 \right\}$
+\\
+tau        & float &1      & - & $\tau = T_i/T_e$
+\\
+beta       & float & 5e-6  & 0 & Plasma beta $5\cdot 10^{-6}$ (TJK), $4\cdot
+10^{-3}$ (Compass), If $0$, then the model is electrostatic
+\\
+nu\_perp   & float &1e-3   & - & perpendicular viscosity $\nu_\perp$, increase
+this or the resolution if you see vertical or horizontal oscillations (likely
+from the advection terms) in your simulation box, decrease if it dampens all
+instabilities
+\\
+perp\_diff & string & "viscous" & "viscous" & "viscous": $\Lambda_\perp\propto
+\nu_\perp\Delta_\perp$ , "hyperviscous": $\Lambda_\perp \propto
+-\nu_\perp\Delta_\perp^2$
+\\
 nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$
 (dimensional analysis reveals there can be a factor $(R_0/\rho_s)^2$ between
 $\nu_\perp $n and $\nu_\parallel$ for $\nu_\parallel$ to become relevant for
-the dynamics)\\
-resistivity & float &1e-4  & - & parallel resistivity parameter Eq.~\eqref{eq:resistivity}\\
+the dynamics)
+\\
+resistivity & float &1e-4  & - & parallel resistivity parameter Eq.~\eqref{eq:resistivity}
+\\
 curvmode  & string & "low beta"  & "toroidal" &
 curvature mode (
 "low beta",
@@ -1136,18 +1178,26 @@ curvature mode (
 communication in z,
 "low beta negative": reversed sign of low beta,
 "toroidal negative": reversed sign of toroidal
-) \\
+)
+\\
 symmetric & bool & false & false & If true, initialize all quantities symmetric
 in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used
-to construct the parallel derivatives and then overwritten to $N_z\equiv 1$. \\
+to construct the parallel derivatives and then overwritten to $N_z\equiv 1$.
+\\
 bc & dict & & & Dictionary of boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
-\qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y for $n_e$ and $N_i$, DIR means
-no outflow by convection while NEU means no outflow by diffusion\\
-\qquad velocity  & char[2] & [NEU,NEU] & - & boundary conditions in x and y for $u_e$ and $U_i$ and $A_\parallel$, DIR is in general not very stable, NEU works better\\
-\qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for $\phi$ and $\psi$, DIR means that
-the $v_{E,\perp}=0$ on the boundary (i.e. no outflow by \ExB drift), NEU can mean lowering the timestep \\
+\qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y
+for $n_e$ and $N_i$, DIR (density 1 on boundary) means both convective and
+    diffusive outflow while NEU (gradient 0) means no outflow by diffusion
+\\
+\qquad velocity  & char[2] & [NEU,NEU] & - & boundary conditions in x and y for
+$u_e$ and $U_i$ and $A_\parallel$, DIR is in general not very stable, NEU works
+better\\
+\qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for
+$\phi$ and $\psi$, DIR means that the $v_{E,\perp}=0$ on the boundary (i.e. no
+outflow by \ExB drift), NEU can can have a detrimental effect on timestep \\
     boxscaleR  & float[2] & [1.1,1.1]     & [1.05,1.05] & $[\varepsilon_{R-}, \varepsilon_{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
-    boxscaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon_{Z-}, \varepsilon_{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box} \\
+    boxscaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon_{Z-}, \varepsilon_{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box}
+\\
 initne    & string & "turbulence"     & "blob"  & initial condition for the
 perturbation $\tilde n$ in \eqref{eq:initial_ne}. "zonal" (Eq.~\eqref{eq:initial_zonal_flow}),
     "blob" = blob simulations (several rounds fieldaligned),
@@ -1155,7 +1205,7 @@ perturbation $\tilde n$ in \eqref{eq:initial_ne}. "zonal" (Eq.~\eqref{eq:initial
     "turbulence" = turbulence simulations ( 1 round fieldaligned, Eq.~\eqref{eq:initial_turbulent})
     "turbulence on gaussian" = Gaussian bg. profile with turbulence perturbation Eq.~\eqref{eq:turbulence_on_gaussian}
     See the file {\tt init.h} to add your own custom condition.
-    \\
+\\
 initphi   & string & "zero"  & "balance" & initial condition for $\phi$ and thus $N_i$ (Eq.~\eqref{eq:initphi}: "zero" : $\phi = 0$, vanishing
 electric potential, "balance": ExB vorticity equals ion diamagnetic vorticity (For $\tau_i =0 $ both are the same)
 \\
@@ -1165,17 +1215,25 @@ posX       & float &0.3    & - & Gaussian R-position in units of $a$\\
 posY       & float &0.0    & - & Gaussian Z-position in units of $a$ \\
 sigma\_z    & float &0.25   & - & toroidal variance in units of $R_0$ of the fieldline-following initialization \\
 k\_psi     & float &0    & - & zonal mode wave number (only for "zonal" initial condition)  \\
-nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}\\
-alpha       & float & 0.2 & - & Width $\alpha$ of the Heaviside Eq.~\eqref{eq:approx_heaviside} in the density and source profiles (should be small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes) \\
-source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source} \\
+nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in
+Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}
+\\
+alpha       & float & 0.2 & - & Width $\alpha$ of the Heaviside
+Eq.~\eqref{eq:approx_heaviside} in the density and source profiles (should be
+small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes)
+\\
+source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}
+\\
 source\_type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
 "influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx},
 "torpex": Torpex inspired source profile Eq.~\eqref{eq:electron_source_torpex},
-    See the file {\tt init.h} to add your own custom source. \\
+    See the file {\tt init.h} to add your own custom source.
+\\
 rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
 %damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
 alpha\_mag   & float & 0.05 & - & Width $\alpha$ of the Heaviside in the modified $\psi_p$ function \eqref{eq:modified_psip}. If zero, then we do not modify the magnetic field and rho\_damping is ignored.\\
-rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0 = (1-\rho_d)\psi_{p,\min}$ in Eq.~\eqref{eq:modified_psip}. \\
+rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0 = (1-\rho_d)\psi_{p,\min}$ in Eq.~\eqref{eq:modified_psip}.
+\\
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
@@ -1323,9 +1381,12 @@ and the time-averaged quantities Y $\in$
     socurvkappai &$z_i\mu_i N_iU_i^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
 \bottomrule
 \end{longtable}
+Even though this might look like a lot of things to compute the actual time spent on diagnostics is negligible if {\tt inner\_loop} parameter is greater than 1.
 \subsection{Restart file} \label{sec:restart_file}
 The program \texttt{feltor\_hpc.cu} has the possibility to initialize time and the fields with
-the results of a previous simulation. This behaviour is enabled by giving an additional file \texttt{initial.nc}
+the results of a previous simulation. In particular, this feature is motivated by chain jobs on a cluster
+(see e.g. the --dependency option in SLURM).
+This behaviour is enabled by giving an additional file \texttt{initial.nc}
 to the command line. In this case the \texttt{initne} and \texttt{initphi} parameters of the input
 file are ignored. Instead, the fields \texttt{electrons, ions, Ue, Ui, induction} at the latest timestep
 are read from the given file to initialize the simulation.
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 76e5ffd1b..bd48fbc66 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -72,7 +72,7 @@ struct Parameters
         stages      = js.get( "stages", 3).asUInt();
         mx          = js["refineDS"].get( 0u, 10).asUInt();
         my          = js["refineDS"].get( 1u, 10).asUInt();
-        rk4eps      = js.get( "rk4eps", 1e-5).asDouble();
+        rk4eps      = js.get( "rk4eps", 1e-6).asDouble();
 
         mu[0]       = js["mu"].asDouble();
         mu[1]       = +1.;
-- 
GitLab


From d85971f6b0bfb1c84ef63c142a3b4d32d880a959 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 2 Oct 2019 11:56:58 +0200
Subject: [PATCH 145/540] Update README with link to troubleshooting

---
 README.adoc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.adoc b/README.adoc
index 4bfe5fc22..a14c1e5b4 100644
--- a/README.adoc
+++ b/README.adoc
@@ -29,6 +29,10 @@ installation. Please read it before you proceed to the https://feltor-dev.github
 We also present how to use the FELTOR software package,
 which requires additional external libraries to be installed on the system.
 ____
+If you run into trouble during the setup, there is a chance that we already know of the problem and have an entry in our https://feltor-dev.github.io/troubleshooting/[troubleshooting] page.
+If you cannot find a solution there, please inform us of the issue.
+____
+____
 The first part assumes a Linux operating system. If you want to work
 on Windows, jump to <<sec_windows,Using Feltor on Windows>>.
 ____
-- 
GitLab


From d8a19607d350c26e445df6064f6e772c8750c6a0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 10 Oct 2019 15:06:29 +0200
Subject: [PATCH 146/540] Make geometries folder compile again

---
 inc/dg/nullstelle.h                           |  9 ++++---
 inc/dg/topology/interpolationX.h              | 25 +++++++++++++++----
 inc/geometries/conformalX_elliptic_b.cu       |  4 +--
 inc/geometries/geometryX_elliptic_b.cu        |  4 +--
 .../geometryX_refined_elliptic_b.cu           |  4 +--
 inc/geometries/geometry_advection_b.cu        |  4 +--
 inc/geometries/geometry_advection_mpib.cu     |  4 +--
 inc/geometries/geometry_diag.cu               |  4 +--
 inc/geometries/hector.h                       | 16 ++++++------
 inc/geometries/ribeiroX_t.cu                  |  2 +-
 inc/geometries/separatrix_orthogonal_t.cu     |  2 +-
 src/feltor/input/default.json                 |  2 +-
 12 files changed, 48 insertions(+), 32 deletions(-)

diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index b526b3d13..77fa25758 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -9,6 +9,7 @@
 #include <exception>
 #include <math.h>
 #include "backend/exceptions.h"
+
 namespace dg{
 
 /*! @brief Exception class, that stores boundaries for 1D root finding
@@ -47,12 +48,13 @@ class NoRoot1d: public std::exception
  * \param op Function or Functor
  * \param x_min left boundary, contains new left boundary on execution
  * \param x_max right boundary, contains new right boundary on execution
- * \param eps accuracy of the root finding
+ * \param eps accuracy of the root finding. Algorithm successful if
+ *  \f$ |x_{\max} - x_{\min}| < \varepsilon |x_{\max}| + \varepsilon \f$
  * \return number of used steps to reach the desired accuracy
  * \throw NoRoot1d if no root lies between the given boundaries
  * \throw std::runtime_error if after 60 steps the accuracy wasn't reached
  *
- * \code nullstelle_1D(funk, x_min, x_max, eps); \endcode
+ * \code bisection1d(f, x_min, x_max, eps); \endcode
  * \note If the root is found exactly the x_min = x_max
  */
 template <typename UnaryOp>
@@ -73,12 +75,11 @@ int bisection1d (UnaryOp& op, double& x_min, double& x_max, const double eps)
         if(wert_mitte==0) 			    {x_min=x_max=mitte; return j+3;}
         else if(wert_mitte*wert_max>0) 	x_max = mitte;
         else 				            x_min = mitte;
-        if((x_max-x_min)<eps)           return j+3;
+        if( fabs(x_max-x_min)<eps*fabs(x_max) + eps)         return j+3;
     }
     throw std::runtime_error("Too many steps in root finding!");
 }
 
-//@}
 }//namespace dg
 #endif //_NULLSTELLE_
 
diff --git a/inc/dg/topology/interpolationX.h b/inc/dg/topology/interpolationX.h
index 1135be067..5349a58f6 100644
--- a/inc/dg/topology/interpolationX.h
+++ b/inc/dg/topology/interpolationX.h
@@ -137,19 +137,34 @@ thrust::host_vector<real_type> forward_transform( const thrust::host_vector<real
 }//namespace create
 
 /**
- * @brief Interpolate a single point
- *
+ * @brief Interpolate a vector on a single point on a 2d Grid
+ *
+ * @param sp Indicate whether the elements of the vector
+ * v are in xspace or lspace
+ *  (choose dg::xspace if you don't know what is going on here,
+ *      It is faster to interpolate in dg::lspace so consider
+ *      transforming v using dg::forward_transform( )
+ *      if you do it very many times)
+ * @param v The vector to interpolate in dg::xspace, or dg::lspace s.a. dg::forward_transform( )
  * @param x X-coordinate of interpolation point
  * @param y Y-coordinate of interpolation point
- * @param v The vector to interpolate in LSPACE
  * @param g The Grid on which to operate
+ * @copydoc hide_bcx_doc
+ * @param bcy analogous to \c bcx, applies to y direction
  *
+ * @ingroup interpolation
  * @return interpolated point
+ * @note \c g.contains(x,y) must return true
  */
 template<class real_type>
-real_type interpolate( real_type x, real_type y,  const thrust::host_vector<real_type>& v, const aRealTopologyX2d<real_type>& g )
+real_type interpolate(
+    dg::space sp,
+    const thrust::host_vector<real_type>& v,
+    real_type x, real_type y,
+    const aRealTopologyX2d<real_type>& g,
+    dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU )
 {
-    return interpolate( x,y,v,g.grid());
+    return interpolate( sp, v, x, y, g.grid(), bcx, bcy);
 }
 
 } //namespace dg
diff --git a/inc/geometries/conformalX_elliptic_b.cu b/inc/geometries/conformalX_elliptic_b.cu
index 198b9ac79..233ebba59 100644
--- a/inc/geometries/conformalX_elliptic_b.cu
+++ b/inc/geometries/conformalX_elliptic_b.cu
@@ -66,8 +66,8 @@ void compute_cellsize( const Geometry& g2d)
     dg::blas1::pointwiseDot( gyy, vol, gyy);
     dg::blas1::scal( gxx, g2d.hx());
     dg::blas1::scal( gyy, g2d.hy());
-    double hxX = dg::interpolate( 0., 0., gxx, g2d);
-    double hyX = dg::interpolate( 0., 0., gyy, g2d);
+    double hxX = dg::interpolate( dg::xspace, gxx, 0., 0., g2d);
+    double hyX = dg::interpolate( dg::xspace, gyy, 0., 0., g2d);
     std::cout << *thrust::max_element( gxx.begin(), gxx.end()) << "\t";
     std::cout << *thrust::max_element( gyy.begin(), gyy.end()) << "\t";
     std::cout << hxX << "\t";
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index 91064c33e..fb7a27f5b 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -141,8 +141,8 @@ int main(int argc, char**argv)
     dg::blas1::pointwiseDot( gyy, vol, gyy);
     dg::blas1::scal( gxx, g2d.hx());
     dg::blas1::scal( gyy, g2d.hy());
-    double hxX = dg::interpolate( 0., 0., (dg::HVec)gxx, g2d);
-    double hyX = dg::interpolate( 0., 0., (dg::HVec)gyy, g2d);
+    double hxX = dg::interpolate( dg::xspace, (dg::HVec)gxx, 0., 0., g2d);
+    double hyX = dg::interpolate( dg::xspace, (dg::HVec)gyy, 0., 0., g2d);
     std::cout << *thrust::max_element( gxx.begin(), gxx.end()) << "\t";
     std::cout << *thrust::max_element( gyy.begin(), gyy.end()) << "\t";
     std::cout << hxX << "\t";
diff --git a/inc/geometries/geometryX_refined_elliptic_b.cu b/inc/geometries/geometryX_refined_elliptic_b.cu
index 4c0121041..242d1b34b 100644
--- a/inc/geometries/geometryX_refined_elliptic_b.cu
+++ b/inc/geometries/geometryX_refined_elliptic_b.cu
@@ -204,8 +204,8 @@ int main(int argc, char**argv)
     dg::blas1::pointwiseDot( gyy, vol, gyy);
     dg::blas1::scal( gxx, g2d_fine.hx());
     dg::blas1::scal( gyy, g2d_fine.hy());
-    double hxX = dg::interpolate( 0., 0., (dg::HVec)gxx, g2d_fine);
-    double hyX = dg::interpolate( 0., 0., (dg::HVec)gyy, g2d_fine);
+    double hxX = dg::interpolate( dg::xspace, (dg::HVec)gxx, 0., 0., g2d_fine);
+    double hyX = dg::interpolate( dg::xspace, (dg::HVec)gyy, 0., 0., g2d_fine);
     std::cout << *thrust::max_element( gxx.begin(), gxx.end()) << "\t";
     std::cout << *thrust::max_element( gyy.begin(), gyy.end()) << "\t";
     std::cout << hxX << "\t";
diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index 1dfac7393..d799543f5 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -87,7 +87,7 @@ struct VariationDirPer
 
 struct CurvatureDirPer
 {
-    CurvatureDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4.), curvR(c), curvZ(c){}
+    CurvatureDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4.), curvR(c,+1), curvZ(c,+1){}
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);}
     double operator()(double R, double Z) const {
@@ -204,7 +204,7 @@ int main(int argc, char** argv)
     std::cout << "TESTING CURVATURE 3D\n";
     dg::DVec curvX, curvY;
     dg::HVec tempX, tempY;
-    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(c), dg::geo::CurvatureNablaBZ(c), tempX, tempY, grid);
+    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(c,+1), dg::geo::CurvatureNablaBZ(c,+1), tempX, tempY, grid);
     dg::blas1::transfer(  tempX, curvX);
     dg::blas1::transfer(  tempY, curvY);
     dg::DMatrix dx, dy;
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index c7cf12076..a36a8dead 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -86,7 +86,7 @@ struct VariationDirPer
 
 struct CurvatureDirPer
 {
-    CurvatureDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4.), curvR(c), curvZ(c){}
+    CurvatureDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4.), curvR(c,+1), curvZ(c,+1){}
     double operator()(double R, double Z, double phi) const {
         return this->operator()(R,Z);}
     double operator()(double R, double Z) const {
@@ -201,7 +201,7 @@ int main(int argc, char** argv)
     if(rank==0)std::cout << "TESTING CURVATURE 3D\n";
     dg::MDVec curvX, curvY;
     dg::MHVec tempX, tempY;
-    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(c), dg::geo::CurvatureNablaBZ(c), tempX, tempY, grid);
+    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(c,+1), dg::geo::CurvatureNablaBZ(c,+1), tempX, tempY, grid);
     dg::blas1::transfer(  tempX, curvX);
     dg::blas1::transfer(  tempY, curvY);
     dg::MDMatrix dx, dy;
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index ef55390e9..a5cdbd764 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -153,7 +153,7 @@ int main( int argc, char* argv[])
     if( gp.hasXpoint())
     {
         dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
-        std::cout <<  "X-point found at "<<R_X << " "<<Z_X<<"\n";
+        std::cout <<  "X-point found at "<<R_X << " "<<Z_X<<" with Psip "<<mag.psip()(R_X, Z_X)<<"\n";
     }
     const double R_H = gp.R_0-gp.triangularity*gp.a;
     const double Z_H = gp.elongation*gp.a;
@@ -439,7 +439,7 @@ int main( int argc, char* argv[])
             pair.first.data(), pair.second.size(), pair.second.data());
 
     int dim1d_ids[1], dim2d_ids[2], dim3d_ids[3] ;
-    err = file::define_dimension( ncid, "psi", &dim1d_ids[0], grid1d);
+    err = file::define_dimension( ncid, &dim1d_ids[0], grid1d, "psi");
     std::string psi_long_name = "Flux surface label";
     err = nc_put_att_text( ncid, dim1d_ids[0], "long_name",
         psi_long_name.size(), psi_long_name.data());
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index b29b0273f..643af9c1c 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -29,13 +29,13 @@ struct Interpolate
     Interpolate( const thrust::host_vector<real_type>& fZeta,
                  const thrust::host_vector<real_type>& fEta,
                  const dg::aTopology2d& g2d ):
-        iter0_( dg::create::forward_transform( fZeta, g2d) ),
-        iter1_( dg::create::forward_transform(  fEta, g2d) ),
+        iter0_( dg::forward_transform( fZeta, g2d) ),
+        iter1_( dg::forward_transform(  fEta, g2d) ),
         g_(g2d), zeta1_(g2d.x1()), eta1_(g2d.y1()){}
     void operator()(real_type t, const thrust::host_vector<real_type>& zeta, thrust::host_vector<real_type>& fZeta)
     {
-        fZeta[0] = interpolate( fmod( zeta[0]+zeta1_, zeta1_), fmod( zeta[1]+eta1_, eta1_), iter0_, g_);
-        fZeta[1] = interpolate( fmod( zeta[0]+zeta1_, zeta1_), fmod( zeta[1]+eta1_, eta1_), iter1_, g_);
+        fZeta[0] = interpolate( dg::lspace, iter0_, fmod( zeta[0]+zeta1_, zeta1_), fmod( zeta[1]+eta1_, eta1_), g_);
+        fZeta[1] = interpolate( dg::lspace, iter1_, fmod( zeta[0]+zeta1_, zeta1_), fmod( zeta[1]+eta1_, eta1_), g_);
         //fZeta[0] = interpolate(  zeta[0], zeta[1], iter0_, g_);
         //fZeta[1] = interpolate(  zeta[0], zeta[1], iter1_, g_);
     }
@@ -43,8 +43,8 @@ struct Interpolate
     {
         for( unsigned i=0; i<zeta[0].size(); i++)
         {
-            fZeta[0][i] = interpolate( fmod( zeta[0][i]+zeta1_, zeta1_), fmod( zeta[1][i]+eta1_, eta1_), iter0_, g_);
-            fZeta[1][i] = interpolate( fmod( zeta[0][i]+zeta1_, zeta1_), fmod( zeta[1][i]+eta1_, eta1_), iter1_, g_);
+            fZeta[0][i] = interpolate( dg::lspace, iter0_, fmod( zeta[0][i]+zeta1_, zeta1_), fmod( zeta[1][i]+eta1_, eta1_), g_);
+            fZeta[1][i] = interpolate( dg::lspace, iter1_, fmod( zeta[0][i]+zeta1_, zeta1_), fmod( zeta[1][i]+eta1_, eta1_), g_);
         }
     }
     private:
@@ -59,13 +59,13 @@ template<class real_type>
 real_type construct_c0( const thrust::host_vector<real_type>& etaVinv, const dg::aRealTopology2d<real_type>& g2d)
 {
     //this is a normal integration:
-    thrust::host_vector<real_type> etaVinvL( dg::create::forward_transform(  etaVinv, g2d) );
+    thrust::host_vector<real_type> etaVinvL( dg::forward_transform(  etaVinv, g2d) );
     dg::Grid1d g1d( 0., 2.*M_PI, g2d.n(), g2d.Ny());
     dg::HVec eta = dg::evaluate(dg::cooX1d, g1d);
     dg::HVec w1d = dg::create::weights( g1d);
     dg::HVec int_etaVinv(eta);
     for( unsigned i=0; i<eta.size(); i++)
-        int_etaVinv[i] = interpolate( 0., eta[i], etaVinvL, g2d);
+        int_etaVinv[i] = interpolate( dg::lspace, etaVinvL, 0., eta[i], g2d);
     real_type c0 = 2.*M_PI/dg::blas1::dot( w1d, int_etaVinv );
     return c0;
 
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index ebcbcfa9b..de0adf427 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -116,7 +116,7 @@ int main( int argc, char* argv[])
     int dim3d[3], dim1d[1];
     err = file::define_dimensions(  ncid, dim3d, g3d_periodic.grid());
     //err = file::define_dimensions(  ncid, dim3d, g2d.grid());
-    err = file::define_dimension(  ncid, "i", dim1d, g1d);
+    err = file::define_dimension(  ncid, dim1d, g1d, "i");
     int coordsID[2], onesID, defID, volID, divBID;
     int coord1D[5];
     err = nc_def_var( ncid, "xc", NC_DOUBLE, 3, dim3d, &coordsID[0]);
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 03ebd3cda..ad1ee32e9 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -143,7 +143,7 @@ int main( int argc, char* argv[])
     int dim3d[3], dim1d[1];
     err = file::define_dimensions(  ncid, dim3d, g3d_periodic.grid());
     //err = file::define_dimensions(  ncid, dim3d, g2d.grid());
-    err = file::define_dimension(  ncid, "i", dim1d, g1d);
+    err = file::define_dimension(  ncid, dim1d, g1d, "i");
     int coordsID[2], defID, volID, divBID, gxxID, gyyID, gxyID;
     err = nc_def_var( ncid, "psi", NC_DOUBLE, 3, dim3d, &gxyID);
     err = nc_def_var( ncid, "deformation", NC_DOUBLE, 3, dim3d, &defID);
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index 470d9cfb8..f2ec6b774 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -32,7 +32,7 @@
     "nprofileamp" : 4,
     "bc" :
     {
-        "density" : ["NEU", "NEU"],
+        "density" : ["DIR", "DIR"],
         "velocity": ["NEU", "NEU"],
         "potential":["DIR", "DIR"]
     },
-- 
GitLab


From ed64d50f644e88c147fe469107c80dc231f3ff18 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 10 Oct 2019 18:02:50 +0200
Subject: [PATCH 147/540] Change boxscale parameters  in geometry_diag

---
 ...metry_params_AUG_Iconst_(Te=70eV,B0=2T).js |  4 +-
 inc/geometries/AUG/FALCHETTO/input.txt        | 50 -------------------
 inc/geometries/geometry_diag.cu               | 12 +++--
 inc/geometries/solovev_parameters.h           |  1 +
 4 files changed, 10 insertions(+), 57 deletions(-)
 delete mode 100644 inc/geometries/AUG/FALCHETTO/input.txt

diff --git a/inc/geometries/AUG/FALCHETTO/geometry_params_AUG_Iconst_(Te=70eV,B0=2T).js b/inc/geometries/AUG/FALCHETTO/geometry_params_AUG_Iconst_(Te=70eV,B0=2T).js
index 15b12cc90..a7d8a44b1 100644
--- a/inc/geometries/AUG/FALCHETTO/geometry_params_AUG_Iconst_(Te=70eV,B0=2T).js
+++ b/inc/geometries/AUG/FALCHETTO/geometry_params_AUG_Iconst_(Te=70eV,B0=2T).js
@@ -21,7 +21,7 @@
 		-0.13835177447024941,
 		0.016216674967513847
 	],
-	"elongation" : 1.77,
+	"elongation" : 1.7,
 	"equilibrium" : "solovev",
 	"inverseaspectratio" : 0.303030303,
 	"psip_max" : 0,
@@ -31,4 +31,4 @@
 	"qampl" : 1,
 	"rk4eps" : 1.0000000000000001e-05,
 	"triangularity" : 0.33000000000000002
-}
\ No newline at end of file
+}
diff --git a/inc/geometries/AUG/FALCHETTO/input.txt b/inc/geometries/AUG/FALCHETTO/input.txt
deleted file mode 100644
index 237e21135..000000000
--- a/inc/geometries/AUG/FALCHETTO/input.txt
+++ /dev/null
@@ -1,50 +0,0 @@
-
-                * Input-File for "FELTOR" *
-                ---------------------------
-
-
-@-----------------------------Space and Time discretization------------
-1)  n  (# of x,y-polynomials)            =   3      (3)
-2)  nx (grid points in x)                =   2000 (192)
-3)  ny (grid points in y)                =   3400 (192)
-4)  nz (grid points in z)                =   15(>16)
-5)  dt (time step in units c_s/rho_s)    =   1e-2 (0.01)
-
-----------------------------------Output parameters--------------------
-6)  n_out (# of x-y polynomials in output)  =  3
-7)  nx_out (# grid points in output field)  =  2000
-8)  ny_out (# grid points in output field)  =  3400
-9)  nz_out (# grid points in output field)  =  15
-10) itstp  (steps between outputs)          =  500  (> 5)
-11) total # of outputs (excluding first)    =  5000
-
--------------------------Algorithmic parameters------------------------
-12)  eps_pol (stop for polarisation)        =   1e-4 (1e-6)
-13)  eps_max (stop for induction)           =   1e-4 (1e-6)
-14)  eps_gamma (stop for Gamma CG)          =   1e-5 (1e-8)
-15)  eps_time ( stop for time inversion )   =   1e-9
--------------------------Physical parameters----------------------------
-16) mu_e (-m_e/m_i)                         = -0.000272121 (-0.000544617,-0.000272121,-0.000181372 )
-17) tau (Ti/Te)                             =  0.0    (0.0)
-18) beta_e0                                 =  2.7e-5    (5.4e-5,8.1e-5,1.35e-4 )
-19) nu_perp                                 =  1e-4
-20) nu_parallel  (viscosity)                =  1   (0.001)
-21) para resistivity (c)                    =  2e-6   (2e-6, 4e-6,6e-6,1e-5)
-
-------------------------Initial perturbation parameters---------------------
-22) amp (blob amplitude)                    =  0.005    (1.0)
-23) sigma (blob variance in units of rho_s) =  2.0     (10)
-24) x-position ( in units of a)             =  0.75   (0.4)
-25) y-position ( in units of a)             =  0.00   (0.5)
-26) sigma_z (variance in units of R_0)      =  0.5       (12.5 for oo)
-27) k_psi (zonal modes)                     =  7.0
-28) Profile amplitude                       =  2.5  (peak amplitude)
-29) Background Prof amplitude               =  1.   (density on the boundary)
-----------------------------------Miscellaneous----------------------------
-30) particle source amplitude               =  0.0
-31) boxscale R+                             =  1.2 (a little larger than 1)
-32) boxscale R-                             =  1.2 (a little larger than 1)
-33) boxscale Z+                             =  1.2 (a little larger than 1)
-34) boxscale Z-                             =  1.2 (a little larger than 1)
-
-@ ------------------------------------------------------------
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index a5cdbd764..7a0760961 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -35,10 +35,10 @@ struct Parameters
         Ny = js.get("Ny",100).asUInt();
         Nz = js.get("Nz", 1).asUInt();
         Npsi = js.get("Npsi", 16).asUInt();
-        boxscaleRm = js.get("boxscaleRm", 1.1).asDouble();
-        boxscaleRp = js.get("boxscaleRp", 1.1).asDouble();
-        boxscaleZm = js.get("boxscaleZm", 1.2).asDouble();
-        boxscaleZp = js.get("boxscaleZp", 1.1).asDouble();
+        boxscaleRm = js["boxscaleR"].get(0u, 1.1).asDouble();
+        boxscaleRp = js["boxscaleR"].get(1u, 1.1).asDouble();
+        boxscaleZm = js["boxscaleZ"].get(0u, 1.2).asDouble();
+        boxscaleZp = js["boxscaleZ"].get(1u, 1.1).asDouble();
         amp = js.get("amplitude", 1.).asDouble();
         k_psi = js.get("k_psi", 1.).asDouble();
         bgprofamp = js.get("bgprofamp", 1.).asDouble();
@@ -154,6 +154,8 @@ int main( int argc, char* argv[])
     {
         dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
         std::cout <<  "X-point found at "<<R_X << " "<<Z_X<<" with Psip "<<mag.psip()(R_X, Z_X)<<"\n";
+        std::cout <<  "     R - Factor "<<(gp.R_0-R_X)/gp.triangularity/gp.a << "   Z - factor "<<-(Z_X/gp.elongation/gp.a)<<std::endl;
+
     }
     const double R_H = gp.R_0-gp.triangularity*gp.a;
     const double Z_H = gp.elongation*gp.a;
@@ -349,7 +351,7 @@ int main( int argc, char* argv[])
         if( gp.hasXpoint() )
             dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
         dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, mag, psipog2d, xpoint_weights);
-        grid1d = dg::Grid1d (psipmin, psipmax, npsi ,Npsi,dg::NEU);
+        grid1d = dg::Grid1d (psipmin<psipmax ? psipmin : psipmax, psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::NEU);
         map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
             "Flux surface average of psi with delta function");
         if( gp.equilibrium == "solovev")
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index 5cdb05139..c4ad22c9b 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -127,6 +127,7 @@ struct Parameters
             os<<" c"<<i+1<<"\t\t = "<<c[i]<<"\n";
 
         os  <<" R0            = "<<R_0<<"\n"
+            <<" a             = "<<a<<"\n"
             <<" epsilon_a     = "<<a/R_0<<"\n"
             <<" elongation    = "<<elongation<<"\n"
             <<" triangularity = "<<triangularity<<"\n"
-- 
GitLab


From 9914a75a109240156c08cc80507e9312a3504e59 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 15 Oct 2019 14:03:58 +0200
Subject: [PATCH 148/540] Update feltor.tex

---
 src/feltor/feltor.tex | 35 +++++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index a7649cff5..1eccb025d 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -254,26 +254,27 @@ for the Solov'ev equilibrium.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%
 
-\subsubsection{Toroidal field line approximation}\label{sec:torfieldlineapprox}
-The toroidal field line approximation applies \(\bhat\approx \ehat_\varphi\) to all perpendicular operators
+\subsubsection{Toroidal (and negative toroidal) field line approximation}\label{sec:torfieldlineapprox}
+The toroidal/negative toroidal field line approximation applies \(\bhat\approx \pm \ehat_\varphi\) to all perpendicular operators
 (e.g.: Poisson bracket, perpendicular elliptic operator and curvature operators)
 but retains the full expression for the magnetic field unit vector \(\bhat\)
 for parallel operators (\(\nabla_\parallel\) and \(\Delta_\parallel\)).
+(Note that we allow the negative sign $-\ehat_\varphi$ to enable a sign reversal of the magnetic field, see Section~\ref{sec:field_reversal}).
 In cylindrical coordinates that is
 \begin{align}
-[f,g]_\perp \equiv [f,g]_{RZ} &= \frac{1}{R} \left(\partial_R f\partial_Z g - \partial_Z f\partial_R g\right) \\
+[f,g]_\perp \equiv [f,g]_{RZ} &= \pm\frac{1}{R} \left(\partial_R f\partial_Z g - \partial_Z f\partial_R g\right) \\
 \nabla_\perp f &= \partial_R f \ehat_R + \partial_Z f \ehat_Z \\
 \Delta_\perp f &= \frac{1}{R}\partial_R \left( R \partial_R f\right) + \partial_Z(\partial_Z f)
 \label{}
 \end{align}
 The curl of $\bhat$ reduces to
 %\begin{align}
- $\nabla\times\bhat \approx -  \frac{1}{R} \ehat_Z$.
+ $\nabla\times\bhat \approx -  \frac{\pm 1}{R} \ehat_Z$.
 %end{align}
 This simplifies the curvature operators to:
 \begin{align}
-\vec{\mathcal{K}}_{{\nabla\times\bhat}}  &\approx  -  \frac{1}{B R} \ehat_Z , &
-\vec{ \mathcal{K} }_{\vec{\nabla}  B}  &\approx  -\frac{1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
+\vec{\mathcal{K}}_{{\nabla\times\bhat}}  &\approx  -  \frac{\pm 1}{B R} \ehat_Z , &
+\vec{ \mathcal{K} }_{\vec{\nabla}  B}  &\approx  -\frac{\pm 1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{\pm 1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
 %\ehat_\varphi \times \vec{\nabla} B, &
 \vec{ \mathcal{K} } &\approx \vec{ \mathcal{K} }_{\vec{\nabla}  B}  +\vec{ \mathcal{K} }_{{\nabla\times\bhat}} ,
 %\\
@@ -283,19 +284,19 @@ This simplifies the curvature operators to:
 \end{align}
 and
 \begin{align}
- \vec{\nabla} \cdot \vec{\mathcal{K}}_{{\nabla\times\bhat}} &\approx \frac{1}{R B^2} \frac{\partial B}{\partial Z},
+ \vec{\nabla} \cdot \vec{\mathcal{K}}_{{\nabla\times\bhat}} &\approx \frac{\pm 1}{R B^2} \frac{\partial B}{\partial Z},
 \end{align}
 which results in a vanishing divergence of the curvature operators \( \vec{\nabla} \cdot \vec{ \mathcal{K} } = 0\).
 
 Note that in an actual toroidal field we have
 \begin{align}
-  \vec B(R) := \frac{R_0}{R} \ehat_\varphi
+  \vec B(R) := \pm \frac{R_0}{R} \ehat_\varphi
   \label{}
 \end{align}
 We then have $\bhat = \ehat_\varphi$ and the curvature operators further
 simplify to
 \begin{align}
-  \vec{ \mathcal K_{\nabla\times\bhat}} = \vec{ \mathcal K_{\nabla B}} = -\frac{1}{R_0} \ehat_Z =
+  \vec{ \mathcal K_{\nabla\times\bhat}} = \vec{ \mathcal K_{\nabla B}} = -\frac{\pm 1}{R_0} \ehat_Z =
 \vec{ \mathcal K}/2\\
   \nabla\cdot\vec{\mathcal K_{{\nabla\times\bhat}}}=
     \nabla_\parallel \ln B = 0
@@ -650,6 +651,18 @@ potential $A_\parallel$.
 We have the continuity equation for the electron density \(n_e\) and the ion gyro-centre
 density \(N_i\) and the momentum conservation equation for
 the parallel electron velocity \(u_e\) and the parallel ion gyro-centre velocity \(U_i\)~\cite{WiesenbergerPhD, HeldPhD}.
+\subsection{ Sign reversals of the magnetic field}\label{sec:field_reversal}
+If we change the direction of the magnetic field vector $\bhat$, we immediately see that all perpendicular
+drifts and $U\bhat$ change directions. On the other side, the diffusive and resistive terms remain unchanged.
+Without resistivity and diffusion a change in direction of the magnetic field thus corresponds to
+a time reversal $t\rightarrow t'=-t$.
+
+Also note that changing the sign of the magnetic field only in the parallel derivatives $\nabla_\parallel \rightarrow -\nabla_\parallel$ does not
+have any effect. This can be seen by simply renormalizing $U'=-U$. This reverts the equations back to the original equations.
+In the code this situation can happen when you change the sign of $\bhat$ using $\mathcal P_\psi$  and $\mathcal P_I$
+and use the toroidal field approximation at the same time.
+To actually change the sign of the magnetic field you need to use the "negative toroidal field approximation".
+
 
 \subsection{Diffusive terms}\label{sec:dissres}
 The dissipative terms can be decomposed into perpendicular and parallel components
@@ -901,7 +914,7 @@ The terms of the particle conservation thus read
 \\
   S_{n_e} =&  S_{n_e}
 \end{align}
-Notice that we used
+Notice that
 \begin{align}
 n_e \vec K = n_e\nabla\times\frac{\bhat}{B} = \nabla\times n_e\frac{\bhat}{B} + \frac{\bhat\times\nabla n_e}{B}
 \label{}
@@ -1086,6 +1099,8 @@ the output file \texttt{output.nc}.
  The output file is described in Section~\ref{sec:output_file}.
  The optional file \texttt{initial.nc} can be used to initialize a simulation from an existing file.
  This behavior is described in Section~\ref{sec:restart_file}.
+ Both programs write unstructured human readable performance information of the running simulation
+ to \texttt{std::cout}.
 
 
 \subsection{Input file structure} \label{sec:input_file}
-- 
GitLab


From 74117fda86d905f22a09cd05dec0ab6632bf2fe3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 2 Nov 2019 13:10:02 +0100
Subject: [PATCH 149/540] Write out novel h02 factor

---
 inc/geometries/average.h        |   8 +--
 inc/geometries/flux_t.cu        | 119 +++++++++++++++++++++-----------
 inc/geometries/geometry_diag.cu |  23 +++++-
 3 files changed, 105 insertions(+), 45 deletions(-)

diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 554d9474b..0400a97e0 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -199,13 +199,13 @@ struct FluxSurfaceAverage
      /**
      * @brief Construct from a field and a grid
      * @param g2d 2d grid
-     * @param c contains psip, psipR and psipZ
-     * @param f container for global safety factor
+     * @param mag contains psip, psipR and psipZ
+     * @param f the function to take the average over (until \c set_container() is called)
      * @param weights Weight function \c H (can be used to cut away parts of the domain e.g. below the X-point and/or contain a volume form without dg weights)
      * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
      */
-    FluxSurfaceAverage( const dg::Grid2d& g2d, const TokamakMagneticField& c, const container& f, container weights, double width_factor = 1.) :
-    m_avg( g2d,c, width_factor), m_area( g2d, c, width_factor)
+    FluxSurfaceAverage( const dg::Grid2d& g2d, const TokamakMagneticField& mag, const container& f, container weights, double width_factor = 1.) :
+    m_avg( g2d,mag, width_factor), m_area( g2d, mag, width_factor)
     {
         m_avg.set_left( f);
         //    container gradpsi  = dg::evaluate( dg::geo::GradPsip( c), g2d);
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 3fdaafdee..ab06e914d 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -80,8 +80,9 @@ int main( int argc, char* argv[])
     dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::DIR);
     dg::Grid2d g2d_periodic(g2d.x0(), g2d.x1(), g2d.y0(), g2d.y1(), g2d.n(), g2d.Nx(), g2d.Ny()+1);
     t.toc();
+    std::cout << "Construction successful!\n";
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
-    //////////////////////////////setup netcdf//////////////////
+    //////////////////////////////setup and write netcdf//////////////////
     int ncid;
     file::NC_Error_Handle err;
     err = nc_create( "flux.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
@@ -93,46 +94,85 @@ int main( int argc, char* argv[])
     dg::HVec X=dg::pullback(dg::cooX2d, g2d), Y=dg::pullback(dg::cooY2d, g2d); //P = dg::pullback( dg::coo3, g);
     err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
     err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
-    //err = nc_put_var_double( ncid, coordsID[2], g.z().data());
-
-    std::string names[] = {"psi", "d", "R", "vol", "divB"};
-    unsigned size=5;
-    int varID[size];
-    for( unsigned i=0; i<size; i++)
-        err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 2, dim2d, &varID[i]);
-    ///////////////////////now fill variables///////////////////
-
-    thrust::host_vector<double> psi_p = dg::pullback( c.psip(), g2d);
-    //g.display();
-    err = nc_put_var_double( ncid, varID[0], periodify(psi_p, g2d_periodic).data());
-    dg::HVec temp0( g2d.size()), temp1(temp0);
-    dg::HVec w3d = dg::create::weights( g2d);
 
-    //compute and write deformation into netcdf
-    dg::SparseTensor<dg::HVec> metric = g2d.metric();
-    dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
-    dg::HVec vol_ = dg::tensor::volume(metric);
-    dg::blas1::pointwiseDivide( g_xy, g_xx, temp0);
-    const dg::HVec ones = dg::evaluate( dg::one, g2d);
-    X=g_yy;
-    err = nc_put_var_double( ncid, varID[1], periodify(X, g2d_periodic).data());
-    //compute and write conformalratio into netcdf
-    dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
-    X=g_xx;
-    err = nc_put_var_double( ncid, varID[2], periodify(X, g2d_periodic).data());
-
-    std::cout << "Construction successful!\n";
+    std::map< std::string, std::function< void( dg::HVec&, dg::geo::CurvilinearGrid2d&, dg::geo::solovev::Parameters&, dg::geo::TokamakMagneticField&)> > output = {
+        { "Psip", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( mag.psip(), g2d);
+        }},
+        { "PsipR", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( mag.psipR(), g2d);
+        }},
+        { "PsipZ", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( mag.psipZ(), g2d);
+        }},
+        { "g_xx", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result=g2d.metric().value(0,0);
+        }},
+        { "g_xy", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result=g2d.metric().value(0,1);
+        }},
+        { "g_yy", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result=g2d.metric().value(1,1);
+        }},
+        { "g_xy_g_xx", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::blas1::pointwiseDivide( g2d.metric().value(0,1),
+                g2d.metric().value(0,0), result);
+        }},
+        { "g_yy_g_xx", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::blas1::pointwiseDivide( g2d.metric().value(1,1),
+                g2d.metric().value(0,0), result);
+        }},
+        { "vol", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result=dg::tensor::volume(g2d.metric());
+        }},
+        { "Bzeta", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::HVec Bzeta, Beta;
+            dg::pushForwardPerp( dg::geo::BFieldR(mag), dg::geo::BFieldZ(mag), Bzeta, Beta, g2d);
+            result=Bzeta;
+        }},
+        { "Beta", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::HVec Bzeta, Beta;
+            dg::pushForwardPerp( dg::geo::BFieldR(mag), dg::geo::BFieldZ(mag), Bzeta, Beta, g2d);
+            result=Beta;
+        }},
+        { "Bphi", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( dg::geo::BFieldP(mag), g2d);
+        }},
+        { "q-profile", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::HVec Bzeta, Beta;
+            dg::pushForwardPerp( dg::geo::BFieldR(mag), dg::geo::BFieldZ(mag), Bzeta, Beta, g2d);
+            result = dg::pullback( dg::geo::BFieldP(mag), g2d);
+            dg::blas1::pointwiseDivide( result, Beta, result); //Bphi / Beta
 
-    dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
-    dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
-    dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
-    dg::blas1::transform( temp0, temp0, dg::SQRT<double>()); //temp0=1/sqrt(g) = sqrt(g^xx g^yy - g^xy^2)
-    dg::blas1::pointwiseDivide( ones, temp0, temp0); //temp0=sqrt(g)
-    X=temp0;
-    err = nc_put_var_double( ncid, varID[3], periodify(X, g2d_periodic).data());
-    dg::blas1::axpby( 1., temp0, -1., vol_, temp0); //temp0 = sqrt(g)-vol
-    double error = sqrt(dg::blas2::dot( temp0, w3d, temp0)/dg::blas2::dot(vol_, w3d, vol_));
-    std::cout << "Rel Consistency  of volume is "<<error<<"\n";
+        }},
+        { "Ipol", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( mag.ipol(), g2d);
+        }}
+    };
+    dg::HVec temp( g2d.size());
+    for( auto pair : output)
+    {
+        int varID;
+        err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 2, dim2d, &varID);
+        pair.second( temp, g2d, gp, c);
+        err = nc_put_var_double( ncid, varID, periodify(temp, g2d_periodic).data());
+    }
+    err = nc_close( ncid);
+    ///////////////////////some further testing//////////////////////
 
     const dg::HVec vol3d = dg::create::volume( g3d);
     dg::HVec ones3d = dg::evaluate( dg::one, g3d);
@@ -160,6 +200,5 @@ int main( int argc, char* argv[])
     std::cout << "relative difference in volume2d is "<<fabs(volumeRZ - volume2d)/volume2d<<std::endl;
     std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
 
-    err = nc_close( ncid);
     return 0;
 }
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 7a0760961..f70d7ce2e 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -82,6 +82,19 @@ struct IPhi
     private:
     double R_0, A;
 };
+//The newly derived h02 factor
+struct Hoo : public dg::geo::aCylindricalFunctor<Hoo>
+{
+    Hoo( dg::geo::TokamakMagneticField mag): mag_(mag){}
+    double do_compute( double R, double Z) const
+    {
+        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
+        double psip2 = psipR*psipR+psipZ*psipZ;
+        return (ipol*ipol + psip2)/R/R/psip2;
+    }
+    private:
+    dg::geo::TokamakMagneticField mag_;
+};
 
 int main( int argc, char* argv[])
 {
@@ -247,7 +260,9 @@ int main( int argc, char* argv[])
         ////
         {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
         {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
-            M_PI, p.sigma, p.sigma, p.sigma, p.amp)}
+            M_PI, p.sigma, p.sigma, p.sigma, p.amp)},
+        { "Hoo", "The novel h02 factor", Hoo( mag) }
+
     };
     dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
     dg::DVec psipog2d   = dg::evaluate( mag.psip(), grid2d);
@@ -396,6 +411,12 @@ int main( int argc, char* argv[])
         dg::blas1::scal(psi_area, 2.*M_PI);
         map1d.emplace_back( "psi_area", psi_area,
             "Flux area with delta function");
+        // h02 factor
+        dg::HVec h02 = dg::evaluate( Hoo(mag), grid2d);
+        fsa.set_container( h02);
+        map1d.emplace_back( "hoo", dg::evaluate( fsa, grid1d),
+            "Flux surface average of novel h02 factor");
+
 
         dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, mag);
         std::cout << "Delta Rho for Flux surface integrals = "<<-fsi.get_deltapsi()/psipmin<<"\n";
-- 
GitLab


From f2655d39a386bab0e75e46c62fc4d19aa8d5b9de Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 6 Nov 2019 23:10:48 +0100
Subject: [PATCH 150/540] Add perp drift-fluid diffusion in feltor writeup

---
 src/feltor/feltor.tex | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 1eccb025d..0a84f3e4d 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -595,19 +595,11 @@ The potential is scaled with $\hat \phi = e/T_e$ and the vector potential with
 $\hat A_\parallel = \rho_s B_0$.
 We introduce the dimensionless parameters
 \begin{align}
-  \tau_a = \frac{T_a}{z_aT_e}~,\quad \mu_a = \frac{m_a}{z_am_i}\text{ and } 
+  \tau_a = \frac{T_a}{z_aT_e}~,\quad \mu_a = \frac{m_a}{z_am_i}\text{ and }
   \beta:=\frac{\mu_0 n_0 T_e}{B_0^2}
   \label{}
 \end{align}
-where $a\in\{e,i\}$ is the species label and $z$ is the charge number. We define with 
-$\eta_\parallel := \frac{0.51 m_e \nu_{ei}}{n_e e^2}$
-\begin{align}
-  \eta:=\frac{en_0\eta_\parallel}{B_0} = 8.45\cdot 10^{-5}\ln \lambda \left(\frac{n_0}{10^{19}\text{m}^3}\right) \left(\frac{T_e}{\text{eV}}\right)^{-3/2} \left(\frac{B_0}{\text{T}}\right)^{-1},
-    \label{eq:resistivity}
-\end{align}
-with $\ln \lambda \approx 10$.
- The approximate Spitzer current \(J_{\parallel,s}:= n_e \left(U_i - u_e\right)\)
- determines the parallel resistive terms to $R_\parallel:= n_e\eta J_{\parallel,s}$.
+where $a\in\{e,i\}$ is the species label and $z$ is the charge number.
 Omitting the species label we arrive at (dividing the density equation by $\Omega_0n_0$ and the velocity equation by $\Omega_0 c_s$)
 \begin{align}
 \frac{\partial}{\partial t} N &+ \vec\nabla\cdot\left( N \left(
@@ -644,7 +636,7 @@ Given $\phi$ we define the generalised electric potential
 \begin{align}
     \psi_e := \phi,\quad \psi_i&:= \Gamma_{1,i} \phi - \frac{\mu_i }{2}\left(\frac{\vec \nabla_\perp\phi}{B}\right)^2
 \end{align}
-In total 
+In total
 we have an isothermal 3d gyro-fluid model with up to 2nd order FLR effects
 on in the electric potential $\phi$ and 0th order FLR effects in the parallel magnetic
 potential $A_\parallel$.
@@ -665,6 +657,16 @@ To actually change the sign of the magnetic field you need to use the "negative
 
 
 \subsection{Diffusive terms}\label{sec:dissres}
+We define with
+$\eta_\parallel := \frac{0.51 m_e \nu_{ei}}{n_e e^2}$ and $\nu_{ei} = \sqrt{2} z^2 e^4 \ln \Lambda/ (12\pi^{3/2} \sqrt{m_e} \epsilon_0^2) n_e /T_e^{3/2}$
+\begin{align}
+  \eta:=\frac{en_0\eta_\parallel}{B_0} = 8.45\cdot 10^{-5}\ln \lambda \left(\frac{n_0}{10^{19}\text{m}^3}\right) \left(\frac{T_e}{\text{eV}}\right)^{-3/2} \left(\frac{B_0}{\text{T}}\right)^{-1},
+    \label{eq:resistivity}
+\end{align}
+with $\ln \lambda \approx 10$.
+ The approximate Spitzer current \(J_{\parallel,s}:= n_e \left(U_i - u_e\right)\)
+ determines the parallel resistive terms to $R_\parallel:= n_e\eta J_{\parallel,s}$.
+
 The dissipative terms can be decomposed into perpendicular and parallel components
 \begin{align}
  \Lambda_{n_e} &= \Lambda_{n_e,\perp}+\Lambda_{n_e,\parallel}, &
@@ -677,7 +679,7 @@ For numerical stabilisation we choose:
 \Lambda_{n_e,\parallel} &= \nu_\parallel \Delta_\parallel n_e &
 \Lambda_{N_i,\parallel} &= \nu_\parallel \Delta_\parallel N_i \\
 \Lambda_{u_e,\parallel} &= \nu_\parallel \Delta_\parallel u_e &
-\Lambda_{U_i,\parallel} &= \nu_\parallel \Delta_\parallel U_i 
+\Lambda_{U_i,\parallel} &= \nu_\parallel \Delta_\parallel U_i
 \end{align}
 Similarly, for the perpendicular dissipation we apply viscous or hyperviscous terms.
 \begin{align}\label{eq:perpdiffNT}
@@ -687,6 +689,12 @@ Similarly, for the perpendicular dissipation we apply viscous or hyperviscous te
  \Lambda_{U_i,\perp} &=  \nu_\perp \Delta_\perp U_i \text{ or } -\nu_\perp \Delta_\perp^2 U_i
 \end{align}
 Here the mass diffusion coefficient coincides with the viscous coefficient, hence we fixed the Schmidt number \(\mathit{Sc}_\parallel:= \frac{\nu_U}{\nu_N}\) to unity.
+The drift-fluid corresponding diffusion gives an order-of-magnitude estimate for $\nu_\perp$.
+We have with $\nu_{ii0} = z^4e^4\ln \Lambda/ (12\pi^{3/2} \sqrt{m_i} \epsilon_0^2) n_{i0} /T_{i0}^{3/2}$ and choose $D_i = \rho_i^2 \nu_{ii}$ and $T_{i0} = T_{e0}$
+\begin{align}
+\nu_\perp = \left( 1+\frac{R_0}{a}q_{95}^2\right)\frac{ m_i \nu_{ii0}}{e B_0}
+\end{align}
+where the prefactor stems from neoclassical corrections and contains the major radius $R_0$ and minor radius $a$.
 
 \subsection{Boundary and initial conditions}
 We define the simulation box as
@@ -944,7 +952,7 @@ The terms of the energy theorem are
 +  S_{\mathcal E}
 +  R_{\mathcal E}
 \end{align}
-with ( $z_e=-1$ and $z_i=+1$)
+with ( $z_e=-1$ and $z_i=+1$) and $\vec u_E := {\bhat\times \nabla\phi}/{B}$
 \begin{align} \label{eq:energy_conservation}
   \mathcal{E}= & z_e\tau_e n_e \ln{(n_e)} +z_i\tau_i N_i\ln{(N_i)}
   +\frac{1}{2\beta}\left(\vec \nabla_\perp A_\parallel\right)^2
-- 
GitLab


From 282148ae01f709cc04ec7085c4d361fb2168e4d1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 7 Nov 2019 17:32:20 +0100
Subject: [PATCH 151/540] Fill in values in feltor collisions

---
 inc/geometries/flux.h |  3 +++
 src/feltor/feltor.tex | 15 +++++++++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index bded4e9f6..0cfb2a7c1 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -127,6 +127,9 @@ struct Fpsi
 /**
  * @brief A symmetry flux generator
  *
+ * Symmetry flux coordinates fulfill the condition \f$\sqrt{g} = \frac{R}{I}\f$
+ * The symmetry refers to the symmetry in the toroidal angle while flux coordinates allow the representation
+ * of the magnetic field in Clebsch form
  * @ingroup generators_geo
  * @snippet flux_t.cu doxygen
  */
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 0a84f3e4d..ab0f03163 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -690,11 +690,18 @@ Similarly, for the perpendicular dissipation we apply viscous or hyperviscous te
 \end{align}
 Here the mass diffusion coefficient coincides with the viscous coefficient, hence we fixed the Schmidt number \(\mathit{Sc}_\parallel:= \frac{\nu_U}{\nu_N}\) to unity.
 The drift-fluid corresponding diffusion gives an order-of-magnitude estimate for $\nu_\perp$.
-We have with $\nu_{ii0} = z^4e^4\ln \Lambda/ (12\pi^{3/2} \sqrt{m_i} \epsilon_0^2) n_{i0} /T_{i0}^{3/2}$ and choose $D_i = \rho_i^2 \nu_{ii}$ and $T_{i0} = T_{e0}$
+We have with $\nu_{ii0} = z^4e^4\ln \Lambda/ (12\pi^{3/2} \sqrt{m_i} \epsilon_0^2) n_{i0} /T_{i0}^{3/2}$ and choose $D_i = \rho_i^2 \nu_{ii}$ and $T_{i0} = T_{e0}$.
+By dividing by $\rho_s^2 \Omega_{ci}$ we arrive at $\nu_\perp = m_i \nu_{ii0}/eB_0$.
+Together with neoclassical corrections we then have
 \begin{align}
-\nu_\perp = \left( 1+\frac{R_0}{a}q_{95}^2\right)\frac{ m_i \nu_{ii0}}{e B_0}
-\end{align}
-where the prefactor stems from neoclassical corrections and contains the major radius $R_0$ and minor radius $a$.
+\nu_\perp =
+5\cdot 10^{-3} \left(1+\frac{R}{a}q_{95}\right) \ln \lambda
+\left(\frac{n_0}{10^{19}\text{m}^3}\right)
+\left(\frac{T_e}{\text{eV}}\right)^{-3/2}
+\left(\frac{B_0}{\text{T}}\right)^{-1}
+\left(\frac{m_i}{m_H}\right)^{1/2},
+\end{align}
+where we use the major radius $R_0$ and minor radius $a$ and the safety factor $q_{95}$~\cite{Madsen2016}.
 
 \subsection{Boundary and initial conditions}
 We define the simulation box as
-- 
GitLab


From f99c2af7e003f254b1bd3fca6ab0503af2fd9493 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 8 Nov 2019 11:35:21 +0100
Subject: [PATCH 152/540] Upgrade error resilience of feltor

with some health checks on user parameters
---
 inc/dg/enums.h           |  2 +-
 src/feltor/feltor.cu     | 15 +++++++++++++--
 src/feltor/feltor.h      |  4 +++-
 src/feltor/feltor.tex    |  4 ++--
 src/feltor/feltor_hpc.cu | 18 +++++++++++++++---
 src/feltor/implicit.h    |  2 ++
 src/feltor/init.h        |  2 +-
 7 files changed, 37 insertions(+), 10 deletions(-)

diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index 8d0ed510d..a6d5c0a18 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -75,7 +75,7 @@ static inline bc str2bc( std::string s)
         return NEU_DIR;
     if( s=="DIR_NEU"||s=="dir_neu" )
         return DIR_NEU;
-    throw std::runtime_error( "No matching boundary condition!");
+    throw std::runtime_error( "Boundary condition '"+s+"' not recognized!");
 }
 
 ///@brief Switch between normalisations
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 396cece59..0a86dff75 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -82,12 +82,23 @@ int main( int argc, char* argv[])
     /////////////////////The initial field///////////////////////////////////////////
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
-    y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+    try{
+        y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+    }catch ( std::out_of_range& error){
+        std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
+        return -1;
+    }
     bool fixed_profile;
 
     HVec profile = dg::evaluate( dg::zero, grid);
-    HVec source_profile = feltor::source_profiles.at(p.source_type)(
+    HVec source_profile;
+    try{
+        source_profile = feltor::source_profiles.at(p.source_type)(
         fixed_profile, profile, grid, p, gp, mag);
+    }catch ( std::out_of_range& error){
+        std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
+        return -1;
+    }
 
     feltor.set_source( fixed_profile, dg::construct<DVec>(profile), p.omega_source, dg::construct<DVec>(source_profile));
 
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 1cbb754fe..904cfcc07 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -400,6 +400,8 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_mag(
         dg::assign(  dg::pullback(dg::geo::DivCurvatureKappa(mag, -1), g),
             m_divCurvKappa);
     }
+    else
+        throw dg::Error(dg::Message(_ping_)<<"Warning! curvmode value '"<<p.curvmode<<"' not recognized!! I don't know what to do! I exit!\n");
     dg::pushForward(curvNabla.x(), curvNabla.y(), curvNabla.z(),
         m_curv[0], m_curv[1], m_curv[2], g);
     dg::pushForward(curvKappa.x(), curvKappa.y(), curvKappa.z(),
@@ -581,7 +583,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::initializeni(
                 MPI_Comm_rank( MPI_COMM_WORLD, &rank);
                 if(rank==0)
             #endif
-            std::cerr <<"WARNING: Unknown initial condition for phi!\n";
+            throw dg::Error(dg::Message(_ping_)<<"Warning! initphi value '"<<initphi<<"' not recognized. I have tau = "<<m_p.tau[1]<<" ! I don't know what to do! I exit!\n");
         }
     }
 }
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index ab0f03163..17bf28760 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1236,8 +1236,8 @@ perturbation $\tilde n$ in \eqref{eq:initial_ne}. "zonal" (Eq.~\eqref{eq:initial
     "turbulence on gaussian" = Gaussian bg. profile with turbulence perturbation Eq.~\eqref{eq:turbulence_on_gaussian}
     See the file {\tt init.h} to add your own custom condition.
 \\
-initphi   & string & "zero"  & "balance" & initial condition for $\phi$ and thus $N_i$ (Eq.~\eqref{eq:initphi}: "zero" : $\phi = 0$, vanishing
-electric potential, "balance": ExB vorticity equals ion diamagnetic vorticity (For $\tau_i =0 $ both are the same)
+initphi   & string & "zero"  & "balance" & (ignored if $\tau_i = 0$, then $\phi=0$) initial condition for $\phi$ and thus $N_i$ (Eq.~\eqref{eq:initphi}: "zero" : $\phi = 0$, vanishing
+electric potential, "balance": ExB vorticity equals ion diamagnetic vorticity
 \\
 amplitude  & float &0.01   & - & amplitude $A$ of initial perturbation (blob, turbulent bath or zonal flow)  \\
 sigma      & float &2      & - & Gaussian variance in units of $\rho_s$ \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index b5315ce31..1b40ce68e 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -188,15 +188,27 @@ int main( int argc, char* argv[])
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
     if( argc == 4)
-        y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+    {
+        try{
+            y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+        }catch ( std::out_of_range& error){
+            MPI_OUT std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
+        return -1;
+        }
+    }
     if( argc == 5)
         y0 = feltor::init_from_file(argv[4], grid, p,time);
 
     bool fixed_profile;
     {
-    HVec profile;
-    HVec source_profile = feltor::source_profiles.at(p.source_type)(
+    HVec profile, source_profile;
+    try{
+        source_profile = feltor::source_profiles.at(p.source_type)(
         fixed_profile, profile, grid, p, gp, mag);
+    }catch ( std::out_of_range& error){
+        std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
+        return -1;
+    }
 
     feltor.set_source( fixed_profile, dg::construct<DVec>(profile), p.omega_source, dg::construct<DVec>(source_profile));
     }
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index b1ebd3ea3..691ee3fa4 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -112,6 +112,8 @@ struct ImplicitVelocity
         m_p=p;
         m_lapM_perpU.construct( g, p.bcxU,p.bcyU,dg::PER,
             dg::normed, dg::centered);
+        if( !(p.perp_diff == "viscous" || p.perp_diff == "hyperviscous") )
+            throw dg::Error(dg::Message(_ping_)<<"Warning! perp_diff value '"<<p.perp_diff<<"' not recognized!! I do not know how to proceed! Exit now!");
         dg::assign( dg::evaluate( dg::zero, g), m_temp);
         m_apar = m_temp;
         m_fields[0][0] = m_fields[0][1] = m_temp;
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 955409ba2..408a1e66f 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -113,7 +113,7 @@ void init_ni(
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
 #endif
-    MPI_OUT std::cout << "initialize ni" << std::endl;
+    MPI_OUT std::cout << "initialize ni with "<<p.initphi << std::endl;
     feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
     double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
     MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
-- 
GitLab


From b2abe4403afe5a2d9704ecad57d6d88c7049d3ae Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 8 Nov 2019 15:14:26 +0100
Subject: [PATCH 153/540] Begin to write netcdf output into multigrid

hopefully brings more clarity into what is going on but does not compile
right now
---
 inc/dg/multigrid_b.cu | 92 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 92 insertions(+)

diff --git a/inc/dg/multigrid_b.cu b/inc/dg/multigrid_b.cu
index e1fbd3193..efb56b2e7 100644
--- a/inc/dg/multigrid_b.cu
+++ b/inc/dg/multigrid_b.cu
@@ -2,6 +2,7 @@
 #include <iomanip>
 
 #include <thrust/device_vector.h>
+#include "dg/file/nc_utilities.h"
 #include "backend/timer.h"
 
 #include "blas.h"
@@ -119,8 +120,99 @@ int main()
     err = sqrt( err/norm);
     std::cout << " Error of Multigrid iterations "<<err<<"\n";
     //should converge to ~2e-7
+    //////////////////////////////setup and write netcdf//////////////////
+    int ncid;
+    file::NC_Error_Handle err;
+    err = nc_create( "multigrid.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
+    std::vector<int> dim2d(2*stages);
+    for( unsigned u=0; u<stages; u++)
+    {
+        dg::Message xs()<<"x"<<u, ys() <<"y"<<u;
+        err = file::define_dimensions(  ncid, *dim2d[2*u], multigrid.grid(u), {xs.str(),ys.str()} );
+    }
+
+    for( auto pair : output)
+    {
+        for( unsigned u=0; u<stages; u++)
+        {
+            dg::HVec temp( grid.size());
+            int varID;
+            err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 2, *dim2d[2*u], &varID);
+            pair.second( temp, g2d, gp, c);
+            err = nc_put_var_double( ncid, varID, grid, temp);
+        }
+    }
+    err = nc_close( ncid);
 
 
     return 0;
 }
 
+    std::map< std::string, std::function< void( dg::HVec&, dg::geo::CurvilinearGrid2d&, dg::geo::solovev::Parameters&, dg::geo::TokamakMagneticField&)> > output = {
+        { "Psip", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( mag.psip(), g2d);
+        }},
+        { "PsipR", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( mag.psipR(), g2d);
+        }},
+        { "PsipZ", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( mag.psipZ(), g2d);
+        }},
+        { "g_xx", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result=g2d.metric().value(0,0);
+        }},
+        { "g_xy", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result=g2d.metric().value(0,1);
+        }},
+        { "g_yy", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result=g2d.metric().value(1,1);
+        }},
+        { "g_xy_g_xx", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::blas1::pointwiseDivide( g2d.metric().value(0,1),
+                g2d.metric().value(0,0), result);
+        }},
+        { "g_yy_g_xx", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::blas1::pointwiseDivide( g2d.metric().value(1,1),
+                g2d.metric().value(0,0), result);
+        }},
+        { "vol", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result=dg::tensor::volume(g2d.metric());
+        }},
+        { "Bzeta", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::HVec Bzeta, Beta;
+            dg::pushForwardPerp( dg::geo::BFieldR(mag), dg::geo::BFieldZ(mag), Bzeta, Beta, g2d);
+            result=Bzeta;
+        }},
+        { "Beta", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::HVec Bzeta, Beta;
+            dg::pushForwardPerp( dg::geo::BFieldR(mag), dg::geo::BFieldZ(mag), Bzeta, Beta, g2d);
+            result=Beta;
+        }},
+        { "Bphi", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( dg::geo::BFieldP(mag), g2d);
+        }},
+        { "q-profile", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            dg::HVec Bzeta, Beta;
+            dg::pushForwardPerp( dg::geo::BFieldR(mag), dg::geo::BFieldZ(mag), Bzeta, Beta, g2d);
+            result = dg::pullback( dg::geo::BFieldP(mag), g2d);
+            dg::blas1::pointwiseDivide( result, Beta, result); //Bphi / Beta
+
+        }},
+        { "Ipol", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
+            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
+            result = dg::pullback( mag.ipol(), g2d);
+        }}
+    };
-- 
GitLab


From f9e05dbb592de01641be53bd22a59214b784d8df Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 9 Nov 2019 01:39:48 +0100
Subject: [PATCH 154/540] Make smoothing on VAx = Vb

---
 inc/dg/Makefile       |   2 +-
 inc/dg/chebyshev.h    |  35 ++++++++++-
 inc/dg/multigrid.h    |  94 ++++++++++++++++++++++------
 inc/dg/multigrid_b.cu | 142 ++++++++++++------------------------------
 4 files changed, 147 insertions(+), 126 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 058deb17a..6d90c889e 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -17,7 +17,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@  -g
 
 %_b: %_b.cu
-	$(CC) $(OPT) $(CFLAGS) -DDG_BENCHMARK $< -o $@ $(INCLUDE) -g
+	$(CC) $(OPT) $(CFLAGS) -DDG_BENCHMARK $< -o $@ $(INCLUDE) $(LIBS) -g
 
 %_mpit: %_mpit.cu
 	$(MPICC) $(OPT) $(INCLUDE) -DDG_DEBUG $(MPICFLAGS) $< -o $@ -g
diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index 0fcb46ffa..f7699de67 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -38,7 +38,7 @@ class Chebyshev
     Chebyshev(){}
     ///@copydoc construct()
     Chebyshev( const ContainerType& copyable):
-        m_ax(copyable), m_xm1(m_ax){}
+        m_ax(copyable), m_xm1(m_ax), m_b( m_ax){}
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_ax;}
@@ -50,7 +50,7 @@ class Chebyshev
      * @param max_iterations Maximum number of iterations to be used
      */
     void construct( const ContainerType& copyable) {
-        m_xm1 = m_ax = copyable;
+        m_xm1 = m_ax = m_b = copyable;
     }
     /**
      * @brief Solve the system A*x = b using Chebyshev iteration
@@ -67,6 +67,35 @@ class Chebyshev
      * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
      */
     template< class MatrixType, class ContainerType0, class ContainerType1>
+    void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b,
+        double min_ev, double max_ev, unsigned num_iter, const ContainerType1& weights)
+    {
+        if( num_iter == 0)
+            return;
+        assert ( min_ev < max_ev);
+        double theta = (min_ev+max_ev)/2., delta = (max_ev-min_ev)/2.;
+        double rhokm1 = delta/theta, rhok=0;
+        dg::blas1::copy( x, m_xm1); //x0
+        dg::blas2::symv( A, x, m_ax);
+        dg::blas1::pointwiseDot( weights, m_ax, m_ax);
+        dg::blas2::symv( weights, b, m_b);
+        dg::blas1::axpbypgz( 1./theta, m_b, -1./theta, m_ax, 1., x); //x1
+        for ( unsigned k=1; k<num_iter; k++)
+        {
+            rhok = 1./(2.*theta/delta - rhokm1);
+            dg::blas2::symv( A, x, m_ax);
+            dg::blas1::pointwiseDot( weights, m_ax, m_ax);
+            dg::blas1::evaluate( m_xm1, dg::equals(), PairSum(),
+                             1.+rhok*rhokm1, x,
+                            -rhok*rhokm1,    m_xm1,
+                             2.*rhok/delta,  m_b,
+                            -2.*rhok/delta,  m_ax
+                            );
+            x.swap(m_xm1);
+            rhokm1 = rhok;
+        }
+    }
+    template< class MatrixType, class ContainerType0, class ContainerType1>
     void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b,
         double min_ev, double max_ev, unsigned num_iter)
     {
@@ -93,7 +122,7 @@ class Chebyshev
         }
     }
   private:
-    ContainerType m_ax, m_xm1;
+    ContainerType m_ax, m_xm1, m_b;
 };
 
 } //namespace dg
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 2efd342d3..cd6cd22d8 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -15,9 +15,43 @@
 #include "topology/mpi_projection.h"
 #endif
 
+size_t start=0;
+int ncid =0;
+int xID=0, bID=0, rID=0, tvarID=0;
+file::NC_Error_Handle err;
+
 namespace dg
 {
 
+template<class Container, class Matrix, class Geometry>
+void file_output( const std::vector<Container>& x,const std::vector<Container>& b, const std::vector<Container>& r, std::vector<Container>& out, unsigned p, std::vector<Matrix>& inter, Geometry& grid )
+{
+    size_t count = 1;
+    double time = start;
+    err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
+    dg::HVec data = out[0];
+
+    dg::blas1::copy( x[p], out[p]);
+    for( int i=(int)p; i>0; i--)
+        dg::blas2::gemv( inter[i-1], out[i], out[i-1]);
+    data = out[0];
+    file::put_vara_double( ncid, xID, start, grid, data);
+
+    dg::blas1::copy( b[p], out[p]);
+    for( int i=(int)p; i>0; i--)
+        dg::blas2::gemv( inter[i-1], out[i], out[i-1]);
+    data = out[0];
+    file::put_vara_double( ncid, bID, start, grid, data);
+
+    dg::blas1::copy( r[p], out[p]);
+    for( int i=(int)p; i>0; i--)
+        dg::blas2::gemv( inter[i-1], out[i], out[i-1]);
+    data = out[0];
+    file::put_vara_double( ncid, rID, start, grid, data);
+
+    start++;
+}
+
 /**
 * @brief Solves the Equation \f[ \frac{1}{W} \hat O \phi = \rho \f]
 *
@@ -102,6 +136,7 @@ struct MultigridCG2d
         for (unsigned u = 0; u < m_stages; u++)
         {
             m_cg[u].construct(m_x[u], 1);
+            m_cg[u].set_max(m_grids[u]->size());
             m_cheby[u].construct(m_x[u]);
         }
     }
@@ -145,7 +180,6 @@ struct MultigridCG2d
 #ifdef DG_BENCHMARK
             t.tic();
 #endif //DG_BENCHMARK
-            m_cg[u].set_max(m_grids[u]->size());
             number[u] = m_cg[u]( op[u], m_x[u], m_r[u], op[u].precond(), op[u].inv_weights(), eps/2, 1.);
             dg::blas2::symv( m_inter[u-1], m_x[u], m_x[u-1]);
 #ifdef DG_BENCHMARK
@@ -165,7 +199,6 @@ struct MultigridCG2d
 
         //update initial guess
         dg::blas1::axpby( 1., m_x[0], 1., x);
-        m_cg[0].set_max(m_grids[0]->size());
         number[0] = m_cg[0]( op[0], x, m_b[0], op[0].precond(),
             op[0].inv_weights(), eps);
 #ifdef DG_BENCHMARK
@@ -238,19 +271,17 @@ struct MultigridCG2d
         dg::blas2::symv(op[0].weights(), b, m_b[0]);
 
         //FULL MULTIGRID
-        dg::blas1::copy( x, m_x[0]);
-        unsigned max_iter = 3;
-        for( unsigned u=0; u<max_iter; u++)
-            full_multigrid( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
-        dg::blas1::copy( m_x[0], x);
+        //solve for residuum ( if not we always get the same solution)
+        dg::blas2::symv(op[0].weights(), b, m_b[0]);
+        dg::blas2::symv( op[0], x, m_r[0]);
+        dg::blas1::axpby( -1., m_r[0], 1., m_b[0]);
+        dg::blas1::copy( 0., m_x[0]);
+        full_multigrid( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
+        dg::blas1::axpby( 1., m_x[0], 1., x);
 
         //MULTIGRID CYCLES
-        //unsigned max_iter = 3;
         //dg::blas1::copy( x, m_x[0]);
-        //for( unsigned u=0; u<max_iter; u++)
-        //{
-        //    multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
-        //}
+        //multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
         //dg::blas1::copy( m_x[0], x);
 
         //PCG WITH MULTIGRID CYCLE AS PRECONDITIONER
@@ -327,10 +358,15 @@ struct MultigridCG2d
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
         //double norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
-        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1);
+
+        std::vector<Container> out( x);
+        file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
+
+        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1, op[p].inv_weights());
         // 2. Residuum
         dg::blas2::symv( op[p], x[p], m_r[p]);
         dg::blas1::axpby( 1., b[p], -1., m_r[p]);
+        file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
         //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum after  "<<norm_res<<"\n";
         // 3. Coarsen
@@ -339,9 +375,23 @@ struct MultigridCG2d
         dg::blas1::scal( x[p+1], 0.);
         if( p+1 == m_stages-1)
         {
-            m_cg[p+1].set_max(m_grids[p+1]->size());
-            m_cg[p+1]( op[p+1], x[p+1], b[p+1], op[p+1].precond(),
-                op[p+1].inv_weights(), 1e-10);
+            file_output( x, b, m_r, out, p+1, m_inter, *m_grids[0] );
+#ifdef DG_BENCHMARK
+            Timer t;
+            t.tic();
+#endif //DG_BENCHMARK
+            int number = m_cg[p+1]( op[p+1], x[p+1], b[p+1], op[p+1].precond(),
+                op[p+1].inv_weights(), eps);
+#ifdef DG_BENCHMARK
+            t.toc();
+#ifdef MPI_VERSION
+            int rank;
+            MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+            if(rank==0)
+#endif //MPI
+            std::cout << "stage: " << p+1 << ", iter: " << number << ", took "<<t.diff()<<"s\n";
+#endif //DG_BENCHMARK
+            file_output( x, b, m_r, out, p+1, m_inter, *m_grids[0] );
             //m_cheby[p+1].solve( op[p+1], x[p+1], b[p+1], 0.1*ev[p+1], 1.1*ev[p+1], nu1+nu2);
             //dg::blas2::symv( op[p+1], x[p+1], m_r[p+1]);
             //dg::blas1::axpby( 1., b[p+1], -1., m_r[p+1]);
@@ -362,7 +412,10 @@ struct MultigridCG2d
         //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
         // 6. Post-Smooth nu2 times
-        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2);
+        file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
+        //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2);
+        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2, op[p].inv_weights());
+        file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
         //dg::blas2::symv( op[p], x[p], m_r[p]);
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
         //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
@@ -375,16 +428,19 @@ struct MultigridCG2d
         std::vector<Container>& x, std::vector<Container>& b, std::vector<double> ev,
         unsigned nu1, unsigned nu2, unsigned gamma, unsigned mu, double eps)
     {
-        //begins on coarsest level and cycles through to highest
         for( unsigned u=0; u<m_stages-1; u++)
         {
             dg::blas2::gemv( m_interT[u], x[u], x[u+1]);
             dg::blas2::gemv( m_interT[u], b[u], b[u+1]);
         }
+        std::vector<Container> out( x);
+        //begins on coarsest level and cycles through to highest
         unsigned s = m_stages-1;
+        file_output( x, b, m_r, out, s, m_inter, *m_grids[0] );
         m_cg[s]( op[s], x[s], b[s], op[s].precond(),
-            op[s].inv_weights(), 1e-10);
+            op[s].inv_weights(), eps);
 
+        file_output( x, b, m_r, out, s, m_inter, *m_grids[0] );
 		for( int p=m_stages-2; p>=0; p--)
         {
             dg::blas2::gemv( m_inter[p], x[p+1],  x[p]);
diff --git a/inc/dg/multigrid_b.cu b/inc/dg/multigrid_b.cu
index efb56b2e7..fc17fcfbf 100644
--- a/inc/dg/multigrid_b.cu
+++ b/inc/dg/multigrid_b.cu
@@ -33,9 +33,9 @@ int main()
     double eps;
     double jfactor;
 
-	n = 3;
-	Nx = Ny = 64;
-	eps = 1e-6;
+	n = 4;
+	Nx = 32, Ny = 64;
+	eps = 1e-8;
 	jfactor = 1;
 
 	/*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
@@ -77,7 +77,7 @@ int main()
     std::cout << "Estimated Eigenvalue Analytical "<<lmax<<"\n";
 
     //invert the elliptic equation
-    double ev_max =0, eps_ev = 1e-2;
+    double ev_max =0, eps_ev = 1e-4;
     unsigned counter;
     counter = eve_cg( pol, x, b, ev_max, eps_ev);
     std::cout << "\nPrecision is "<<eps_ev<<"\n";
@@ -97,122 +97,58 @@ int main()
     std::vector<double> multi_ev(stages);
     for(unsigned u=0; u<stages; u++)
     {
-        multi_pol[u].construct( multigrid.grid(u), dg::not_normed, dg::centered, jfactor);
+        multi_pol[u].construct( multigrid.grid(u), dg::normed, dg::centered, 10*jfactor);
         multi_eve[u].construct( multi_chi[u]);
         multi_pol[u].set_chi( multi_chi[u]);
         counter = multi_eve[u]( multi_pol[u], multi_x[u], multi_b[u],
             multi_ev[u], eps_ev);
+        multi_pol[u].construct( multigrid.grid(u), dg::not_normed, dg::centered, 10*jfactor);
+        multi_pol[u].set_chi( multi_chi[u]);
         std::cout << "Eigenvalue estimate eve: "<<multi_ev[u]<<"\n";
     }
-    std::cout << "Type nu1 (3), nu2 (3) gamma (1)\n";
+    //////////////////////////////setup and write netcdf//////////////////
+    err = nc_create( "multigrid.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
+    int dim3d[3];
+    err = file::define_dimensions(  ncid, dim3d, &tvarID, grid, {"step", "y", "x"} );
+
+    err = nc_def_var( ncid, "x_num", NC_DOUBLE, 3, dim3d, &xID);
+    err = nc_def_var( ncid, "b_num", NC_DOUBLE, 3, dim3d, &bID);
+    err = nc_def_var( ncid, "r_num", NC_DOUBLE, 3, dim3d, &rID);
+    ////////////////////////////////////////////////////
+    std::cout << "Type nu1 (3), nu2 (3) gamma (1) max_iter (3)\n";
     unsigned nu1, nu2, gamma;
-    std::cin >> nu1 >> nu2 >> gamma;
+    int max_iter = 3;
+    std::cin >> nu1 >> nu2 >> gamma >> max_iter;
     x = dg::evaluate( initial, grid);
-    multigrid.solve(multi_pol, x, b, multi_ev, nu1, nu2, gamma, eps);
-    //CURRENTLY BEST METHOD:
-    //multigrid.direct_solve(multi_pol, x, b, eps);
-
     const dg::DVec solution = dg::evaluate( sol, grid);
+    for( int i=0; i<max_iter; i++)
+    {
+        multigrid.solve(multi_pol, x, b, multi_ev, nu1, nu2, gamma, eps);
+        //CURRENTLY BEST METHOD:
+        //multigrid.direct_solve(multi_pol, x, b, eps);
+        const double norm = dg::blas2::dot( w2d, solution);
+        dg::DVec error( solution);
+        dg::blas1::axpby( 1.,x,-1., solution, error);
+        double err = dg::blas2::dot( w2d, error);
+        err = sqrt( err/norm);
+        std::cout << " At iteration "<<i<<"\n";
+        std::cout << " Error of Multigrid iterations "<<err<<"\n";
+    }
+    x = dg::evaluate( initial, grid);
+    multigrid.direct_solve(multi_pol, x, b, eps);
     const double norm = dg::blas2::dot( w2d, solution);
     dg::DVec error( solution);
     dg::blas1::axpby( 1.,x,-1., solution, error);
     double err = dg::blas2::dot( w2d, error);
     err = sqrt( err/norm);
-    std::cout << " Error of Multigrid iterations "<<err<<"\n";
-    //should converge to ~2e-7
-    //////////////////////////////setup and write netcdf//////////////////
-    int ncid;
-    file::NC_Error_Handle err;
-    err = nc_create( "multigrid.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
-    std::vector<int> dim2d(2*stages);
-    for( unsigned u=0; u<stages; u++)
-    {
-        dg::Message xs()<<"x"<<u, ys() <<"y"<<u;
-        err = file::define_dimensions(  ncid, *dim2d[2*u], multigrid.grid(u), {xs.str(),ys.str()} );
-    }
+    std::cout << " Error of nested iterations "<<err<<"\n";
 
-    for( auto pair : output)
-    {
-        for( unsigned u=0; u<stages; u++)
-        {
-            dg::HVec temp( grid.size());
-            int varID;
-            err = nc_def_var( ncid, pair.first.data(), NC_DOUBLE, 2, *dim2d[2*u], &varID);
-            pair.second( temp, g2d, gp, c);
-            err = nc_put_var_double( ncid, varID, grid, temp);
-        }
-    }
+    int varID;
+    err = nc_def_var( ncid, "x_ana", NC_DOUBLE, 2, &dim3d[1], &varID);
+    dg::HVec data = solution;
+    file::put_var_double( ncid, varID, grid, data);
     err = nc_close( ncid);
 
 
     return 0;
 }
-
-    std::map< std::string, std::function< void( dg::HVec&, dg::geo::CurvilinearGrid2d&, dg::geo::solovev::Parameters&, dg::geo::TokamakMagneticField&)> > output = {
-        { "Psip", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            result = dg::pullback( mag.psip(), g2d);
-        }},
-        { "PsipR", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            result = dg::pullback( mag.psipR(), g2d);
-        }},
-        { "PsipZ", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            result = dg::pullback( mag.psipZ(), g2d);
-        }},
-        { "g_xx", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            result=g2d.metric().value(0,0);
-        }},
-        { "g_xy", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            result=g2d.metric().value(0,1);
-        }},
-        { "g_yy", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            result=g2d.metric().value(1,1);
-        }},
-        { "g_xy_g_xx", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            dg::blas1::pointwiseDivide( g2d.metric().value(0,1),
-                g2d.metric().value(0,0), result);
-        }},
-        { "g_yy_g_xx", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            dg::blas1::pointwiseDivide( g2d.metric().value(1,1),
-                g2d.metric().value(0,0), result);
-        }},
-        { "vol", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            result=dg::tensor::volume(g2d.metric());
-        }},
-        { "Bzeta", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            dg::HVec Bzeta, Beta;
-            dg::pushForwardPerp( dg::geo::BFieldR(mag), dg::geo::BFieldZ(mag), Bzeta, Beta, g2d);
-            result=Bzeta;
-        }},
-        { "Beta", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            dg::HVec Bzeta, Beta;
-            dg::pushForwardPerp( dg::geo::BFieldR(mag), dg::geo::BFieldZ(mag), Bzeta, Beta, g2d);
-            result=Beta;
-        }},
-        { "Bphi", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            result = dg::pullback( dg::geo::BFieldP(mag), g2d);
-        }},
-        { "q-profile", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            dg::HVec Bzeta, Beta;
-            dg::pushForwardPerp( dg::geo::BFieldR(mag), dg::geo::BFieldZ(mag), Bzeta, Beta, g2d);
-            result = dg::pullback( dg::geo::BFieldP(mag), g2d);
-            dg::blas1::pointwiseDivide( result, Beta, result); //Bphi / Beta
-
-        }},
-        { "Ipol", []( dg::HVec& result, dg::geo::CurvilinearGrid2d& g2d,
-            dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag){
-            result = dg::pullback( mag.ipol(), g2d);
-        }}
-    };
-- 
GitLab


From e497b41500f8ef0b94fdb58939e29bb53bf1b965 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 9 Nov 2019 12:37:38 +0100
Subject: [PATCH 155/540] First convergence results with multigrid look
 promising

---
 inc/dg/multigrid_b.cu | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/inc/dg/multigrid_b.cu b/inc/dg/multigrid_b.cu
index fc17fcfbf..700ee8d5a 100644
--- a/inc/dg/multigrid_b.cu
+++ b/inc/dg/multigrid_b.cu
@@ -26,6 +26,17 @@ double rhs( double x, double y) { return 2.*sin(x)*sin(y)*(amp*sin(x)*sin(y)+1)-
 double sol(double x, double y)  { return sin( x)*sin(y);}
 double der(double x, double y)  { return cos( x)*sin(y);}
 
+//TODO list
+//1. Show that the thing reliably converges (the first trick seems to be that
+// the Chebyshev smoother smoothes on VAx=Vb instead of Ax=b even though this
+// should be the same? Maybe has something to do with how the weights cancel
+// the second trick was to increase the jumps but take care to have enough
+// smoothing steps)
+//2. Can we use this with fixed number of smooting and 1 FMG cycle in simulations?
+// (it seems that the number of smoothing steps influences execution time very
+// little which means that the CG on the coarse grid dominates time; but why
+// does the number of stages influence the error in both nested iterations and
+// the multigrid? )
 
 int main()
 {
@@ -33,18 +44,20 @@ int main()
     double eps;
     double jfactor;
 
-	n = 4;
-	Nx = 32, Ny = 64;
-	eps = 1e-8;
-	jfactor = 1;
+    n = 3;
+    Nx = 32, Ny = 64;
+    eps = 1e-10;
+    std::cout << "Type n(3) Nx(32) Ny(64)!\n";
+    std::cin >> n >> Nx >> Ny;
+    jfactor = 1;
 
-	/*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
+    /*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
     std::cin >> n >> Nx >> Ny; //more N means less iterations for same error
     std::cin >> eps >> jfactor;*/
     std::cout << "Computation on: "<< n <<" x "<< Nx <<" x "<< Ny << std::endl;
     //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
 
-	dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
+    dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
     dg::DVec w2d = dg::create::weights( grid);
     dg::DVec v2d = dg::create::inv_weights( grid);
     dg::DVec one = dg::evaluate( dg::one, grid);
@@ -97,14 +110,15 @@ int main()
     std::vector<double> multi_ev(stages);
     for(unsigned u=0; u<stages; u++)
     {
-        multi_pol[u].construct( multigrid.grid(u), dg::normed, dg::centered, 10*jfactor);
+        multi_pol[u].construct( multigrid.grid(u), dg::not_normed, dg::centered, 10*jfactor);
         multi_eve[u].construct( multi_chi[u]);
         multi_pol[u].set_chi( multi_chi[u]);
         counter = multi_eve[u]( multi_pol[u], multi_x[u], multi_b[u],
             multi_ev[u], eps_ev);
-        multi_pol[u].construct( multigrid.grid(u), dg::not_normed, dg::centered, 10*jfactor);
-        multi_pol[u].set_chi( multi_chi[u]);
+        w2d = dg::create::weights( multigrid.grid(u));
+        multi_ev[u]/=hxhy;
         std::cout << "Eigenvalue estimate eve: "<<multi_ev[u]<<"\n";
+        hxhy*=4;
     }
     //////////////////////////////setup and write netcdf//////////////////
     err = nc_create( "multigrid.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
@@ -115,12 +129,13 @@ int main()
     err = nc_def_var( ncid, "b_num", NC_DOUBLE, 3, dim3d, &bID);
     err = nc_def_var( ncid, "r_num", NC_DOUBLE, 3, dim3d, &rID);
     ////////////////////////////////////////////////////
-    std::cout << "Type nu1 (3), nu2 (3) gamma (1) max_iter (3)\n";
+    std::cout << "Type nu1 (20), nu2 (20) gamma (1) max_iter (1)\n";
     unsigned nu1, nu2, gamma;
     int max_iter = 3;
     std::cin >> nu1 >> nu2 >> gamma >> max_iter;
     x = dg::evaluate( initial, grid);
     const dg::DVec solution = dg::evaluate( sol, grid);
+    t.tic();
     for( int i=0; i<max_iter; i++)
     {
         multigrid.solve(multi_pol, x, b, multi_ev, nu1, nu2, gamma, eps);
@@ -134,14 +149,19 @@ int main()
         std::cout << " At iteration "<<i<<"\n";
         std::cout << " Error of Multigrid iterations "<<err<<"\n";
     }
+    t.toc();
+    std::cout << "Took "<<t.diff()<<"s\n";
     x = dg::evaluate( initial, grid);
+    t.tic();
     multigrid.direct_solve(multi_pol, x, b, eps);
+    t.toc();
     const double norm = dg::blas2::dot( w2d, solution);
     dg::DVec error( solution);
     dg::blas1::axpby( 1.,x,-1., solution, error);
     double err = dg::blas2::dot( w2d, error);
     err = sqrt( err/norm);
     std::cout << " Error of nested iterations "<<err<<"\n";
+    std::cout << "Took "<<t.diff()<<"s\n";
 
     int varID;
     err = nc_def_var( ncid, "x_ana", NC_DOUBLE, 2, &dim3d[1], &varID);
-- 
GitLab


From c79dff95e47c6484cb9115551f1a3d334d4a5c65 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 12 Nov 2019 01:19:54 +0100
Subject: [PATCH 156/540] Further tests and debugging on multigrid

- apparently it did not matter that the chebyshev smoothes without weights
- but convergence is sensitive to jump factor (~5-10)
- built in stopping criterion
---
 inc/dg/elliptic2d_b.cu |   1 -
 inc/dg/multigrid.h     | 182 ++++++++++++++++++++++++-----------------
 inc/dg/multigrid_b.cu  |  94 ++++++++-------------
 3 files changed, 140 insertions(+), 137 deletions(-)

diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 30cb3d183..2a1eaa893 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -50,7 +50,6 @@ int main()
 	dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
     dg::DVec w2d = dg::create::weights( grid);
     dg::DVec v2d = dg::create::inv_weights( grid);
-    dg::DVec one = dg::evaluate( dg::one, grid);
     //create functions A(chi) x = b
     dg::DVec x =    dg::evaluate( initial, grid);
     dg::DVec b =    dg::evaluate( rhs, grid);
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index cd6cd22d8..6b842c94d 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -15,6 +15,7 @@
 #include "topology/mpi_projection.h"
 #endif
 
+#include "dg/file/nc_utilities.h"
 size_t start=0;
 int ncid =0;
 int xID=0, bID=0, rID=0, tvarID=0;
@@ -180,7 +181,8 @@ struct MultigridCG2d
 #ifdef DG_BENCHMARK
             t.tic();
 #endif //DG_BENCHMARK
-            number[u] = m_cg[u]( op[u], m_x[u], m_r[u], op[u].precond(), op[u].inv_weights(), eps/2, 1.);
+            number[u] = m_cg[u]( op[u], m_x[u], m_r[u], op[u].precond(),
+                op[u].inv_weights(), eps/2, 1.);
             dg::blas2::symv( m_inter[u-1], m_x[u], m_x[u-1]);
 #ifdef DG_BENCHMARK
             t.toc();
@@ -189,7 +191,7 @@ struct MultigridCG2d
             MPI_Comm_rank(MPI_COMM_WORLD, &rank);
             if(rank==0)
 #endif //MPI
-            std::cout << "stage: " << u << ", iter: " << number[u] << ", took "<<t.diff()<<"s\n";
+            std::cout << "# Nested iterations stage: " << u << ", iter: " << number[u] << ", took "<<t.diff()<<"s\n";
 #endif //DG_BENCHMARK
 
         }
@@ -208,7 +210,7 @@ struct MultigridCG2d
         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
         if(rank==0)
 #endif //MPI
-        std::cout << "stage: " << 0 << ", iter: " << number[0] << ", took "<<t.diff()<<"s\n";
+        std::cout << "# Nested iterations stage: " << 0 << ", iter: " << number[0] << ", took "<<t.diff()<<"s\n";
 #endif //DG_BENCHMARK
 
         return number;
@@ -279,59 +281,76 @@ struct MultigridCG2d
         full_multigrid( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
         dg::blas1::axpby( 1., m_x[0], 1., x);
 
-        //MULTIGRID CYCLES
-        //dg::blas1::copy( x, m_x[0]);
-        //multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
-        //dg::blas1::copy( m_x[0], x);
+        value_type nrmb = sqrt( blas2::dot( op[0].inv_weights(), m_b[0]));
+        blas2::symv( op[0],x,m_cgr);
+        blas1::axpby( 1., m_b[0], -1., m_cgr);
+        //if x happens to be the solution
+        value_type error = sqrt( blas2::dot(op[0].inv_weights(),m_cgr) );
+        std::cout<< "# Relative Residual error is  "<<error<<"\n";
+        while ( error >  eps*(nrmb + 1))
+        {
+            //MULTIGRID CYCLES
+
+            dg::blas1::copy( x, m_x[0]);
+            multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
+            dg::blas1::copy( m_x[0], x);
+
+            blas2::symv( op[0],x,m_cgr);
+            blas1::axpby( 1., m_b[0], -1., m_cgr);
+            error = sqrt( blas2::dot(op[0].inv_weights(), m_cgr));
+            std::cout<< "# Relative Residual error is  "<<error/(nrmb+1)<<"\n";
+        }
+    }
+	template<class SymmetricOp, class ContainerType0, class ContainerType1>
+    void pcg_solve( std::vector<SymmetricOp>& op,
+    ContainerType0& x, const ContainerType1& b, std::vector<double> ev, unsigned nu_pre, unsigned
+    nu_post, unsigned gamma, double eps)
+    {
 
+        dg::blas2::symv(op[0].weights(), b, m_b[0]);
         //PCG WITH MULTIGRID CYCLE AS PRECONDITIONER
-        //unsigned max_iter_ = m_grids[0]->size();
-        //value_type nrmb = sqrt( blas2::dot( op[0].inv_weights(), m_b[0]));
-        //if( nrmb == 0)
-        //{
-        //    blas1::copy( m_b[0], x);
-        //    return;
-        //}
-        //blas2::symv( op[0],x,m_cgr);
-        //blas1::axpby( 1., m_b[0], -1., m_cgr);
-        ////if x happens to be the solution
-        //if( sqrt( blas2::dot(op[0].inv_weights(),m_cgr) )
-        //        < eps*(nrmb + 1))
-        //    return;
-        ////blas2::symv( P, r, p );//<-- compute p_0
-        ////dg::blas2::symv( op[0].precond(), m_cgr, m_p);
-        ////dg::blas1::copy( 0, m_p);
-        ////m_cheby[0].solve( op[0], m_p, m_cgr, 0.1*ev[0], 1.1*ev[0], nu_post+nu_pre);
-
-        //dg::blas1::copy( 0, m_x[0]);
-        //dg::blas1::copy( m_cgr, m_b[0]);
-        //full_multigrid( op,m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
-        //dg::blas1::copy( m_x[0], m_p);
-
-        ////and store the scalar product
-        //value_type nrmzr_old = blas1::dot( m_p,m_cgr);
-        //value_type alpha, nrmzr_new;
-        //for( unsigned i=2; i<max_iter_; i++)
-        //{
-        //    blas2::symv( op[0], m_p, m_x[0]);
-        //    alpha =  nrmzr_old/blas1::dot( m_p, m_x[0]);
-        //    blas1::axpby( alpha, m_p, 1., x);
-        //    blas1::axpby( -alpha, m_x[0], 1., m_cgr);
-        //    value_type error = sqrt( blas2::dot(op[0].inv_weights(), m_cgr))/(nrmb+1);
-        //    std::cout << "\t\t\tError at "<<i<<" is "<<error<<"\n";
-        //    if( error < eps)
-        //        return;
-        //    dg::blas2::symv( op[0].precond(), m_cgr, m_x[0]);
-        //    dg::blas1::copy( 0, m_x[0]);
-        //    m_cheby[0].solve( op[0], m_x[0], m_cgr, 0.1*ev[0], 1.1*ev[0], nu_post+nu_pre);
-        ////dg::blas1::copy( 0, m_x[0]);
-        ////dg::blas1::copy( m_cgr, m_b[0]);
-        ////multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
-
-        //    nrmzr_new = blas1::dot( m_x[0], m_cgr);
-        //    blas1::axpby(1., m_x[0], nrmzr_new/nrmzr_old, m_p );
-        //    nrmzr_old=nrmzr_new;
-        //}
+        unsigned max_iter_ = m_grids[0]->size();
+        value_type nrmb = sqrt( blas2::dot( op[0].inv_weights(), m_b[0]));
+        if( nrmb == 0)
+        {
+            blas1::copy( m_b[0], x);
+            return;
+        }
+        blas2::symv( op[0],x,m_cgr);
+        blas1::axpby( 1., m_b[0], -1., m_cgr);
+        //if x happens to be the solution
+        if( sqrt( blas2::dot(op[0].inv_weights(),m_cgr) )
+                < eps*(nrmb + 1))
+            return;
+
+        dg::blas1::copy( 0, m_x[0]);
+        dg::blas1::copy( m_cgr, m_b[0]);
+        //multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
+        full_multigrid( op,m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
+        dg::blas1::copy( m_x[0], m_p);
+
+        //and store the scalar product
+        value_type nrmzr_old = blas1::dot( m_p,m_cgr);
+        value_type alpha, nrmzr_new;
+        for( unsigned i=2; i<max_iter_; i++)
+        {
+            blas2::symv( op[0], m_p, m_x[0]);
+            alpha =  nrmzr_old/blas1::dot( m_p, m_x[0]);
+            blas1::axpby( alpha, m_p, 1., x);
+            blas1::axpby( -alpha, m_x[0], 1., m_cgr);
+            value_type error = sqrt( blas2::dot(op[0].inv_weights(), m_cgr))/(nrmb+1);
+            std::cout << "\t\t\tError at "<<i<<" is "<<error<<"\n";
+            if( error < eps)
+                return;
+        dg::blas1::copy( 0, m_x[0]);
+        dg::blas1::copy( m_cgr, m_b[0]);
+        //multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
+        full_multigrid( op,m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
+
+            nrmzr_new = blas1::dot( m_x[0], m_cgr);
+            blas1::axpby(1., m_x[0], nrmzr_new/nrmzr_old, m_p );
+            nrmzr_old=nrmzr_new;
+        }
 
     }
 
@@ -358,15 +377,19 @@ struct MultigridCG2d
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
         //double norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
+#ifdef DG_BENCHMARK
+        Timer t;
+#endif //DG_BENCHMARK
 
-        std::vector<Container> out( x);
-        file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
+        //std::vector<Container> out( x);
+        //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
 
-        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1, op[p].inv_weights());
+        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1);
+        //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1, op[p].inv_weights());
         // 2. Residuum
         dg::blas2::symv( op[p], x[p], m_r[p]);
         dg::blas1::axpby( 1., b[p], -1., m_r[p]);
-        file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
+        //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
         //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum after  "<<norm_res<<"\n";
         // 3. Coarsen
@@ -375,13 +398,12 @@ struct MultigridCG2d
         dg::blas1::scal( x[p+1], 0.);
         if( p+1 == m_stages-1)
         {
-            file_output( x, b, m_r, out, p+1, m_inter, *m_grids[0] );
+            //file_output( x, b, m_r, out, p+1, m_inter, *m_grids[0] );
 #ifdef DG_BENCHMARK
-            Timer t;
             t.tic();
 #endif //DG_BENCHMARK
             int number = m_cg[p+1]( op[p+1], x[p+1], b[p+1], op[p+1].precond(),
-                op[p+1].inv_weights(), eps);
+                op[p+1].inv_weights(), eps/2.);
 #ifdef DG_BENCHMARK
             t.toc();
 #ifdef MPI_VERSION
@@ -389,10 +411,9 @@ struct MultigridCG2d
             MPI_Comm_rank(MPI_COMM_WORLD, &rank);
             if(rank==0)
 #endif //MPI
-            std::cout << "stage: " << p+1 << ", iter: " << number << ", took "<<t.diff()<<"s\n";
+            std::cout << "# Multigrid stage: " << p+1 << ", iter: " << number << ", took "<<t.diff()<<"s\n";
 #endif //DG_BENCHMARK
-            file_output( x, b, m_r, out, p+1, m_inter, *m_grids[0] );
-            //m_cheby[p+1].solve( op[p+1], x[p+1], b[p+1], 0.1*ev[p+1], 1.1*ev[p+1], nu1+nu2);
+            //file_output( x, b, m_r, out, p+1, m_inter, *m_grids[0] );
             //dg::blas2::symv( op[p+1], x[p+1], m_r[p+1]);
             //dg::blas1::axpby( 1., b[p+1], -1., m_r[p+1]);
             //double norm_res = sqrt(dg::blas1::dot( m_r[p+1], m_r[p+1]));
@@ -412,13 +433,13 @@ struct MultigridCG2d
         //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
         // 6. Post-Smooth nu2 times
-        file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
-        //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2);
-        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2, op[p].inv_weights());
-        file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
+        //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
+        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2);
+        //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2, op[p].inv_weights());
+        //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
         //dg::blas2::symv( op[p], x[p], m_r[p]);
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
-        //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
+        //double norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum after "<<norm_res<<"\n";
     }
 
@@ -433,14 +454,27 @@ struct MultigridCG2d
             dg::blas2::gemv( m_interT[u], x[u], x[u+1]);
             dg::blas2::gemv( m_interT[u], b[u], b[u+1]);
         }
-        std::vector<Container> out( x);
+        //std::vector<Container> out( x);
         //begins on coarsest level and cycles through to highest
         unsigned s = m_stages-1;
-        file_output( x, b, m_r, out, s, m_inter, *m_grids[0] );
-        m_cg[s]( op[s], x[s], b[s], op[s].precond(),
-            op[s].inv_weights(), eps);
+        //file_output( x, b, m_r, out, s, m_inter, *m_grids[0] );
+#ifdef DG_BENCHMARK
+        dg::Timer t;
+        t.tic();
+#endif //DG_BENCHMARK
+        int number = m_cg[s]( op[s], x[s], b[s], op[s].precond(),
+            op[s].inv_weights(), eps/2.);
+#ifdef DG_BENCHMARK
+        t.toc();
+#ifdef MPI_VERSION
+        int rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        if(rank==0)
+#endif //MPI
+        std::cout << "# Multigrid stage: " << s << ", iter: " << number << ", took "<<t.diff()<<"s\n";
+#endif //DG_BENCHMARK
 
-        file_output( x, b, m_r, out, s, m_inter, *m_grids[0] );
+        //file_output( x, b, m_r, out, s, m_inter, *m_grids[0] );
 		for( int p=m_stages-2; p>=0; p--)
         {
             dg::blas2::gemv( m_inter[p], x[p+1],  x[p]);
diff --git a/inc/dg/multigrid_b.cu b/inc/dg/multigrid_b.cu
index 700ee8d5a..a3db16084 100644
--- a/inc/dg/multigrid_b.cu
+++ b/inc/dg/multigrid_b.cu
@@ -2,7 +2,7 @@
 #include <iomanip>
 
 #include <thrust/device_vector.h>
-#include "dg/file/nc_utilities.h"
+
 #include "backend/timer.h"
 
 #include "blas.h"
@@ -40,86 +40,56 @@ double der(double x, double y)  { return cos( x)*sin(y);}
 
 int main()
 {
-    unsigned n, Nx, Ny;
-    double eps;
-    double jfactor;
+    unsigned n = 3, Nx = 32, Ny = 64;
+    double eps = 1e-8;
+    double jfactor = 1;
 
-    n = 3;
-    Nx = 32, Ny = 64;
-    eps = 1e-10;
     std::cout << "Type n(3) Nx(32) Ny(64)!\n";
     std::cin >> n >> Nx >> Ny;
-    jfactor = 1;
 
-    /*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
-    std::cin >> n >> Nx >> Ny; //more N means less iterations for same error
-    std::cin >> eps >> jfactor;*/
     std::cout << "Computation on: "<< n <<" x "<< Nx <<" x "<< Ny << std::endl;
-    //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
 
     dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
     dg::DVec w2d = dg::create::weights( grid);
     dg::DVec v2d = dg::create::inv_weights( grid);
-    dg::DVec one = dg::evaluate( dg::one, grid);
     //create functions A(chi) x = b
     dg::DVec x =    dg::evaluate( initial, grid);
     dg::DVec b =    dg::evaluate( rhs, grid);
-    dg::DVec chi =  dg::evaluate( pol, grid);
-    dg::DVec chi_inv(chi);
-    dg::blas1::transform( chi, chi_inv, dg::INVERT<double>());
-    dg::blas1::pointwiseDot( chi_inv, v2d, chi_inv);
-    dg::DVec temp0( x), temp1(x), temp2(x), temp3(x);
-
-    dg::Timer t;
-    t.tic();
-
-    //create an Elliptic object without volume form (not normed)
-    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol( grid, dg::not_normed, dg::centered, jfactor);
-
-    //Set the chi function (chi is a dg::DVec of size grid.size())
-    pol.set_chi( chi);
-
-    //construct an invert object
-    dg::EVE<dg::DVec> eve_cg( x, n*n*Nx*Ny);
-    //dg::CG<dg::DVec> eve_cg( x, n*n*Nx*Ny);
-    double lmax = M_PI*M_PI*(n*n*Nx*Nx/lx/lx + n*n*Ny*Ny/ly/ly); //Eigenvalues of Laplace
-    double hxhy = lx*ly/(n*n*Nx*Ny);
-    double chi_max = dg::blas1::reduce( chi, 0, thrust::maximum<double>());
-    lmax *= chi_max;
-    lmax *= hxhy; //we multiplied the matrix by w2d
-    std::cout << "Estimated Eigenvalue Analytical "<<lmax<<"\n";
+    const dg::DVec chi =  dg::evaluate( pol, grid);
+    const dg::DVec solution = dg::evaluate( sol, grid);
 
-    //invert the elliptic equation
-    double ev_max =0, eps_ev = 1e-4;
-    unsigned counter;
-    counter = eve_cg( pol, x, b, ev_max, eps_ev);
-    std::cout << "\nPrecision is "<<eps_ev<<"\n";
-    std::cout << "\nEstimated EigenValue Eve is "<<ev_max<<"\n";
-    std::cout << " with "<<counter<<" iterations\n";
 
-    //  Now test multigrid with estimated eigenvalues
     unsigned stages = 3;
+    std::cout<< "Type number of stages (3) and jfactor (10) !\n";
+    std::cin >> stages >> jfactor;
     dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid(
         grid, stages);
     const std::vector<dg::DVec> multi_chi = multigrid.project( chi);
-    x = dg::evaluate( initial, grid);
+
     std::vector<dg::DVec> multi_x = multigrid.project( x);
     const std::vector<dg::DVec> multi_b = multigrid.project( b);
     std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
     std::vector<dg::EVE<dg::DVec> > multi_eve(stages);
     std::vector<double> multi_ev(stages);
+    double eps_ev = 1e-4;
+    double hxhy = lx*ly/(n*n*Nx*Ny);
+    unsigned counter;
+    std::cout << "\nPrecision EVE is "<<eps_ev<<"\n";
     for(unsigned u=0; u<stages; u++)
     {
-        multi_pol[u].construct( multigrid.grid(u), dg::not_normed, dg::centered, 10*jfactor);
-        multi_eve[u].construct( multi_chi[u]);
+        multi_pol[u].construct( multigrid.grid(u), dg::not_normed,
+            dg::centered, jfactor);
         multi_pol[u].set_chi( multi_chi[u]);
+        //estimate EVs
+        multi_eve[u].construct( multi_chi[u]);
         counter = multi_eve[u]( multi_pol[u], multi_x[u], multi_b[u],
             multi_ev[u], eps_ev);
-        w2d = dg::create::weights( multigrid.grid(u));
-        multi_ev[u]/=hxhy;
+        //multi_ev[u]/=hxhy;
         std::cout << "Eigenvalue estimate eve: "<<multi_ev[u]<<"\n";
+        std::cout << " with "<<counter<<" iterations\n";
         hxhy*=4;
     }
+    std::cout << "\n\n";
     //////////////////////////////setup and write netcdf//////////////////
     err = nc_create( "multigrid.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim3d[3];
@@ -134,23 +104,23 @@ int main()
     int max_iter = 3;
     std::cin >> nu1 >> nu2 >> gamma >> max_iter;
     x = dg::evaluate( initial, grid);
-    const dg::DVec solution = dg::evaluate( sol, grid);
-    t.tic();
-    for( int i=0; i<max_iter; i++)
+    dg::Timer t;
+    //for( int i=0; i<max_iter; i++)
     {
-        multigrid.solve(multi_pol, x, b, multi_ev, nu1, nu2, gamma, eps);
-        //CURRENTLY BEST METHOD:
-        //multigrid.direct_solve(multi_pol, x, b, eps);
+        t.tic();
+        multigrid.pcg_solve(multi_pol, x, b, multi_ev, nu1, nu2, gamma, eps);
+        t.toc();
+        std::cout << "Took "<<t.diff()<<"s\n";
         const double norm = dg::blas2::dot( w2d, solution);
         dg::DVec error( solution);
         dg::blas1::axpby( 1.,x,-1., solution, error);
         double err = dg::blas2::dot( w2d, error);
         err = sqrt( err/norm);
-        std::cout << " At iteration "<<i<<"\n";
-        std::cout << " Error of Multigrid iterations "<<err<<"\n";
+        //std::cout << " At iteration "<<i<<"\n";
+        std::cout << " Error of Multigrid iterations "<<err<<"\n\n";
     }
-    t.toc();
-    std::cout << "Took "<<t.diff()<<"s\n";
+    ////////////////////////////////////////////////////
+
     x = dg::evaluate( initial, grid);
     t.tic();
     multigrid.direct_solve(multi_pol, x, b, eps);
@@ -161,8 +131,8 @@ int main()
     double err = dg::blas2::dot( w2d, error);
     err = sqrt( err/norm);
     std::cout << " Error of nested iterations "<<err<<"\n";
-    std::cout << "Took "<<t.diff()<<"s\n";
-
+    std::cout << "Took "<<t.diff()<<"s\n\n";
+    /////////////////////////////////////////////////////
     int varID;
     err = nc_def_var( ncid, "x_ana", NC_DOUBLE, 2, &dim3d[1], &varID);
     dg::HVec data = solution;
-- 
GitLab


From ff19002d35bb1b534761a5fbc428c2e4776c854c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Nov 2019 15:18:15 +0100
Subject: [PATCH 157/540] Add Gaussian shaped source profile

---
 src/feltor/feltor.tex |  1 +
 src/feltor/init.h     | 19 ++++++++++++++++---
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 17bf28760..9dece55f9 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1257,6 +1257,7 @@ source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq
 source\_type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
 "influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx},
 "torpex": Torpex inspired source profile Eq.~\eqref{eq:electron_source_torpex},
+"gaussian": Gaussian shaped source profile - uses \texttt{posX}, \texttt{posY} and \texttt{sigma},
     See the file {\tt init.h} to add your own custom source.
 \\
 rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 408a1e66f..9079f5af3 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -75,17 +75,19 @@ HVec profile_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
+    double psip0 = mag.psip()( mag.R0(), 0);
     HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
-        mag.psip(), -p.alpha, p.alpha, -1), grid);
+        mag.psip(), -p.alpha, p.alpha, ((psip0>0)-(psip0<0))), grid); //sign operator!!
     dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), profile_damping, profile_damping);
     return profile_damping;
 }
 HVec profile(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag ){
+    double psip0 = mag.psip()( mag.R0(), 0);
     //First the profile and the source (on the host since we want to output those)
     HVec profile = dg::pullback( dg::geo::Compose<dg::LinearX>( mag.psip(),
-        p.nprofamp/mag.psip()(mag.R0(), 0.), 0.), grid);
+        p.nprofamp/psip0, 0.), grid);
     dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), profile, profile);
     return profile;
 }
@@ -277,7 +279,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
 
 std::map<std::string, std::function< HVec(
     bool& fixed_profile, //indicate whether a profile should be forced (yes or no)
-    HVec& ne_profile, //construct profile if yes, do nothing or construct (determines what is written in output fiele) if no
+    HVec& ne_profile,    // if fixed_profile is yes you need to construct something here, if no then you can ignore the parameter; if you construct something it will show in the output file in any case
     Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
 > > source_profiles =
@@ -320,6 +322,17 @@ std::map<std::string, std::function< HVec(
             return source_profile;
         }
     },
+    {"gaussian",
+        []( bool& fixed_profile, HVec& ne_profile,
+        Geometry& grid, const feltor::Parameters& p,
+        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        {
+            fixed_profile = false;
+            dg::Gaussian prof( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
+                p.sigma, 1.);
+            return dg::pullback( prof, grid);
+        }
+    },
 };
 
 } //namespace feltor
-- 
GitLab


From f06424896d76025ec1bccbcd41756e83b617ae18 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Nov 2019 18:26:40 +0100
Subject: [PATCH 158/540] Widen range of EV in multigrid

---
 inc/dg/chebyshev.h    | 31 +------------------------------
 inc/dg/multigrid.h    | 41 +++++++++++++++++++++++------------------
 inc/dg/multigrid_b.cu | 18 ++++++++----------
 3 files changed, 32 insertions(+), 58 deletions(-)

diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index f7699de67..a306ddd47 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -60,42 +60,13 @@ class Chebyshev
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
      * @param min_ev the minimum Eigenvalue
-     * @param max_ev the minimum Eigenvalue
+     * @param max_ev the maximum Eigenvalue
      * @param num_iter the number of iterations k (equals the number of times A is applied)
      *
      * @copydoc hide_matrix
      * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
      */
     template< class MatrixType, class ContainerType0, class ContainerType1>
-    void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b,
-        double min_ev, double max_ev, unsigned num_iter, const ContainerType1& weights)
-    {
-        if( num_iter == 0)
-            return;
-        assert ( min_ev < max_ev);
-        double theta = (min_ev+max_ev)/2., delta = (max_ev-min_ev)/2.;
-        double rhokm1 = delta/theta, rhok=0;
-        dg::blas1::copy( x, m_xm1); //x0
-        dg::blas2::symv( A, x, m_ax);
-        dg::blas1::pointwiseDot( weights, m_ax, m_ax);
-        dg::blas2::symv( weights, b, m_b);
-        dg::blas1::axpbypgz( 1./theta, m_b, -1./theta, m_ax, 1., x); //x1
-        for ( unsigned k=1; k<num_iter; k++)
-        {
-            rhok = 1./(2.*theta/delta - rhokm1);
-            dg::blas2::symv( A, x, m_ax);
-            dg::blas1::pointwiseDot( weights, m_ax, m_ax);
-            dg::blas1::evaluate( m_xm1, dg::equals(), PairSum(),
-                             1.+rhok*rhokm1, x,
-                            -rhok*rhokm1,    m_xm1,
-                             2.*rhok/delta,  m_b,
-                            -2.*rhok/delta,  m_ax
-                            );
-            x.swap(m_xm1);
-            rhokm1 = rhok;
-        }
-    }
-    template< class MatrixType, class ContainerType0, class ContainerType1>
     void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b,
         double min_ev, double max_ev, unsigned num_iter)
     {
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 6b842c94d..0c3cec01f 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -270,34 +270,39 @@ struct MultigridCG2d
     ContainerType0& x, const ContainerType1& b, std::vector<double> ev, unsigned nu_pre, unsigned
     nu_post, unsigned gamma, double eps)
     {
-        dg::blas2::symv(op[0].weights(), b, m_b[0]);
-
         //FULL MULTIGRID
         //solve for residuum ( if not we always get the same solution)
         dg::blas2::symv(op[0].weights(), b, m_b[0]);
+        value_type nrmb = sqrt( blas1::dot( m_b[0], b));
+
         dg::blas2::symv( op[0], x, m_r[0]);
         dg::blas1::axpby( -1., m_r[0], 1., m_b[0]);
         dg::blas1::copy( 0., m_x[0]);
         full_multigrid( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
         dg::blas1::axpby( 1., m_x[0], 1., x);
 
-        value_type nrmb = sqrt( blas2::dot( op[0].inv_weights(), m_b[0]));
-        blas2::symv( op[0],x,m_cgr);
-        blas1::axpby( 1., m_b[0], -1., m_cgr);
-        //if x happens to be the solution
-        value_type error = sqrt( blas2::dot(op[0].inv_weights(),m_cgr) );
-        std::cout<< "# Relative Residual error is  "<<error<<"\n";
+        dg::blas2::symv(op[0].weights(), b, m_b[0]);
+        blas2::symv( op[0],x,m_r[0]);
+        dg::blas1::axpby( -1., m_r[0], 1., m_b[0]);
+        dg::blas1::copy( 0., m_x[0]);
+        value_type error = sqrt( blas2::dot(op[0].inv_weights(),m_b[0]) );
+        std::cout<< "# Relative Residual error is  "<<error/(nrmb+1)<<"\n";
+
         while ( error >  eps*(nrmb + 1))
         {
             //MULTIGRID CYCLES
-
-            dg::blas1::copy( x, m_x[0]);
-            multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
-            dg::blas1::copy( m_x[0], x);
-
-            blas2::symv( op[0],x,m_cgr);
-            blas1::axpby( 1., m_b[0], -1., m_cgr);
-            error = sqrt( blas2::dot(op[0].inv_weights(), m_cgr));
+            //dg::blas1::copy( x, m_x[0]);
+            //multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
+            //dg::blas1::copy( m_x[0], x);
+            //FMG cycles
+            full_multigrid( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
+            dg::blas1::axpby( 1., m_x[0], 1., x);
+
+            dg::blas2::symv(op[0].weights(), b, m_b[0]);
+            blas2::symv( op[0],x,m_r[0]);
+            dg::blas1::axpby( -1., m_r[0], 1., m_b[0]);
+            dg::blas1::copy( 0., m_x[0]);
+            error = sqrt( blas2::dot(op[0].inv_weights(),m_b[0]) );
             std::cout<< "# Relative Residual error is  "<<error/(nrmb+1)<<"\n";
         }
     }
@@ -384,7 +389,7 @@ struct MultigridCG2d
         //std::vector<Container> out( x);
         //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
 
-        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1);
+        m_cheby[p].solve( op[p], x[p], b[p], 1e-2*ev[p], 1.1*ev[p], nu1);
         //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1, op[p].inv_weights());
         // 2. Residuum
         dg::blas2::symv( op[p], x[p], m_r[p]);
@@ -434,7 +439,7 @@ struct MultigridCG2d
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
         // 6. Post-Smooth nu2 times
         //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
-        m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2);
+        m_cheby[p].solve( op[p], x[p], b[p], 1e-2*ev[p], 1.1*ev[p], nu2);
         //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2, op[p].inv_weights());
         //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
         //dg::blas2::symv( op[p], x[p], m_r[p]);
diff --git a/inc/dg/multigrid_b.cu b/inc/dg/multigrid_b.cu
index a3db16084..58c79edfe 100644
--- a/inc/dg/multigrid_b.cu
+++ b/inc/dg/multigrid_b.cu
@@ -27,21 +27,20 @@ double sol(double x, double y)  { return sin( x)*sin(y);}
 double der(double x, double y)  { return cos( x)*sin(y);}
 
 //TODO list
-//1. Show that the thing reliably converges (the first trick seems to be that
-// the Chebyshev smoother smoothes on VAx=Vb instead of Ax=b even though this
-// should be the same? Maybe has something to do with how the weights cancel
-// the second trick was to increase the jumps but take care to have enough
-// smoothing steps)
+//1. Show that the thing reliably converges ( I think we should focus on showing that a single
+//FMG sweep reliably produces an acceptable solution even if the discretiation error has not yet been reached)
 //2. Can we use this with fixed number of smooting and 1 FMG cycle in simulations?
 // (it seems that the number of smoothing steps influences execution time very
-// little which means that the CG on the coarse grid dominates time; but why
-// does the number of stages influence the error in both nested iterations and
-// the multigrid? )
+// little which means that the CG on the coarse grid dominates time;)
+//3. The eps on the lowest grid can be larger than on the fine grids because the discretization error on the coarse grid is h^3 times higher
+//4. With forward discretization there seems to be a sweet spot on how many smoothing steps to choose
+//5. The relevant errors for us are the gradient in phi errors
+//6. The range that the Chebyshev solver smoothes influences the error in the end
 
 int main()
 {
     unsigned n = 3, Nx = 32, Ny = 64;
-    double eps = 1e-8;
+    double eps = 1e-6;
     double jfactor = 1;
 
     std::cout << "Type n(3) Nx(32) Ny(64)!\n";
@@ -58,7 +57,6 @@ int main()
     const dg::DVec chi =  dg::evaluate( pol, grid);
     const dg::DVec solution = dg::evaluate( sol, grid);
 
-
     unsigned stages = 3;
     std::cout<< "Type number of stages (3) and jfactor (10) !\n";
     std::cin >> stages >> jfactor;
-- 
GitLab


From 94fb6f15e6eda06bacdd661a3d6de59904e47d2b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 14 Nov 2019 00:22:39 +0100
Subject: [PATCH 159/540] Add potential FLR source correction

- and correct signs in vorticity equation in writeup
---
 src/feltor/feltor.h   | 10 ++++++-
 src/feltor/feltor.tex | 69 ++++++++++++++++++++++++++++++-------------
 2 files changed, 58 insertions(+), 21 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 904cfcc07..ce5ff0152 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -258,6 +258,8 @@ struct Explicit
     }
     const Container& lapMperpP (int i)
     {
+        dg::blas1::copy( 1., m_temp1);
+        m_lapperpP.set_chi( m_temp1);
         dg::blas2::gemv( m_lapperpP, m_phi[i], m_temp1);
         return m_temp1;
     }
@@ -863,9 +865,15 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
                 m_profne, m_source, m_omega_source);
         else
             dg::blas1::axpby( m_omega_source, m_source, 0., m_s[0][0]);
-        //compute FLR correction
+        //compute FLR corrections
         dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
         dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
+        // potential part of FLR correction
+        dg::blas1::subroutine( routines::ComputeChi(),
+            m_temp0, y[1], m_binv, m_p.mu[1]);
+        m_lapperpP.set_chi( m_temp0);
+        dg::blas2::gemv( m_lapperpP, m_phi[0], m_temp0);//negative!!
+        dg::blas1::axpby( -1., m_temp0, 1., m_s[0][0]);
 
         dg::blas1::axpby( 1., m_s[0][0], 1.0, yp[0][0]);
         dg::blas1::axpby( 1., m_s[0][1], 1.0, yp[0][1]);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 9dece55f9..49d4a98bd 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -832,7 +832,7 @@ We can choose the constant influx
 \end{align}
 or a Torpex inspired source profile
 \begin{align} \label{eq:electron_source_torpex}
-  S_{prof}(R,Z) &= 
+  S_{prof}(R,Z) &=
   \begin{cases}
     \exp\left( - \frac{(R-R_0)^2}{a^2 }- \frac{(Z-Z_0)^2}{b^2}\right) \text{ if} R > R_0 \\
     \frac{1}{2}\exp\left( - \frac{(R-R_0)^2}{a^2} -2c(R-R_0)(Z-Z_0)- \frac{(Z-Z_0)^2}{b^2} \right) \\
@@ -844,10 +844,14 @@ with $a=0.0335$m, $b=0.05$m, $c=565m^{-2}$, $R_0=0.98$m and $Z_0=-0.02$m.
 
 For ions we use
 \begin{align}
-    S_{N_i} = \Gamma_{1,i}^{-1} S_{n_e} = \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S_{n_e}
+    S_{N_i} = \Gamma_{1,i}^{-1} S_{n_e} = \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S_{n_e} \\
+    \delta S_{n_e} = \nabla\cdot\left( \frac{\mu_i S_{N_i}}{B^2}\nabla_\perp \phi\right)
   \label{eq:ion_source}
 \end{align}
-Note that Eq.~\eqref{eq:ion_source} is explicitly chosen as to avoid vorticity generation
+and finally add $\delta S_{n_e}$ to $S_{n_e}$ as the potential FLR correction.
+Note that the correction $\delta S_{n_e}$ is a total divergence which means
+it does not change the volume integrated "total" particle number created by the source.
+Note that Eq.~\eqref{eq:ion_source} is explicitly chosen as to avoid potential generation
 by the particle source (cf.~Section~\ref{sec:conservation}). $S_{n_e}$ needs to be smooth
 so that $\nabla_\perp^2 S_{n_e}$ is well defined.
 
@@ -998,7 +1002,7 @@ We have the energy flux through a flux surface
 \label{eq:energy_flux}
 \end{align}
 
-\subsection{ Force balance equation}
+\subsection{ Vorticity and momentum balance equation}
 In order to discuss poloidal flows let us first define orthogonal vectors $\{\vec{ \hat\zeta}, \vec{\hat\eta},\bhat\}$
 \begin{align}
 \vec{\hat\zeta} := \nabla\psi_p,\quad
@@ -1013,35 +1017,60 @@ Notice that $\vec{\hat \eta}$ in general has a (small) toroidal component in add
 Furthermore, from $\vec u_E = \bhat\times \vec\nabla\phi/B$ and $\vec u_d = \tau_i \bhat\times\nabla \ln N_i$
 we can derive
 \begin{align}
-u_E^{\hat\eta} &:= \vec u_E\cdot \vec{\hat\eta} = \frac{\partial_{\hat\zeta} \phi}{B^2}
-&&u_d^{\hat\eta} := \vec u_d\cdot \vec{\hat\eta} =  \frac{\tau_i \partial_{\hat\zeta} \ln N_i}{B} \\
-u_E^{\hat\zeta}&:= \vec u_E\cdot \vec{\hat\zeta}  = -\partial_{\hat\eta} \phi
-&&u_d^{\hat\zeta}:= \vec u_d\cdot \vec{\hat\zeta} = -B \tau_i \partial_{\hat\eta} \ln N_i
-\end{align}
-With this we write the force balance equation (in the LWL)
-\begin{align}
+u_E^{\hat\eta} &:= \vec u_E\cdot \vec{\hat\eta} = \frac{\partial_{\hat\zeta} \phi}{B^2} \sim u_{E,\varphi}
+&&u_d^{\hat\eta} := \vec u_d\cdot \vec{\hat\eta} =  \frac{\tau_i \partial_{\hat\zeta} \ln N_i}{B} \sim u_{D,\varphi} \\
+u_E^{\hat\zeta}&:= \vec u_E\cdot \vec{\hat\zeta}  = -\partial_{\hat\eta} \phi \sim u_E^v \sim -\partial_\varphi\phi
+&&u_d^{\hat\zeta}:= \vec u_d\cdot \vec{\hat\zeta} = -B \tau_i \partial_{\hat\eta} \ln N_i \sim u_D^v
+\end{align}
+where we write $\partial_{\hat\zeta} := \nabla\psi_p\cdot\nabla$ and $\partial_{\hat\eta} := \vec{\hat\eta}\cdot\nabla$.
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\subsubsection{Vorticity equation}
+With this we write the vorticity equation (in the LWL)
+\begin{align} \label{eq:vorticity_average}
 &\frac{\partial}{\partial t} \left\langle \mu_i N_i \left(
 \frac{\partial_{\hat\zeta}\phi\,}{B^2} + \tau_i \partial_{\hat\zeta} \ln N_i\right) \right\rangle
 \nonumber\\
-&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle \mu_i N_i\partial_{\hat\eta} \phi \left(\frac{\partial_{\hat\zeta}\phi }{B^2}
-+ \tau_i \partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel \right\rangle
+&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle -\mu_i N_i\partial_{\hat\eta} \phi \left(\frac{\partial_{\hat\zeta}\phi }{B^2}
++ \tau_i \partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel + \frac{1}{2}\partial_{\hat\eta} A_\parallel \partial_{\hat\zeta} \tau_i N_i U_i \right\rangle
 \nonumber\\
 &= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\nabla\times\bhat}\cdot\nabla\psi_p \right\rangle
 \end{align}
-where the right hand side represents the Lorentz force $\vec j\times\vec B$.
-Notice that 
+where we neglect the second order derivative $\partial_{\hat\eta}\partial_{\hat\zeta}A_\parallel$ in the magnetization density term and
+where the right hand side represents the negative toroidal component of the Lorentz force $-(\vec j\times\vec B)_\varphi$.
+Notice that
 \begin{align}
 \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p\right\rangle
 = \left\langle \frac{ \bhat\times \nabla (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cdot\nabla\psi_p\right\rangle
 = -\left\langle\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)\right\rangle
 \end{align}
-We can interpret the force balance as the flux surface average of a
-continuity equation
+We can interpret Eq.~\eqref{eq:vorticity_average} as the flux surface average of
+the vorticity equation
 \begin{align}
-&\partial_t \Omega + \nabla\cdot \vec j_\Omega = S_\Omega \\
+&\partial_t \GA{\Omega} + \GA{\nabla\cdot \vec j_\Omega} = \GA{S_\Omega} \\
 \Omega &:= \mu_i N_i \left(\frac{\nabla\psi_p\cdot\nabla\phi}{B^2} + \tau_i \nabla\psi_p \cdot\nabla \ln N_i\right) \\
-\vec j_{\Omega} &:= -\Omega \vec u_E 
-    - \frac{1}{\beta} \nabla\psi_p\cdot\nabla A_\parallel \frac{\bhat\times\nabla A_\parallel}{B} \\
+\vec j_{\Omega} &:= \Omega \vec u_E
+    - \left(\frac{1}{\beta} \nabla\psi_p\cdot\nabla A_\parallel +\frac{1}{2} \nabla\psi_p\cdot\nabla \tau_i N_iU_i\right)\frac{\bhat\times\nabla A_\parallel}{B} \\
+    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\nabla\times\bhat}(\psi_p)
+\end{align}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\subsubsection{Toroidal ExB momentum equation}
+Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB velocity
+\begin{align} \label{eq:exb_average}
+&\frac{\partial}{\partial t} \left\langle \mu_i N_i \left(
+\frac{\partial_{\hat\zeta}\phi\,}{B^2}\right) \right\rangle
+\nonumber\\
+&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle -\mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}\left(\partial_{\hat\eta} \phi + \tau_i \partial_{\hat\eta} \ln N_i \right)
+ + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel + \frac{1}{2}\partial_{\hat\zeta} A_\parallel \partial_{\hat\eta} \tau_i N_i U_i\right \rangle
+\nonumber\\
+&= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\nabla\times\bhat}\cdot\nabla\psi_p \right\rangle
+\end{align}
+We can interpret Eq.~\eqref{eq:exb_average} as the flux surface average of
+the \ExB vorticity equation
+\begin{align}
+&\partial_t \GA{\Omega_E} + \GA{\nabla\cdot \vec j_{\Omega_E}} = \GA{S_\Omega} \\
+\Omega_E &:= \mu_i N_i \frac{\nabla\psi_p\cdot\nabla\phi}{B^2} \\
+\vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
+    - \frac{1}{\beta} \nabla\psi_p\cdot\nabla A_\parallel \left(\frac{\bhat\times\nabla A_\parallel}{B} +\frac{1}{2} \bhat \times \nabla \tau_i N_iU_i\right) \\
     S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\nabla\times\bhat}(\psi_p)
 \end{align}
 
-- 
GitLab


From 284b4eab7da97b0f87a319c75c5479650c7f8aa4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 14 Nov 2019 13:16:38 +0100
Subject: [PATCH 160/540] Update feltordiag with diamagnetic vorticity

---
 src/feltor/feltor.tex            | 14 +++---
 src/feltor/feltordiag.h          | 76 ++++++++++++++++++++++++++------
 src/feltor/geometry/compass.json | 43 +++++++++---------
 3 files changed, 93 insertions(+), 40 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 49d4a98bd..8ebede76b 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1030,8 +1030,8 @@ With this we write the vorticity equation (in the LWL)
 &\frac{\partial}{\partial t} \left\langle \mu_i N_i \left(
 \frac{\partial_{\hat\zeta}\phi\,}{B^2} + \tau_i \partial_{\hat\zeta} \ln N_i\right) \right\rangle
 \nonumber\\
-&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle -\mu_i N_i\partial_{\hat\eta} \phi \left(\frac{\partial_{\hat\zeta}\phi }{B^2}
-+ \tau_i \partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel + \frac{1}{2}\partial_{\hat\eta} A_\parallel \partial_{\hat\zeta} \tau_i N_i U_i \right\rangle
+&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle -\partial_{\hat\eta} \phi \left(\mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}
++ \mu_i \tau_i N_i\partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\eta}A_\parallel\partial_{\hat\zeta} A_\parallel  + \frac{1}{2}\partial_{\hat\eta} A_\parallel \partial_{\hat\zeta} \tau_i N_i U_i \right\rangle
 \nonumber\\
 &= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\nabla\times\bhat}\cdot\nabla\psi_p \right\rangle
 \end{align}
@@ -1059,7 +1059,7 @@ Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continui
 &\frac{\partial}{\partial t} \left\langle \mu_i N_i \left(
 \frac{\partial_{\hat\zeta}\phi\,}{B^2}\right) \right\rangle
 \nonumber\\
-&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle -\mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}\left(\partial_{\hat\eta} \phi + \tau_i \partial_{\hat\eta} \ln N_i \right)
+&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle \mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}\left(-\partial_{\hat\eta} \phi - \tau_i \partial_{\hat\eta} \ln N_i \right)
  + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel + \frac{1}{2}\partial_{\hat\zeta} A_\parallel \partial_{\hat\eta} \tau_i N_i U_i\right \rangle
 \nonumber\\
 &= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\nabla\times\bhat}\cdot\nabla\psi_p \right\rangle
@@ -1432,8 +1432,12 @@ and the time-averaged quantities Y $\in$
     leiperp &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_i \nu_\perp \Delta_\perp U_i$ \\
     leeparallel &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_e \nu_\parallel \Delta_\parallel u_e$ \\
     leiparallel &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_i \nu_\parallel \Delta_\parallel U_i$ \\
-    jsoexbi &$-(\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i)  \vec u_E\cdot\nabla \psi_p$ \\
-    jsoexbe &$-(\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2+\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e) \vec u_E\cdot\nabla \psi_p$ \\
+    jsoexbi &$(\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2) (\bhat\times\nabla\phi)\cdot\nabla \psi_p/B$ \\
+    jsoexbe &$(\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2) (\bhat\times\nabla\phi)\cdot\nabla \psi_p/B$ \\
+    jsodiaiUE &$(\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i) (\bhat\times\nabla\phi)\cdot\nabla \psi_p/B$ \\
+    jsodiaeUE &$(\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e) (\bhat\times\nabla\phi)\cdot\nabla \psi_p/B$ \\
+    jsoexbiUD &$(\mu_i\tau_i \nabla\psi_p\cdot\nabla\phi/B^2) (\bhat\times\nabla N_i)\cdot\nabla \psi_p/B$ \\
+    jsoexbeUD &$(\mu_i\tau_i \nabla\psi_p\cdot\nabla\phi/B^2) (\bhat\times\nabla n_e)\cdot\nabla \psi_p/B$ \\
     jsoapar &$ \nabla\psi_p\cdot\nabla A_\parallel \bhat\times\nabla A_\parallel\cdot\nabla \psi_p/B/\beta$ \\
     socurve &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
     socurvi &$z_i\tau_i N_i \mathcal K(\psi_p)$ \\
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 34d4dafab..09fbc3fdf 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -741,33 +741,81 @@ std::vector<Record> diagnostics2d_list = {
     /// --------------------- Vorticity flux terms ---------------------------//
     {"jsoexbi_tt", "ExB vorticity flux term with ion density (Time average)", true,
         []( DVec& result, Variables& v){
-            // - ExB Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
 
-            // Omega
+            // Omega_E
             routines::dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
             dg::blas1::pointwiseDot( 1., v.tmp[0], v.f.binv(), v.f.binv(), 0., v.tmp[0]);
             dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], v.f.density(1), 0., v.tmp[0]);
 
-            routines::dot( v.f.gradN(1), v.gradPsip, v.tmp[1]);
-            dg::blas1::axpby( v.p.mu[1]*v.p.tau[1], v.tmp[1], 1., v.tmp[0] );
-
             // Multiply everything
             dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
         }
     },
     {"jsoexbe_tt", "ExB vorticity flux term with electron density (Time average)", true,
         []( DVec& result, Variables& v){
-            // - ExB Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradP(0), result);
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
 
-            // Omega
+            // Omega_E
             routines::dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
             dg::blas1::pointwiseDot( 1., v.tmp[0], v.f.binv(), v.f.binv(), 0., v.tmp[0]);
             dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], v.f.density(0), 0., v.tmp[0]);
 
-            routines::dot( v.f.gradN(0), v.gradPsip, v.tmp[1]);
-            dg::blas1::axpby( v.p.mu[1]*v.p.tau[1], v.tmp[1], 1., v.tmp[0] );
+            // Multiply everything
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
+        }
+    },
+    {"jsodiaiUE_tt", "Diamagnetic vorticity flux by ExB veloctiy term with ion density (Time average)", true,
+        []( DVec& result, Variables& v){
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
+
+            // Omega_D,phi
+            routines::dot( v.f.gradN(1), v.gradPsip, v.tmp[0]);
+            dg::blas1::scal( v.tmp[0], v.p.mu[1]*v.p.tau[1]);
+
+            // Multiply everything
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
+        }
+    },
+    {"jsodiaeUE_tt", "Diamagnetic vorticity flux by ExB velocity term with electron density (Time average)", true,
+        []( DVec& result, Variables& v){
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
+
+            // Omega_D,phi
+            routines::dot( v.f.gradN(0), v.gradPsip, v.tmp[0]);
+            dg::blas1::scal( v.tmp[0], v.p.mu[1]*v.p.tau[1]);
+
+            // Multiply everything
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
+        }
+    },
+    {"jsoexbiUD_tt", "ExB vorticity flux term by diamagnetic velocity with ion density (Time average)", true,
+        []( DVec& result, Variables& v){
+            // bxGradN/B Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradN(1), v.gradPsip, result);
+            dg::blas1::scal( result, v.p.tau[1]);
+
+            // m Omega_E,phi
+            routines::dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], v.f.binv(), v.f.binv(), 0., v.tmp[0]);
+
+            // Multiply everything
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
+        }
+    },
+    {"jsoexbeUD_tt", "ExB vorticity flux term by diamagnetic velocity with electron density (Time average)", true,
+        []( DVec& result, Variables& v){
+            // bxGradN/B Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradN(0), v.gradPsip, result);
+            dg::blas1::scal( result, v.p.tau[1]);
+
+            // m Omega_E,phi
+            routines::dot( v.f.gradP(0), v.gradPsip, v.tmp[0]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], v.f.binv(), v.f.binv(), 0., v.tmp[0]);
 
             // Multiply everything
             dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
@@ -817,9 +865,9 @@ std::vector<Record> diagnostics2d_list = {
 
 };
 
-///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 std::vector<Record> restart3d_list = {
     {"restart_electrons", "electron density", false,
         []( DVec& result, Variables& v ) {
diff --git a/src/feltor/geometry/compass.json b/src/feltor/geometry/compass.json
index ed5f7e807..df90b406d 100644
--- a/src/feltor/geometry/compass.json
+++ b/src/feltor/geometry/compass.json
@@ -1,23 +1,24 @@
 {
-	"A" : 1,
-	"R_0" : 178.941,
-	"c" :
-	[
-		0.1837793492930699307570501845612999759808,
-        0.003053802605442159220296950705726992590288,
-        -0.1882242965994715344669754151680388020196,
-        -0.08356260086850207612663444582876793886111,
-        0.1400828823828941556097064593061472639108,
-        -0.1464795892953832541951287559065793362435,
-        -0.006527941916323078493138471704332722876669,
-        0.05608898864420130592521710379149509407494,
-        0.3846091415179281423168465616340378123959,
-        -0.1950822491569537061735371149653311041446,
-        -0.04578101700693340619317565018736939517299,
-        0.006936460124168526675792408803022789327478
-	],
-	"equilibrium" : "solovev",
-	"elongation" : 1.6594,
-	"inverseaspectratio" : 0.35714285714,
-	"triangularity" : 0.4
+    "A": 0,
+    "R_0": 1.789406847665635e2,
+    "PP": 1,
+    "PI": 1,
+    "c":[
+           0.081203566774753699,
+           -0.068223110133006717,
+           -0.15733151525792472,
+           -0.070753041538505742,
+           0.10414465434569499,
+           -0.10994533867714237,
+           -0.0047245091245514442,
+           0.039697138423254924,
+           0.26895337916962100,
+           -0.13814872783212580,
+           -0.033095116055736015,
+           0.0047665391068593747
+    ],
+    "elongation": 1.6594,
+    "equilibrium": "solovev",
+    "inverseaspectratio": 0.35714285714285715,
+    "triangularity": 0.4
 }
-- 
GitLab


From 294b6612bcbfb34d9b1cf33ec5520e1bf31d22ff Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 14 Nov 2019 13:55:55 +0100
Subject: [PATCH 161/540] Add a steady state solver for feltor

---
 inc/dg/andersonacc.h      |   2 +
 src/feltor/Makefile       |  11 ++--
 src/feltor/feltor.h       |   3 +-
 src/feltor/steadystate.cu | 115 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 126 insertions(+), 5 deletions(-)
 create mode 100644 src/feltor/steadystate.cu

diff --git a/inc/dg/andersonacc.h b/inc/dg/andersonacc.h
index 0bd3b8d38..036aa98d7 100644
--- a/inc/dg/andersonacc.h
+++ b/inc/dg/andersonacc.h
@@ -79,6 +79,8 @@ struct AndersonAcceleration
     const ContainerType& copyable() const{ return m_gval;}
 
     /*!@brief Solve the system \f$ f(x) = b \f$ in the given norm
+     *
+     * Iterates until \f$ ||f(x)-b|| < a_{\mathrm{tol}} + r_{\mathrm{tol}} ||b||\f$
      *
      * @param f The function \c y=f(x) in the form \c f(x,y). The first argument is the input and the second the output.
      * @param x Contains an initial guess on input and the solution on output.
diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index 384e91c1c..0b58ab143 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -10,19 +10,22 @@ INCLUDE+= -I../../inc   # other project libraries
 
 all: feltor_hpc manufactured
 
-manufactured: manufactured.cu manufactured.h feltor.h
+manufactured: manufactured.cu manufactured.h feltor.h implicit.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
 
 feltordiag: feltordiag.cu
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g
 
-feltor: feltor.cu feltor.h
+feltor: feltor.cu feltor.h implicit.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
-feltor_hpc: feltor_hpc.cu feltor.h
+steadystate: steadystate.cu feltor.h implicit.h
 	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
 
-feltor_mpi: feltor_hpc.cu feltor.h
+feltor_hpc: feltor_hpc.cu feltor.h implicit.h
+	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
+
+feltor_mpi: feltor_hpc.cu feltor.h implicit.h
 	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DFELTOR_MPI -DDG_BENCHMARK
 
 .PHONY: clean
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index ce5ff0152..b5c7749ce 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -161,6 +161,7 @@ struct ComputeSource{
 template< class Geometry, class IMatrix, class Matrix, class Container >
 struct Explicit
 {
+    using vector = std::array<std::array<Container,2>,2>;
     Explicit( const Geometry& g, feltor::Parameters p,
         dg::geo::TokamakMagneticField mag);
 
@@ -870,7 +871,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
         // potential part of FLR correction
         dg::blas1::subroutine( routines::ComputeChi(),
-            m_temp0, y[1], m_binv, m_p.mu[1]);
+            m_temp0, y[0][1], m_binv, m_p.mu[1]);
         m_lapperpP.set_chi( m_temp0);
         dg::blas2::gemv( m_lapperpP, m_phi[0], m_temp0);//negative!!
         dg::blas1::axpby( -1., m_temp0, 1., m_s[0][0]);
diff --git a/src/feltor/steadystate.cu b/src/feltor/steadystate.cu
new file mode 100644
index 000000000..0baa4d98e
--- /dev/null
+++ b/src/feltor/steadystate.cu
@@ -0,0 +1,115 @@
+#include <iostream>
+#include <iomanip>
+#include <vector>
+#include <sstream>
+#include <cmath>
+
+#include "feltor.h"
+#include "implicit.h"
+using HVec = dg::HVec;
+using DVec = dg::DVec;
+using DMatrix = dg::DMatrix;
+using IDMatrix = dg::IDMatrix;
+using IHMatrix = dg::IHMatrix;
+using Geometry = dg::CylindricalGrid3d;
+#define MPI_OUT
+
+#include "init.h"
+#include "feltordiag.h"
+
+namespace detail{
+template<class Explicit, class Implicit, class Container >
+struct FullSystem
+{
+    FullSystem() = default;
+    FullSystem( Explicit exp, Implicit imp, Container temp):
+            m_exp( exp), m_imp( imp), m_temp(temp){}
+
+    template<class Container2>
+    void operator()( const Container2& y, Container2& yp)
+    {
+        m_exp( 0, y, m_temp);
+        m_imp( 0, y, yp);
+        dg::blas1::axpby( 1., m_temp, 1., yp);
+    }
+    private:
+    Explicit m_exp;
+    Implicit m_imp;
+    Container m_temp;
+};
+}//namespace detail
+
+template<class Explicit, class Implicit, class Container, class ContainerType2>
+void solve_steady_state( Explicit& ex, Implicit& im, Container& x, const Container& b, const ContainerType2& weights)
+{
+    Container tmp(x);
+    detail::FullSystem<Explicit&, Implicit&, Container&> full(ex,im,tmp);
+    // allocate memory
+    unsigned mMax =3, restart = mMax;
+    dg::AndersonAcceleration<Container> acc( x, mMax);
+    // Evaluate right hand side and solution on the grid
+    const double eps = 1e-1;
+    unsigned max_iter =10;
+    double damping = 1e-2;
+    std::cout << "Number of iterations "<< acc.solve( full, x, b, weights, eps, eps, max_iter, damping, restart, true)<<std::endl;
+
+}
+
+int main( int argc, char* argv[])
+{
+    ////Parameter initialisation ////////////////////////////////////////////
+    Json::Value js, gs;
+    if( argc == 1)
+    {
+        std::ifstream is("input.json");
+        std::ifstream ks("geometry_params.json");
+        is >> js;
+        ks >> gs;
+    }
+    else if( argc == 3)
+    {
+        std::ifstream is(argv[1]);
+        std::ifstream ks(argv[2]);
+        is >> js;
+        ks >> gs;
+    }
+    else
+    {
+        std::cerr << "ERROR: Too many arguments!\nUsage: "
+                  << argv[0]<<" [inputfile] [geomfile] \n";
+        return -1;
+    }
+    const feltor::Parameters p( js);
+    const dg::geo::solovev::Parameters gp(gs);
+    p.display( std::cout);
+    gp.display( std::cout);
+    /////////////////////////////////////////////////////////////////////////
+    double Rmin=gp.R_0-p.boxscaleRm*gp.a;
+    double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
+    double Rmax=gp.R_0+p.boxscaleRp*gp.a;
+    double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    //Make grid
+    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
+        p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    if( p.alpha_mag > 0.)
+        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
+
+    //create RHS
+    std::cout << "Constructing Explicit...\n";
+    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
+    std::cout << "Constructing Implicit...\n";
+    feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
+    std::cout << "Done!\n";
+
+    DVec result = dg::evaluate( dg::zero, grid);
+    std::array<std::array<DVec,2>,2> y0;
+    y0[0][0] = y0[0][1] =y0[1][0] =y0[1][1] = result;
+    std::array<std::array<DVec,2>,2> b(y0);
+    DVec weights = dg::create::weights( grid);
+    solve_steady_state( feltor, im, y0, b, weights);
+
+    ////////////////////////////////////////////////////////////////////
+    return 0;
+
+}
-- 
GitLab


From 50031aeb3a9ed1d5a809eaee86f03691956536b6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 14 Nov 2019 15:26:41 +0100
Subject: [PATCH 162/540] Tryouts with steadystate unsuccesful

---
 src/feltor/Makefile       |  2 +-
 src/feltor/feltor.cu      |  4 +-
 src/feltor/steadystate.cu | 96 +++++++++++++++++++++++++++++++++++----
 3 files changed, 90 insertions(+), 12 deletions(-)

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index 0b58ab143..b02405068 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -20,7 +20,7 @@ feltor: feltor.cu feltor.h implicit.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
 steadystate: steadystate.cu feltor.h implicit.h
-	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
+	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
 
 feltor_hpc: feltor_hpc.cu feltor.h implicit.h
 	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 0a86dff75..9ad2d69c3 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -88,8 +88,8 @@ int main( int argc, char* argv[])
         std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
         return -1;
     }
-    bool fixed_profile;
 
+    bool fixed_profile;
     HVec profile = dg::evaluate( dg::zero, grid);
     HVec source_profile;
     try{
@@ -144,7 +144,6 @@ int main( int argc, char* argv[])
     std::cout << std::scientific << std::setprecision( 2);
     dg::Average<dg::HVec> toroidal_average( grid, dg::coo3d::z);
     title << std::setprecision(2) << std::scientific;
-    //unsigned failed_counter = 0;
     while ( !glfwWindowShouldClose( w ))
     {
         title << std::fixed;
@@ -153,7 +152,6 @@ int main( int argc, char* argv[])
         {
             if(pair.first == "Ome / ")
             {
-                //dg::blas2::gemv( laplacianM, *pair.second, dvisual);
                 dg::assign( feltor.lapMperpP(0), hvisual);
                 dg::assign( *pair.second, hvisual);
             }
diff --git a/src/feltor/steadystate.cu b/src/feltor/steadystate.cu
index 0baa4d98e..12ac61e3a 100644
--- a/src/feltor/steadystate.cu
+++ b/src/feltor/steadystate.cu
@@ -4,6 +4,7 @@
 #include <sstream>
 #include <cmath>
 
+#include "draw/host_window.h"
 #include "feltor.h"
 #include "implicit.h"
 using HVec = dg::HVec;
@@ -14,8 +15,7 @@ using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 
-#include "init.h"
-#include "feltordiag.h"
+#include "init.h" //for the source profiles
 
 namespace detail{
 template<class Explicit, class Implicit, class Container >
@@ -40,17 +40,18 @@ struct FullSystem
 }//namespace detail
 
 template<class Explicit, class Implicit, class Container, class ContainerType2>
-void solve_steady_state( Explicit& ex, Implicit& im, Container& x, const Container& b, const ContainerType2& weights)
+void solve_steady_state( Explicit& ex, Implicit& im, Container& x, const Container& b, const ContainerType2& weights, double damping)
 {
     Container tmp(x);
     detail::FullSystem<Explicit&, Implicit&, Container&> full(ex,im,tmp);
     // allocate memory
-    unsigned mMax =3, restart = mMax;
+    unsigned mMax =10, restart = mMax;
     dg::AndersonAcceleration<Container> acc( x, mMax);
     // Evaluate right hand side and solution on the grid
-    const double eps = 1e-1;
-    unsigned max_iter =10;
-    double damping = 1e-2;
+    const double eps = 1e-3;
+    unsigned max_iter =1000;
+    std::cout << "Type maximum iteration number\n";
+    std::cin >> max_iter;
     std::cout << "Number of iterations "<< acc.solve( full, x, b, weights, eps, eps, max_iter, damping, restart, true)<<std::endl;
 
 }
@@ -79,6 +80,7 @@ int main( int argc, char* argv[])
                   << argv[0]<<" [inputfile] [geomfile] \n";
         return -1;
     }
+    js["symmetric"] =  true; //overwrite symmetric parameter
     const feltor::Parameters p( js);
     const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
@@ -102,12 +104,90 @@ int main( int argc, char* argv[])
     feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     std::cout << "Done!\n";
 
+    bool fixed_profile;
+    HVec profile = dg::evaluate( dg::zero, grid);
+    HVec source_profile;
+    try{
+        source_profile = feltor::source_profiles.at(p.source_type)(
+        fixed_profile, profile, grid, p, gp, mag);
+    }catch ( std::out_of_range& error){
+        std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
+        return -1;
+    }
+
+    feltor.set_source( fixed_profile, dg::construct<DVec>(profile), p.omega_source, dg::construct<DVec>(source_profile));
+
     DVec result = dg::evaluate( dg::zero, grid);
     std::array<std::array<DVec,2>,2> y0;
     y0[0][0] = y0[0][1] =y0[1][0] =y0[1][1] = result;
     std::array<std::array<DVec,2>,2> b(y0);
     DVec weights = dg::create::weights( grid);
-    solve_steady_state( feltor, im, y0, b, weights);
+
+
+    /////////////////////////////////////////////////
+    while(true)
+    {
+
+    /////////////////////////set up transfer for glfw
+    dg::DVec dvisual( grid.size(), 0.);
+    dg::HVec hvisual( grid.size(), 0.), visual(hvisual), avisual(hvisual);
+    dg::IHMatrix equi = dg::create::backscatter( grid);
+    draw::ColorMapRedBlueExtMinMax colors(-1.0, 1.0);
+    std::map<std::string, const dg::DVec* > v4d;
+    v4d["ne-1 / "] = &y0[0][0],               v4d["ni-1 / "] = &y0[0][1];
+    v4d["Ue / "]   = &feltor.fields()[1][0],  v4d["Ui / "]   = &feltor.fields()[1][1];
+    v4d["Ome / "] = &feltor.potential(0); v4d["Apar / "] = &feltor.induction();
+    /////////glfw initialisation ////////////////////////////////////////////
+    //
+    std::stringstream title;
+    std::ifstream is( "window_params.js");
+    is >> js;
+    is.close();
+    unsigned red = js.get("reduction", 1).asUInt();
+    double rows = js["rows"].asDouble(), cols = p.Nz/red,
+           width = js["width"].asDouble(), height = js["height"].asDouble();
+    if ( p.symmetric ) cols = rows, rows = 1;
+    GLFWwindow* w = draw::glfwInitAndCreateWindow( cols*width, rows*height, "");
+    draw::RenderHostData render(rows, cols);
+
+    std::cout << "Begin computation \n";
+    std::cout << std::scientific << std::setprecision( 2);
+    title << std::setprecision(2) << std::scientific;
+    while ( !glfwWindowShouldClose( w ))
+    {
+        title << std::scientific;
+        solve_steady_state( feltor, im, y0, b, weights, p.dt);
+        for( auto pair : v4d)
+        {
+            if(pair.first == "Ome / ")
+            {
+                dg::assign( feltor.lapMperpP(0), hvisual);
+                dg::assign( *pair.second, hvisual);
+            }
+            else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
+            {
+                dg::assign( *pair.second, hvisual);
+                dg::blas1::axpby( 1., hvisual, -1., profile, hvisual);
+            }
+            else
+                dg::assign( *pair.second, hvisual);
+            dg::blas2::gemv( equi, hvisual, visual);
+            colors.scalemax() = (double)thrust::reduce(
+                visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
+            colors.scalemin() = -colors.scalemax();
+            title <<pair.first << colors.scalemax()<<"   ";
+            render.renderQuad( hvisual, grid.n()*grid.Nx(),
+                                        grid.n()*grid.Ny(), colors);
+        }
+        glfwSetWindowTitle(w,title.str().c_str());
+        title.str("");
+        glfwPollEvents();
+        glfwSwapBuffers( w);
+
+    }
+    }
+    glfwTerminate();
+    ////////////////////////////////////////////////////////////////////
 
     ////////////////////////////////////////////////////////////////////
     return 0;
-- 
GitLab


From c79809dfb6d1431cb4a01efad1177af0608a5237 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 15 Nov 2019 13:15:11 +0100
Subject: [PATCH 163/540] Add source profile in geometry_diag

---
 inc/geometries/geometry_diag.cu | 11 ++++++++---
 src/feltor/init.h               |  3 ++-
 src/feltor/input/default.json   |  5 +++--
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index f70d7ce2e..25fe48a5a 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -28,7 +28,7 @@ struct Parameters
     double boxscaleZm, boxscaleZp;
     double amp, k_psi, bgprofamp, nprofileamp;
     double sigma, posX, posY;
-    double rho_damping, alpha_mag;
+    double rho_damping, alpha, alpha_mag, rho_source;
     Parameters( const Json::Value& js){
         n = js.get("n",3).asUInt();
         Nx = js.get("Nx",100).asUInt();
@@ -48,6 +48,8 @@ struct Parameters
         posY = js.get("posY", 0.5).asDouble();
         rho_damping = js.get("rho_damping", 1.2).asDouble();
         alpha_mag = js.get("alpha_mag", 0.05).asDouble();
+        alpha = js.get("alpha", 0.05).asDouble();
+        rho_source = js.get("rho_source", 0.2).asDouble();
     }
     void display( std::ostream& os = std::cout ) const
     {
@@ -251,12 +253,15 @@ int main( int argc, char* argv[])
         //////////////////////////////////
         {"Iris", "A flux aligned Heaviside", dg::geo::Iris(mag.psip(), gp.psipmin, gp.psipmax)},
         {"Pupil", "A flux aligned Heaviside", dg::geo::Pupil(mag.psip(), gp.psipmaxcut)},
-        {"GaussianDamping", "A flux aligned Heaviside with Gaussian damping", dg::geo::GaussianDamping(mag.psip(), gp.psipmaxcut, gp.alpha)},
+        {"GaussianDamping", "A flux aligned Heaviside with Gaussian damping", dg::geo::GaussianDamping(mag.psip(), gp.psipmaxcut, p.alpha)},
         {"ZonalFlow",  "Flux aligned zonal flows", dg::geo::ZonalFlow(mag.psip(), p.amp, 0., 2.*M_PI*p.k_psi )},
         {"PsiLimiter", "A flux aligned Heaviside", dg::geo::PsiLimiter(mag.psip(), gp.psipmaxlim)},
+        {"SourceProfile", "A source profile", dg::geo::Compose<dg::PolynomialHeaviside>(
+            dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
+            p.rho_source, p.alpha, ((psip0>0)-(psip0<0)) ) },
         {"Nprofile", "A flux aligned profile", dg::geo::Nprofile(mag.psip(), p.nprofileamp/mag.psip()(mag.R0(),0.), p.bgprofamp )},
         {"Delta", "A flux aligned delta function", dg::geo::DeltaFunction( mag, gp.alpha*gp.alpha, psip0*0.2)},
-        {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::geo::TanhDamping(mag.psip(), -3*gp.alpha, gp.alpha, -1)},
+        {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::geo::TanhDamping(mag.psip(), -3*p.alpha, p.alpha, -1)},
         ////
         {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
         {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 9079f5af3..8e7699a85 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -95,11 +95,12 @@ HVec source_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
+    double psip0 = mag.psip()( mag.R0(), 0);
     HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
         //first change coordinate from psi to (psi_0 - psip)/psi_0
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
         //then shift
-        p.rho_source, p.alpha, -1), grid);
+        p.rho_source, p.alpha, ((psip0>0)-(psip0<0)), grid);
     dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), source_damping, source_damping);
     return source_damping;
 }
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index f2ec6b774..c0e86a0c2 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -39,11 +39,12 @@
     "boxscaleR" :  [1.15,1.2],
     "boxscaleZ" :  [1.2,1.15],
     "initne"     : "turbulence",
-    "initphi"    : "balance",
-    "curvmode"   : "true",
+    "initphi"    : "zero",
+    "curvmode"   : "toroidal",
     "symmetric"  : false,
     "alpha_mag"  : 0.05,
     "alpha"      : 0.2,
+    "source_type": "influx",
     "source"     : 0,
     "rho_source" : 0.2,
     "rho_damping" : 1.2
-- 
GitLab


From 291c3e91e617ebef44497bfb4fa7202367de2c55 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 15 Nov 2019 13:33:01 +0100
Subject: [PATCH 164/540] Corrected bug in init.h

---
 src/feltor/init.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/feltor/init.h b/src/feltor/init.h
index 8e7699a85..69151a611 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -100,7 +100,7 @@ HVec source_damping(const Geometry& grid,
         //first change coordinate from psi to (psi_0 - psip)/psi_0
         dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
         //then shift
-        p.rho_source, p.alpha, ((psip0>0)-(psip0<0)), grid);
+        p.rho_source, p.alpha, ((psip0>0)-(psip0<0)) ), grid);
     dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), source_damping, source_damping);
     return source_damping;
 }
-- 
GitLab


From b82570235e70c3e3682c8a24e3069fbdb34a129a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 15 Nov 2019 22:19:44 +0100
Subject: [PATCH 165/540] HOTFIX nan in dot function produces segfault

---
 inc/dg/backend/exblas/accumulate.cuh | 2 +-
 inc/dg/backend/exblas/accumulate.h   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/dg/backend/exblas/accumulate.cuh b/inc/dg/backend/exblas/accumulate.cuh
index 86d8d0ffd..1f6f867d8 100644
--- a/inc/dg/backend/exblas/accumulate.cuh
+++ b/inc/dg/backend/exblas/accumulate.cuh
@@ -104,7 +104,7 @@ static inline void Accumulate( int64_t* accumulator, double x, int stride = 1) {
     double xscaled = ldexp(x, -DIGITS * exp_word);
 
     int i;
-    for (i = iup; xscaled != 0; --i) {
+    for (i = iup; xscaled != 0 && i>=0; --i) {
         double xrounded = rint(xscaled);
         int64_t xint = (int64_t) xrounded;
         AccumulateWord(accumulator, i, xint, stride);
diff --git a/inc/dg/backend/exblas/accumulate.h b/inc/dg/backend/exblas/accumulate.h
index 6bf1a9252..4a745f60d 100644
--- a/inc/dg/backend/exblas/accumulate.h
+++ b/inc/dg/backend/exblas/accumulate.h
@@ -130,7 +130,7 @@ static inline void Accumulate( int64_t* accumulator, double x) {
     double xscaled = cpu::myldexp(x, -DIGITS * exp_word);
 
     int i;
-    for (i = iup; xscaled != 0; --i) {
+    for (i = iup; xscaled != 0 && i>=0; --i) {
         double xrounded = cpu::myrint(xscaled);
         int64_t xint = cpu::myllrint(xscaled);
         AccumulateWord(accumulator, i, xint);
-- 
GitLab


From d439c7a2f466a89bd42432a4b5bd435cbe4a5c20 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 16 Nov 2019 12:26:12 +0100
Subject: [PATCH 166/540] FIX Implicit solver copies right hand side

- and now use Anderson implicit step for feltor
---
 inc/dg/backend/exblas/accumulate.cuh |  1 +
 inc/dg/backend/exblas/accumulate.h   |  1 +
 inc/dg/backend/predicate.h           |  1 +
 inc/dg/blas1.h                       |  2 +-
 inc/dg/blas2.h                       |  2 +-
 inc/dg/implicit.h                    | 11 ++---
 inc/dg/multistep.h                   |  1 +
 src/feltor/feltor.cu                 | 36 ++++++++++------
 src/feltor/feltor.h                  | 64 ++++++++++++++++++++++++++--
 src/feltor/feltor_hpc.cu             | 29 ++++++++-----
 src/feltor/implicit.h                | 21 +--------
 11 files changed, 117 insertions(+), 52 deletions(-)

diff --git a/inc/dg/backend/exblas/accumulate.cuh b/inc/dg/backend/exblas/accumulate.cuh
index 1f6f867d8..f6f91e578 100644
--- a/inc/dg/backend/exblas/accumulate.cuh
+++ b/inc/dg/backend/exblas/accumulate.cuh
@@ -95,6 +95,7 @@ __device__
 static inline void Accumulate( int64_t* accumulator, double x, int stride = 1) { //transposed accumulation
     if (x == 0)
         return;
+    assert( !std::isnan(x) && "Detected NaN in dot product!!");
 
     int e;
     frexp(x, &e); //extract the exponent of x (lies in -1024;1023 ?)
diff --git a/inc/dg/backend/exblas/accumulate.h b/inc/dg/backend/exblas/accumulate.h
index 4a745f60d..44ccaa875 100644
--- a/inc/dg/backend/exblas/accumulate.h
+++ b/inc/dg/backend/exblas/accumulate.h
@@ -121,6 +121,7 @@ static inline void AccumulateWord( int64_t *accumulator, int i, int64_t x) {
 static inline void Accumulate( int64_t* accumulator, double x) {
     if (x == 0)
         return;
+    assert( !std::isnan(x) && "Detected NaN in dot product!!");
 
 
     int e = cpu::exponent(x);
diff --git a/inc/dg/backend/predicate.h b/inc/dg/backend/predicate.h
index fd94029b7..650ddbccf 100644
--- a/inc/dg/backend/predicate.h
+++ b/inc/dg/backend/predicate.h
@@ -92,6 +92,7 @@ template < bool...> struct bool_pack;
 template<bool... v>
 using all_true = std::is_same<bool_pack<true,v...>, bool_pack<v..., true>>;
 
+template< typename ...> struct WhichType; //!< This is a utility class to get type information at compile time for debugging purposes Use like @code dg::WhichType<T>{};@endcode
 
 
 }//namespace dg
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 8ab2148f5..91749e05a 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -64,7 +64,7 @@ double result = dg::blas1::dot( two, three); // result = 600 (100*(2*3))
  * @param y Right Container may alias x
  * @return Scalar product as defined above
  * @note This routine is always executed synchronously due to the
-        implicit memcpy of the result. With mpi the result is broadcasted to all processes
+        implicit memcpy of the result. With mpi the result is broadcasted to all processes. Also note that the behaviour is undefined when one of the containers contains \c nan
  * @copydoc hide_ContainerType
  */
 template< class ContainerType1, class ContainerType2>
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index 80ccf303c..38c8ce8b4 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -71,7 +71,7 @@ inline std::vector<int64_t> doDot_superacc( const ContainerType1& x, const Matri
  * @param y Right input (may alias \c x)
  * @return Generalized scalar product. If \c x and \c y are vectors of containers and \c m is not, then we sum the results of \c dg::blas2::dot( x[i], m, y[i])
  * @note This routine is always executed synchronously due to the
-    implicit memcpy of the result.
+    implicit memcpy of the result. With mpi the result is broadcasted to all processes. Also note that the behaviour is undefined when one of the containers contains \c nan
  * @tparam MatrixType \c MatrixType has to have a category derived from \c AnyVectorTag and must be compatible with the \c ContainerTypes
  * @copydoc hide_ContainerType
  */
diff --git a/inc/dg/implicit.h b/inc/dg/implicit.h
index 9021753ec..2154e73ae 100644
--- a/inc/dg/implicit.h
+++ b/inc/dg/implicit.h
@@ -93,8 +93,8 @@ struct DefaultSolver
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_rhs;}
 
-    template< class Implicit> //going to be a reference type
-    void solve( value_type alpha, Implicit im, value_type t, ContainerType& y, const ContainerType& rhs)
+    template< class Implicit>
+    void solve( value_type alpha, Implicit& im, value_type t, ContainerType& y, const ContainerType& rhs)
     {
         detail::Implicit<Implicit, ContainerType> implicit( alpha, t, im);
         blas2::symv( im.weights(), rhs, m_rhs);
@@ -154,7 +154,7 @@ struct FixedPointSolver
     const ContainerType& copyable()const{ return m_current;}
 
     template< class Implicit>
-    void solve( value_type alpha, Implicit im, value_type t, ContainerType& y, const ContainerType& rhs)
+    void solve( value_type alpha, Implicit& im, value_type t, ContainerType& y, const ContainerType& rhs)
     {
 #ifdef DG_BENCHMARK
 #ifdef MPI_VERSION
@@ -223,9 +223,10 @@ struct AndersonSolver
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_acc.copyable();}
 
-    template< class Implicit> //going to be a reference type
-    void solve( value_type alpha, Implicit im, value_type t, ContainerType& y, const ContainerType& rhs)
+    template< class Implicit>
+    void solve( value_type alpha, Implicit& im, value_type t, ContainerType& y, const ContainerType& rhs)
     {
+        //dg::WhichType<Implicit> {};
         detail::Implicit<Implicit, ContainerType> implicit( alpha, t, im);
 #ifdef DG_BENCHMARK
 #ifdef MPI_VERSION
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index b1d81524b..07120811a 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -418,6 +418,7 @@ template< class ContainerType, class SolverType>
 template<class RHS>
 void BDF<ContainerType, SolverType>::step(RHS& rhs, value_type& t, container_type& u)
 {
+    //dg::WhichType<RHS> {};
     dg::blas1::axpby( m_bdf[0], m_u[0], 0., m_f);
     for (unsigned i = 1; i < m_k; i++){
         dg::blas1::axpby( m_bdf[i], m_u[i], 1., m_f);
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 9ad2d69c3..3926eca52 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -61,10 +61,12 @@ int main( int argc, char* argv[])
         mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
     //create RHS
-    std::cout << "Constructing Explicit...\n";
-    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
-    std::cout << "Constructing Implicit...\n";
-    feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
+    std::cout << "Constructing RHS...\n";
+    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, true);
+    //std::cout << "Constructing Explicit...\n";
+    //feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
+    //std::cout << "Constructing Implicit...\n";
+    //feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     std::cout << "Done!\n";
 
     DVec result = dg::evaluate( dg::zero, grid);
@@ -107,17 +109,26 @@ int main( int argc, char* argv[])
     //
     dg::Timer t;
     unsigned step = 0;
-    dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
-        feltor::FeltorSpecialSolver<
-            Geometry, IDMatrix, DMatrix, DVec>
-        > karniadakis( grid, p, mag);
+    //dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
+    //    feltor::FeltorSpecialSolver<
+    //        Geometry, IDMatrix, DMatrix, DVec>
+    //    > karniadakis( grid, p, mag);
+    unsigned mMax = 3, restart = 3, max_iter = 100;
+    double damping = 1e-3;
+    dg::BDF< std::array<std::array<dg::DVec,2>,2 >,
+        dg::AndersonSolver< std::array<std::array<dg::DVec,2>,2> >
+        > bdf( 3, y0, mMax, p.rtol, max_iter, damping, restart);
+    //dg::AdamsBashforth< std::array<std::array<dg::DVec,2>,2 >
+    //    > bdf( 3, y0);
+
     std::cout << "Initialize Timestepper" << std::endl;
-    karniadakis.init( feltor, im, time, y0, p.dt);
+    //karniadakis.init( feltor, im, time, y0, p.dt);
+    bdf.init( feltor, time, y0, p.dt);
     std::cout << "Done!" << std::endl;
 
     std::map<std::string, const dg::DVec* > v4d;
-    v4d["ne-1 / "] = &y0[0][0],               v4d["ni-1 / "] = &y0[0][1];
-    v4d["Ue / "]   = &feltor.fields()[1][0],  v4d["Ui / "]   = &feltor.fields()[1][1];
+    v4d["ne-1 / "] = &y0[0][0],  v4d["ni-1 / "] = &y0[0][1];
+    v4d["Ue / "]   = &feltor.velocity(0), v4d["Ui / "]   = &feltor.velocity(1);
     v4d["Ome / "] = &feltor.potential(0); v4d["Apar / "] = &feltor.induction();
     double dEdt = 0, accuracy = 0;
     double E0 = 0.;
@@ -199,7 +210,8 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    karniadakis.step( feltor, im, time, y0);
+                    //karniadakis.step( feltor, im, time, y0);
+                    bdf.step( feltor, time, y0);
                 }
                 catch( dg::Fail& fail) {
                     std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index b5c7749ce..a36e71c0d 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -156,14 +156,32 @@ struct ComputeSource{
         result = temp;
     }
 };
+//Resistivity (consistent density dependency,
+//parallel momentum conserving, quadratic current energy conservation dependency)
+struct AddResistivity{
+    AddResistivity( double eta, std::array<double,2> mu): m_eta(eta){
+        m_mu[0] = mu[0], m_mu[1] = mu[1];
+    }
+    DG_DEVICE
+    void operator()( double ne, double ni, double ue,
+        double ui, double& dtUe, double& dtUi) const{
+        double current = (ne)*(ui-ue);
+        dtUe += -m_eta/m_mu[0] * current;
+        dtUi += -m_eta/m_mu[1] * (ne)/(ni) * current;
+    }
+    private:
+    double m_eta;
+    double m_mu[2];
+};
 }//namespace routines
 
 template< class Geometry, class IMatrix, class Matrix, class Container >
 struct Explicit
 {
     using vector = std::array<std::array<Container,2>,2>;
+    using container = Container;
     Explicit( const Geometry& g, feltor::Parameters p,
-        dg::geo::TokamakMagneticField mag);
+        dg::geo::TokamakMagneticField mag, bool full_system );
 
     //Given N_i-1 initialize n_e-1 such that phi=0
     void initializene( const Container& ni, Container& ne);
@@ -253,6 +271,7 @@ struct Explicit
     const Container& binv( ) const { return m_binv; }
     const Container& divb( ) const { return m_divb; }
     const Container& vol3d() const { return m_vol3d;}
+    const Container& weights() const { return m_lapperpN.weights();}
     //bhat / sqrt{g} / B
     const std::array<Container, 3> & bhatgB () const {
         return m_b;
@@ -363,6 +382,7 @@ struct Explicit
     const feltor::Parameters m_p;
     double m_omega_source = 0.;
     bool m_fixed_profile = true;
+    bool m_full_system = false;
 
 };
 
@@ -370,6 +390,8 @@ template<class Grid, class IMatrix, class Matrix, class Container>
 void Explicit<Grid, IMatrix, Matrix, Container>::construct_mag(
     const Grid& g, feltor::Parameters p, dg::geo::TokamakMagneticField mag)
 {
+    if( !(p.perp_diff == "viscous" || p.perp_diff == "hyperviscous") )
+        throw dg::Error(dg::Message(_ping_)<<"Warning! perp_diff value '"<<p.perp_diff<<"' not recognized!! I do not know how to proceed! Exit now!");
     //due to the various approximations bhat and mag not always correspond
     dg::geo::CylindricalVectorLvl0 curvNabla, curvKappa;
     if( p.curvmode == "true" )
@@ -507,7 +529,7 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
 Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
-    feltor::Parameters p, dg::geo::TokamakMagneticField mag):
+    feltor::Parameters p, dg::geo::TokamakMagneticField mag, bool full_system):
 #ifdef DG_MANUFACTURED
     m_R( dg::pullback( dg::cooX3d, g)),
     m_Z( dg::pullback( dg::cooY3d, g)),
@@ -523,7 +545,7 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     m_multigrid( g, p.stages),
     m_old_phi( 2, dg::evaluate( dg::zero, g)),
     m_old_psi( m_old_phi), m_old_gammaN( m_old_phi), m_old_apar( m_old_phi),
-    m_p(p)
+    m_p(p), m_full_system(full_system)
 {
     //--------------------------init vectors to 0-----------------//
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
@@ -880,6 +902,42 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         dg::blas1::axpby( 1., m_s[0][1], 1.0, yp[0][1]);
         //currently we ignore m_s[1]
     }
+    if( m_full_system)
+    {
+#if FELTORPERP == 1
+        /* y[0] := n_e - 1
+           y[1] := N_i - 1
+        */
+        for( unsigned i=0; i<2; i++)
+        {
+            if( m_p.perp_diff == "hyperviscous")
+            {
+                dg::blas2::symv( m_lapperpN, y[0][i], m_temp0);
+                dg::blas2::symv( -m_p.nu_perp, m_lapperpN, m_temp0, 1., yp[0][i]);
+            }
+            else // m_p.perp_diff == "viscous"
+                dg::blas2::symv( -m_p.nu_perp, m_lapperpN, y[0][i],  1., yp[0][i]);
+        }
+        /* fields[1][0] := u_e
+           fields[1][1] := U_i
+        */
+        for( unsigned i=0; i<2; i++)
+        {
+            if( m_p.perp_diff == "hyperviscous")
+            {
+                dg::blas2::symv( m_lapperpU, m_fields[1][i], m_temp0);
+                dg::blas2::symv( -m_p.nu_perp, m_lapperpU, m_temp0, 1., yp[1][i]);
+            }
+            else // m_p.perp_diff == "viscous"
+                dg::blas2::symv( -m_p.nu_perp, m_lapperpU,
+                    m_fields[1][i],  1., yp[1][i]);
+        }
+#endif
+        //------------------Add Resistivity--------------------------//
+        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+            m_fields[0][0], m_fields[0][1],
+            m_fields[1][0], m_fields[1][1], yp[1][0], yp[1][1]);
+    }
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( yp[0][0], dg::plus_equals(), manufactured::SNe{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 1b40ce68e..07cbdc69b 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -153,10 +153,12 @@ int main( int argc, char* argv[])
         mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
     //create RHS
-    MPI_OUT std::cout << "Constructing Explicit...\n";
-    feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
-    MPI_OUT std::cout << "Constructing Implicit...\n";
-    feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
+    MPI_OUT std::cout << "Constructing RHS...\n";
+    feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, true);
+    //MPI_OUT std::cout << "Constructing Explicit...\n";
+    //feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
+    //MPI_OUT std::cout << "Constructing Implicit...\n";
+    //feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
     // helper variables for output computations
@@ -385,11 +387,17 @@ int main( int argc, char* argv[])
     MPI_OUT err = nc_close(ncid);
     MPI_OUT std::cout << "First write successful!\n";
     ///////////////////////////////////////Timeloop/////////////////////////////////
-    dg::Karniadakis< std::array<std::array<DVec,2>,2 >,
-        feltor::FeltorSpecialSolver<
-            Geometry, IDMatrix, DMatrix, DVec>
-        > karniadakis( grid, p, mag);
-    karniadakis.init( feltor, im, time, y0, p.dt);
+    //dg::Karniadakis< std::array<std::array<DVec,2>,2 >,
+    //    feltor::FeltorSpecialSolver<
+    //        Geometry, IDMatrix, DMatrix, DVec>
+    //    > karniadakis( grid, p, mag);
+    //karniadakis.init( feltor, im, time, y0, p.dt);
+    unsigned mMax = 3, restart = 3, max_iter = 100;
+    double damping = 1e-3;
+    dg::BDF< std::array<std::array<dg::DVec,2>,2 >,
+        dg::AndersonSolver< std::array<std::array<dg::DVec,2>,2> >
+        > bdf( 3, y0, mMax, p.rtol, max_iter, damping, restart);
+    bdf.init( feltor, time, y0, p.dt);
     dg::Timer t;
     t.tic();
     unsigned step = 0;
@@ -404,7 +412,8 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    karniadakis.step( feltor, im, time, y0);
+                    //karniadakis.step( feltor, im, time, y0);
+                    bdf.step( feltor, time, y0);
                 }
                 catch( dg::Fail& fail) {
                     MPI_OUT std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 691ee3fa4..3d2a971c9 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -3,32 +3,13 @@
 #include "dg/algorithm.h"
 #include "parameters.h"
 #include "dg/geometries/geometries.h"
+#include "feltor.h"
 
 //This contains the implicit part of the feltor equations
 //We even write a custom solver object for it
 
 namespace feltor
 {
-namespace routines
-{
-//Resistivity (consistent density dependency,
-//parallel momentum conserving, quadratic current energy conservation dependency)
-struct AddResistivity{
-    AddResistivity( double eta, std::array<double,2> mu): m_eta(eta){
-        m_mu[0] = mu[0], m_mu[1] = mu[1];
-    }
-    DG_DEVICE
-    void operator()( double ne, double ni, double ue,
-        double ui, double& dtUe, double& dtUi) const{
-        double current = (ne)*(ui-ue);
-        dtUe += -m_eta/m_mu[0] * current;
-        dtUi += -m_eta/m_mu[1] * (ne)/(ni) * current;
-    }
-    private:
-    double m_eta;
-    double m_mu[2];
-};
-}//namespace routines
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
 struct ImplicitDensity
-- 
GitLab


From 3734146d740cbb6b7b8507a62409c8a332e83e8a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 16 Nov 2019 12:35:17 +0100
Subject: [PATCH 167/540] Fixed dg::DVec issue

---
 src/feltor/feltor_hpc.cu | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 07cbdc69b..645e57075 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -394,8 +394,8 @@ int main( int argc, char* argv[])
     //karniadakis.init( feltor, im, time, y0, p.dt);
     unsigned mMax = 3, restart = 3, max_iter = 100;
     double damping = 1e-3;
-    dg::BDF< std::array<std::array<dg::DVec,2>,2 >,
-        dg::AndersonSolver< std::array<std::array<dg::DVec,2>,2> >
+    dg::BDF< std::array<std::array<DVec,2>,2 >,
+        dg::AndersonSolver< std::array<std::array<DVec,2>,2> >
         > bdf( 3, y0, mMax, p.rtol, max_iter, damping, restart);
     bdf.init( feltor, time, y0, p.dt);
     dg::Timer t;
-- 
GitLab


From 7480c763620325ea06bad6039a40924cfecfaa7b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 16 Nov 2019 16:01:53 +0100
Subject: [PATCH 168/540] Revert back to karniadakis

---
 inc/geometries/geometry_diag.cu |  2 ++
 src/feltor/feltor.cu            | 38 ++++++++++++++++-----------------
 src/feltor/feltor_hpc.cu        | 38 ++++++++++++++++-----------------
 3 files changed, 40 insertions(+), 38 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 25fe48a5a..e4df387b4 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -92,6 +92,8 @@ struct Hoo : public dg::geo::aCylindricalFunctor<Hoo>
     {
         double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
         double psip2 = psipR*psipR+psipZ*psipZ;
+        if( psip2 == 0)
+            psip2 = 1e-16;
         return (ipol*ipol + psip2)/R/R/psip2;
     }
     private:
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 3926eca52..acfb04ef0 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -61,12 +61,12 @@ int main( int argc, char* argv[])
         mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
     //create RHS
-    std::cout << "Constructing RHS...\n";
-    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, true);
-    //std::cout << "Constructing Explicit...\n";
-    //feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
-    //std::cout << "Constructing Implicit...\n";
-    //feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
+    //std::cout << "Constructing RHS...\n";
+    //feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, true);
+    std::cout << "Constructing Explicit...\n";
+    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, false);
+    std::cout << "Constructing Implicit...\n";
+    feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     std::cout << "Done!\n";
 
     DVec result = dg::evaluate( dg::zero, grid);
@@ -109,21 +109,21 @@ int main( int argc, char* argv[])
     //
     dg::Timer t;
     unsigned step = 0;
-    //dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
-    //    feltor::FeltorSpecialSolver<
-    //        Geometry, IDMatrix, DMatrix, DVec>
-    //    > karniadakis( grid, p, mag);
-    unsigned mMax = 3, restart = 3, max_iter = 100;
-    double damping = 1e-3;
-    dg::BDF< std::array<std::array<dg::DVec,2>,2 >,
-        dg::AndersonSolver< std::array<std::array<dg::DVec,2>,2> >
-        > bdf( 3, y0, mMax, p.rtol, max_iter, damping, restart);
+    dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
+        feltor::FeltorSpecialSolver<
+            Geometry, IDMatrix, DMatrix, DVec>
+        > karniadakis( grid, p, mag);
+    //unsigned mMax = 3, restart = 3, max_iter = 100;
+    //double damping = 1e-3;
+    //dg::BDF< std::array<std::array<dg::DVec,2>,2 >,
+    //    dg::AndersonSolver< std::array<std::array<dg::DVec,2>,2> >
+    //    > bdf( 3, y0, mMax, p.rtol, max_iter, damping, restart);
     //dg::AdamsBashforth< std::array<std::array<dg::DVec,2>,2 >
     //    > bdf( 3, y0);
 
     std::cout << "Initialize Timestepper" << std::endl;
-    //karniadakis.init( feltor, im, time, y0, p.dt);
-    bdf.init( feltor, time, y0, p.dt);
+    karniadakis.init( feltor, im, time, y0, p.dt);
+    //bdf.init( feltor, time, y0, p.dt);
     std::cout << "Done!" << std::endl;
 
     std::map<std::string, const dg::DVec* > v4d;
@@ -210,8 +210,8 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    //karniadakis.step( feltor, im, time, y0);
-                    bdf.step( feltor, time, y0);
+                    karniadakis.step( feltor, im, time, y0);
+                    //bdf.step( feltor, time, y0);
                 }
                 catch( dg::Fail& fail) {
                     std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 645e57075..9a8697a2a 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -153,12 +153,12 @@ int main( int argc, char* argv[])
         mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
 
     //create RHS
-    MPI_OUT std::cout << "Constructing RHS...\n";
-    feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, true);
-    //MPI_OUT std::cout << "Constructing Explicit...\n";
-    //feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
-    //MPI_OUT std::cout << "Constructing Implicit...\n";
-    //feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
+    //MPI_OUT std::cout << "Constructing RHS...\n";
+    //feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, true);
+    MPI_OUT std::cout << "Constructing Explicit...\n";
+    feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, false);
+    MPI_OUT std::cout << "Constructing Implicit...\n";
+    feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
     // helper variables for output computations
@@ -387,17 +387,17 @@ int main( int argc, char* argv[])
     MPI_OUT err = nc_close(ncid);
     MPI_OUT std::cout << "First write successful!\n";
     ///////////////////////////////////////Timeloop/////////////////////////////////
-    //dg::Karniadakis< std::array<std::array<DVec,2>,2 >,
-    //    feltor::FeltorSpecialSolver<
-    //        Geometry, IDMatrix, DMatrix, DVec>
-    //    > karniadakis( grid, p, mag);
-    //karniadakis.init( feltor, im, time, y0, p.dt);
-    unsigned mMax = 3, restart = 3, max_iter = 100;
-    double damping = 1e-3;
-    dg::BDF< std::array<std::array<DVec,2>,2 >,
-        dg::AndersonSolver< std::array<std::array<DVec,2>,2> >
-        > bdf( 3, y0, mMax, p.rtol, max_iter, damping, restart);
-    bdf.init( feltor, time, y0, p.dt);
+    dg::Karniadakis< std::array<std::array<DVec,2>,2 >,
+        feltor::FeltorSpecialSolver<
+            Geometry, IDMatrix, DMatrix, DVec>
+        > karniadakis( grid, p, mag);
+    karniadakis.init( feltor, im, time, y0, p.dt);
+    //unsigned mMax = 3, restart = 3, max_iter = 100;
+    //double damping = 1e-3;
+    //dg::BDF< std::array<std::array<DVec,2>,2 >,
+    //    dg::AndersonSolver< std::array<std::array<DVec,2>,2> >
+    //    > bdf( 3, y0, mMax, p.rtol, max_iter, damping, restart);
+    //bdf.init( feltor, time, y0, p.dt);
     dg::Timer t;
     t.tic();
     unsigned step = 0;
@@ -412,8 +412,8 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    //karniadakis.step( feltor, im, time, y0);
-                    bdf.step( feltor, time, y0);
+                    karniadakis.step( feltor, im, time, y0);
+                    //bdf.step( feltor, time, y0);
                 }
                 catch( dg::Fail& fail) {
                     MPI_OUT std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-- 
GitLab


From e5e36d2d9881439820e23c69dc2cd0b2f9bd52d9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 17 Nov 2019 14:20:18 +0100
Subject: [PATCH 169/540] Make dot products capture NaN

---
 inc/dg/backend/average_cpu.h           |  7 +++--
 inc/dg/backend/average_gpu.cuh         |  7 +++--
 inc/dg/backend/average_omp.h           | 10 +++++--
 inc/dg/backend/blas1_array.h           |  5 +++-
 inc/dg/backend/blas1_cuda.cuh          | 11 +++++--
 inc/dg/backend/blas1_dispatch_scalar.h |  6 +++-
 inc/dg/backend/blas1_omp.h             | 14 ++++++---
 inc/dg/backend/blas1_serial.h          | 11 +++++--
 inc/dg/backend/blas2_dispatch_scalar.h |  5 +++-
 inc/dg/backend/exblas/accumulate.cuh   |  5 ++--
 inc/dg/backend/exblas/accumulate.h     |  4 +--
 inc/dg/backend/exblas/exdot_cuda.cuh   | 30 ++++++++++++++-----
 inc/dg/backend/exblas/exdot_omp.h      | 40 ++++++++++++++++++++------
 inc/dg/backend/exblas/exdot_serial.h   | 32 +++++++++++++++------
 inc/dg/backend/exblas/mylibm.hpp       |  1 +
 inc/dg/topology/evaluation_t.cu        |  5 ++++
 16 files changed, 149 insertions(+), 44 deletions(-)

diff --git a/inc/dg/backend/average_cpu.h b/inc/dg/backend/average_cpu.h
index 3420eaaaf..e95e04329 100644
--- a/inc/dg/backend/average_cpu.h
+++ b/inc/dg/backend/average_cpu.h
@@ -35,10 +35,13 @@ void average( SerialTag, unsigned nx, unsigned ny, const value_type* in0, const
     static_assert( std::is_same<value_type, double>::value, "Value type must be double!");
     static thrust::host_vector<int64_t> h_accumulator;
     h_accumulator.resize( ny*exblas::BIN_COUNT);
+    int status;
     for( unsigned i=0; i<ny; i++)
-        exblas::exdot_cpu(nx, &in0[i*nx], &in1[i*nx], &h_accumulator[i*exblas::BIN_COUNT]);
+        exblas::exdot_cpu(nx, &in0[i*nx], &in1[i*nx], &h_accumulator[i*exblas::BIN_COUNT], &status);
     for( unsigned i=0; i<ny; i++)
-        out[i] = exblas::cpu::Round( &h_accumulator[i*exblas::BIN_COUNT]);
+        out[i] = exblas::cpu::Round( &h_accumulator[i*exblas::BIN_COUNT], *status);
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"CPU Average failed since one of the inputs contains NaN or Inf");
 }
 
 #ifdef MPI_VERSION
diff --git a/inc/dg/backend/average_gpu.cuh b/inc/dg/backend/average_gpu.cuh
index 2fec6b746..fe5d189eb 100644
--- a/inc/dg/backend/average_gpu.cuh
+++ b/inc/dg/backend/average_gpu.cuh
@@ -74,13 +74,16 @@ void average( CudaTag, unsigned nx, unsigned ny, const value_type* in0, const va
     static thrust::host_vector<value_type> h_round;
     d_accumulator.resize( ny*exblas::BIN_COUNT);
     int64_t* d_ptr = thrust::raw_pointer_cast( d_accumulator.data());
+    int status;
     for( unsigned i=0; i<ny; i++)
-        exblas::exdot_gpu(nx, &in0[i*nx], &in1[i*nx], &d_ptr[i*exblas::BIN_COUNT]);
+        exblas::exdot_gpu(nx, &in0[i*nx], &in1[i*nx], &d_ptr[i*exblas::BIN_COUNT], &status);
     h_accumulator = d_accumulator;
     h_round.resize( ny);
     for( unsigned i=0; i<ny; i++)
-        h_round[i] = exblas::cpu::Round( &h_accumulator[i*exblas::BIN_COUNT]);
+        h_round[i] = exblas::cpu::Round( &h_accumulator[i*exblas::BIN_COUNT], &status);
     cudaMemcpy( out, &h_round[0], ny*sizeof(value_type), cudaMemcpyHostToDevice);
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"GPU Average failed since one of the inputs contains NaN or Inf");
 }
 
 #ifdef MPI_VERSION
diff --git a/inc/dg/backend/average_omp.h b/inc/dg/backend/average_omp.h
index 2574c98b3..484d0ba31 100644
--- a/inc/dg/backend/average_omp.h
+++ b/inc/dg/backend/average_omp.h
@@ -40,8 +40,11 @@ void average( OmpTag, unsigned nx, unsigned ny, const value_type* in0, const val
     static_assert( std::is_same<value_type, double>::value, "Value type must be double!");
     static thrust::host_vector<int64_t> h_accumulator;
     h_accumulator.resize( ny*exblas::BIN_COUNT);
+    int status;
     for( unsigned i=0; i<ny; i++)
-        exblas::exdot_omp(nx, &in0[i*nx], &in1[i*nx], &h_accumulator[i*exblas::BIN_COUNT]);
+        exblas::exdot_omp(nx, &in0[i*nx], &in1[i*nx], &h_accumulator[i*exblas::BIN_COUNT], &status);
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"OMP Average failed since one of the inputs contains NaN or Inf");
     for( unsigned i=0; i<ny; i++)
         out[i] = exblas::cpu::Round( &h_accumulator[i*exblas::BIN_COUNT]);
 }
@@ -55,8 +58,11 @@ void average_mpi( OmpTag, unsigned nx, unsigned ny, const value_type* in0, const
     static thrust::host_vector<int64_t> h_accumulator;
     static thrust::host_vector<int64_t> h_accumulator2;
     h_accumulator2.resize( ny*exblas::BIN_COUNT);
+    int status;
     for( unsigned i=0; i<ny; i++)
-        exblas::exdot_omp(nx, &in0[i*nx], &in1[i*nx], &h_accumulator2[i*exblas::BIN_COUNT]);
+        exblas::exdot_omp(nx, &in0[i*nx], &in1[i*nx], &h_accumulator2[i*exblas::BIN_COUNT], &status);
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"MPI OMP Average failed since one of the inputs contains NaN or Inf");
     h_accumulator.resize( h_accumulator2.size());
     exblas::reduce_mpi_cpu( ny, &h_accumulator2[0], &h_accumulator[0], comm, comm_mod, comm_mod_reduce);
     for( unsigned i=0; i<ny; i++)
diff --git a/inc/dg/backend/blas1_array.h b/inc/dg/backend/blas1_array.h
index 48db4f82f..56f86b6f0 100644
--- a/inc/dg/backend/blas1_array.h
+++ b/inc/dg/backend/blas1_array.h
@@ -25,7 +25,10 @@ template< class T, std::size_t N>
 std::vector<int64_t> doDot_superacc( const std::array<T,N>& x, const std::array<T,N>& y, StdArrayTag)
 {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    exblas::exdot_cpu( N, x.begin(),y.begin(), &h_superacc[0]) ;
+    int status;
+    exblas::exdot_cpu( N, x.begin(),y.begin(), &h_superacc[0], &status) ;
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"CPU Dot failed since one of the inputs contains NaN or Inf");
     return h_superacc;
 }
 
diff --git a/inc/dg/backend/blas1_cuda.cuh b/inc/dg/backend/blas1_cuda.cuh
index 307f7a27c..be904c366 100644
--- a/inc/dg/backend/blas1_cuda.cuh
+++ b/inc/dg/backend/blas1_cuda.cuh
@@ -2,6 +2,7 @@
 #define _DG_BLAS_CUDA_
 #include <thrust/reduce.h>
 #include <thrust/system/cuda/execution_policy.h>
+#include "exceptions.h"
 #include "exblas/exdot_cuda.cuh"
 namespace dg
 {
@@ -14,7 +15,10 @@ template<class PointerOrValue1, class PointerOrValue2>
 inline std::vector<int64_t> doDot_dispatch( CudaTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr) {
     static thrust::device_vector<int64_t> d_superacc(exblas::BIN_COUNT);
     int64_t * d_ptr = thrust::raw_pointer_cast( d_superacc.data());
-    exblas::exdot_gpu( size, x_ptr,y_ptr, d_ptr);
+    int status;
+    exblas::exdot_gpu( size, x_ptr,y_ptr, d_ptr, &status );
+    if( status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"GPU Dot product failed since one of the inputs contains NaN or Inf");
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
     cudaMemcpy( &h_superacc[0], d_ptr, exblas::BIN_COUNT*sizeof(int64_t), cudaMemcpyDeviceToHost);
     return h_superacc;
@@ -23,7 +27,10 @@ template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3>
 inline std::vector<int64_t> doDot_dispatch( CudaTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr, PointerOrValue3 z_ptr) {
     static thrust::device_vector<int64_t> d_superacc(exblas::BIN_COUNT);
     int64_t * d_ptr = thrust::raw_pointer_cast( d_superacc.data());
-    exblas::exdot_gpu( size, x_ptr,y_ptr,z_ptr, d_ptr);
+    int status;
+    exblas::exdot_gpu( size, x_ptr,y_ptr,z_ptr, d_ptr, &status);
+    if( status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"GPU Dot product failed since one of the inputs contains NaN or Inf");
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
     cudaMemcpy( &h_superacc[0], d_ptr, exblas::BIN_COUNT*sizeof(int64_t), cudaMemcpyDeviceToHost);
     return h_superacc;
diff --git a/inc/dg/backend/blas1_dispatch_scalar.h b/inc/dg/backend/blas1_dispatch_scalar.h
index 6e5e11f8d..58db82c5c 100644
--- a/inc/dg/backend/blas1_dispatch_scalar.h
+++ b/inc/dg/backend/blas1_dispatch_scalar.h
@@ -25,7 +25,11 @@ std::vector<int64_t> doDot_superacc( const Vector1& x, const Vector2& y, AnyScal
     const get_value_type<Vector2>* y_ptr = &y;
     //since we only accumulate up to two values (multiplication and rest) reduce the size of the FPE
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    exblas::exdot_cpu<const get_value_type<Vector1>*, const get_value_type<Vector2>*, 2>( 1, x_ptr,y_ptr, &h_superacc[0]) ;
+    int status;
+    exblas::exdot_cpu<const get_value_type<Vector1>*, const get_value_type<Vector2>*, 2>( 1, x_ptr,y_ptr, &h_superacc[0], &status) ;
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"CPU Dot failed since one of the inputs contains NaN or Inf");
+
     return h_superacc;
 }
 
diff --git a/inc/dg/backend/blas1_omp.h b/inc/dg/backend/blas1_omp.h
index 262379933..18f4c3326 100644
--- a/inc/dg/backend/blas1_omp.h
+++ b/inc/dg/backend/blas1_omp.h
@@ -17,19 +17,25 @@ const int MIN_SIZE=100;//don't parallelize if work is too small
 template<class PointerOrValue1, class PointerOrValue2>
 inline std::vector<int64_t> doDot_dispatch( OmpTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr) {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
+    int status;
     if(size<MIN_SIZE)
-        exblas::exdot_cpu( size, x_ptr,y_ptr, &h_superacc[0]);
+        exblas::exdot_cpu( size, x_ptr,y_ptr, &h_superacc[0], &status);
     else
-        exblas::exdot_omp( size, x_ptr,y_ptr, &h_superacc[0]);
+        exblas::exdot_omp( size, x_ptr,y_ptr, &h_superacc[0], &status);
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"OMP Dot failed since one of the inputs contains NaN or Inf");
     return h_superacc;
 }
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3>
 inline std::vector<int64_t> doDot_dispatch( OmpTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr, PointerOrValue3 z_ptr) {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
+    int status;
     if(size<MIN_SIZE)
-        exblas::exdot_cpu( size, x_ptr,y_ptr,z_ptr, &h_superacc[0]);
+        exblas::exdot_cpu( size, x_ptr,y_ptr,z_ptr, &h_superacc[0], &status);
     else
-        exblas::exdot_omp( size, x_ptr,y_ptr,z_ptr, &h_superacc[0]);
+        exblas::exdot_omp( size, x_ptr,y_ptr,z_ptr, &h_superacc[0], &status);
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"OMP Dot failed since one of the inputs contains NaN or Inf");
     return h_superacc;
 }
 
diff --git a/inc/dg/backend/blas1_serial.h b/inc/dg/backend/blas1_serial.h
index 59163749b..26f8ac2b7 100644
--- a/inc/dg/backend/blas1_serial.h
+++ b/inc/dg/backend/blas1_serial.h
@@ -1,6 +1,7 @@
 #ifndef _DG_BLAS_SERIAL_
 #define _DG_BLAS_SERIAL_
 #include "config.h"
+#include "exceptions.h"
 #include "execution_policy.h"
 #include "exblas/exdot_serial.h"
 
@@ -13,13 +14,19 @@ namespace detail
 template<class PointerOrValue1, class PointerOrValue2>
 inline std::vector<int64_t> doDot_dispatch( SerialTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr) {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    exblas::exdot_cpu( size, x_ptr,y_ptr, &h_superacc[0]) ;
+    int status;
+    exblas::exdot_cpu( size, x_ptr,y_ptr, &h_superacc[0],&status) ;
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"CPU Dot failed since one of the inputs contains NaN or Inf");
     return h_superacc;
 }
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3>
 inline std::vector<int64_t> doDot_dispatch( SerialTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr, PointerOrValue3 z_ptr) {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    exblas::exdot_cpu( size, x_ptr,y_ptr,z_ptr, &h_superacc[0]) ;
+    int status;
+    exblas::exdot_cpu( size, x_ptr,y_ptr,z_ptr, &h_superacc[0], &status) ;
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"CPU Dot failed since one of the inputs contains NaN or Inf");
     return h_superacc;
 }
 
diff --git a/inc/dg/backend/blas2_dispatch_scalar.h b/inc/dg/backend/blas2_dispatch_scalar.h
index 888647d38..8dfce433a 100644
--- a/inc/dg/backend/blas2_dispatch_scalar.h
+++ b/inc/dg/backend/blas2_dispatch_scalar.h
@@ -23,7 +23,10 @@ inline std::vector<int64_t> doDot_superacc( const Vector1& x, const Matrix& m, c
     const get_value_type<Vector2>* y_ptr = &y;
     //since we only accumulate up to three values (multiplication and rest) reduce the size of the FPE
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    exblas::exdot_cpu<const get_value_type<Vector1>*, const get_value_type<Matrix>*, const get_value_type<Vector2>*, 3>( 1, x_ptr,m_ptr,y_ptr, &h_superacc[0]) ;
+    int status;
+    exblas::exdot_cpu<const get_value_type<Vector1>*, const get_value_type<Matrix>*, const get_value_type<Vector2>*, 3>( 1, x_ptr,m_ptr,y_ptr, &h_superacc[0], &status) ;
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"Scalar Dot failed since one of the inputs contains NaN or Inf");
     return h_superacc;
 }
 template< class Vector1, class Matrix, class Vector2 >
diff --git a/inc/dg/backend/exblas/accumulate.cuh b/inc/dg/backend/exblas/accumulate.cuh
index f6f91e578..75166926b 100644
--- a/inc/dg/backend/exblas/accumulate.cuh
+++ b/inc/dg/backend/exblas/accumulate.cuh
@@ -95,7 +95,8 @@ __device__
 static inline void Accumulate( int64_t* accumulator, double x, int stride = 1) { //transposed accumulation
     if (x == 0)
         return;
-    assert( !std::isnan(x) && "Detected NaN in dot product!!");
+    //MW: This assert does not help very much in finding out where the nan originates
+    //assert( !std::isnan(x) && "Detected NaN in dot product!!");
 
     int e;
     frexp(x, &e); //extract the exponent of x (lies in -1024;1023 ?)
@@ -105,7 +106,7 @@ static inline void Accumulate( int64_t* accumulator, double x, int stride = 1) {
     double xscaled = ldexp(x, -DIGITS * exp_word);
 
     int i;
-    for (i = iup; xscaled != 0 && i>=0; --i) {
+    for (i = iup; xscaled != 0 && i>=0; --i) { //MW: i>=0 protects agains NaN
         double xrounded = rint(xscaled);
         int64_t xint = (int64_t) xrounded;
         AccumulateWord(accumulator, i, xint, stride);
diff --git a/inc/dg/backend/exblas/accumulate.h b/inc/dg/backend/exblas/accumulate.h
index 44ccaa875..1cbfb747a 100644
--- a/inc/dg/backend/exblas/accumulate.h
+++ b/inc/dg/backend/exblas/accumulate.h
@@ -121,7 +121,7 @@ static inline void AccumulateWord( int64_t *accumulator, int i, int64_t x) {
 static inline void Accumulate( int64_t* accumulator, double x) {
     if (x == 0)
         return;
-    assert( !std::isnan(x) && "Detected NaN in dot product!!");
+    //assert( !std::isnan(x) && "Detected NaN in dot product!!");
 
 
     int e = cpu::exponent(x);
@@ -131,7 +131,7 @@ static inline void Accumulate( int64_t* accumulator, double x) {
     double xscaled = cpu::myldexp(x, -DIGITS * exp_word);
 
     int i;
-    for (i = iup; xscaled != 0 && i>=0; --i) {
+    for (i = iup; xscaled != 0 && i>=0; --i) { //MW: i>=0 protects against NaN
         double xrounded = cpu::myrint(xscaled);
         int64_t xint = cpu::myllrint(xscaled);
         AccumulateWord(accumulator, i, xint);
diff --git a/inc/dg/backend/exblas/exdot_cuda.cuh b/inc/dg/backend/exblas/exdot_cuda.cuh
index a32a6da67..f5cdc01c3 100644
--- a/inc/dg/backend/exblas/exdot_cuda.cuh
+++ b/inc/dg/backend/exblas/exdot_cuda.cuh
@@ -49,7 +49,8 @@ __global__ void ExDOT(
     int64_t *d_PartialSuperaccs,
     PointerOrValue1 d_a,
     PointerOrValue2 d_b,
-    const uint NbElements
+    const uint NbElements,
+    volatile bool* error
 ) {
     __shared__ int64_t l_sa[WARP_COUNT * BIN_COUNT]; //shared variables live for a thread block (39 rows, 16 columns!)
     int64_t *l_workingBase = l_sa + (threadIdx.x & (WARP_COUNT - 1)); //the bitwise & with 15 is a modulo operation: threadIdx.x % 16
@@ -65,6 +66,9 @@ __global__ void ExDOT(
         double x = TwoProductFMA(get_element(d_a,pos), get_element(d_b,pos), &r);
         //double x = d_a[pos]*d_b[pos];//ATTENTION: if we write it like this, cpu compiler might generate an fma from this while nvcc does not...
 
+        //Check if the input is sane
+        if( !isfinite(x) ) *error = true;
+
         #pragma unroll
         for(uint i = 0; i != NBFPE; ++i) {
             double s;
@@ -135,7 +139,8 @@ __global__ void ExDOT(
     PointerOrValue1 d_a,
     PointerOrValue2 d_b,
     PointerOrValue3 d_c,
-    const uint NbElements
+    const uint NbElements,
+    volatile bool *error
 ) {
     __shared__ int64_t l_sa[WARP_COUNT * BIN_COUNT]; //shared variables live for a thread block (39 rows, 16 columns!)
     int64_t *l_workingBase = l_sa + (threadIdx.x & (WARP_COUNT - 1)); //the bitwise & with 15 is a modulo operation: threadIdx.x % 16
@@ -154,7 +159,10 @@ __global__ void ExDOT(
         double x1 = __fma_rn( get_element(d_a,pos), get_element(d_b,pos), 0);
         double x2 = __fma_rn( x1                  , get_element(d_c,pos), 0);
 
-        if( x2 != 0.0) {//accumulate x2
+        //Check if the input is sane
+        if( !isfinite(x2) ) *error = true;
+
+        if( x2 != 0.0 ) {//accumulate x2
             #pragma unroll
             for(uint i = 0; i != NBFPE; ++i) {
                 double s;
@@ -321,14 +329,18 @@ void ExDOTComplete(
 */
 template<class PointerOrValue1, class PointerOrValue2, size_t NBFPE=3>
 __host__
-void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, int64_t* d_superacc)
+void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, int64_t* d_superacc, int* status)
 {
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue2>::value, "PointerOrValue2 needs to be T or T* with T one of (const) float or (const) double");
     static thrust::device_vector<int64_t> d_PartialSuperaccsV( gpu::PARTIAL_SUPERACCS_COUNT*BIN_COUNT, 0.0); //39 columns and PSC rows
     int64_t *d_PartialSuperaccs = thrust::raw_pointer_cast( d_PartialSuperaccsV.data());
-    gpu::ExDOT<NBFPE, gpu::WARP_COUNT><<<gpu::PARTIAL_SUPERACCS_COUNT, gpu::WORKGROUP_SIZE>>>( d_PartialSuperaccs, x1_ptr, x2_ptr,size);
+    thrust::device_vector<bool> d_errorV(1, false);
+    bool *d_error = thrust::raw_pointer_cast( d_errorV.data());
+    gpu::ExDOT<NBFPE, gpu::WARP_COUNT><<<gpu::PARTIAL_SUPERACCS_COUNT, gpu::WORKGROUP_SIZE>>>( d_PartialSuperaccs, x1_ptr, x2_ptr,size, d_error);
     gpu::ExDOTComplete<gpu::MERGE_SUPERACCS_SIZE><<<gpu::PARTIAL_SUPERACCS_COUNT/gpu::MERGE_SUPERACCS_SIZE, gpu::MERGE_WORKGROUP_SIZE>>>( d_PartialSuperaccs, d_superacc );
+    *status = 0;
+    if( d_errorV[0] ) *status = 1;
 }
 
 /*!@brief gpu version of exact triple dot product
@@ -346,15 +358,19 @@ void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
  */
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3, size_t NBFPE=3>
 __host__
-void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, PointerOrValue3 x3_ptr, int64_t* d_superacc)
+void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, PointerOrValue3 x3_ptr, int64_t* d_superacc, int* status)
 {
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue2>::value, "PointerOrValue2 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue3>::value, "PointerOrValue3 needs to be T or T* with T one of (const) float or (const) double");
     static thrust::device_vector<int64_t> d_PartialSuperaccsV( gpu::PARTIAL_SUPERACCS_COUNT*BIN_COUNT, 0); //39 columns and PSC rows
     int64_t *d_PartialSuperaccs = thrust::raw_pointer_cast( d_PartialSuperaccsV.data());
-    gpu::ExDOT<NBFPE, gpu::WARP_COUNT><<<gpu::PARTIAL_SUPERACCS_COUNT, gpu::WORKGROUP_SIZE>>>( d_PartialSuperaccs, x1_ptr, x2_ptr, x3_ptr,size);
+    thrust::device_vector<bool> d_errorV(1, false);
+    bool *d_error = thrust::raw_pointer_cast( d_errorV.data());
+    gpu::ExDOT<NBFPE, gpu::WARP_COUNT><<<gpu::PARTIAL_SUPERACCS_COUNT, gpu::WORKGROUP_SIZE>>>( d_PartialSuperaccs, x1_ptr, x2_ptr, x3_ptr,size,d_error);
     gpu::ExDOTComplete<gpu::MERGE_SUPERACCS_SIZE><<<gpu::PARTIAL_SUPERACCS_COUNT/gpu::MERGE_SUPERACCS_SIZE, gpu::MERGE_WORKGROUP_SIZE>>>( d_PartialSuperaccs, d_superacc );
+    *status = 0;
+    if( d_errorV[0] ) *status = 1;
 }
 
 }//namespace exblas
diff --git a/inc/dg/backend/exblas/exdot_omp.h b/inc/dg/backend/exblas/exdot_omp.h
index 156820c2a..4972cb834 100644
--- a/inc/dg/backend/exblas/exdot_omp.h
+++ b/inc/dg/backend/exblas/exdot_omp.h
@@ -90,12 +90,13 @@ inline static void Reduction(unsigned int tid, unsigned int tnum, std::vector<in
 }
 
 template<typename CACHE, typename PointerOrValue1, typename PointerOrValue2>
-void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* h_superacc) {
+void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* h_superacc, bool* err) {
     // OpenMP sum+reduction
     int const linesize = 16;    // * sizeof(int32_t)
     int maxthreads = omp_get_max_threads();
     std::vector<int64_t> acc(maxthreads*BIN_COUNT,0);
     std::vector<int32_t> ready(maxthreads * linesize);
+    std::vector<bool> error( maxthreads, false);
 
     #pragma omp parallel
     {
@@ -117,6 +118,10 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* h_superacc)
             vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,i), make_vcl_vec8d(b,i), r1);
             //vcl::Vec8d x  = TwoProductFMA(vcl::Vec8d().load(a+i), vcl::Vec8d().load(b+i), r1);
             //vcl::Vec8d x  = vcl::mul_add( vcl::Vec8d().load(a+i),vcl::Vec8d().load(b+i),0);
+            //MW: check sanity of input
+            vcl::Vec8db finite = vcl::is_finite( x);
+            if( !vcl::horizontal_and( finite) ) error[tid] = true;
+
             cache.Accumulate(x);
             cache.Accumulate(r1); //MW: exact product but halfs the speed
         }
@@ -127,6 +132,10 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* h_superacc)
             vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,r,N-r), make_vcl_vec8d(b,r,N-r), r1);
             //vcl::Vec8d x  = TwoProductFMA(vcl::Vec8d().load_partial(N-r, a+r), vcl::Vec8d().load_partial(N-r,b+r), r1);
             //vcl::Vec8d x  = vcl::mul_add( vcl::Vec8d().load_partial(N-r,a+r),vcl::Vec8d().load_partial(N-r,b+r),0);
+
+            //MW: check sanity of input
+            vcl::Vec8db finite = vcl::is_finite( x);
+            if( !vcl::horizontal_and( finite) ) error[tid] = true;
             cache.Accumulate(x);
             cache.Accumulate(r1);
         }
@@ -148,15 +157,18 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* h_superacc)
     }
     for( int i=IMIN; i<=IMAX; i++)
         h_superacc[i] = acc[i];
+    for ( int i=0; i<maxthreads; i++)
+        if( error[i] == true) *err = true;
 }
 
 template<typename CACHE, typename PointerOrValue1, typename PointerOrValue2, typename PointerOrValue3>
-void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, int64_t* h_superacc) {
+void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, int64_t* h_superacc, bool* err) {
     // OpenMP sum+reduction
     int const linesize = 16;    // * sizeof(int32_t) (MW avoid false sharing?)
     int maxthreads = omp_get_max_threads();
     std::vector<int64_t> acc(maxthreads*BIN_COUNT,0);
     std::vector<int32_t> ready(maxthreads * linesize);
+    std::vector<bool> error( maxthreads, false);
 
     #pragma omp parallel
     {
@@ -181,6 +193,8 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, in
             //vcl::Vec8d x2  = vcl::mul_add( x1                   ,vcl::Vec8d().load(c+i), 0);
             vcl::Vec8d x1  = vcl::mul_add(make_vcl_vec8d(a,i),make_vcl_vec8d(b,i), 0);
             vcl::Vec8d x2  = vcl::mul_add( x1                ,make_vcl_vec8d(c,i), 0);
+            vcl::Vec8db finite = vcl::is_finite( x2);
+            if( !vcl::horizontal_and( finite) ) error[tid] = true;
             cache.Accumulate(x2);
             //cache.Accumulate(r2);
             //x2 = TwoProductFMA(r1, cvec, r2);
@@ -197,6 +211,8 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, in
             //vcl::Vec8d x2  = vcl::mul_add( x1                   ,vcl::Vec8d().load_partial(N-r,c+r), 0);
             vcl::Vec8d x1  = vcl::mul_add(make_vcl_vec8d(a,r,N-r),make_vcl_vec8d(b,r,N-r), 0);
             vcl::Vec8d x2  = vcl::mul_add( x1                    ,make_vcl_vec8d(c,r,N-r), 0);
+            vcl::Vec8db finite = vcl::is_finite( x2);
+            if( !vcl::horizontal_and( finite) ) error[tid] = true;
             cache.Accumulate(x2);
             //cache.Accumulate(r2);
             //x2 = TwoProductFMA(r1, cvec, r2);
@@ -222,6 +238,8 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, in
     }
     for( int i=IMIN; i<=IMAX; i++)
         h_superacc[i] = acc[i];
+    for ( int i=0; i<maxthreads; i++)
+        if( error[i] == true) *err = true;
 }
 }//namespace cpu
 ///@endcond
@@ -239,14 +257,17 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, in
  * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
 */
 template<class PointerOrValue1, class PointerOrValue2, size_t NBFPE=8>
-void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, int64_t* h_superacc){
+void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, int64_t* h_superacc, int* status){
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue2>::value, "PointerOrValue2 needs to be T or T* with T one of (const) float or (const) double");
+    bool error = false;
 #ifndef _WITHOUT_VCL
-    cpu::ExDOTFPE<cpu::FPExpansionVect<vcl::Vec8d, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, h_superacc);
+    cpu::ExDOTFPE<cpu::FPExpansionVect<vcl::Vec8d, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, h_superacc, &error);
 #else
-    cpu::ExDOTFPE<cpu::FPExpansionVect<double, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, h_superacc);
+    cpu::ExDOTFPE<cpu::FPExpansionVect<double, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, h_superacc, &error);
 #endif//_WITHOUT_VCL
+    *status = 0;
+    if( error ) *status = 1;
 }
 /*!@brief OpenMP parallel version of exact triple dot product
  *
@@ -262,15 +283,18 @@ void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
  * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
  */
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3, size_t NBFPE=8>
-void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, PointerOrValue3 x3_ptr, int64_t* h_superacc) {
+void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, PointerOrValue3 x3_ptr, int64_t* h_superacc, int* status) {
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue2>::value, "PointerOrValue2 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue3>::value, "PointerOrValue3 needs to be T or T* with T one of (const) float or (const) double");
+    bool error = false;
 #ifndef _WITHOUT_VCL
-    cpu::ExDOTFPE<cpu::FPExpansionVect<vcl::Vec8d, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, x3_ptr, h_superacc);
+    cpu::ExDOTFPE<cpu::FPExpansionVect<vcl::Vec8d, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, x3_ptr, h_superacc, &error);
 #else
-    cpu::ExDOTFPE<cpu::FPExpansionVect<double, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, x3_ptr, h_superacc);
+    cpu::ExDOTFPE<cpu::FPExpansionVect<double, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, x3_ptr, h_superacc, &error);
 #endif//_WITHOUT_VCL
+    *status = 0;
+    if( error ) *status = 1;
 }
 
 }//namespace exblas
diff --git a/inc/dg/backend/exblas/exdot_serial.h b/inc/dg/backend/exblas/exdot_serial.h
index b9848713d..d9f626f4e 100644
--- a/inc/dg/backend/exblas/exdot_serial.h
+++ b/inc/dg/backend/exblas/exdot_serial.h
@@ -30,7 +30,7 @@ namespace exblas{
 namespace cpu{
 
 template<typename CACHE, typename PointerOrValue1, typename PointerOrValue2>
-void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* acc) {
+void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* acc, bool* error) {
     CACHE cache(acc);
 #ifndef _WITHOUT_VCL
     int r = (( int64_t(N) ) & ~7ul);
@@ -42,6 +42,8 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* acc) {
         vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,i), make_vcl_vec8d(b,i), r1);
         //vcl::Vec8d x  = TwoProductFMA(vcl::Vec8d().load(a+i), vcl::Vec8d().load(b+i), r1);
         //vcl::Vec8d x  = vcl::Vec8d().load(a+i)*vcl::Vec8d().load(b+i);
+        vcl::Vec8db finite = vcl::is_finite( x);
+        if( !vcl::horizontal_and( finite) ) *error = true;
         cache.Accumulate(x);
         cache.Accumulate(r1);
     }
@@ -51,6 +53,8 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* acc) {
         vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,r,N-r), make_vcl_vec8d(b,r,N-r), r1);
         //vcl::Vec8d x  = TwoProductFMA(vcl::Vec8d().load_partial(N-r, a+r), vcl::Vec8d().load_partial(N-r,b+r), r1);
         //vcl::Vec8d x  = vcl::Vec8d().load_partial(N-r, a+r)*vcl::Vec8d().load_partial(N-r,b+r);
+        vcl::Vec8db finite = vcl::is_finite( x);
+        if( !vcl::horizontal_and( finite) ) *error = true;
         cache.Accumulate(x);
         cache.Accumulate(r1);
     }
@@ -58,6 +62,7 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* acc) {
     for(int i = 0; i < N; i++) {
         double r1;
         double x = TwoProductFMA(get_element(a,i),get_element(b,i),r1);
+        if( !std::isfinite(x) ) *error = true;
         cache.Accumulate(x);
         cache.Accumulate(r1);
     }
@@ -66,7 +71,7 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* acc) {
 }
 
 template<typename CACHE, typename PointerOrValue1, typename PointerOrValue2, typename PointerOrValue3>
-void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, int64_t* acc) {
+void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, int64_t* acc, bool* error) {
     CACHE cache(acc);
 #ifndef _WITHOUT_VCL
     int r = (( int64_t(N))  & ~7ul);
@@ -81,6 +86,8 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c
         //vcl::Vec8d x2  = vcl::mul_add( x1                   ,vcl::Vec8d().load(c+i), 0);
         vcl::Vec8d x1  = vcl::mul_add(make_vcl_vec8d(a,i),make_vcl_vec8d(b,i), 0);
         vcl::Vec8d x2  = vcl::mul_add( x1                ,make_vcl_vec8d(c,i), 0);
+        vcl::Vec8db finite = vcl::is_finite( x2);
+        if( !vcl::horizontal_and( finite) ) *error = true;
         cache.Accumulate(x2);
         //cache.Accumulate(r2);
         //x2 = TwoProductFMA(r1, cvec, r2);
@@ -94,6 +101,8 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c
         //vcl::Vec8d x2 = TwoProductFMA(x , cvec, r2);
         vcl::Vec8d x1  = vcl::mul_add(make_vcl_vec8d(a,r,N-r),make_vcl_vec8d(b,r,N-r), 0);
         vcl::Vec8d x2  = vcl::mul_add( x1                    ,make_vcl_vec8d(c,r,N-r), 0);
+        vcl::Vec8db finite = vcl::is_finite( x2);
+        if( !vcl::horizontal_and( finite) ) *error = true;
         cache.Accumulate(x2);
         //cache.Accumulate(r2);
         //x2 = TwoProductFMA(r1, cvec, r2);
@@ -104,6 +113,7 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c
     for(int i = 0; i < N; i++) {
         double x1 = get_element(a,i)*get_element(b,i);
         double x2 = x1*get_element(c,i);
+        if( !std::isfinite(x2) ) *error = true;
         cache.Accumulate(x2);
     }
 #endif// _WITHOUT_VCL
@@ -125,16 +135,19 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c
  * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
 */
 template<class PointerOrValue1, class PointerOrValue2, size_t NBFPE=8>
-void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, int64_t* h_superacc){
+void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, int64_t* h_superacc, int* status){
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue2>::value, "PointerOrValue2 needs to be T or T* with T one of (const) float or (const) double");
     for( int i=0; i<exblas::BIN_COUNT; i++)
         h_superacc[i] = 0;
+    bool error = false;
 #ifndef _WITHOUT_VCL
-    cpu::ExDOTFPE_cpu<cpu::FPExpansionVect<vcl::Vec8d, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, h_superacc);
+    cpu::ExDOTFPE_cpu<cpu::FPExpansionVect<vcl::Vec8d, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, h_superacc, &error);
 #else
-    cpu::ExDOTFPE_cpu<cpu::FPExpansionVect<double, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, h_superacc);
+    cpu::ExDOTFPE_cpu<cpu::FPExpansionVect<double, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, h_superacc, &error);
 #endif//_WITHOUT_VCL
+    *status = 0;
+    if( error ) *status = 1;
 }
 
 /*!@brief gpu version of exact triple dot product
@@ -151,17 +164,20 @@ void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
  * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
  */
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3, size_t NBFPE=8>
-void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, PointerOrValue3 x3_ptr, int64_t* h_superacc) {
+void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, PointerOrValue3 x3_ptr, int64_t* h_superacc, int* status) {
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue2>::value, "PointerOrValue2 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue3>::value, "PointerOrValue3 needs to be T or T* with T one of (const) float or (const) double");
     for( int i=0; i<exblas::BIN_COUNT; i++)
         h_superacc[i] = 0;
+    bool error = false;
 #ifndef _WITHOUT_VCL
-    cpu::ExDOTFPE_cpu<cpu::FPExpansionVect<vcl::Vec8d, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, x3_ptr, h_superacc);
+    cpu::ExDOTFPE_cpu<cpu::FPExpansionVect<vcl::Vec8d, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, x3_ptr, h_superacc, &error);
 #else
-    cpu::ExDOTFPE_cpu<cpu::FPExpansionVect<double, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, x3_ptr, h_superacc);
+    cpu::ExDOTFPE_cpu<cpu::FPExpansionVect<double, NBFPE, cpu::FPExpansionTraits<true> > >((int)size,x1_ptr,x2_ptr, x3_ptr, h_superacc, &error);
 #endif//_WITHOUT_VCL
+    *status = 0;
+    if( error ) *status = 1;
 }
 
 
diff --git a/inc/dg/backend/exblas/mylibm.hpp b/inc/dg/backend/exblas/mylibm.hpp
index 90786a014..906b87712 100644
--- a/inc/dg/backend/exblas/mylibm.hpp
+++ b/inc/dg/backend/exblas/mylibm.hpp
@@ -86,6 +86,7 @@ static inline int biased_exponent(double x)
     return e;
 }
 
+//MW: if x is a NaN this thing still returns a number??
 static inline double myldexp(double x, int e)
 {
     // Scale x by e
diff --git a/inc/dg/topology/evaluation_t.cu b/inc/dg/topology/evaluation_t.cu
index d4417a4d2..7918afcc3 100644
--- a/inc/dg/topology/evaluation_t.cu
+++ b/inc/dg/topology/evaluation_t.cu
@@ -6,6 +6,7 @@
 #include <thrust/device_vector.h>
 
 #include "dg/blas.h"
+#include "dg/functors.h"
 
 #include "evaluation.h"
 #include "weights.h"
@@ -112,6 +113,10 @@ int main()
     dg::blas1::axpby( 1., integral_ana, -1., integral_num);
     norm = dg::blas2::dot( integral_num, dg::create::weights( g1d), integral_num);
     std::cout << " Error norm of  1d integral function "<<norm<<"\n";
+    // TEST if dot throws on NaN
+    dg::blas1::transform( x,x, dg::LN<double>());
+    dg::blas1::dot( x,x);
+
     std::cout << "\nFINISHED! Continue with topology/derivatives_t.cu !\n\n";
     return 0;
 }
-- 
GitLab


From 3216eb77c7230b7a10d81f857064828145503be5 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 17 Nov 2019 17:14:39 +0100
Subject: [PATCH 170/540] Add multiply_sigma function in elliptic

This is to enable the modified source term in feltor
---
 inc/dg/backend/average_cpu.h           |  6 +-
 inc/dg/backend/average_gpu.cuh         |  8 +--
 inc/dg/backend/average_omp.h           |  4 +-
 inc/dg/backend/blas1_array.h           |  2 +-
 inc/dg/backend/blas1_cuda.cuh          |  4 +-
 inc/dg/backend/blas1_dispatch_scalar.h |  2 +-
 inc/dg/backend/blas1_omp.h             |  4 +-
 inc/dg/backend/blas1_serial.h          |  4 +-
 inc/dg/backend/blas2_dispatch_scalar.h |  2 +-
 inc/dg/elliptic.h                      | 82 +++++++++++++++++++++++++-
 src/feltor/feltor.cu                   | 13 ++--
 src/feltor/feltor.h                    |  9 +--
 src/feltor/feltor_hpc.cu               |  3 +
 13 files changed, 112 insertions(+), 31 deletions(-)

diff --git a/inc/dg/backend/average_cpu.h b/inc/dg/backend/average_cpu.h
index e95e04329..fb4793c05 100644
--- a/inc/dg/backend/average_cpu.h
+++ b/inc/dg/backend/average_cpu.h
@@ -35,13 +35,13 @@ void average( SerialTag, unsigned nx, unsigned ny, const value_type* in0, const
     static_assert( std::is_same<value_type, double>::value, "Value type must be double!");
     static thrust::host_vector<int64_t> h_accumulator;
     h_accumulator.resize( ny*exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     for( unsigned i=0; i<ny; i++)
         exblas::exdot_cpu(nx, &in0[i*nx], &in1[i*nx], &h_accumulator[i*exblas::BIN_COUNT], &status);
-    for( unsigned i=0; i<ny; i++)
-        out[i] = exblas::cpu::Round( &h_accumulator[i*exblas::BIN_COUNT], *status);
     if(status != 0)
         throw dg::Error(dg::Message(_ping_)<<"CPU Average failed since one of the inputs contains NaN or Inf");
+    for( unsigned i=0; i<ny; i++)
+        out[i] = exblas::cpu::Round( &h_accumulator[i*exblas::BIN_COUNT]);
 }
 
 #ifdef MPI_VERSION
diff --git a/inc/dg/backend/average_gpu.cuh b/inc/dg/backend/average_gpu.cuh
index fe5d189eb..db9704624 100644
--- a/inc/dg/backend/average_gpu.cuh
+++ b/inc/dg/backend/average_gpu.cuh
@@ -74,16 +74,16 @@ void average( CudaTag, unsigned nx, unsigned ny, const value_type* in0, const va
     static thrust::host_vector<value_type> h_round;
     d_accumulator.resize( ny*exblas::BIN_COUNT);
     int64_t* d_ptr = thrust::raw_pointer_cast( d_accumulator.data());
-    int status;
+    int status = 0;
     for( unsigned i=0; i<ny; i++)
         exblas::exdot_gpu(nx, &in0[i*nx], &in1[i*nx], &d_ptr[i*exblas::BIN_COUNT], &status);
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"GPU Average failed since one of the inputs contains NaN or Inf");
     h_accumulator = d_accumulator;
     h_round.resize( ny);
     for( unsigned i=0; i<ny; i++)
-        h_round[i] = exblas::cpu::Round( &h_accumulator[i*exblas::BIN_COUNT], &status);
+        h_round[i] = exblas::cpu::Round( &h_accumulator[i*exblas::BIN_COUNT]);
     cudaMemcpy( out, &h_round[0], ny*sizeof(value_type), cudaMemcpyHostToDevice);
-    if(status != 0)
-        throw dg::Error(dg::Message(_ping_)<<"GPU Average failed since one of the inputs contains NaN or Inf");
 }
 
 #ifdef MPI_VERSION
diff --git a/inc/dg/backend/average_omp.h b/inc/dg/backend/average_omp.h
index 484d0ba31..a516e4c23 100644
--- a/inc/dg/backend/average_omp.h
+++ b/inc/dg/backend/average_omp.h
@@ -40,7 +40,7 @@ void average( OmpTag, unsigned nx, unsigned ny, const value_type* in0, const val
     static_assert( std::is_same<value_type, double>::value, "Value type must be double!");
     static thrust::host_vector<int64_t> h_accumulator;
     h_accumulator.resize( ny*exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     for( unsigned i=0; i<ny; i++)
         exblas::exdot_omp(nx, &in0[i*nx], &in1[i*nx], &h_accumulator[i*exblas::BIN_COUNT], &status);
     if(status != 0)
@@ -58,7 +58,7 @@ void average_mpi( OmpTag, unsigned nx, unsigned ny, const value_type* in0, const
     static thrust::host_vector<int64_t> h_accumulator;
     static thrust::host_vector<int64_t> h_accumulator2;
     h_accumulator2.resize( ny*exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     for( unsigned i=0; i<ny; i++)
         exblas::exdot_omp(nx, &in0[i*nx], &in1[i*nx], &h_accumulator2[i*exblas::BIN_COUNT], &status);
     if(status != 0)
diff --git a/inc/dg/backend/blas1_array.h b/inc/dg/backend/blas1_array.h
index 56f86b6f0..a8b036953 100644
--- a/inc/dg/backend/blas1_array.h
+++ b/inc/dg/backend/blas1_array.h
@@ -25,7 +25,7 @@ template< class T, std::size_t N>
 std::vector<int64_t> doDot_superacc( const std::array<T,N>& x, const std::array<T,N>& y, StdArrayTag)
 {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     exblas::exdot_cpu( N, x.begin(),y.begin(), &h_superacc[0], &status) ;
     if(status != 0)
         throw dg::Error(dg::Message(_ping_)<<"CPU Dot failed since one of the inputs contains NaN or Inf");
diff --git a/inc/dg/backend/blas1_cuda.cuh b/inc/dg/backend/blas1_cuda.cuh
index be904c366..14728c1c7 100644
--- a/inc/dg/backend/blas1_cuda.cuh
+++ b/inc/dg/backend/blas1_cuda.cuh
@@ -15,7 +15,7 @@ template<class PointerOrValue1, class PointerOrValue2>
 inline std::vector<int64_t> doDot_dispatch( CudaTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr) {
     static thrust::device_vector<int64_t> d_superacc(exblas::BIN_COUNT);
     int64_t * d_ptr = thrust::raw_pointer_cast( d_superacc.data());
-    int status;
+    int status = 0;
     exblas::exdot_gpu( size, x_ptr,y_ptr, d_ptr, &status );
     if( status != 0)
         throw dg::Error(dg::Message(_ping_)<<"GPU Dot product failed since one of the inputs contains NaN or Inf");
@@ -27,7 +27,7 @@ template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3>
 inline std::vector<int64_t> doDot_dispatch( CudaTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr, PointerOrValue3 z_ptr) {
     static thrust::device_vector<int64_t> d_superacc(exblas::BIN_COUNT);
     int64_t * d_ptr = thrust::raw_pointer_cast( d_superacc.data());
-    int status;
+    int status = 0;
     exblas::exdot_gpu( size, x_ptr,y_ptr,z_ptr, d_ptr, &status);
     if( status != 0)
         throw dg::Error(dg::Message(_ping_)<<"GPU Dot product failed since one of the inputs contains NaN or Inf");
diff --git a/inc/dg/backend/blas1_dispatch_scalar.h b/inc/dg/backend/blas1_dispatch_scalar.h
index 58db82c5c..140419c9f 100644
--- a/inc/dg/backend/blas1_dispatch_scalar.h
+++ b/inc/dg/backend/blas1_dispatch_scalar.h
@@ -25,7 +25,7 @@ std::vector<int64_t> doDot_superacc( const Vector1& x, const Vector2& y, AnyScal
     const get_value_type<Vector2>* y_ptr = &y;
     //since we only accumulate up to two values (multiplication and rest) reduce the size of the FPE
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     exblas::exdot_cpu<const get_value_type<Vector1>*, const get_value_type<Vector2>*, 2>( 1, x_ptr,y_ptr, &h_superacc[0], &status) ;
     if(status != 0)
         throw dg::Error(dg::Message(_ping_)<<"CPU Dot failed since one of the inputs contains NaN or Inf");
diff --git a/inc/dg/backend/blas1_omp.h b/inc/dg/backend/blas1_omp.h
index 18f4c3326..0bd35f2cc 100644
--- a/inc/dg/backend/blas1_omp.h
+++ b/inc/dg/backend/blas1_omp.h
@@ -17,7 +17,7 @@ const int MIN_SIZE=100;//don't parallelize if work is too small
 template<class PointerOrValue1, class PointerOrValue2>
 inline std::vector<int64_t> doDot_dispatch( OmpTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr) {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     if(size<MIN_SIZE)
         exblas::exdot_cpu( size, x_ptr,y_ptr, &h_superacc[0], &status);
     else
@@ -29,7 +29,7 @@ inline std::vector<int64_t> doDot_dispatch( OmpTag, unsigned size, PointerOrValu
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3>
 inline std::vector<int64_t> doDot_dispatch( OmpTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr, PointerOrValue3 z_ptr) {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     if(size<MIN_SIZE)
         exblas::exdot_cpu( size, x_ptr,y_ptr,z_ptr, &h_superacc[0], &status);
     else
diff --git a/inc/dg/backend/blas1_serial.h b/inc/dg/backend/blas1_serial.h
index 26f8ac2b7..0a4f913f3 100644
--- a/inc/dg/backend/blas1_serial.h
+++ b/inc/dg/backend/blas1_serial.h
@@ -14,7 +14,7 @@ namespace detail
 template<class PointerOrValue1, class PointerOrValue2>
 inline std::vector<int64_t> doDot_dispatch( SerialTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr) {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     exblas::exdot_cpu( size, x_ptr,y_ptr, &h_superacc[0],&status) ;
     if(status != 0)
         throw dg::Error(dg::Message(_ping_)<<"CPU Dot failed since one of the inputs contains NaN or Inf");
@@ -23,7 +23,7 @@ inline std::vector<int64_t> doDot_dispatch( SerialTag, unsigned size, PointerOrV
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3>
 inline std::vector<int64_t> doDot_dispatch( SerialTag, unsigned size, PointerOrValue1 x_ptr, PointerOrValue2 y_ptr, PointerOrValue3 z_ptr) {
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     exblas::exdot_cpu( size, x_ptr,y_ptr,z_ptr, &h_superacc[0], &status) ;
     if(status != 0)
         throw dg::Error(dg::Message(_ping_)<<"CPU Dot failed since one of the inputs contains NaN or Inf");
diff --git a/inc/dg/backend/blas2_dispatch_scalar.h b/inc/dg/backend/blas2_dispatch_scalar.h
index 8dfce433a..c6f7325de 100644
--- a/inc/dg/backend/blas2_dispatch_scalar.h
+++ b/inc/dg/backend/blas2_dispatch_scalar.h
@@ -23,7 +23,7 @@ inline std::vector<int64_t> doDot_superacc( const Vector1& x, const Matrix& m, c
     const get_value_type<Vector2>* y_ptr = &y;
     //since we only accumulate up to three values (multiplication and rest) reduce the size of the FPE
     std::vector<int64_t> h_superacc(exblas::BIN_COUNT);
-    int status;
+    int status = 0;
     exblas::exdot_cpu<const get_value_type<Vector1>*, const get_value_type<Matrix>*, const get_value_type<Vector2>*, 3>( 1, x_ptr,m_ptr,y_ptr, &h_superacc[0], &status) ;
     if(status != 0)
         throw dg::Error(dg::Message(_ping_)<<"Scalar Dot failed since one of the inputs contains NaN or Inf");
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 52f13f5c2..115008eea 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -23,8 +23,8 @@ namespace dg
  *
  * @ingroup matrixoperators
  *
- * The term discretized is \f[ -\nabla \cdot ( \chi \nabla_\perp ) \f]
- * where \f$ \nabla_\perp \f$ is the two-dimensional gradient and \f$\chi\f$ is a
+ * The term discretized is \f[ -\nabla \cdot ( \chi \nabla ) \f]
+ * where \f$ \nabla \f$ is the two-dimensional nabla and \f$\chi\f$ is a
  * (possibly spatially dependent) tensor.
  * In general coordinates that means
  * \f[ -\frac{1}{\sqrt{g}}\left(
@@ -250,6 +250,46 @@ class Elliptic
             dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
     }
 
+    /**
+     * @brief Compute elliptic term with a possibly zero prefactor and add to output
+     *
+     * i.e this function computes \f[ y = -\alpha\nabla \cdot ( \sigma\chi \nabla x )  + \beta y\f]
+     * This is in principle possible also with the \c set_chi() and \c symv() functions
+     * however sometimes you have a \c sigma with explicit zeros or negative values.
+     * Then you need to use this function because \c set_chi() won't allow a \c sigma with zeros
+     * @note This function does not change the internal \c chi tensor
+     * @param alpha a scalar
+     * @param sigma The prefactor for the \c chi tensor
+     * @param x left-hand-side
+     * @param beta a scalar
+     * @param y result
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1, class ContainerType2>
+    void multiply_sigma( value_type alpha, const ContainerType2& sigma, const ContainerType0& x, value_type beta, ContainerType1& y)
+    {
+        //compute gradient
+        dg::blas2::gemv( m_rightx, x, m_tempx); //R_x*f
+        dg::blas2::gemv( m_righty, x, m_tempy); //R_y*f
+
+        //multiply with tensor (note the alias)
+        dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
+        //sigma is possibly zero so we don't multiply it to m_chi
+        dg::blas1::pointwiseDot( m_tempx, sigma, m_tempx); ///////
+        dg::blas1::pointwiseDot( m_tempy, sigma, m_tempy); ///////
+
+        //now take divergence
+        dg::blas2::symv( m_lefty, m_tempy, m_temp);
+        dg::blas2::symv( -1., m_leftx, m_tempx, -1., m_temp);
+
+        //add jump terms
+        dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
+        dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        if( m_no == normed)
+            dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
+        if( m_no == not_normed)//multiply weights without volume
+            dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
+    }
     private:
     bc inverse( bc bound)
     {
@@ -489,6 +529,44 @@ class Elliptic3d
         if( m_no == not_normed)//multiply weights without volume
             dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
     }
+    ///@copydoc Elliptic::multiply_sigma(value_type,const ContainerType2&,const ContainerType0&,value_type,ContainerType1&)
+    template<class ContainerType0, class ContainerType1, class ContainerType2>
+    void multiply_sigma( value_type alpha, const ContainerType2& sigma, const ContainerType0& x, value_type beta, ContainerType1& y)
+    {
+        //compute gradient
+        dg::blas2::gemv( m_rightx, x, m_tempx); //R_x*f
+        dg::blas2::gemv( m_righty, x, m_tempy); //R_y*f
+        if( m_multiplyZ )
+        {
+            dg::blas2::gemv( m_rightz, x, m_tempz); //R_z*f
+
+            //multiply with tensor (note the alias)
+            dg::tensor::multiply3d(m_chi, m_tempx, m_tempy, m_tempz, m_tempx, m_tempy, m_tempz);
+            //sigma is possibly zero so we don't multiply it to m_chi
+            dg::blas1::pointwiseDot( m_tempx, sigma, m_tempx); ///////
+            dg::blas1::pointwiseDot( m_tempy, sigma, m_tempy); ///////
+            dg::blas1::pointwiseDot( m_tempz, sigma, m_tempz); ///////
+            //now take divergence
+            dg::blas2::symv( -1., m_leftz, m_tempz, 0., m_temp);
+            dg::blas2::symv( -1., m_lefty, m_tempy, 1., m_temp);
+        }
+        else
+        {
+            dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
+            dg::blas1::pointwiseDot( m_tempx, sigma, m_tempx); ///////
+            dg::blas1::pointwiseDot( m_tempy, sigma, m_tempy); ///////
+            dg::blas2::symv( -1.,m_lefty, m_tempy, 0., m_temp);
+        }
+        dg::blas2::symv( -1., m_leftx, m_tempx, 1., m_temp);
+
+        //add jump terms
+        dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
+        dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        if( m_no == normed)
+            dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
+        if( m_no == not_normed)//multiply weights without volume
+            dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
+    }
 
     private:
     bc inverse( bc bound)
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index acfb04ef0..fc0e0ee56 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -174,8 +174,8 @@ int main( int argc, char* argv[])
             else
                 dg::assign( *pair.second, hvisual);
             dg::blas2::gemv( equi, hvisual, visual);
-            colors.scalemax() = (double)thrust::reduce(
-                visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
+            colors.scalemax() = dg::blas1::reduce(
+                visual, 0., dg::AbsMax<double>() );
             colors.scalemin() = -colors.scalemax();
             title <<pair.first << colors.scalemax()<<"   ";
             if ( p.symmetric )
@@ -227,18 +227,20 @@ int main( int argc, char* argv[])
             {
                 if( std::find( feltor::energies.begin(), feltor::energies.end(), record.name) != feltor::energies.end())
                 {
+                    std::cout << record.name<<" : ";
                     record.function( result, var);
                     double norm = dg::blas1::dot( result, feltor.vol3d());
                     energy += norm;
-                    std::cout << record.name<<" : "<<norm<<std::endl;
+                    std::cout << norm<<std::endl;
 
                 }
                 if( std::find( feltor::energy_diff.begin(), feltor::energy_diff.end(), record.name) != feltor::energy_diff.end())
                 {
+                    std::cout << record.name<<" : ";
                     record.function( result, var);
                     double norm = dg::blas1::dot( result, feltor.vol3d());
                     ediff += norm;
-                    std::cout << record.name<<" : "<<norm<<std::endl;
+                    std::cout << norm<<std::endl;
                 }
 
             }
@@ -250,6 +252,9 @@ int main( int argc, char* argv[])
             std::cout <<"\td E/dt = " << dEdt
               <<" Lambda = " << ediff
               <<" -> Accuracy: " << accuracy << "\n";
+            double max_ue = dg::blas1::reduce(
+                feltor.velocity(0), 0., dg::AbsMax<double>() );
+            MPI_OUT std::cout << "\tMaximum ue "<<max_ue<<"\n";
             //----------------Test if induction equation holds
             if( p.beta != 0)
             {
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index a36e71c0d..f75193251 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -278,8 +278,6 @@ struct Explicit
     }
     const Container& lapMperpP (int i)
     {
-        dg::blas1::copy( 1., m_temp1);
-        m_lapperpP.set_chi( m_temp1);
         dg::blas2::gemv( m_lapperpP, m_phi[i], m_temp1);
         return m_temp1;
     }
@@ -892,11 +890,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
         dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
         // potential part of FLR correction
-        dg::blas1::subroutine( routines::ComputeChi(),
-            m_temp0, y[0][1], m_binv, m_p.mu[1]);
-        m_lapperpP.set_chi( m_temp0);
-        dg::blas2::gemv( m_lapperpP, m_phi[0], m_temp0);//negative!!
-        dg::blas1::axpby( -1., m_temp0, 1., m_s[0][0]);
+        dg::blas1::pointwiseDot( m_p.mu[1], m_s[0][1], m_binv, m_binv, 0., m_temp0);
+        m_lapperpP.multiply_sigma( -1., m_temp0, m_phi[0], 1., m_s[0][0]);
 
         dg::blas1::axpby( 1., m_s[0][0], 1.0, yp[0][0]);
         dg::blas1::axpby( 1., m_s[0][1], 1.0, yp[0][1]);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 9a8697a2a..f71be7e46 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -463,6 +463,9 @@ int main( int argc, char* argv[])
             MPI_OUT std::cout <<"\td E/dt = " << dEdt
                       <<" Lambda = " << ediff
                       <<" -> Accuracy: " << accuracy << "\n";
+            double max_ue = dg::blas1::reduce(
+                feltor.velocity(0), 0., dg::AbsMax<double>() );
+            MPI_OUT std::cout << "\tMaximum ue "<<max_ue<<"\n";
             //----------------Test if induction equation holds
             if( p.beta != 0)
             {
-- 
GitLab


From 978219308a6399cd200da497a12b0dc6fd6ef8d3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 17 Nov 2019 17:26:00 +0100
Subject: [PATCH 171/540] Fix average_cpu with new status

---
 inc/dg/backend/average_cpu.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/inc/dg/backend/average_cpu.h b/inc/dg/backend/average_cpu.h
index fb4793c05..cc3f59972 100644
--- a/inc/dg/backend/average_cpu.h
+++ b/inc/dg/backend/average_cpu.h
@@ -53,8 +53,11 @@ void average_mpi( SerialTag, unsigned nx, unsigned ny, const value_type* in0, co
     static thrust::host_vector<int64_t> h_accumulator;
     static thrust::host_vector<int64_t> h_accumulator2;
     h_accumulator2.resize( ny*exblas::BIN_COUNT);
+    int status = 0;
     for( unsigned i=0; i<ny; i++)
-        exblas::exdot_cpu(nx, &in0[i*nx], &in1[i*nx], &h_accumulator2[i*exblas::BIN_COUNT]);
+        exblas::exdot_cpu(nx, &in0[i*nx], &in1[i*nx], &h_accumulator2[i*exblas::BIN_COUNT], &status);
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"MPI CPU Average failed since one of the inputs contains NaN or Inf");
     h_accumulator.resize( h_accumulator2.size());
     exblas::reduce_mpi_cpu( ny, &h_accumulator2[0], &h_accumulator[0], comm, comm_mod, comm_mod_reduce);
     for( unsigned i=0; i<ny; i++)
-- 
GitLab


From fa5ca747139fc6394b40d3b892c773ae492ff27a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 18 Nov 2019 14:16:32 +0100
Subject: [PATCH 172/540] Reduce accuracy of turbulence init

---
 src/feltor/Makefile |  8 ++++----
 src/feltor/init.h   | 12 ++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index b02405068..c97025b2c 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -16,16 +16,16 @@ manufactured: manufactured.cu manufactured.h feltor.h implicit.h
 feltordiag: feltordiag.cu
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g
 
-feltor: feltor.cu feltor.h implicit.h
+feltor: feltor.cu feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
-steadystate: steadystate.cu feltor.h implicit.h
+steadystate: steadystate.cu feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
 
-feltor_hpc: feltor_hpc.cu feltor.h implicit.h
+feltor_hpc: feltor_hpc.cu feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
 
-feltor_mpi: feltor_hpc.cu feltor.h implicit.h
+feltor_mpi: feltor_hpc.cu feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DFELTOR_MPI -DDG_BENCHMARK
 
 .PHONY: clean
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 69151a611..64a8be795 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -155,7 +155,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
                     fieldaligned( mag, grid, p.bcxN, p.bcyN,
                     dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-                //evaluate should always be used with mx,my > 1
+                //evaluate should always be used with mx,my > 1 (but this takes a lot of memory)
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 3);
             }
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
@@ -183,7 +183,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
                     fieldaligned( mag, grid, p.bcxN, p.bcyN,
                     dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-                //evaluate should always be used with mx,my > 1
+                //evaluate should always be used with mx,my > 1 (but this takes a lot of memory)
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
@@ -210,8 +210,8 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             {
                 dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
                     fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-                //evaluate should always be used with mx,my > 1
+                    dg::geo::NoLimiter(), p.rk4eps, 1, 1);
+                //For turbulence the exact evaluate is maybe not so important (thus takes less memory)
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
             dg::blas1::pointwiseDot( detail::profile_damping(grid,p,gp,mag), ntilde, ntilde);
@@ -262,8 +262,8 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             {
                 dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
                     fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                    dg::geo::NoLimiter(), p.rk4eps, 5, 5);
-                //evaluate should always be used with mx,my > 1
+                    dg::geo::NoLimiter(), p.rk4eps, 1, 1);
+                //For turbulence the exact evaluate is maybe not so important (thus takes less memory)
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0] );
-- 
GitLab


From ae9793eeb7c39b20a580d2be06f1f44d766647b0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 18 Nov 2019 15:26:04 +0100
Subject: [PATCH 173/540] Aslaks BICGSTAB and LGMRES methods

---
 inc/dg/bicgstabl.h | 224 +++++++++++++++++++++++++++++++++
 inc/dg/cg.h        |   2 +-
 inc/dg/cg2d_t.cu   |  82 ++++++++----
 inc/dg/lgmres.h    | 301 +++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 582 insertions(+), 27 deletions(-)
 create mode 100644 inc/dg/bicgstabl.h
 create mode 100644 inc/dg/lgmres.h

diff --git a/inc/dg/bicgstabl.h b/inc/dg/bicgstabl.h
new file mode 100644
index 000000000..cb4ec30ac
--- /dev/null
+++ b/inc/dg/bicgstabl.h
@@ -0,0 +1,224 @@
+#ifndef _DG_BICGSTABl_
+#define _DG_BICGSTABl_
+
+#include <iostream>
+#include <cstring>
+#include <cmath>
+#include <algorithm>
+
+#include "blas.h"
+#include "functors.h"
+
+/*!@file
+ * BICGSTABl class
+ *
+ * @author Aslak Poulsen
+ */
+
+namespace dg{
+
+/**
+* @brief Preconditioned BICGSTAB(l) method to solve
+* \f[ Ax=b\f]
+*
+* @ingroup invert
+*
+* @note BICGSTAB(l) is a method for solving non-symmetrical linear systems.
+* BICGSTAB(l) is a modification of BICGSTAB that aims to improve convergence.
+* See a paper here
+* https://pdfs.semanticscholar.org/c185/7ceab3c9ab4dbcb6a52fb62916f5757c0b38.pdf
+*
+*/
+template< class ContainerType>
+class BICGSTABl
+{
+  public:
+    using container_type = ContainerType;
+    using value_type = dg::get_value_type<ContainerType>; //!< value type of the ContainerType class
+    ///@brief Allocate nothing, Call \c construct method before usage
+    BICGSTABl(){}
+    ///@copydoc construct()
+    BICGSTABl( const ContainerType& copyable, unsigned max_iterations, unsigned l_input){
+        construct(copyable, max_iterations, l_input);
+    }
+    ///@brief Set the maximum number of iterations
+    ///@param new_max New maximum number
+    void set_max( unsigned new_max) {max_iter = new_max;}
+    ///@brief Get the current maximum number of iterations
+    ///@return the current maximum
+    unsigned get_max() const {return max_iter;}
+    /**
+     * @brief Allocate memory for the preconditioned BICGSTABl method
+     *
+     * @param copyable A ContainerType must be copy-constructible from this
+     * @param max_iterations Maximum number of iterations
+     * @param l_input Size of polynomial used for stabilisation. Usually 2 or 4 is a good number.
+     */
+    void construct(const ContainerType& copyable, unsigned max_iterations, unsigned l_input){
+        max_iter = max_iterations;
+        l = l_input;
+        xhat = r = rtilde = u = copyable;
+        rhat.assign(l+1,copyable);
+        uhat.assign(l+1,copyable);
+        sigma.assign(l+1,0);
+        gamma.assign(l+1,0);
+        gammap.assign(l+1,0);
+        gammapp.assign(l+1,0);
+        for(unsigned i = 0; i < l; i++){
+            tau.push_back(std::vector<value_type>());
+            for(unsigned j = 0; j < l; j++){
+                tau[i].push_back(0);
+            }
+        }
+    }
+
+    /**
+     * @brief Solve \f$ Ax = b\f$ using a preconditioned BICGSTABl method
+     *
+     * The iteration stops if \f$ ||Ax||_S < \epsilon( ||b||_S + C) \f$ where \f$C\f$ is
+     * the absolute error in units of \f$ \epsilon\f$ and \f$ S \f$ defines a square norm
+     * @param A A matrix
+     * @param x Contains an initial value on input and the solution on output.
+     * @param b The right hand side vector. x and b may be the same vector.
+     * @param P The preconditioner to be used
+     * @param S (Inverse) Weights used to compute the norm for the error condition
+     * @param eps The relative error to be respected
+     * @param nrmb_correction the absolute error \c C in units of \c eps to be respected
+     *
+     * @return Number of iterations used to achieve desired precision
+     * @copydoc hide_matrix
+     * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
+     * @tparam Preconditioner A type for which the blas2::symv(Preconditioner&, ContainerType&, ContainerType&) function is callable.
+     * @tparam SquareNorm A type for which the blas2::dot( const SquareNorm&, const ContainerType&) function is callable. This can e.g. be one of the ContainerType types.
+     */
+    template< class MatrixType, class ContainerType0, class ContainerType1, class Preconditioner, class SquareNorm >
+    unsigned solve( MatrixType& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
+
+  private:
+    unsigned max_iter, l;
+    ContainerType r, rtilde, u, xhat;
+    std::vector<ContainerType> rhat;
+    std::vector<ContainerType> uhat;
+    std::vector<value_type> sigma, gamma, gammap, gammapp;
+    std::vector<std::vector<value_type>> tau;
+
+};
+///@cond
+
+template< class ContainerType>
+template< class Matrix, class ContainerType0, class ContainerType1, class Preconditioner, class SquareNorm>
+unsigned BICGSTABl< ContainerType>::solve( Matrix& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction)
+{
+    value_type normb = sqrt(dg::blas2::dot(S,b));
+
+    dg::blas1::scal(u,0);
+
+    dg::blas2::symv(A,x,r);
+    dg::blas1::axpby(1.,b,-1.,r);
+    dg::blas2::symv(P,r,r);
+
+    dg::blas1::copy(b,rtilde);
+
+    value_type rho_0 = 1;
+    value_type alpha = 0;
+    value_type omega = 1;
+
+    for (unsigned k = 0; k < max_iter; k++){
+        dg::blas1::copy(u,uhat[0]);
+        dg::blas1::copy(r,rhat[0]);
+        dg::blas1::copy(x,xhat);
+
+        rho_0 = -omega*rho_0;
+
+        /// Bi-CG part ///
+        for(unsigned j = 0; j<l;j++)
+        {
+            value_type rho_1 = dg::blas1::dot(rhat[j],rtilde);
+            value_type beta = alpha*rho_1/rho_0;
+            rho_0 = rho_1;
+            for(unsigned i = 0; i<=j;i++)
+            {
+                dg::blas1::axpby(1.,rhat[i],-1.0*beta,uhat[i]);
+            }
+            dg::blas2::symv(A,uhat[j],uhat[j+1]);
+            dg::blas2::symv(P,uhat[j+1],uhat[j+1]);
+            alpha = rho_0/dg::blas1::dot(uhat[j+1],rtilde);
+            for(unsigned i = 0; i<=j; i++)
+            {
+                dg::blas1::axpby(-1.0*alpha,uhat[i+1],1.,rhat[i]);
+            }
+            dg::blas2::symv(A,rhat[j],rhat[j+1]);
+            dg::blas2::symv(P,rhat[j+1],rhat[j+1]);
+            dg::blas1::axpby(alpha,uhat[0],1.,xhat);
+        }
+
+        /// MR part ///
+        for(unsigned j = 1; j<=l; j++){
+            for(unsigned i = 1; i<j;i++){
+                tau[i][j] = 1.0/sigma[i]*dg::blas1::dot(rhat[j],rhat[i]);
+                dg::blas1::axpby(-tau[i][j],rhat[i],1.,rhat[j]);
+            }
+            sigma[j] = dg::blas1::dot(rhat[j],rhat[j]);
+            gammap[j] = 1.0/sigma[j]*dg::blas1::dot(rhat[0],rhat[j]);
+        }
+
+        gamma[l] = gammap[l];
+        omega = gamma[l];
+
+        for(unsigned j=l-1;j>=1;j--){
+            value_type tmp = 0;
+            for(unsigned i=j+1;i<=l;i++){
+                tmp += tau[j][i]*gamma[i];
+            }
+            gamma[j] = gammap[j]-tmp;
+        }
+        for(unsigned j=1;j<=l-1;j++){
+            value_type tmp = 0.;
+            for(unsigned i=j+1;i<=l-1;i++){
+                tmp += tau[j][i]*gamma[i+1];
+            }
+            gammapp[j] = gamma[j+1]+tmp;
+        }
+        dg::blas1::axpby(gamma[1],rhat[0],1.,xhat);
+        dg::blas1::axpby(-gammap[l],rhat[l],1.,rhat[0]);
+        dg::blas1::axpby(-gamma[l],uhat[l],1.,uhat[0]);
+        for(unsigned j = 1; j<=l-1; j++){
+            dg::blas1::axpby(gammapp[j],rhat[j],1.,xhat);
+            dg::blas1::axpby(-gamma[j],uhat[j],1.,uhat[0]);
+            dg::blas1::axpby(-gammap[j],rhat[j],1.,rhat[0]);
+        }
+        dg::blas1::copy(uhat[0],u);
+        dg::blas1::copy(rhat[0],r);
+        dg::blas1::copy(xhat,x);
+
+        value_type err = dg::blas2::dot(S,r);
+#ifdef DG_DEBUG
+#ifdef MPI_VERSION
+    int rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    if(rank==0)
+#endif //MPI
+        std::cout << "# Error is now : " << sqrt(err) << " Against " << sqrt(eps*eps*normb*normb) << std::endl;
+#endif //DG_DEBUG
+        if(err < eps*eps*normb*normb){
+#ifdef DG_DEBUG
+#ifdef MPI_VERSION
+    if(rank==0)
+#endif //MPI
+            std::cout << "# Exited with error : " << sqrt(err) << " After " << k << " Iterations." << std::endl;
+#endif //DG_DEBUG
+            return k;
+        }
+    }
+#ifdef DG_DEBUG
+#ifdef MPI_VERSION
+    if(rank==0)
+#endif //MPI
+    std::cout << "# Failed to converge within max_iter" << std::endl;
+#endif //DG_DEBUG
+    return max_iter;
+}
+///@endcond
+
+}//namespace dg
+#endif
diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 1539c992f..3ee16208c 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -19,7 +19,7 @@ namespace dg{
 //// TO DO: check for better stopping criteria using condition number estimates?
 
 /**
-* @brief Functor class for the preconditioned conjugate gradient method to solve
+* @brief Preconditioned conjugate gradient method to solve
 * \f[ Ax=b\f]
 *
 * @ingroup invert
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index aa85daf23..15057853e 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -2,6 +2,8 @@
 #include <iomanip>
 
 #include "cg.h"
+#include "bicgstabl.h"
+#include "lgmres.h"
 #include "elliptic.h"
 
 const double lx = 2.*M_PI;
@@ -12,6 +14,37 @@ double fct(double x, double y){ return sin(y)*sin(x);}
 double laplace_fct( double x, double y) { return 2*sin(y)*sin(x);}
 double initial( double x, double y) {return sin(0);}
 
+template<class Matrix, class Container>
+void solve( std::string solver, Matrix& A, Container& x, const Container& b, const dg::Grid2d& grid)
+{
+    unsigned n = grid.n(), Nx = grid.Nx(), Ny = grid.Ny();
+    if( "cheby" == solver)
+    {
+        dg::ChebyshevIteration<Container> cheby( x);
+        double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
+        double hxhy = lx*ly/(n*n*Nx*Ny);
+        lmin *= hxhy, lmax *= hxhy; //we multiplied the matrix by w2d
+        //std::cout << "Type number of Chebyshev iterations\n";
+        unsigned num_iter =100;
+        //std::cin >> num_iter;
+        cheby.solve( A, x, b, lmin, lmax, num_iter);
+        std::cout << "After "<<num_iter<<" Chebyshev iterations we have:\n";
+    }
+    if( "bicgstabl" == solver)
+    {
+        dg::BICGSTABl<Container> bicg( x, 100, 2);
+        unsigned num_iter = bicg.solve( A, x, b, A.precond(), A.inv_weights(), 1e-6);
+        std::cout << "After "<<num_iter<<" BICGSTABl iterations we have:\n";
+    }
+    if( "lgmres" == solver)
+    {
+        dg::LGMRES<Container> lgmres( x, 30, 4, 10000);
+        unsigned num_iter = lgmres.solve( A, x, b, A.precond(), A.inv_weights(), 1e-6);
+        std::cout << "After "<<num_iter<<" LGMRES iterations we have:\n";
+    }
+
+}
+
 int main()
 {
     //global relative error in L2 norm is O(h^P)
@@ -46,8 +79,9 @@ int main()
 
     // use inverse volume as preconditioner in solution method
     const double eps = 1e-6;
-    std::cout << "Number of pcg iterations "<< pcg( A, x, b, v2d, eps)<<std::endl;
+    unsigned num_iter = pcg( A, x, b, v2d, eps);
 //! [doxygen]
+    std::cout << "Number of pcg iterations "<< num_iter<<std::endl;
     std::cout << "For a precision of "<< eps<<std::endl;
     //compute error
     dg::HVec error( solution);
@@ -67,32 +101,28 @@ int main()
     res.d = sqrt(dg::blas2::dot( w2d, resi));
     std::cout << "L2 Norm of Residuum is        " << res.d<<"\t"<<res.i << std::endl;
     //Fehler der Integration des Sinus ist vernachlässigbar (vgl. evaluation_t)
-    dg::blas1::copy( 0., x);
-    dg::ChebyshevIteration<dg::HVec> cheby( copyable_vector);
-    double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
-    double hxhy = lx*ly/(n*n*Nx*Ny);
-    lmin *= hxhy, lmax *= hxhy; //we multiplied the matrix by w2d
-    std::cout << "Type number of Chebyshev iterations\n";
-    unsigned num_iter;
-    std::cin >> num_iter;
-    cheby.solve( A, x, b, lmin, lmax, num_iter);
-    std::cout << "After "<<num_iter<<" Chebyshev iterations we have:\n";
-
-    dg::blas1::copy( solution, error);
-    dg::blas1::axpby( 1.,x,-1.,error);
-
-    dg::blas1::copy( b, resi);
-    dg::blas2::symv(  A, x, Ax);
-    dg::blas1::axpby( 1.,Ax,-1.,resi);
 
-    res.d = sqrt(dg::blas2::dot( w2d, x));
-    std::cout << "L2 Norm of x0 is              " << res.d<<"\n";
-    res.d = sqrt(dg::blas2::dot(w2d , solution));
-    std::cout << "L2 Norm of Solution is        " << res.d<<"\n";
-    res.d = sqrt(dg::blas2::dot(w2d , error));
-    std::cout << "L2 Norm of Error is           " << res.d<<"\n";
-    res.d = sqrt(dg::blas2::dot( w2d, resi));
-    std::cout << "L2 Norm of Residuum is        " << res.d<<"\n";
+    std::vector<std::string> solvers{ "cheby", "bicgstabl", "lgmres"};
+    for(auto solver : solvers)
+    {
+        dg::blas1::copy( 0., x);
+        solve( solver, A, x, b, grid);
+        dg::blas1::copy( solution, error);
+        dg::blas1::axpby( 1.,x,-1.,error);
+
+        dg::blas1::copy( b, resi);
+        dg::blas2::symv(  A, x, Ax);
+        dg::blas1::axpby( 1.,Ax,-1.,resi);
+
+        res.d = sqrt(dg::blas2::dot( w2d, x));
+        std::cout << "L2 Norm of x0 is              " << res.d<<"\n";
+        res.d = sqrt(dg::blas2::dot(w2d , solution));
+        std::cout << "L2 Norm of Solution is        " << res.d<<"\n";
+        res.d = sqrt(dg::blas2::dot(w2d , error));
+        std::cout << "L2 Norm of Error is           " << res.d<<"\n";
+        res.d = sqrt(dg::blas2::dot( w2d, resi));
+        std::cout << "L2 Norm of Residuum is        " << res.d<<"\n\n";
+    }
 
 
     return 0;
diff --git a/inc/dg/lgmres.h b/inc/dg/lgmres.h
new file mode 100644
index 000000000..ff4dc2b99
--- /dev/null
+++ b/inc/dg/lgmres.h
@@ -0,0 +1,301 @@
+#ifndef _DG_LGMRES_
+#define _DG_LGMRES_
+
+#include <iostream>
+#include <cstring>
+#include <cmath>
+#include <algorithm>
+
+#include "blas.h"
+#include "functors.h"
+/*!@file
+ * LGMRES class
+ *
+ * @author Aslak Poulsen
+ */
+
+namespace dg{
+
+/**
+* @brief Functor class for the preconditioned LGMRES method to solve
+* \f[ Ax=b\f]
+*
+* @ingroup invert
+*
+* @note GMRES is a method for solving non-symmetrical linear systems.
+* LGMRES is a modification of restarted GMRES that aims to improve convergence.
+* This implementation is adapted from:
+* https://github.com/KellyBlack/GMRES/blob/master/GMRES.h
+* with LGMRES elements from
+* https://github.com/haranjackson/NewtonKrylov/blob/master/lgmres.cpp
+* A paper can be found at
+* https://www.cs.colorado.edu/~jessup/SUBPAGES/PS/lgmres.pdf
+*
+*/
+template< class ContainerType>
+class LGMRES
+{
+  public:
+    using container_type = ContainerType;
+    using value_type = dg::get_value_type<ContainerType>; //!< value type of the ContainerType class
+    ///@brief Allocate nothing, Call \c construct method before usage
+    LGMRES(){}
+    ///@copydoc construct()
+    LGMRES( const ContainerType& copyable, unsigned max_outer, unsigned max_inner, unsigned Restarts){
+        construct(copyable, max_outer, max_inner, Restarts);
+    }
+    ///@brief Set the number of restarts
+    ///@param new_Restarts New maximum number of restarts
+    void set_max( unsigned new_Restarts) {numberRestarts = new_Restarts;}
+    ///@brief Get the current maximum number of iterations
+    ///@return the current maximum
+    unsigned get_numberRestarts() const {return numberRestarts;}
+    /**
+     * @brief Allocate memory for the preconditioned LGMRES method
+     *
+     * @param copyable A ContainerType must be copy-constructible from this
+     * @param max_inner Maximum number of vectors to be saved in gmres. Usually 30 seems to be a decent number.
+     * @param max_outer Maximum number of solutions saved for restart. Usually 3-10 seems to be a good number.
+     * @param Restarts Maximum number of restarts. This can be set high just in case. Like e.g. gridsize/max_outer.
+     */
+    void construct(const ContainerType& copyable, unsigned max_outer, unsigned max_inner, unsigned Restarts){
+        outer_k = max_outer;
+        inner_m = max_inner;
+        numberRestarts = Restarts;
+        krylovDimension = inner_m + outer_k;
+        //Declare Hessenberg matrix
+        for(unsigned i = 0; i < krylovDimension+1; i++){
+            H.push_back(std::vector<value_type>());
+            for(unsigned j = 0; j < krylovDimension; j++){
+                H[i].push_back(0);
+            }
+        }
+        //Declare givens rotation matrix
+        for(unsigned i = 0; i < krylovDimension+1; i++){
+            givens.push_back(std::vector<value_type>());
+            for(unsigned j = 0; j < 2; j++){
+                givens[i].push_back(0);
+            }
+        }
+        //Declare s that minimizes the residual... something like that.
+        //s(krylovDimension+1);
+        s.assign(krylovDimension,0);
+
+        //The residual which will be used to calculate the solution.
+        V.assign(krylovDimension+1,copyable);
+        W.assign(krylovDimension,copyable);
+        //In principle we don't need this many... but just to be on board with the algorithm
+        outer_v.assign(outer_k,copyable);
+        z = copyable;
+        dx = copyable;
+        residual = copyable;
+    }
+
+    /**
+     * @brief Solve \f$ Ax = b\f$ using a preconditioned LGMRES method
+     *
+     * The iteration stops if \f$ ||Ax||_S < \epsilon( ||b||_S + C) \f$ where \f$C\f$ is
+     * the absolute error in units of \f$ \epsilon\f$ and \f$ S \f$ defines a square norm
+     * @param A A matrix
+     * @param x Contains an initial value on input and the solution on output.
+     * @param b The right hand side vector. x and b may be the same vector.
+     * @param P The preconditioner to be used
+     * @param S (Inverse) Weights used to compute the norm for the error condition
+     * @param eps The relative error to be respected
+     * @param nrmb_correction the absolute error \c C in units of \c eps to be respected
+     *
+     * @return Number of iterations used to achieve desired precision
+     * @copydoc hide_matrix
+     * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
+     * @tparam Preconditioner A type for which the blas2::symv(Preconditioner&, ContainerType&, ContainerType&) function is callable.
+     * @tparam SquareNorm A type for which the blas2::dot( const SquareNorm&, const ContainerType&) function is callable. This can e.g. be one of the ContainerType types.
+     */
+    template< class MatrixType, class ContainerType0, class ContainerType1, class Preconditioner, class SquareNorm >
+    unsigned solve( MatrixType& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
+
+  private:
+    template < class Hess, class HessContainerType1, class HessContainerType2, class HessContainerType3  >
+    void Update(HessContainerType1 &dx, HessContainerType1 &x, unsigned dimension, Hess &H, HessContainerType2 &s, HessContainerType3 &V);
+    value_type tolerance;
+    std::vector<std::vector<value_type>> H, givens;
+    ContainerType z, dx, residual;
+    std::vector<ContainerType> V, W, outer_v;
+    std::vector<value_type> s;
+    unsigned numberRestarts, inner_m, outer_k, krylovDimension;
+};
+///@cond
+
+template< class ContainerType>
+template < class Hess, class HessContainerType1, class HessContainerType2, class HessContainerType3  >
+void LGMRES< ContainerType>::Update(HessContainerType1 &dx, HessContainerType1 &x, unsigned dimension, Hess &H, HessContainerType2 &s, HessContainerType3 &V)
+{
+    // Solve for the coefficients, i.e. solve for c in
+    // H*c=s, but we do it in place.
+    int lupe;
+    for (lupe = dimension; lupe >= 0; --lupe)
+	    {
+		    s[lupe] = s[lupe]/H[lupe][lupe];
+            if(lupe > 0){
+                for (int innerLupe = lupe - 1; innerLupe >= 0; --innerLupe)
+                {
+                    // Subtract off the parts from the upper diagonal of the matrix.
+                    s[innerLupe] -=  s[lupe]*H[innerLupe][lupe];
+                }
+            }
+	}
+
+    // Finally update the approximation.
+    dg::blas1::scal(dx,0.);
+    for (lupe = 0; lupe <= (int)dimension; lupe++)
+        dg::blas1::axpby(s[lupe],V[lupe],1.,dx);
+    dg::blas1::axpby(1.,dx,1.,x);
+}
+
+template< class ContainerType>
+template< class Matrix, class ContainerType0, class ContainerType1, class Preconditioner, class SquareNorm>
+unsigned LGMRES< ContainerType>::solve( Matrix& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction)
+{
+    dg::blas2::symv(A,x,residual);
+    dg::blas1::axpby(1.,b,-1.,residual);
+    dg::blas2::symv(P,residual,residual);
+    value_type rho = sqrt(dg::blas1::dot(residual,residual));
+    value_type normres = sqrt(dg::blas2::dot(S,residual));
+    value_type normRHS = sqrt(dg::blas1::dot(b,b));
+    value_type normedRHS = sqrt(dg::blas2::dot(S,b));
+
+    tolerance = eps;
+
+    unsigned totalRestarts = 0;
+
+    if(normRHS < 1.0E-5)
+        normRHS = 1.0;
+
+    // Go through the requisite number of restarts.
+	unsigned iteration = 0;
+
+    while( (totalRestarts < numberRestarts) && (normres > tolerance*normedRHS))
+	{
+        // The first vector in the Krylov subspace is the normalized residual.
+        dg::blas1::axpby(1.0/rho,residual,0.,V[0]);
+
+        for(unsigned lupe=0;lupe<=krylovDimension;++lupe)
+			s[lupe] = 0.0;
+		s[0] = rho;
+
+		// Go through and generate the pre-determined number of vectors for the Krylov subspace.
+		for( iteration=0;iteration<krylovDimension;++iteration)
+		{
+            unsigned outer_v_count = std::min(totalRestarts,outer_k);
+            if(iteration < outer_v_count){
+                //dg::blas1::copy(outer_v[totalRestarts-outer_v_count+iteration],z);
+                dg::blas1::copy(outer_v[iteration],z);
+            } else if (iteration == outer_v_count) {
+                dg::blas1::copy(V[0],z);
+            } else {
+                dg::blas1::copy(V[iteration],z);
+            }
+
+			// Get the next entry in the vectors that form the basis for the Krylov subspace.
+            dg::blas2::symv(A,z,V[iteration+1]);
+            dg::blas2::symv(P,V[iteration+1],V[iteration+1]);
+            unsigned row;
+
+            for(row=0;row<=iteration;++row)
+			{
+                H[row][iteration] = dg::blas1::dot(V[iteration+1],V[row]);
+                dg::blas1::axpby(-H[row][iteration],V[row],1.,V[iteration+1]);
+			}
+            H[iteration+1][iteration] = sqrt(dg::blas1::dot(V[iteration+1],V[iteration+1]));
+            dg::blas1::scal(V[iteration+1],1.0/H[iteration+1][iteration]);
+            dg::blas1::copy(z,W[iteration]);
+
+			// Apply the Givens Rotations to insure that H is
+			// an upper diagonal matrix. First apply previous
+			// rotations to the current matrix.
+			value_type tmp;
+			for (row = 0; row < iteration; row++)
+			{
+				tmp = givens[row][0]*H[row][iteration] +
+					givens[row][1]*H[row+1][iteration];
+				H[row+1][iteration] = -givens[row][1]*H[row][iteration]
+					+ givens[row][0]*H[row+1][iteration];
+				H[row][iteration]  = tmp;
+			}
+
+			// Figure out the next Givens rotation.
+			if(H[iteration+1][iteration] == 0.0)
+			{
+				// It is already lower diagonal. Just leave it be....
+				givens[iteration][0] = 1.0;
+				givens[iteration][1] = 0.0;
+			}
+			else if (fabs(H[iteration+1][iteration]) > fabs(H[iteration][iteration]))
+			{
+				// The off diagonal entry has a larger
+				// magnitude. Use the ratio of the
+				// diagonal entry over the off diagonal.
+				tmp = H[iteration][iteration]/H[iteration+1][iteration];
+				givens[iteration][1] = 1.0/sqrt(1.0+tmp*tmp);
+				givens[iteration][0] = tmp*givens[iteration][1];
+			}
+			else
+			{
+				// The off diagonal entry has a smaller
+				// magnitude. Use the ratio of the off
+				// diagonal entry to the diagonal entry.
+				tmp = H[iteration+1][iteration]/H[iteration][iteration];
+				givens[iteration][0] = 1.0/sqrt(1.0+tmp*tmp);
+				givens[iteration][1] = tmp*givens[iteration][0];
+			}
+            // Apply the new Givens rotation on the new entry in the upper Hessenberg matrix.
+			tmp = givens[iteration][0]*H[iteration][iteration] +
+				  givens[iteration][1]*H[iteration+1][iteration];
+			H[iteration+1][iteration] = -givens[iteration][1]*H[iteration][iteration] +
+				  givens[iteration][0]*H[iteration+1][iteration];
+			H[iteration][iteration] = tmp;
+			// Finally apply the new Givens rotation on the s vector
+			tmp = givens[iteration][0]*s[iteration] + givens[iteration][1]*s[iteration+1];
+			s[iteration+1] = -givens[iteration][1]*s[iteration] + givens[iteration][1]*s[iteration+1];
+			s[iteration] = tmp;
+
+            rho = fabs(s[iteration+1]);
+#ifdef DG_DEBUG
+#ifdef MPI_VERSION
+    int rank;
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    if(rank==0)
+#endif //MPI
+            std::cout << "# rho = " << rho << std::endl;
+#endif //DG_DEBUG
+			if(rho < tolerance*normRHS)
+			{
+
+                Update(dx,x,iteration,H,s,W);
+                dg::blas2::symv(A,x,residual);
+                dg::blas1::axpby(1.,b,-1.,residual);
+                std::cout << sqrt(dg::blas2::dot(S,residual) )<< std::endl;
+                return(iteration+totalRestarts*krylovDimension);
+            }
+        }
+        Update(dx,x,iteration-1,H,s,W);
+        value_type nx = sqrt(dg::blas1::dot(dx,dx));
+        if(nx>0.){
+            if (totalRestarts<outer_k){
+                dg::blas1::axpby(1.0/nx,dx,0.,outer_v[totalRestarts]); //new outer entry = dx/nx
+            } else {
+                std::rotate(outer_v.begin(),outer_v.begin()+1,outer_v.end()); //rotate one to the left.
+                dg::blas1::axpby(1.0/nx,dx,0.,outer_v[outer_k]);
+            }
+        }
+        dg::blas2::symv(A,x,residual);
+        dg::blas1::axpby(1.,b,-1.,residual);
+        normres = sqrt(dg::blas2::dot(S,residual));
+        dg::blas2::symv(P,residual,residual);
+        value_type rho = sqrt(dg::blas1::dot(residual,residual));
+        totalRestarts += 1;
+    }
+    return totalRestarts*krylovDimension;
+}
+///@endcond
+}//namespace dg
+#endif
-- 
GitLab


From 390ee9337c011378553644dece5f244aa9df4e10 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 Nov 2019 13:43:37 +0100
Subject: [PATCH 174/540] Set source jfactor 0 in feltor

---
 inc/dg/elliptic.h   | 14 ++++++++++----
 src/feltor/feltor.h |  1 +
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 115008eea..f21bd499d 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -283,8 +283,11 @@ class Elliptic
         dg::blas2::symv( -1., m_leftx, m_tempx, -1., m_temp);
 
         //add jump terms
-        dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
-        dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        if( 0 != m_jfactor )
+        {
+            dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
+            dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        }
         if( m_no == normed)
             dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
         if( m_no == not_normed)//multiply weights without volume
@@ -560,8 +563,11 @@ class Elliptic3d
         dg::blas2::symv( -1., m_leftx, m_tempx, 1., m_temp);
 
         //add jump terms
-        dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
-        dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        if( 0 != m_jfactor )
+        {
+            dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
+            dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        }
         if( m_no == normed)
             dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
         if( m_no == not_normed)//multiply weights without volume
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index f75193251..360baafeb 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -483,6 +483,7 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
         m_lapperpU.set_compute_in_2d(true);
         m_lapperpP.set_compute_in_2d(true);
     }
+    m_lapperpP.set_jfactor(0); //we don't want jump terms in source
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
 void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
-- 
GitLab


From bf04a3bdbd53cb9612a5329b9d561483e57b9911 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 21 Nov 2019 00:15:25 +0100
Subject: [PATCH 175/540] Start ds interpolation routine

---
 inc/geometries/fieldaligned.h | 44 +++++++++++++++++++++++++++++++----
 1 file changed, 39 insertions(+), 5 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index bde927fe8..92f19d55a 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -26,10 +26,10 @@ namespace geo{
 ///@ingroup fieldaligned
 enum whichMatrix
 {
-    einsPlus = 0,   /// plus interpolation in next plane
-    einsPlusT = 1,  /// transposed plus interpolation in previous plane
-    einsMinus = 2,  /// minus interpolation in previous plane
-    einsMinusT = 3, /// transposed minus interpolation in next plane
+    einsPlus = 0,   //!< plus interpolation in next plane
+    einsPlusT = 1,  //!< transposed plus interpolation in previous plane
+    einsMinus = 2,  //!< minus interpolation in previous plane
+    einsMinusT = 3, //!< transposed minus interpolation in next plane
 };
 
 ///@brief Full Limiter means there is a limiter everywhere
@@ -327,7 +327,9 @@ struct Fieldaligned
     container evaluate( const BinaryOp& binary, const UnaryOp& unary, unsigned p0, unsigned rounds) const;
 
     /**
-    * @brief Applies the interpolation
+    * @brief Apply the interpolation to three-dimensional vectors
+    *
+    * computes \f$  y = 1^\pm \otimes \mathcal T x\f$
     * @param which specify what interpolation should be applied
     * @param in input
     * @param out output may not equal input
@@ -351,6 +353,20 @@ struct Fieldaligned
     }
     ///Grid used for construction
     const ProductGeometry& grid()const{return *m_g;}
+
+    /**
+    * @brief Interpolate along fieldlines from a coarse to a fine grid in phi
+    *
+    * In this function we assume that the Fieldaligned object lives on the fine
+    * grid and we now want to interpolate values from a vector living on a coarse grid along the fieldlines onto the fine grid.
+    * Here, coarse and fine are with respect to the phi direction. The perpendicular directions need to have the same resolution in both input and output, i.e. there
+    * is no interpolation in those directions.
+    * @param cphi The compression factor in phi (must integer divide \c Nz in input grid)
+    * @param in the coarse input vector
+    *
+    * @return the input interpolated onto the grid given in the constructor
+    */
+    container interpolate_from_coarse_grid( unsigned cphi, const container& in);
     private:
     void ePlus( enum whichMatrix which, const container& in, container& out);
     void eMinus(enum whichMatrix which, const container& in, container& out);
@@ -534,6 +550,24 @@ container Fieldaligned<G, I,container>::evaluate( const BinaryOp& binary, const
     return vec3d;
 }
 
+template<class G, class I, class container>
+container interpolate_from_coarse_grid( unsigned cphi, const container& in)
+{
+    //I think we need grid as input to split input vector and we need to interpret
+    //the grid nodes as node centered not cell-centered!
+    //idea: apply I+/I- cphi - 1 times in each direction and then apply interpolation formula
+    assert( m_g->Nz() % cphi == 0);
+    unsigned Nz_coarse = m_g.Nz() % cphi;
+
+    container out = dg::evaluate( dg::zero, *m_g);
+    dg::split( out, m_temp, *m_g);
+    //1. copy input vector to appropriate place in output
+    for ( int i=0; i<(int)Nz_coarse; i++)
+    {
+    }
+
+
+}
 
 template<class G, class I, class container>
 void Fieldaligned<G, I, container >::operator()(enum whichMatrix which, const container& f, container& fe)
-- 
GitLab


From 2c4964b8345b622c90437e23d118afd5bdbdebce Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 21 Nov 2019 19:41:36 +0100
Subject: [PATCH 176/540] New COMPASS equilibrium

---
 inc/dg/elliptic.h                |  2 ++
 inc/geometries/geometry_diag.cu  |  2 ++
 src/feltor/feltor.tex            |  2 +-
 src/feltor/geometry/compass.json | 48 ++++++++++++++++----------------
 4 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index f21bd499d..4e3e806d6 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -456,6 +456,8 @@ class Elliptic3d
      * a scalar part \f$ \sigma\f$ and a tensor part \f$ \tau\f$ and you can
      * set each part seperately. This functions sets the tensor part.
      *
+     * @note The class will take care of the volume element in the divergence so do not multiply it to \c tau yourself
+     *
      * @param tau The new tensor part in \f$\chi\f$ (must be positive definite)
      * @tparam ContainerType0 must be usable in \c dg::assign to \c Container
      */
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index e4df387b4..c6992124a 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -63,6 +63,8 @@ struct Parameters
             <<" boxscaleRp    = "<<boxscaleRp<<"\n"
             <<" boxscaleZm    = "<<boxscaleZm<<"\n"
             <<" boxscaleZp    = "<<boxscaleZp<<"\n"
+            <<" alpha         = "<<alpha<<"\n"
+            <<" alpha_mag     = "<<alpha_mag<<"\n"
             <<" amp           = "<<amp<<"\n"
             <<" k_psi         = "<<k_psi<<"\n"
             <<" bgprofamp     = "<<bgprofamp<<"\n"
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 8ebede76b..7eb843974 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1313,7 +1313,7 @@ File format: json
     R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ in Eq.~\eqref{eq:solovev} (This is the only geometry quantity to change if $\rho_s$ changes)\\
     elongation    & float & 1 & - & Elongation $e$, used in determining the box size Eq.~\eqref{eq:box} and the initial guess for the location of the X-point $Z_X = -1.1 ea$ \\
     triangularity & float & 0 & - & Triangularity $\delta$, used in the initial guess for the location of the X-point $R_X = R_0-1.1\delta a$ \\
-    inverseaspectratio & float & 0.16667 & - & minor to major radius $a/R_0$ \\
+    inverseaspectratio & float & 0.16667 & - & minor to major radius $a/R_0$ (used to compute $a$ from $R_0$) \\
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
diff --git a/src/feltor/geometry/compass.json b/src/feltor/geometry/compass.json
index df90b406d..a487c4869 100644
--- a/src/feltor/geometry/compass.json
+++ b/src/feltor/geometry/compass.json
@@ -1,24 +1,24 @@
-{
-    "A": 0,
-    "R_0": 1.789406847665635e2,
-    "PP": 1,
-    "PI": 1,
-    "c":[
-           0.081203566774753699,
-           -0.068223110133006717,
-           -0.15733151525792472,
-           -0.070753041538505742,
-           0.10414465434569499,
-           -0.10994533867714237,
-           -0.0047245091245514442,
-           0.039697138423254924,
-           0.26895337916962100,
-           -0.13814872783212580,
-           -0.033095116055736015,
-           0.0047665391068593747
-    ],
-    "elongation": 1.6594,
-    "equilibrium": "solovev",
-    "inverseaspectratio": 0.35714285714285715,
-    "triangularity": 0.4
-}
+{
+    "A": 0,
+   	"R_0": 2.191566859511476e2,
+   	"PP": 0.8,
+   	"PI": 1,
+   	"c":[
+   	    0.10587834884347505,
+   	    0.18967653484360376,
+   	    -0.41197076060549486,
+   	    -0.15733959980607443,
+   	    0.29149170772913691,
+   	    -0.29079253027214297,
+   	    -0.012380284150980477,
+   	    0.092294585192278152,
+   	    0.58087399002810055,
+   	    -0.30962849734343311,
+   	    -0.075321444509503473,
+   	    0.010040461725044062
+   	],
+   	"elongation": 1.6594,
+   	"equilibrium": "solovev",
+   	"inverseaspectratio": 0.2857142857142857,
+   	"triangularity": 0.4
+}
-- 
GitLab


From 475c1f7fbfa0d5bc0a9df1627c6e46acae0169cc Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 21 Nov 2019 22:58:20 +0100
Subject: [PATCH 177/540] Add parallel interpolation method to fieldaligned

---
 inc/geometries/fieldaligned.h | 42 ++++++++++++++++++++++++++---------
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 92f19d55a..3e2c38675 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -361,12 +361,13 @@ struct Fieldaligned
     * grid and we now want to interpolate values from a vector living on a coarse grid along the fieldlines onto the fine grid.
     * Here, coarse and fine are with respect to the phi direction. The perpendicular directions need to have the same resolution in both input and output, i.e. there
     * is no interpolation in those directions.
-    * @param cphi The compression factor in phi (must integer divide \c Nz in input grid)
-    * @param in the coarse input vector
+    * @param grid_coarse The coarse grid (\c coarse_grid.Nz() must integer divide \c Nz from input grid) The x and y dimensions must be equal
+    * @param coarse the coarse input vector
     *
     * @return the input interpolated onto the grid given in the constructor
+    * @note the interpolation weights are taken in the phi distance not the s-distancewhich makes the interpolation linear in phi
     */
-    container interpolate_from_coarse_grid( unsigned cphi, const container& in);
+    container interpolate_from_coarse_grid( const ProductGeometry& grid_coarse, const container& coarse);
     private:
     void ePlus( enum whichMatrix which, const container& in, container& out);
     void eMinus(enum whichMatrix which, const container& in, container& out);
@@ -551,22 +552,43 @@ container Fieldaligned<G, I,container>::evaluate( const BinaryOp& binary, const
 }
 
 template<class G, class I, class container>
-container interpolate_from_coarse_grid( unsigned cphi, const container& in)
+container Fieldaligned<G, I,container>::interpolate_from_coarse_grid( const G& grid, const container& in)
 {
     //I think we need grid as input to split input vector and we need to interpret
     //the grid nodes as node centered not cell-centered!
     //idea: apply I+/I- cphi - 1 times in each direction and then apply interpolation formula
-    assert( m_g->Nz() % cphi == 0);
-    unsigned Nz_coarse = m_g.Nz() % cphi;
+    assert( m_g->Nz() % grid.Nz() == 0);
+    unsigned Nz_coarse = grid.Nz(), Nz = m_g->Nz();
+    unsigned cphi = Nz / Nz_coarse;
 
     container out = dg::evaluate( dg::zero, *m_g);
-    dg::split( out, m_temp, *m_g);
-    //1. copy input vector to appropriate place in output
+    container helper = dg::evaluate( dg::zero, *m_g);
+    dg::split( out, m_f, *m_g);
+    dg::split( helper, m_temp, *m_g);
+    std::vector<dg::View< container>> in_split = dg::split( in, grid);
     for ( int i=0; i<(int)Nz_coarse; i++)
     {
+        //1. copy input vector to appropriate place in output
+        dg::blas1::copy( in_split[i], m_f[i*cphi]);
+        dg::blas1::copy( in_split[i], m_temp[i*cphi]);
+        //2. Now apply plus and minus T to fill in the rest
+        for( int j=1; j<(int)cphi; j++)
+        {
+            //!!! The value of f at the plus plane is I^- of the current plane
+            dg::blas2::symv( m_minus, m_f[i*cphi+j-1], m_f[i*cphi+j]);
+            //!!! The value of f at the minus plane is I^+ of the current plane
+            dg::blas2::symv( m_plus, m_temp[(i*cphi+cphi+1-j)%Nz], m_temp[i*cphi+cphi-j]);
+        }
     }
-
-
+    //3. Now add up with appropriate weights
+    for( int i=0; i<(int)Nz_coarse; i++)
+        for( int j=1; j<(int)cphi; j++)
+        {
+            double alpha = (double)(cphi-j)/(double)cphi;
+            double beta = (double)j/(double)cphi;
+            dg::blas1::axpby( alpha, m_f[i*cphi+j], beta, m_temp[i*cphi+j], m_f[i*cphi+j]);
+        }
+    return out;
 }
 
 template<class G, class I, class container>
-- 
GitLab


From 7af27f51e2150945f5639b8586869b98afaa85c2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 4 Dec 2019 16:13:21 +0100
Subject: [PATCH 178/540] Add newcommands and parallel momentum balance

to writeup
---
 doc/related_pages/newcommands.tex |  11 +-
 src/feltor/feltor.tex             | 413 ++++++++++++++++--------------
 2 files changed, 222 insertions(+), 202 deletions(-)

diff --git a/doc/related_pages/newcommands.tex b/doc/related_pages/newcommands.tex
index 594221907..f72318e7f 100644
--- a/doc/related_pages/newcommands.tex
+++ b/doc/related_pages/newcommands.tex
@@ -15,7 +15,7 @@
 \newcommand{\GKI}{\int d^6 \bm{Z} \BSP}
 \newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}
 \newcommand{\BSP}{B_{\|}^*}
-\newcommand{\GA}[1]{\langle #1	 \rangle}
+\newcommand{\GA}[1]{\left\langle #1	 \right\rangle}
 
 \newcommand{\Abar}{\langle A_\parallel \rangle}
 %Vectors
@@ -41,9 +41,14 @@
 \newcommand{\ffrac}[2]{\frac{\delta#1}{\delta#2}}
 \newcommand{\fixd}[1]{\Big{\arrowvert}_{#1}}
 \newcommand{\curl}[1]{\nabla \times #1}
-\newcommand{\np}{\nabla_{\perp}}
+
+\newcommand{\np}{\vec{\nabla}_{\perp}}
 \newcommand{\npc}{\nabla_{\perp} \cdot }
-\newcommand{\nc}{\nabla\cdot }
+\newcommand{\nc}{\vec\nabla\cdot}
+\newcommand{\cn}{\cdot\vec\nabla}
+\newcommand{\vn}{\vec{\nabla}}
+\newcommand{\npar}{\nabla_\parallel}
+
 \newcommand{\GAI}{\Gamma_{1}^{\dagger}}
 \newcommand{\GAII}{\Gamma_{1}^{\dagger -1}}
 \newcommand{\T}{\mathrm{T}}
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 7eb843974..17f8b5762 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -36,7 +36,7 @@ Given a vector field $\vec B(\vec x)$ with unit vector $\bhat(\vec x) := (\vec B
 we can define various differential operations.
 %Let us further assume that $\bhat$ is perturbed by the parallel
 %vector potential $A_\parallel$ via
-%$\tilde{ \vec b }_\perp := ({\nabla \times A_\parallel \bhat)}/{B}$
+%$\tilde{ \vec b }_\perp := ({\vn \times A_\parallel \bhat)}/{B}$
 \rowcolors{2}{gray!25}{white}
 %\begin{longtable}{>{\RaggedRight}p{7cm}>{\RaggedRight}p{7cm}}
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
@@ -45,39 +45,39 @@ we can define various differential operations.
 \midrule
     Perpendicular Poisson bracket&
     $\left[.,.\right]_\perp$ &
-    $\left[f,g\right]_\perp := \bhat \cdot \left(\vec{\nabla} f \times \vec\nabla g\right) =
+    $\left[f,g\right]_\perp := \bhat \cdot \left(\vn f \times \vn g\right) =
     b_i \varepsilon^{ijk}\partial_j f\partial_k g/\sqrt{g}$  \\
     Projection Tensor&
     $h $ & $h^{ij} := g^{ij} - b^ib^j $\\
     %Alignment Tensor&
     %$t $ & $ t^{ij} := b^ib^j$\\
     Perpendicular Gradient&
-    $\vec \nabla_\perp $&
-    $ \vec \nabla_\perp f := \bhat\times(\vec \nabla f\times \bhat ) \equiv
-    h \cdot \nabla f$ \\
+    $\np $&
+    $ \np f := \bhat\times(\vn f\times \bhat ) \equiv
+    h \cn f$ \\
     Perpendicular Laplacian&
     $\Delta_\perp $&
-    $ \Delta_\perp f:= \vec \nabla\cdot (\vec \nabla_\perp f)
-    = \nabla\cdot( h\cdot\nabla f)$  \\
+    $ \Delta_\perp f:= \vec \nc (\np f)
+    = \nc( h\cn f)$  \\
     Curl-b Curvature &
-    $\mathcal K_{\nabla\times\bhat}$ &
-    $\mathcal K_{\nabla\times\bhat}(f) := \vec{ \mathcal K_{\nabla\times\bhat} }\cdot \vec \nabla f = \frac{1}{B}(\nabla \times \bhat)\cdot \vec \nabla f$ \\[4pt]
+    $\mathcal K_{\vn\times\bhat}$ &
+    $\mathcal K_{\vn\times\bhat}(f) := \vec{ \mathcal K_{\vn\times\bhat} }\cn f = \frac{1}{B}(\vn \times \bhat)\cn f$ \\[4pt]
     Grad-B Curvature &
-    $\mathcal K_{\nabla B} $ &
-    $\mathcal K_{\nabla B}(f) := \vec{\mathcal K_{\nabla B}} \cdot \vec \nabla f = \frac{1}{B}(\bhat \times \vec \nabla \ln B)\cdot \vec \nabla f$ \\[4pt]
+    $\mathcal K_{\vn B} $ &
+    $\mathcal K_{\vn B}(f) := \vec{\mathcal K_{\vn B}} \cn f = \frac{1}{B}(\bhat \times \vn \ln B)\cn f$ \\[4pt]
     Curvature &
     $\mathcal K$ &
-    $\mathcal{K}(f):=\vec{\mathcal K} \cdot \vec \nabla f =
-     \vec{\nabla}\cdot\left(\frac{\bhat\times\vec{\nabla} f}{B}\right)$,\\[4pt]
+    $\mathcal{K}(f):=\vec{\mathcal K} \cn f =
+     \nc\left(\frac{\bhat\times\vn f}{B}\right)$,\\[4pt]
     Parallel derivative&
-    $\nabla_\parallel $&
-    $ \nabla_\parallel f := \bhat\cdot\vec{\nabla} f$ \\
+    $\npar $&
+    $ \npar f := \bhat\cn f$ \\
     %Perturbed parallel Derivative&
-    %$\bar\nabla_\parallel$ &
-    %$\bar\nabla_\parallel f := (\bhat + \tilde{\vec b }_\perp)\cdot \nabla f = \nabla_\parallel f + A_\parallel \mathcal K_{\nabla\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$ \\
+    %$\bar\npar$ &
+    %$\bar\npar f := (\bhat + \tilde{\vec b }_\perp)\cn f = \npar f + A_\parallel \mathcal K_{\vn\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$ \\
     Parallel Laplacian&
     $\Delta_\parallel $&
-    $\Delta_\parallel f:= \vec{\nabla} \cdot ( \bhat\bhat\cdot\vec{\nabla} f )$\\
+    $\Delta_\parallel f:= \nc ( \bhat\bhat\cn f )$\\
 \bottomrule
 \end{longtable}
 with $b^i$ the contra- and $b_i$ the co-variant components of $\bhat$, and
@@ -86,21 +86,21 @@ Explicit expressions for the above expressions
 depend on the choice of the magnetic field and the underlying coordinate system.
 Note that we have
 \begin{align}
-    \vec \nabla \cdot \vec{\mathcal K_{\nabla\times\bhat}}
-    &= -\vec\nabla \cdot \vec{\mathcal K_{\nabla B}} = -\vec{ \mathcal K_{\nabla\times\bhat}}\cdot\nabla\ln B, \\
-    \vec\nabla\cdot\vec{ \mathcal K} &= 0, \\
+    \nc \vec{\mathcal K_{\vn\times\bhat}}
+    &= -\nc \vec{\mathcal K_{\vn B}} = -\vec{ \mathcal K_{\vn\times\bhat}}\cn\ln B, \\
+    \vec\nc\vec{ \mathcal K} &= 0, \\
     \vec{\mathcal K} &=
-     \nabla\times\frac{\bhat}{B}\cdot\nabla f
-    = \mathcal K_{\nabla\times\bhat}(f) + \mathcal K_{\nabla B}(f),\\
-    \vec{ \mathcal K_{\nabla\times\bhat}} - \vec{ \mathcal K_{\nabla B}} &= \frac{1}{B^2} (\vec \nabla \times \vec B), \\
-    \nabla_\parallel \ln B &= -\vec\nabla\cdot\bhat.
+     \vn\times\frac{\bhat}{B}\cn f
+    = \mathcal K_{\vn\times\bhat}(f) + \mathcal K_{\vn B}(f),\\
+    \vec{ \mathcal K_{\vn\times\bhat}} - \vec{ \mathcal K_{\vn B}} &= \frac{1}{B^2} (\vn \times \vec B), \\
+    \npar \ln B &= -\vec\nc\bhat.
     \label{eq:curl_curvature}
 \end{align}
-The last equality holds if $\vec\nabla\cdot \vec B = 0$.
+The last equality holds if $\vec\nc \vec B = 0$.
 Note that in any arbitrary coordinate system we have
 \begin{align}
-(\vec \nabla f)^i = g^{ij}\partial_j f ~, \quad
-\vec \nabla \cdot \vec v = \frac{1}{\sqrt{g}}\partial_i \left(\sqrt{g} v^i\right) ~, \quad
+(\vn f)^i = g^{ij}\partial_j f ~, \quad
+\nc \vec v = \frac{1}{\sqrt{g}}\partial_i \left(\sqrt{g} v^i\right) ~, \quad
 (\vec v \times \vec w)^i = \frac{1}{\sqrt{g}}\varepsilon^{ijk} v_jw_k ~.
 %\label{}
 \end{align}
@@ -127,9 +127,9 @@ basis vectors and (covariant) metric tensor are:
   0 & 1 & 0 \\
   0 & 0 & R^2
    \end{pmatrix}
-% \vec{\nabla} R &= (\sin{(\varphi)} ,   \cos{(\varphi)},0 )^T , &
-%  \vec{\nabla}Z &= ( 0 ,0 ,1 )^T,  &
-%  \vec{\nabla}{\varphi} &= \frac{1}{R} ( \cos{(\varphi)} , -\sin{(\varphi)} , 0 )^T .
+% \vn R &= (\sin{(\varphi)} ,   \cos{(\varphi)},0 )^T , &
+%  \vnZ &= ( 0 ,0 ,1 )^T,  &
+%  \vn{\varphi} &= \frac{1}{R} ( \cos{(\varphi)} , -\sin{(\varphi)} , 0 )^T .
 \end{align}
 With the help of the metric elements we get a well behaved volume element \(\sqrt{g} = R\). However, we have a coordinate singularity at \(R=0\).
 The cylindrical coordinate basis vectors are mutually orthogonal to each other.
@@ -157,15 +157,15 @@ Cartesian coordinates while straightening the field lines for large $R_0$
 we recover the familiar slab geometry.
 
 We have the equilibrium equations in toroidally symmetric, ideal MHD (
-$\vec\nabla p = \vec j\times \vec B$ and $\vec \nabla\times\vec B = \vec j$ normalized with $p_0 = B_0^2/\mu_0$, and $j_0 = B_0/\rho_s/\mu_0$ )
+$\vn p = \vec j\times \vec B$ and $\vn\times\vec B = \vec j$ normalized with $p_0 = B_0^2/\mu_0$, and $j_0 = B_0/\rho_s/\mu_0$ )
 \begin{align}
-    \vec\nabla\times \vec B &= \frac{R_0}{R}\left[ -\Delta^*\psi_p\ehat_\varphi + I_Z \ehat_R - I_R\ehat_Z \right]\equiv \vec j\\
+    \vn\times \vec B &= \frac{R_0}{R}\left[ -\Delta^*\psi_p\ehat_\varphi + I_Z \ehat_R - I_R\ehat_Z \right]\equiv \vec j\\
  j_\parallel &= \vec j\cdot \bhat = \frac{\d p}{\d\psi_p} \frac{I(\psi_p)}{B} +
  \frac{\d I}{\d\psi_p} B \quad \text{  Pfirsch-Schl\"uter \& Bootstrap current } \\
  \vec j_\perp &= \bhat\times\left(\vec j\times\bhat\right)=
- \frac{\bhat \times \nabla p}{B} \quad\quad\quad \text{ diamagnetic current} \\
+ \frac{\bhat \times \vn p}{B} \quad\quad\quad \text{ diamagnetic current} \\
  \vec j\times\vec B &= \frac{R_0^2}{R^2}\left[ -\Delta^* \psi_p - I
-     \frac{\d I}{\d \psi_p} \right]\vec\nabla\psi_p \equiv \frac{\d p}{\d\psi_p}\vec\nabla\psi_p =\vec \nabla p
+     \frac{\d I}{\d \psi_p} \right]\vn\psi_p \equiv \frac{\d p}{\d\psi_p}\vn\psi_p =\vn p
 \end{align}
 from where we recover the Grad-Shafranov equation
 \begin{align}\label{eq:GSEdimless}
@@ -242,12 +242,12 @@ Note that
 (contra- and covariant components of $\vec B$).
 By construction we have $\partial_\varphi B = 0$ with
 \begin{align}
-  B = \frac{R_0}{R}\sqrt{ {I^2 + |\nabla \psi_p|^2}}.
+  B = \frac{R_0}{R}\sqrt{ {I^2 + |\vn \psi_p|^2}}.
     \label{}
 \end{align}
 Furthermore, we have
 \begin{align}
-  \nabla_\parallel f(R,Z) = \frac{R_0}{RB}[f,\psi_p]_{RZ}\Rightarrow \nabla_\parallel \ln B = \frac{R_0}{RB^2}\left[B, \psi_p\right]_{RZ} = -\vec\nabla\cdot\bhat.
+  \npar f(R,Z) = \frac{R_0}{RB}[f,\psi_p]_{RZ}\Rightarrow \npar \ln B = \frac{R_0}{RB^2}\left[B, \psi_p\right]_{RZ} = -\vec\nc\bhat.
 \end{align}
 We allow various simplifications to the curvature operator
 for the Solov'ev equilibrium.
@@ -258,35 +258,35 @@ for the Solov'ev equilibrium.
 The toroidal/negative toroidal field line approximation applies \(\bhat\approx \pm \ehat_\varphi\) to all perpendicular operators
 (e.g.: Poisson bracket, perpendicular elliptic operator and curvature operators)
 but retains the full expression for the magnetic field unit vector \(\bhat\)
-for parallel operators (\(\nabla_\parallel\) and \(\Delta_\parallel\)).
+for parallel operators (\(\npar\) and \(\Delta_\parallel\)).
 (Note that we allow the negative sign $-\ehat_\varphi$ to enable a sign reversal of the magnetic field, see Section~\ref{sec:field_reversal}).
 In cylindrical coordinates that is
 \begin{align}
 [f,g]_\perp \equiv [f,g]_{RZ} &= \pm\frac{1}{R} \left(\partial_R f\partial_Z g - \partial_Z f\partial_R g\right) \\
-\nabla_\perp f &= \partial_R f \ehat_R + \partial_Z f \ehat_Z \\
+\np f &= \partial_R f \ehat_R + \partial_Z f \ehat_Z \\
 \Delta_\perp f &= \frac{1}{R}\partial_R \left( R \partial_R f\right) + \partial_Z(\partial_Z f)
 \label{}
 \end{align}
 The curl of $\bhat$ reduces to
 %\begin{align}
- $\nabla\times\bhat \approx -  \frac{\pm 1}{R} \ehat_Z$.
+ $\vn\times\bhat \approx -  \frac{\pm 1}{R} \ehat_Z$.
 %end{align}
 This simplifies the curvature operators to:
 \begin{align}
-\vec{\mathcal{K}}_{{\nabla\times\bhat}}  &\approx  -  \frac{\pm 1}{B R} \ehat_Z , &
-\vec{ \mathcal{K} }_{\vec{\nabla}  B}  &\approx  -\frac{\pm 1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{\pm 1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
-%\ehat_\varphi \times \vec{\nabla} B, &
-\vec{ \mathcal{K} } &\approx \vec{ \mathcal{K} }_{\vec{\nabla}  B}  +\vec{ \mathcal{K} }_{{\nabla\times\bhat}} ,
+\vec{\mathcal{K}}_{{\vn\times\bhat}}  &\approx  -  \frac{\pm 1}{B R} \ehat_Z , &
+\vec{ \mathcal{K} }_{\vn  B}  &\approx  -\frac{\pm 1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{\pm 1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
+%\ehat_\varphi \times \vn B, &
+\vec{ \mathcal{K} } &\approx \vec{ \mathcal{K} }_{\vn  B}  +\vec{ \mathcal{K} }_{{\vn\times\bhat}} ,
 %\\
-%\mathcal{K}_{{\nabla\times\bhat}}(f)   &\approx  -  \frac{1}{B R} \frac{\partial f}{\partial Z},&
-%\mathcal{K}_{\vec{\nabla}  B} (f)  &= \frac{1}{B} \left[\ln B, f \right]_{RZ},&
+%\mathcal{K}_{{\vn\times\bhat}}(f)   &\approx  -  \frac{1}{B R} \frac{\partial f}{\partial Z},&
+%\mathcal{K}_{\vn  B} (f)  &= \frac{1}{B} \left[\ln B, f \right]_{RZ},&
 %\mathcal{K} (f) &\approx\frac{1}{B} \left[\ln B, f \right]_{RZ}-  \frac{1}{B R} \frac{\partial f}{\partial Z} ,
 \end{align}
 and
 \begin{align}
- \vec{\nabla} \cdot \vec{\mathcal{K}}_{{\nabla\times\bhat}} &\approx \frac{\pm 1}{R B^2} \frac{\partial B}{\partial Z},
+ \nc \vec{\mathcal{K}}_{{\vn\times\bhat}} &\approx \frac{\pm 1}{R B^2} \frac{\partial B}{\partial Z},
 \end{align}
-which results in a vanishing divergence of the curvature operators \( \vec{\nabla} \cdot \vec{ \mathcal{K} } = 0\).
+which results in a vanishing divergence of the curvature operators \( \nc \vec{ \mathcal{K} } = 0\).
 
 Note that in an actual toroidal field we have
 \begin{align}
@@ -296,10 +296,10 @@ Note that in an actual toroidal field we have
 We then have $\bhat = \ehat_\varphi$ and the curvature operators further
 simplify to
 \begin{align}
-  \vec{ \mathcal K_{\nabla\times\bhat}} = \vec{ \mathcal K_{\nabla B}} = -\frac{\pm 1}{R_0} \ehat_Z =
+  \vec{ \mathcal K_{\vn\times\bhat}} = \vec{ \mathcal K_{\vn B}} = -\frac{\pm 1}{R_0} \ehat_Z =
 \vec{ \mathcal K}/2\\
-  \nabla\cdot\vec{\mathcal K_{{\nabla\times\bhat}}}=
-    \nabla_\parallel \ln B = 0
+  \nc\vec{\mathcal K_{{\vn\times\bhat}}}=
+    \npar \ln B = 0
     \label{}
 \end{align}
 
@@ -307,49 +307,49 @@ simplify to
 In this approximation we apply the toroidal field line approximation
 as in Section
 \ref{sec:torfieldlineapprox}
-but approximate the curvature operator $\mathcal K_{\nabla\times\bhat} \approx \bhat\times\vec \kappa$
+but approximate the curvature operator $\mathcal K_{\vn\times\bhat} \approx \bhat\times\vec \kappa$
   with
-  $\vec \kappa := \bhat \cdot \vec \nabla\bhat = -\bhat \times(\vec \nabla\times \bhat)$.
+  $\vec \kappa := \bhat \cn\bhat = -\bhat \times( \vn\times \bhat)$.
 For an isotropic pressure plasma \(\vec{P} = \vec{I} P_\perp + \vec{b} \vec{b} P_\Delta \approx \vec{I} P_\perp\) and with the definition of the plasma beta parameter
 \(\beta = \frac{P}{B^2/(2 \mu_0) } \)
 we can rewrite the curvature to
 \begin{align}
-    \vec{\kappa} &\approx \frac{\beta}{2} \vec{\nabla} \ln(P) +\vec{\nabla}_\perp \ln{B} .
+    \vec{\kappa} &\approx \frac{\beta}{2} \vn \ln(P) +\np \ln{B} .
 \end{align}
 In low beta plasmas \(\beta\ll1\) the curvature reduces to:
 \begin{align}
-    \vec{\kappa} & \approx \vec{\nabla}_\perp \ln{B} .
+    \vec{\kappa} & \approx \np \ln{B} .
 \end{align}
 This simplifies the curvature operators to:
 \begin{align}
-\vec{\mathcal{K}_{{\nabla\times\bhat}}}(f) \approx
-\vec{ \mathcal{K} }_{\vec{\nabla}  B}  &\approx  -\frac{1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
-\mathcal{K} (f) &\approx 2\mathcal{K}_{\vec{\nabla}  B} (f) , &
-    \vec{{\nabla\times\bhat}} \cdot \vec{\mathcal{K}}_{\vec{\nabla}  B} &= 0.
+\vec{\mathcal{K}_{{\vn\times\bhat}}}(f) \approx
+\vec{ \mathcal{K} }_{\vn  B}  &\approx  -\frac{1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
+\mathcal{K} (f) &\approx 2\mathcal{K}_{\vn  B} (f) , &
+    \vn\times\bhat \cdot \vec{\mathcal{K}}_{\vn  B} &= 0.
 \end{align}
-The divergence over the curvature vanishes \( \vec{\nabla} \cdot \vec{ \mathcal{K} } = 0\) only if \( \vec{\nabla} \cdot \vec{ \mathcal{K}}_{\vec{\nabla}  B}   = 0\).
-In general, the divergence \( \vec{\nabla} \cdot \vec{ \mathcal{K} } \approx 0\) is only approximately vanishing.
+The divergence over the curvature vanishes \( \nc \vec{ \mathcal{K} } = 0\) only if \( \nc \vec{ \mathcal{K}}_{\vn  B}   = 0\).
+In general, the divergence \( \nc \vec{ \mathcal{K} } \approx 0\) is only approximately vanishing.
 \subsubsection{True perpendicular terms}
 
 Without any approximations we have
 \begin{align}
-b^R = {\frac{\partial \psi}{\partial Z}}\left(I^2+|\nabla\psi|^2\right)^{-1/2} \quad
-b^Z = -{\frac{\partial \psi}{\partial R}}\left(I^2+|\nabla\psi|^2\right)^{-1/2} \quad 
-b^\varphi = \frac{I}{R}\left(I^2+|\nabla\psi|^2\right)^{-1/2} \\
-\vec\nabla\cdot\bhat = -\nabla_\parallel \ln B = -\frac{R_0}{R B^2}[B,\psi_p]_{RZ} \\
-\left({\nabla\times\bhat}\right) \cdot\bhat =
-    (I'(\nabla\psi_p)^2 - I \Delta_\perp^* \psi_p)\frac{ R_0^2}{R^2B^2} \propto 1/R_0
+b^R = {\frac{\partial \psi}{\partial Z}}\left(I^2+|\vn\psi|^2\right)^{-1/2} \quad
+b^Z = -{\frac{\partial \psi}{\partial R}}\left(I^2+|\vn\psi|^2\right)^{-1/2} \quad 
+b^\varphi = \frac{I}{R}\left(I^2+|\vn\psi|^2\right)^{-1/2} \\
+\vec\nc\bhat = -\npar \ln B = -\frac{R_0}{R B^2}[B,\psi_p]_{RZ} \\
+\left({\vn\times\bhat}\right) \cdot\bhat =
+    (I'(\vn\psi_p)^2 - I \Delta_\perp^* \psi_p)\frac{ R_0^2}{R^2B^2} \propto 1/R_0
 \label{}
 \end{align}
 where for the last
 estimate we inserted the Grad-Shafranov equation and the Solov'ev assumptions.
-We can then insert $\bhat$ into the exact definitions for $[.,.]_\perp$, $\nabla_\perp$ and $\Delta_\perp$ from Section~\ref{sec:magnetic}.
+We can then insert $\bhat$ into the exact definitions for $[.,.]_\perp$, $\np$ and $\Delta_\perp$ from Section~\ref{sec:magnetic}.
 
 For the curvature terms we can explicitly write
 \begin{align}
-K_{\nabla B}^R &= -\frac{R_0 I}{B^3R}\frac{\partial B}{\partial Z} \equiv -\frac{1}{B^2}\frac{\partial B}{\partial Z}b^\varphi \\
-K_{\nabla B}^Z &= \frac{R_0 I}{B^3R}\frac{\partial B}{\partial R}\equiv \frac{1}{B^2}\frac{\partial B}{\partial R}b^\varphi \\
-K_{\nabla B}^\varphi &= \frac{R_0}{B^3R^2}\left(
+K_{\vn B}^R &= -\frac{R_0 I}{B^3R}\frac{\partial B}{\partial Z} \equiv -\frac{1}{B^2}\frac{\partial B}{\partial Z}b^\varphi \\
+K_{\vn B}^Z &= \frac{R_0 I}{B^3R}\frac{\partial B}{\partial R}\equiv \frac{1}{B^2}\frac{\partial B}{\partial R}b^\varphi \\
+K_{\vn B}^\varphi &= \frac{R_0}{B^3R^2}\left(
       \frac{\partial \psi}{\partial Z} \frac{\partial B}{\partial Z}
     + \frac{\partial \psi}{\partial R}\frac{\partial B}{\partial R}\right)
 %\equiv \frac{1}{B^2R}\left(\bhat^R \frac{\partial B}{\partial Z} - \bhat^Z \frac{\partial B}{\partial R}\right)\quad %contravariant phi component
@@ -357,16 +357,16 @@ K_{\nabla B}^\varphi &= \frac{R_0}{B^3R^2}\left(
 \end{align}
 and
 \begin{align}
-K_{\nabla\times\bhat}^R &= \frac{R_0 }{RB^3}\left( B\frac{\partial I}{\partial Z} -I\frac{\partial B}{\partial Z}\right) \\
-K_{\nabla\times\bhat}^Z &= \frac{R_0 }{RB^3} \left( I\frac{\partial B}{\partial R} - B\frac{\partial I}{\partial R} \right)\\
-K_{\nabla\times\bhat}^\varphi &= \frac{R_0}{R^2B^2}\left(
+K_{\vn\times\bhat}^R &= \frac{R_0 }{RB^3}\left( B\frac{\partial I}{\partial Z} -I\frac{\partial B}{\partial Z}\right) \\
+K_{\vn\times\bhat}^Z &= \frac{R_0 }{RB^3} \left( I\frac{\partial B}{\partial R} - B\frac{\partial I}{\partial R} \right)\\
+K_{\vn\times\bhat}^\varphi &= \frac{R_0}{R^2B^2}\left(
 + \frac{1}{B}\frac{\partial\psi}{\partial Z} \frac{\partial B}{\partial Z}
 + \frac{1}{B}\frac{\partial \psi}{\partial R}\frac{\partial B}{\partial R}
 -R\frac{\partial}{\partial R}\left(\frac{1}{R}\frac{\partial\psi}{\partial R}\right) 
 - \frac{\partial^2 \psi}{\partial Z^2}
 \right) \\
-\vec\nabla\cdot\vec{\mathcal K_{\nabla\times\bhat}} &= -\vec\nabla\cdot\vec{\mathcal K_{\nabla B}}=
-    -\vec{\mathcal K_{\nabla\times\bhat}}\cdot \vec \nabla\ln B = \frac{R_0}{RB^3}[I,B]_{RZ}
+\vec\nc\vec{\mathcal K_{\vn\times\bhat}} &= -\vec\nc\vec{\mathcal K_{\vn B}}=
+    -\vec{\mathcal K_{\vn\times\bhat}}\cn\ln B = \frac{R_0}{RB^3}[I,B]_{RZ}
 %contravariant phi component
 \label{}
 \end{align}
@@ -420,7 +420,7 @@ Note that $\Theta_\alpha(0) = 0.5$ and $\theta_\alpha(0) = 35\alpha/256$.
 \subsection{Preliminary}
 Recall that the {\bf Dirac delta-function} has the property (in any dimension):
 \begin{align} \label{eq:dirac_delta}
-\int_V f(\vec x) \delta(h(\vec x) - h') \dV = \int_{h=h'} \frac{f(\vec x)}{|\nabla h|} \dA
+\int_V f(\vec x) \delta(h(\vec x) - h') \dV = \int_{h=h'} \frac{f(\vec x)}{|\vn h|} \dA
 \end{align}
 which means that the delta-function can be used to express area integrals of the
 submanifold given as a contour of the function $h(\vec x)$.
@@ -442,7 +442,7 @@ over neighboring contour lines which is given by the grid distance.
 Furthermore, recall the {\bf co-area formula}
 \begin{align} \label{eq:coarea}
 \int_{\Omega_0} f(\vec x) \dV =
-\int_0^{h_0} \left( \int_{h=h'} \frac{f(\vec x)}{|\nabla h|}  \dA  \right) \d h'
+\int_0^{h_0} \left( \int_{h=h'} \frac{f(\vec x)}{|\vn h|}  \dA  \right) \d h'
 \end{align}
 where $\Omega_0$ is the volume enclosed by the contour $h=h_0$.
 The co-area formula can be viewed as a change of variables in the
@@ -457,17 +457,17 @@ In arbitrary coordinates the area integral is defined by the pull back
 of the flux 2-form and the metric
 \begin{align}
 \label{}
-\dA^2 = i_{\hat \psi_p} vol^3 \quad \hat \psi_p = \frac{\nabla \psi_p}{|\nabla \psi_p|}
+\dA^2 = i_{\hat \psi_p} vol^3 \quad \hat \psi_p = \frac{\vn \psi_p}{|\vn \psi_p|}
 \end{align}
 to a parameterization of the flux-surface.
 In a flux-aligned coordinate system $\{\zeta, \eta, \varphi\}$ the pull-back is trivial ($\zeta=const$) and we have
 \begin{align}
-\dA &= \sqrt{g^{\zeta\zeta}} \sqrt{g} \d\eta\d\varphi = f_0|\nabla\psi_p|\sqrt{g}\d\eta\d\varphi,
+\dA &= \sqrt{g^{\zeta\zeta}} \sqrt{g} \d\eta\d\varphi = f_0|\vn\psi_p|\sqrt{g}\d\eta\d\varphi,
 \\
-\vec\dA &:= \hat\psi_p \dA = f_0 (\nabla\psi_p) \sqrt{g}\d\eta\d\varphi,\quad
+\vec\dA &:= \hat\psi_p \dA = f_0 (\vn\psi_p) \sqrt{g}\d\eta\d\varphi,\quad
 \label{}
 \end{align}
-where we used that $g^{\zeta\zeta} = (\nabla\zeta)^2 = f_0^2(\nabla\psi_p)^2$.
+where we used that $g^{\zeta\zeta} = (\vn\zeta)^2 = f_0^2(\vn\psi_p)^2$.
 Notice that numerically we can integrate in flux-aligned coordinates by generating a corresponding
 grid and pulling back (interpolating) the relevant fields to this grid. This is the second method
 to numerically compute area integrals.
@@ -482,8 +482,8 @@ of a function $f(R,Z,\varphi)$ is given by the formula
 \begin{align}\label{eq:fsa_area}
 \langle f \rangle^{area}_{\psi_{p}} :=&
 \frac{ \oint_{\psi_p  } f(R,Z,\varphi)\dA}{\oint_{\psi_p } \dA} \nonumber \\
-=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vec\nabla\psi_p| \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
-{\int_\Omega |\vec\nabla\psi_p|\delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z} \nonumber\\
+=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vn\psi_p| \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
+{\int_\Omega |\vn\psi_p|\delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z} \nonumber\\
 =& \frac{\oint \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g g^{\zeta\zeta}}\d\eta}
          { \oint \sqrt{g g^{\zeta\zeta}}\d\eta}
 \end{align}
@@ -499,9 +499,9 @@ meaning while the coordinate $\zeta(\psi_p)$ is an arbitrary choice) we define
 \begin{align} \label{eq:fsa_vol}
 v(\psi_p) :=& \int_{\psi_{p,\min}}^\psi \dV = \int^{\zeta(\psi_p)} \sqrt{g}\d\zeta\d\eta\d\varphi,
 \\
-\frac{\d v}{\d\psi_p} =& \int\dA |\nabla\psi_p|^{-1} = 2\pi f_0\oint_{\zeta(\psi_p)} \sqrt{g}\d\eta \\
+\frac{\d v}{\d\psi_p} =& \int\dA |\vn\psi_p|^{-1} = 2\pi f_0\oint_{\zeta(\psi_p)} \sqrt{g}\d\eta \\
 \langle f \rangle^{vol}_\psi :=& \frac{\partial}{\partial v} \int \dV f
- = \frac{1}{\int \dA |\nabla\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\nabla\psi_p|} \dA \nonumber\\
+ = \frac{1}{\int \dA |\vn\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\vn\psi_p|} \dA \nonumber\\
 =& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
 {\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}\nonumber\\
  =& \left(\frac{\d v}{\d\psi_p }\right)^{-1} 2\pi f_0 \oint_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
@@ -509,7 +509,7 @@ v(\psi_p) :=& \int_{\psi_{p,\min}}^\psi \dV = \int^{\zeta(\psi_p)} \sqrt{g}\d\ze
 \end{align}
 where we used the co-area formula Eq.~\eqref{eq:coarea} for the second
 identity. We immediately see that this definition differs from the first
-Eq.~\eqref{eq:fsa_area} by the weight factor $|\nabla\psi_p|$ and that it is particularly easy to compute
+Eq.~\eqref{eq:fsa_area} by the weight factor $|\vn\psi_p|$ and that it is particularly easy to compute
 in a flux-aligned coordinate system. Notice however that the volume element does appear (unlike e.g. Tokam3X papers)
 
 Both averages fulfill the basic identities
@@ -522,27 +522,27 @@ Both averages fulfill the basic identities
 
 The volume average is better suited for density-like quantities
 than the area average as we can see with the following identity.
-Assume we have a quantity $X$ with $\partial_t X + \nabla \cdot \vec j_X = \Lambda_X$. Then we can use the volume average to write
+Assume we have a quantity $X$ with $\partial_t X + \nc \vec j_X = \Lambda_X$. Then we can use the volume average to write
 \begin{align}
 \frac{\partial}{\partial t} \langle X \rangle^{vol} + \frac{\partial}{
-  \partial v} \langle \vec j_X\cdot \nabla v\rangle^{vol}  = \langle \Lambda_X\rangle^{vol}
+  \partial v} \langle \vec j_X\cn v\rangle^{vol}  = \langle \Lambda_X\rangle^{vol}
 \label{eq:fsa_balance}
 \end{align}
 where again $v=v(\psi_p)$ is the volume flux label.
 The {\bf total flux} of a given flux density $\vec j_X$ though the
 flux surface $\psi_p = \psi_{p0}$ is given by
 \begin{align}
-\left\langle\vec j_X\cdot\nabla v\right\rangle^{vol} &:= J_X=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
- \frac{\d v}{\d\psi_p} \langle \vec j_X\cdot\nabla\psi_p \rangle^{vol}\\
+\left\langle\vec j_X\cn v\right\rangle^{vol} &:= J_X=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
+ \frac{\d v}{\d\psi_p} \langle \vec j_X\cn\psi_p \rangle^{vol}\\
  &=
-   2\pi f_0 \oint_0^{2\pi} \langle \vec j_X\cdot\nabla\psi_p\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
-%2\pi\int_\Omega \vec \langle \vec j\cdot \vec\nabla\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
+   2\pi f_0 \oint_0^{2\pi} \langle \vec j_X\cn\psi_p\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
+%2\pi\int_\Omega \vec \langle \vec j\cn\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
 \label{eq:total_flux}
 \end{align}
 Once we have the flux-surface averaged equation we can easily get the volume integrated version (again with the help of the co-area formula)
 \begin{align}
 \frac{\partial}{\partial t} \int_0^{v(\psi_p)}\langle X \rangle^{vol} \d v 
-+ \langle \vec j_X\cdot \nabla v\rangle^{vol}(v(\psi_p))  = \int_0^{v(\psi_p)}\langle \Lambda_X\rangle^{vol}\d v
++ \langle \vec j_X\cn v\rangle^{vol}(v(\psi_p))  = \int_0^{v(\psi_p)}\langle \Lambda_X\rangle^{vol}\d v
 \label{eq:integral_balance}
 \end{align}
 
@@ -559,18 +559,18 @@ full poloidal turn this definition is independent of which
 fieldline we pick on a given flux surface.
 
 %Let us define the poloidal length $s$ as the fieldline following
-%parameter i.e. $\vec B\cdot \nabla s \equiv B_p = R_0|\nabla \psi_p|/R$
+%parameter i.e. $\vec B\cn s \equiv B_p = R_0|\vn \psi_p|/R$
 %and $\d\varphi/\d s = B^\varphi(R(s), Z(s)) / B_p(R(s),Z(s))$.
 %We can then express the safety factor as the line integral
 %\begin{align}
-%q=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vec\nabla\psi_p|} \d s
+%q=\frac{1}{2\pi}\oint \frac{B^\varphi}{B_p} \d s = \frac{1}{2\pi}\oint_{\psi_p=\psi_{p0}}\frac{I(\psi_p)}{R|\vn\psi_p|} \d s
 %= \frac{1}{2\pi}\int \frac{I(\psi_p)}{R}\delta(\psi_p-\psi_{p0}) H(Z-Z_X) \d R\d Z
 %\end{align}
 %where we made use of Eq.~\eqref{eq:dirac_delta} in two dimensions in the
 %last equality and thus arrive at a numerical tractable expression
 %to evaluate the safety factor.
 Let us define the geometric poloidal angle $\Theta$ as the fieldline following
-parameter i.e. $\vec B\cdot\nabla\Theta = R_0(\psi_R (R-R_0) + \psi_Z Z)/r^2R$.
+parameter i.e. $\vec B\cn\Theta = R_0(\psi_R (R-R_0) + \psi_Z Z)/r^2R$.
 We can then directly integrate the safety factor as
 \begin{align}\label{eq:safety_factor}
 \frac{\d R}{\d\Theta} = \frac{B^R}{B^\Theta}\quad 
@@ -584,7 +584,7 @@ and refine the stepsize until machine-precision is reached.
 
 Notice that the safety factor diverges on the last closed flux
 surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa_area}
-remain finite due to the $\nabla\psi$ factor.
+remain finite due to the $\vn\psi$ factor.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
@@ -602,39 +602,39 @@ We introduce the dimensionless parameters
 where $a\in\{e,i\}$ is the species label and $z$ is the charge number.
 Omitting the species label we arrive at (dividing the density equation by $\Omega_0n_0$ and the velocity equation by $\Omega_0 c_s$)
 \begin{align}
-\frac{\partial}{\partial t} N &+ \vec\nabla\cdot\left( N \left(
+\frac{\partial}{\partial t} N &+ \vec\nc\left( N \left(
     \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + \tilde{\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
 \mu N \frac{\partial}{\partial t} U &+ \mu N \left(
     \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + \tilde{\vec b}_\perp\right)
-    \right)\cdot \vec\nabla U  \nonumber \\
-    &+ 2\mu \vec \nabla \cdot ( NU \vec v_{\nabla\times\bhat})
-    -\mu NU\vec \nabla\cdot \vec v_{\nabla\times\bhat}
-    + \mu NU\mathcal K_{\nabla\times\bhat}(\psi) \nonumber\\
-    &= -\tau \left(\bhat + \tilde{\vec b}_\perp\right)\cdot \nabla N 
-    -N \left( \left(\bhat+\tilde{\vec b}_\perp\right)\cdot \nabla \psi + \frac{\partial A_\parallel}{\partial t}\right) 
+    \right)\cn U  \nonumber \\
+    &+ 2\mu \nc ( NU \vec v_{\vn\times\bhat})
+    -\mu NU\nc \vec v_{\vn\times\bhat}
+    + \mu NU\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
+    &= -\tau \left(\bhat + \tilde{\vec b}_\perp\right)\cn N 
+    -N \left( \left(\bhat+\tilde{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right) 
     - \eta n_e^2(U_i-u_e) + \mu N\Lambda_U
 \label{}
 \end{align}
 with
 \begin{align}
-\vec v_E := \frac{\bhat\times\nabla\psi}{B},\quad
-\vec v_{K} := \tau \left(\vec{\mathcal K_{\nabla B}} + \vec{\mathcal K_{\nabla\times\bhat}}\right)=\tau\vec{\mathcal K}  ,\nonumber\\
-\vec v_C := \mu U^2\vec{\mathcal K_{\nabla\times\bhat}},\quad
-\vec v_{\nabla\times\bhat} := \tau\vec{\mathcal K_{\nabla\times\bhat}},\quad
-\tilde{\vec b}_\perp = \frac{\nabla\times A_\parallel \bhat}{B}.
+\vec v_E := \frac{\bhat\times\vn\psi}{B},\quad
+\vec v_{K} := \tau \left(\vec{\mathcal K_{\vn B}} + \vec{\mathcal K_{\vn\times\bhat}}\right)=\tau\vec{\mathcal K}  ,\nonumber\\
+\vec v_C := \mu U^2\vec{\mathcal K_{\vn\times\bhat}},\quad
+\vec v_{\vn\times\bhat} := \tau\vec{\mathcal K_{\vn\times\bhat}},\quad
+\tilde{\vec b}_\perp = \frac{\vn\times A_\parallel \bhat}{B}.
 \label{}
 \end{align}
 
 The electric potential \(\phi\) and parallel magnetic vector potential \(A_\parallel\) are
 computed by the polarisation and induction equations (with $q_e=-e$ and $q_i=+e$)
 \begin{align}
- -\vec{\nabla} \cdot\left(\frac{\mu_iN_i}{B^2} \vec{\nabla}_\perp \phi\right) &=  \Gamma_{1,i} N_i -n_e, \quad \Gamma_{1,i}^{-1} := 1-\frac{1}{2}\mu_i\tau_i\Delta_\perp , \\
+ -\nc\left(\frac{\mu_iN_i}{B^2} \np \phi\right) &=  \Gamma_{1,i} N_i -n_e, \quad \Gamma_{1,i}^{-1} := 1-\frac{1}{2}\mu_i\tau_i\Delta_\perp , \\
   -\frac{1}{\beta} \Delta_\perp A_\parallel &= \left(N_i U_i-n_e u_e \right)
   \label{eq:polarisation_dimensional}
 \end{align}
 Given $\phi$ we define the generalised electric potential
 \begin{align}
-    \psi_e := \phi,\quad \psi_i&:= \Gamma_{1,i} \phi - \frac{\mu_i }{2}\left(\frac{\vec \nabla_\perp\phi}{B}\right)^2
+    \psi_e := \phi,\quad \psi_i&:= \Gamma_{1,i} \phi - \frac{\mu_i }{2}\left(\frac{\np\phi}{B}\right)^2
 \end{align}
 In total
 we have an isothermal 3d gyro-fluid model with up to 2nd order FLR effects
@@ -649,7 +649,7 @@ drifts and $U\bhat$ change directions. On the other side, the diffusive and resi
 Without resistivity and diffusion a change in direction of the magnetic field thus corresponds to
 a time reversal $t\rightarrow t'=-t$.
 
-Also note that changing the sign of the magnetic field only in the parallel derivatives $\nabla_\parallel \rightarrow -\nabla_\parallel$ does not
+Also note that changing the sign of the magnetic field only in the parallel derivatives $\npar \rightarrow -\npar$ does not
 have any effect. This can be seen by simply renormalizing $U'=-U$. This reverts the equations back to the original equations.
 In the code this situation can happen when you change the sign of $\bhat$ using $\mathcal P_\psi$  and $\mathcal P_I$
 and use the toroidal field approximation at the same time.
@@ -725,7 +725,7 @@ in the same way as $U$.
 Typically,
 \begin{align}
 n_e = n_0, \quad u_e = \phi = 0
-\text{ or } \hat n \cdot \nabla n_e = \hat n \cdot \nabla u_e = 0
+\text{ or } \hat n \cn n_e = \hat n \cn u_e = 0
 \end{align}
 where $\hat n$ is the normal vector to the boundary.
 
@@ -845,7 +845,7 @@ with $a=0.0335$m, $b=0.05$m, $c=565m^{-2}$, $R_0=0.98$m and $Z_0=-0.02$m.
 For ions we use
 \begin{align}
     S_{N_i} = \Gamma_{1,i}^{-1} S_{n_e} = \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S_{n_e} \\
-    \delta S_{n_e} = \nabla\cdot\left( \frac{\mu_i S_{N_i}}{B^2}\nabla_\perp \phi\right)
+    \delta S_{n_e} = \nc\left( \frac{\mu_i S_{N_i}}{B^2}\np \phi\right)
   \label{eq:ion_source}
 \end{align}
 and finally add $\delta S_{n_e}$ to $S_{n_e}$ as the potential FLR correction.
@@ -853,7 +853,7 @@ Note that the correction $\delta S_{n_e}$ is a total divergence which means
 it does not change the volume integrated "total" particle number created by the source.
 Note that Eq.~\eqref{eq:ion_source} is explicitly chosen as to avoid potential generation
 by the particle source (cf.~Section~\ref{sec:conservation}). $S_{n_e}$ needs to be smooth
-so that $\nabla_\perp^2 S_{n_e}$ is well defined.
+so that $\np^2 S_{n_e}$ is well defined.
 
 %The idea for the terms $S_U$ is mainly to provide more numerical stability
 %in the corner regions of the domain, where the parallel derivative may lead
@@ -872,23 +872,23 @@ two functions for which we have no boundary conditions
     \begin{align}
     \frac{\partial}{\partial t} N =&
         - \frac{1}{B}[\psi, N]_{\perp}%\nonumber\\
-        - \bar \nabla_\parallel \left( NU\right)
-        - NU\left(\vec \nabla\cdot\bhat+\vec \nabla\cdot\tilde{\vec b}_\perp\right)
+        - \bar \npar \left( NU\right)
+        - NU\left(\vec \nc\bhat+\vec \nc\tilde{\vec b}_\perp\right)
         - \tau \mathcal K(N) \nonumber \\&
         - N \mathcal K(\psi)
-        -\mu \mathcal K_{\nabla\times\bhat}(NU^2)
-        -\mu NU^2\nabla\cdot \vec{ \mathcal K_{\nabla\times\bhat}}
+        -\mu \mathcal K_{\vn\times\bhat}(NU^2)
+        -\mu NU^2\nc \vec{ \mathcal K_{\vn\times\bhat}}
         + \nu_\perp\Delta_\perp N + \nu_\parallel \Delta_\parallel N + S_N, \\
     \frac{\partial}{\partial t} W =&
         - \frac{1}{B}\left[\psi, U\right]_{\perp}%& \nonumber\\
-        - \frac{1}{\mu} \bar \nabla_\parallel \psi% \nonumber\\
-        - \frac{1}{2}\bar \nabla_\parallel U^2
-        -\frac{\tau}{\mu} \bar \nabla_\parallel \ln N
-        - U\mathcal K_{\nabla\times\bhat}(\psi)
+        - \frac{1}{\mu} \bar \npar \psi% \nonumber\\
+        - \frac{1}{2}\bar \npar U^2
+        -\frac{\tau}{\mu} \bar \npar \ln N
+        - U\mathcal K_{\vn\times\bhat}(\psi)
         - \tau \mathcal K(U)
-        -\tau U\nabla\cdot\vec{ \mathcal K_{\nabla\times\bhat}}\nonumber\\&
-        - \left(2\tau + {\mu}U^2\right) \mathcal K_{\nabla\times\bhat} (U)
-        -2\tau U\mathcal K_{\nabla\times\bhat}(\ln N)
+        -\tau U\nc\vec{ \mathcal K_{\vn\times\bhat}}\nonumber\\&
+        - \left(2\tau + {\mu}U^2\right) \mathcal K_{\vn\times\bhat} (U)
+        -2\tau U\mathcal K_{\vn\times\bhat}(\ln N)
         - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
         + \nu_\perp\Delta_\perp U
         + \nu_\parallel \Delta_\parallel U,
@@ -898,14 +898,14 @@ two functions for which we have no boundary conditions
     \label{eq:Egyrofluid}
 \end{subequations}
 together with
-$\bar\nabla_\parallel f = \nabla_\parallel f + A_\parallel \mathcal K_{\nabla\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$
-and $\vec \nabla \cdot \tilde{ \vec b}_\perp = A_\parallel \vec \nabla\cdot\vec{ \mathcal{ K}_{\nabla\times\bhat}} - \mathcal K_{\nabla B}(A_\parallel) $
+$\bar\npar f = \npar f + A_\parallel \mathcal K_{\vn\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$
+and $\nc \tilde{ \vec b}_\perp = A_\parallel \vec \nc\vec{ \mathcal{ K}_{\vn\times\bhat}} - \mathcal K_{\vn B}(A_\parallel) $
 and
 \begin{subequations} \label{eq:elliptic}
   \begin{align}
-    -\nabla\cdot\left( \frac{N_i}{B^2}\nabla_\perp \phi \right) &= \Gamma_{1,i} N_i - n_e, \quad\quad
+    -\nc\left( \frac{N_i}{B^2}\np \phi \right) &= \Gamma_{1,i} N_i - n_e, \quad\quad
     \Gamma_{1,i}^{-1} = 1-\frac{1}{2}\tau_i\mu_i \Delta_\perp \\
-    \psi_e = \phi, \quad \psi_i &= \Gamma_{1,i}\phi -\frac{\mu_i}{2}\frac{(\nabla_\perp\phi)^2}{B^2} \\
+    \psi_e = \phi, \quad \psi_i &= \Gamma_{1,i}\phi -\frac{\mu_i}{2}\frac{(\np\phi)^2}{B^2} \\
     \left(\frac{\beta}{\mu_i}N_i - \frac{\beta}{\mu_e}n_e-\Delta_\perp\right)
     A_\parallel &= \beta\left(N_iW_i-n_e w_e\right)
   \end{align}
@@ -916,7 +916,7 @@ Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} posit
 The density equation directly yields the particle conservation
 \begin{align} \label{eq:mass_theorem}
   \frac{\partial}{\partial t} n_e
-  + \nabla\cdot\vec{ j_{n_e}}
+  + \nc\vec{ j_{n_e}}
   =  \Lambda_{n_e}+S_{n_e}
 \end{align}
 The terms of the particle conservation thus read
@@ -924,9 +924,9 @@ The terms of the particle conservation thus read
   n_e= & n_e,\\
   \vec j_{n_e} =& n_e\left(
   \vec v_E + \vec v_C + \vec v_{K} +u_e\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \nonumber\\
-  =& n_e \left(\frac{\bhat\times \nabla\phi}{B} 
-  + \tau_e \frac{\bhat\times\nabla n_e}{n_eB} 
-  + \mu_e u_e^2\vec K_{\nabla\times\bhat} 
+  =& n_e \left(\frac{\bhat\times \vn\phi}{B} 
+  + \tau_e \frac{\bhat\times\vn n_e}{n_eB} 
+  + \mu_e u_e^2\vec K_{\vn\times\bhat} 
   + u_e(\bhat + \tilde{\vec b}_\perp) \right), \\
   \Lambda_{n_e} =&
   \nu_\perp\Delta_\perp n_e + \nu_\parallel\Delta_\parallel n_e
@@ -935,7 +935,7 @@ The terms of the particle conservation thus read
 \end{align}
 Notice that
 \begin{align}
-n_e \vec K = n_e\nabla\times\frac{\bhat}{B} = \nabla\times n_e\frac{\bhat}{B} + \frac{\bhat\times\nabla n_e}{B}
+n_e \vec K = n_e\vn\times\frac{\bhat}{B} = \vn\times n_e\frac{\bhat}{B} + \frac{\bhat\times\vn n_e}{B}
 \label{}
 \end{align}
 such that we can define the diamagnetic flux in the particle flux since
@@ -943,14 +943,13 @@ the rotation vanishes under the divergence.
 
 Let us here also derive the particle flux \eqref{eq:mass_conservation} through a flux surface
 \begin{align} \label{eq:particle_flux}
- \vec j_{N}\cdot \vec \nabla v %=& N\left( \vec v_E + \vec v_C + \vec v_{\nabla
- %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cdot \vec
- %\nabla\psi_p \nonumber\\
+ \vec j_{N}\cn v %=& N\left( \vec v_E + \vec v_C + \vec v_{\vn
+ %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cn \psi_p \nonumber\\
  =&
   \frac{\d v}{\d \psi_p} N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
-   \mathcal K_{\nabla\times\bhat}(\psi_p) + \tau  \mathcal K_{\nabla B}(\psi_p) \right] \nonumber\\
+   \mathcal K_{\vn\times\bhat}(\psi_p) + \tau  \mathcal K_{\vn B}(\psi_p) \right] \nonumber\\
  &+ NU\frac{\d v}{\d \psi_p}\left [\left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
+ K_{\vn\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
 \end{align}
 
 
@@ -958,15 +957,15 @@ Let us here also derive the particle flux \eqref{eq:mass_conservation} through a
 The terms of the energy theorem are
 \begin{align} \label{eq:energy_theorem}
 \partial_t \mathcal E +
-\nabla \cdot \vec j_{\mathcal E}
+\nc \vec j_{\mathcal E}
 = \Lambda_{\mathcal E}
 +  S_{\mathcal E}
 +  R_{\mathcal E}
 \end{align}
-with ( $z_e=-1$ and $z_i=+1$) and $\vec u_E := {\bhat\times \nabla\phi}/{B}$
+with ( $z_e=-1$ and $z_i=+1$) and $\vec u_E := {\bhat\times \vn\phi}/{B}$
 \begin{align} \label{eq:energy_conservation}
   \mathcal{E}= & z_e\tau_e n_e \ln{(n_e)} +z_i\tau_i N_i\ln{(N_i)}
-  +\frac{1}{2\beta}\left(\vec \nabla_\perp A_\parallel\right)^2
+  +\frac{1}{2\beta}\left(\np A_\parallel\right)^2
    +  \frac{1}{2} z_i \mu_i N_i u_E^2  \nonumber\\
    & +\frac{1}{2} z_e\mu_e  n_e u_e^2
   +\frac{1}{2} z_i\mu_i  N_i U_i^2,\\
@@ -974,7 +973,7 @@ with ( $z_e=-1$ and $z_i=+1$) and $\vec u_E := {\bhat\times \nabla\phi}/{B}$
   \left(\tau \ln N + \frac{1}{2}\mu U^2 + \psi \right)N\left(
   \vec v_E + \vec v_C + \vec v_{K} +U\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \right]
   \nonumber\\
-  &+ \sum_z z\left[\mu \tau NU^2\vec K_{\nabla\times\bhat} + \tau NU \left(\bhat + \tilde{\vec b}_\perp\right)\right], \\
+  &+ \sum_z z\left[\mu \tau NU^2\vec K_{\vn\times\bhat} + \tau NU \left(\bhat + \tilde{\vec b}_\perp\right)\right], \\
   \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U^2 \right)
   \left(\nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N\right)  +  \mu NU\left(\nu_\perp\Delta_\perp U + \nu_\parallel\Delta_\parallel U\right) \right]
 \nonumber \\
@@ -993,28 +992,28 @@ for the diffusion terms in the above equations.
 
 We have the energy flux through a flux surface
 \begin{align}
- \vec j_{\mathcal E}\cdot \vec \nabla v =&%\frac{\d v}{\d \psi_p} \vec j_{\mathcal E}\cdot \vec \nabla \psi_p  =
-\frac{\d v}{\d \psi_p}\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cdot\vec\nabla\psi_p
-+ z \mu\tau NU^2 \mathcal K_{\nabla\times\bhat}(\psi_p) \nonumber\\
+ \vec j_{\mathcal E}\cn v =&%\frac{\d v}{\d \psi_p} \vec j_{\mathcal E}\cn \psi_p  =
+\frac{\d v}{\d \psi_p}\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cn\psi_p
++ z \mu\tau NU^2 \mathcal K_{\vn\times\bhat}(\psi_p) \nonumber\\
 &+ z \tau NU
  \left( A_\parallel \mathcal
- K_{\nabla\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
+ K_{\vn\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
 \label{eq:energy_flux}
 \end{align}
 
 \subsection{ Vorticity and momentum balance equation}
 In order to discuss poloidal flows let us first define orthogonal vectors $\{\vec{ \hat\zeta}, \vec{\hat\eta},\bhat\}$
 \begin{align}
-\vec{\hat\zeta} := \nabla\psi_p,\quad
+\vec{\hat\zeta} := \vn\psi_p,\quad
 \vec{\hat \eta} := \frac{\bhat \times\hat \zeta}{B}
 \end{align}
 where $\vec{\hat\eta}$
 points in the counter-clockwise poloidal direction with our choice of the magnetic field direction.
 Notice that $\vec{\hat \eta}$ in general has a (small) toroidal component in addition to the dominant poloidal component.
 
-%We then have $\nabla\phi = \hat \zeta\partial_{\hat\zeta}\phi + \hat\eta\partial_{\hat\eta} \phi$
-%with $\partial_{\hat\zeta}:= \hat\zeta\cdot\nabla$ and $\partial_{\hat\eta}:=\hat\eta\cdot\nabla$.
-Furthermore, from $\vec u_E = \bhat\times \vec\nabla\phi/B$ and $\vec u_d = \tau_i \bhat\times\nabla \ln N_i$
+%We then have $\vn\phi = \hat \zeta\partial_{\hat\zeta}\phi + \hat\eta\partial_{\hat\eta} \phi$
+%with $\partial_{\hat\zeta}:= \hat\zeta\cn$ and $\partial_{\hat\eta}:=\hat\eta\cn$.
+Furthermore, from $\vec u_E = \bhat\times \vn\phi/B$ and $\vec u_d = \tau_i \bhat\times\vn \ln N_i$
 we can derive
 \begin{align}
 u_E^{\hat\eta} &:= \vec u_E\cdot \vec{\hat\eta} = \frac{\partial_{\hat\zeta} \phi}{B^2} \sim u_{E,\varphi}
@@ -1022,7 +1021,7 @@ u_E^{\hat\eta} &:= \vec u_E\cdot \vec{\hat\eta} = \frac{\partial_{\hat\zeta} \ph
 u_E^{\hat\zeta}&:= \vec u_E\cdot \vec{\hat\zeta}  = -\partial_{\hat\eta} \phi \sim u_E^v \sim -\partial_\varphi\phi
 &&u_d^{\hat\zeta}:= \vec u_d\cdot \vec{\hat\zeta} = -B \tau_i \partial_{\hat\eta} \ln N_i \sim u_D^v
 \end{align}
-where we write $\partial_{\hat\zeta} := \nabla\psi_p\cdot\nabla$ and $\partial_{\hat\eta} := \vec{\hat\eta}\cdot\nabla$.
+where we write $\partial_{\hat\zeta} := \vn\psi_p\cn$ and $\partial_{\hat\eta} := \vec{\hat\eta}\cn$.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Vorticity equation}
 With this we write the vorticity equation (in the LWL)
@@ -1033,24 +1032,24 @@ With this we write the vorticity equation (in the LWL)
 &+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle -\partial_{\hat\eta} \phi \left(\mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}
 + \mu_i \tau_i N_i\partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\eta}A_\parallel\partial_{\hat\zeta} A_\parallel  + \frac{1}{2}\partial_{\hat\eta} A_\parallel \partial_{\hat\zeta} \tau_i N_i U_i \right\rangle
 \nonumber\\
-&= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\nabla\times\bhat}\cdot\nabla\psi_p \right\rangle
+&= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p \right\rangle
 \end{align}
 where we neglect the second order derivative $\partial_{\hat\eta}\partial_{\hat\zeta}A_\parallel$ in the magnetization density term and
 where the right hand side represents the negative toroidal component of the Lorentz force $-(\vec j\times\vec B)_\varphi$.
 Notice that
 \begin{align}
-\left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p\right\rangle
-= \left\langle \frac{ \bhat\times \nabla (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cdot\nabla\psi_p\right\rangle
+\left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p\right\rangle
+= \left\langle \frac{ \bhat\times \vn (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cn\psi_p\right\rangle
 = -\left\langle\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)\right\rangle
 \end{align}
 We can interpret Eq.~\eqref{eq:vorticity_average} as the flux surface average of
 the vorticity equation
 \begin{align}
-&\partial_t \GA{\Omega} + \GA{\nabla\cdot \vec j_\Omega} = \GA{S_\Omega} \\
-\Omega &:= \mu_i N_i \left(\frac{\nabla\psi_p\cdot\nabla\phi}{B^2} + \tau_i \nabla\psi_p \cdot\nabla \ln N_i\right) \\
+&\partial_t \GA{\Omega} + \GA{\nc \vec j_\Omega} = \GA{S_\Omega} \\
+\Omega &:= \mu_i N_i \left(\frac{\vn\psi_p\cn\phi}{B^2} + \tau_i \vn\psi_p \cn \ln N_i\right) \\
 \vec j_{\Omega} &:= \Omega \vec u_E
-    - \left(\frac{1}{\beta} \nabla\psi_p\cdot\nabla A_\parallel +\frac{1}{2} \nabla\psi_p\cdot\nabla \tau_i N_iU_i\right)\frac{\bhat\times\nabla A_\parallel}{B} \\
-    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\nabla\times\bhat}(\psi_p)
+    - \left(\frac{1}{\beta} \vn\psi_p\cn A_\parallel +\frac{1}{2} \vn\psi_p\cn \tau_i N_iU_i\right)\frac{\bhat\times\vn A_\parallel}{B} \\
+    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p)
 \end{align}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Toroidal ExB momentum equation}
@@ -1062,19 +1061,35 @@ Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continui
 &+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle \mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}\left(-\partial_{\hat\eta} \phi - \tau_i \partial_{\hat\eta} \ln N_i \right)
  + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel + \frac{1}{2}\partial_{\hat\zeta} A_\parallel \partial_{\hat\eta} \tau_i N_i U_i\right \rangle
 \nonumber\\
-&= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cdot\nabla\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\nabla\times\bhat}\cdot\nabla\psi_p \right\rangle
+&= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p \right\rangle
 \end{align}
 We can interpret Eq.~\eqref{eq:exb_average} as the flux surface average of
 the \ExB vorticity equation
 \begin{align}
-&\partial_t \GA{\Omega_E} + \GA{\nabla\cdot \vec j_{\Omega_E}} = \GA{S_\Omega} \\
-\Omega_E &:= \mu_i N_i \frac{\nabla\psi_p\cdot\nabla\phi}{B^2} \\
+&\partial_t \GA{\Omega_E} + \GA{\nc \vec j_{\Omega_E}} = \GA{S_\Omega} \\
+\Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \\
 \vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
-    - \frac{1}{\beta} \nabla\psi_p\cdot\nabla A_\parallel \left(\frac{\bhat\times\nabla A_\parallel}{B} +\frac{1}{2} \bhat \times \nabla \tau_i N_iU_i\right) \\
-    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\nabla\times\bhat}(\psi_p)
+    - \frac{1}{\beta} \vn\psi_p\cn A_\parallel \left(\frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
+    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p)
 \end{align}
 
-
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\subsubsection{Parallel momentum balance}
+The flux surface average over the parallel momentum equation under species summation and the LWL yields
+\begin{align}
+  \frac{\partial}{\partial t}\GA{m_iN_iU_{i} }
+    % \nonumber\\
+    + \frac{\partial}{\partial v} \GA{m_iN_iU_i u_E^{\;v} + \sum_s (N_sT_s + mN_sU_s^2) \widetilde{ b_{\perp}^{\;v} } }
+   % \nonumber\\
+   = \sum_s\GA{-N_sT_s\npar \ln B}
+   \label{eq:parallel_momentum}
+\end{align}
+while the toroidal parallel angular momentum contribution reads
+\begin{align}\label{eq:parallel_momentum_direction}
+    \frac{\partial}{\partial t}  \GA{m_iNU_i b_\varphi}
+    + \frac{\partial}{\partial v} \GA{m_iN_iU_i b_\varphi u_E^{\;v} + \sum_s (N_sT_s + m_sN_sU_s^2) b_\varphi\widetilde{ b_{\perp}^{\;v} }}
+   = -\GA{S_\Omega}
+\end{align}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
@@ -1114,7 +1129,7 @@ discontinuous Galerkin on structured grid
 Advection terms & direct DG & DG approximation with centered flux of derivatives \\
 Elliptic terms & local DG & The local DG approximation with centered flux \\
 Helmholtz and Elliptic matrix inversions & multigrid/ conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
-Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017}. The terms $\nabla_\parallel N$ and $\nabla_\parallel \phi$ in the velocity equation use a forward difference, while the term $\nabla_\parallel U$ in the
+Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017}. The terms $\npar N$ and $\npar \phi$ in the velocity equation use a forward difference, while the term $\npar U$ in the
 density equation uses backward difference. This is to avoid a too wide stencil for the diverence of the current and increases stability for low resistivity. \\
 time & Multistep "Karniadakis" & \\
 \qquad explicit & Multistep "Karniadakis" & $3$rd order explicit\\
@@ -1380,7 +1395,7 @@ and X $\in$
     psi &$\psi$ \\
     induction &$A_\parallel$ &
     vorticity &$-\Delta_\perp\phi$ \\
-    dssue & $\nabla_\parallel^2 u_e$&
+    dssue & $\npar^2 u_e$&
     dppue & $\partial_\varphi^2 u_e$\\
     dpue2 & $(\partial_\varphi u_e)^2$&
     apar\_vorticity &$-\Delta_\perp A_\parallel$ \\
@@ -1388,10 +1403,10 @@ and X $\in$
     niui &$N_i U_i$ \\
     neuebphi &$n_eu_eb_\varphi$ &
     niuibphi &$N_iU_ib_\varphi$ \\
-    lperpinv &$L_\perp^{-1} := |\vec\nabla_\perp n_e|/n_e$ &
-    perpaligned &$(\vec\nabla_\perp n_e)^2/n_e$ \\
-    lparallelinv &$L_\parallel^{-1} := |\nabla_\parallel n_e|/n_e$ &
-    aligned &$ (\nabla_\parallel n_e)^2/n_e$ \\
+    lperpinv &$L_\perp^{-1} := |\vec\np n_e|/n_e$ &
+    perpaligned &$(\vec\np n_e)^2/n_e$ \\
+    lparallelinv &$L_\parallel^{-1} := |\npar n_e|/n_e$ &
+    aligned &$ (\npar n_e)^2/n_e$ \\
     ne2 & $n_e^2$ &
     phi2 & $\phi^2$ \\
     nephi & $n_e\phi$ &
@@ -1400,14 +1415,14 @@ and X $\in$
     sei & $z_i(\tau_i (1+\ln N_i) + \psi + \frac{1}{2}\mu_i U_i^2) S_{N_i} $ \\
     nelnne &$ z_e\tau_e n_e \ln n_e$ &
     nilnni &$ z_i\tau_i N_i \ln N_i$ \\
-    aperp2 &$ (\nabla_\perp A_\parallel)^2/2/\beta$ &
+    aperp2 &$ (\np A_\parallel)^2/2/\beta$ &
     ue2   &$z_i\mu_i N_i u_E^2 /2$ \\
     neue2 &$ z_e\mu_e n_e u_e^2/2$ &
     niui2 &$ z_i\mu_i N_i U_i^2/2$ \\
-    oexbe &$\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2$ &
-    oexbi &$\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2$ \\
-    odiae &$\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e$ &
-    odiai &$\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i$ \\
+    oexbe &$\mu_i n_e \vn\psi_p\cn\phi/B^2$ &
+    oexbi &$\mu_i N_i \vn\psi_p\cn\phi/B^2$ \\
+    odiae &$\mu_i \tau_i\vn\psi_p\cn n_e$ &
+    odiai &$\mu_i \tau_i\vn\psi_p\cn N_i$ \\
 \bottomrule
 \end{longtable}
 and the time-averaged quantities Y $\in$
@@ -1415,34 +1430,34 @@ and the time-averaged quantities Y $\in$
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}\\
 \midrule
-    jsne &$ n_e (\vec v_E + \vec v_K + \vec v_C )\cdot\nabla \psi_p$ \\
-    jsneA &$ n_e u_e \vec{\tilde b}_\perp  \cdot\nabla \psi_p$ \\
+    jsne &$ n_e (\vec v_E + \vec v_K + \vec v_C )\cn \psi_p$ \\
+    jsneA &$ n_e u_e \vec{\tilde b}_\perp  \cn \psi_p$ \\
     lneperp &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ \\
     lneparallel &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
     resistivity &-$\eta_\parallel n_e^2 (U_i-u_e)^2$ \\
-    jsee &$z_e(\tau_e \ln n_e + \mu_e u_e^2/2 + \phi)n_e(\vec v_E + \vec v_C + \vec v_K)\cdot\nabla \psi_p
-        + z_e \tau_e n_e u_e^2 \vec K_{\nabla\times\bhat}\cdot\nabla \psi_p$ \\
-    jsei &$z_i(\tau_i \ln N_i + \mu_i U_i^2/2 + \psi_i)N_i(\vec v_E^i + \vec v_C + \vec v_K)\cdot\nabla \psi_p
-        + z_i \tau_i N_i U_i^2 \vec K_{\nabla\times\bhat}\cdot\nabla \psi_p$ \\
-    jseea &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec {\tilde b}_\perp\cdot\nabla \psi_p
-        + z_e \tau_e n_e u_e \vec{\tilde b}_\perp \cdot \nabla \psi_p $ \\
-    jseia &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec {\tilde b}_\perp\cdot\nabla \psi_p
-        + z_i \tau_i N_i U_i \vec{\tilde b}_\perp \cdot \nabla \psi_p $ \\
+    jsee &$z_e(\tau_e \ln n_e + \mu_e u_e^2/2 + \phi)n_e(\vec v_E + \vec v_C + \vec v_K)\cn \psi_p
+        + z_e \tau_e n_e u_e^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
+    jsei &$z_i(\tau_i \ln N_i + \mu_i U_i^2/2 + \psi_i)N_i(\vec v_E^i + \vec v_C + \vec v_K)\cn \psi_p
+        + z_i \tau_i N_i U_i^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
+    jseea &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec {\tilde b}_\perp\cn \psi_p
+        + z_e \tau_e n_e u_e \vec{\tilde b}_\perp \cn \psi_p $ \\
+    jseia &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec {\tilde b}_\perp\cn \psi_p
+        + z_i \tau_i N_i U_i \vec{\tilde b}_\perp \cn \psi_p $ \\
     leeperp &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_e \nu_\perp \Delta_\perp u_e$ \\
     leiperp &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_i \nu_\perp \Delta_\perp U_i$ \\
     leeparallel &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_e \nu_\parallel \Delta_\parallel u_e$ \\
     leiparallel &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_i \nu_\parallel \Delta_\parallel U_i$ \\
-    jsoexbi &$(\mu_i N_i \nabla\psi_p\cdot\nabla\phi/B^2) (\bhat\times\nabla\phi)\cdot\nabla \psi_p/B$ \\
-    jsoexbe &$(\mu_i n_e \nabla\psi_p\cdot\nabla\phi/B^2) (\bhat\times\nabla\phi)\cdot\nabla \psi_p/B$ \\
-    jsodiaiUE &$(\mu_i \tau_i\nabla\psi_p\cdot\nabla N_i) (\bhat\times\nabla\phi)\cdot\nabla \psi_p/B$ \\
-    jsodiaeUE &$(\mu_i \tau_i\nabla\psi_p\cdot\nabla n_e) (\bhat\times\nabla\phi)\cdot\nabla \psi_p/B$ \\
-    jsoexbiUD &$(\mu_i\tau_i \nabla\psi_p\cdot\nabla\phi/B^2) (\bhat\times\nabla N_i)\cdot\nabla \psi_p/B$ \\
-    jsoexbeUD &$(\mu_i\tau_i \nabla\psi_p\cdot\nabla\phi/B^2) (\bhat\times\nabla n_e)\cdot\nabla \psi_p/B$ \\
-    jsoapar &$ \nabla\psi_p\cdot\nabla A_\parallel \bhat\times\nabla A_\parallel\cdot\nabla \psi_p/B/\beta$ \\
+    jsoexbi &$(\mu_i N_i \vn\psi_p\cn\phi/B^2) (\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    jsoexbe &$(\mu_i n_e \vn\psi_p\cn\phi/B^2) (\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    jsodiaiUE &$(\mu_i \tau_i\vn\psi_p\cn N_i) (\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    jsodiaeUE &$(\mu_i \tau_i\vn\psi_p\cn n_e) (\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    jsoexbiUD &$(\mu_i\tau_i \vn\psi_p\cn\phi/B^2) (\bhat\times\vn N_i)\cn \psi_p/B$ \\
+    jsoexbeUD &$(\mu_i\tau_i \vn\psi_p\cn\phi/B^2) (\bhat\times\vn n_e)\cn \psi_p/B$ \\
+    jsoapar &$ \vn\psi_p\cn A_\parallel \bhat\times\vn A_\parallel\cn \psi_p/B/\beta$ \\
     socurve &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
     socurvi &$z_i\tau_i N_i \mathcal K(\psi_p)$ \\
-    socurvkappae &$z_e\mu_e n_eu_e^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
-    socurvkappai &$z_i\mu_i N_iU_i^2 \mathcal K_{\nabla\times\bhat}(\psi_p)$ \\
+    socurvkappae &$z_e\mu_e n_eu_e^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
+    socurvkappai &$z_i\mu_i N_iU_i^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
 \bottomrule
 \end{longtable}
 Even though this might look like a lot of things to compute the actual time spent on diagnostics is negligible if {\tt inner\_loop} parameter is greater than 1.
@@ -1491,7 +1506,7 @@ zc           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} \\
 psi\_psi         & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
 rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
-psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\nabla\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
+psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\vn\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
 psi\_vol         & Dataset & 1 (psi) & The volume enclosed by the flux surfaces $v(\psi_p) = \int_{\psi_p} \dV $ \\
 dvdpsip          & Dataset & 1 (psi) & $\d v/\d\psi_p$ \\
 Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \langle Z\rangle_{\psi_{p}}$ \\
@@ -1505,7 +1520,7 @@ Z\_ifs\_norm     & Dataset & 1 (time) & Volume integrated square flux surface av
 where Z $\in$ \{X, Y\_tt\}
 Note that feltoridag converts all $jsX$ quantities into $jvX$
 by multiplying $\d v/\d \psi_p$
-in the sense that $\vec j\cdot \nabla v  = \vec j \cdot \nabla \psi_p \d v/\d\psi_p$.
+in the sense that $\vec j\cn v  = \vec j \cn \psi_p \d v/\d\psi_p$.
 
 
 
-- 
GitLab


From 494102e873d0315f35d25378779b631b0c2c9fc2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 4 Dec 2019 23:53:24 +0100
Subject: [PATCH 179/540]  Include toroidal flux label

---
 inc/dg/algorithm.h                       |  2 ++
 inc/geometries/geometry_diag.cu          | 26 ++++++++++++++++--------
 inc/geometries/geometry_params_Xpoint.js |  1 +
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 3b9741893..5e301f065 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -18,6 +18,8 @@
 #include "blas.h"
 #include "helmholtz.h"
 #include "cg.h"
+#include "bicgstabl.h"
+#include "lgmres.h"
 #include "functors.h"
 #include "multistep.h"
 #include "elliptic.h"
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index c6992124a..8467c8cb1 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -378,20 +378,28 @@ int main( int argc, char* argv[])
         grid1d = dg::Grid1d (psipmin<psipmax ? psipmin : psipmax, psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::NEU);
         map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
             "Flux surface average of psi with delta function");
+        dg::HVec qprofile;
         if( gp.equilibrium == "solovev")
         {
-            dg::geo::SafetyFactor     qprof( mag);
-            map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
+            dg::geo::SafetyFactor qprof( mag);
+            qprofile = dg::evaluate( qprof, grid1d);
+            map1d.emplace_back("q-profile", qprofile,
                 "q-profile (Safety factor) using direct integration");
         }
-        else
-        {
-            dg::geo::SafetyFactorAverage     qprof( grid2d, mag);
-            map1d.emplace_back("q-profile", dg::evaluate( qprof,    grid1d),
-                "q-profile (Safety factor) using average integration");
-        }
+        dg::geo::SafetyFactorAverage qprof( grid2d, mag);
+        qprofile = dg::evaluate( qprof, grid1d);
+        map1d.emplace_back("q-profile_fsa", qprofile,
+            "q-profile (Safety factor) using average integration");
+        dg::HVec psit = dg::integrate( qprofile, grid1d);
+        map1d.emplace_back("psit1d", psit,
+            "Toroidal flux label psi_t evaluated on grid1d");
+        double psit_tot = dg::interpolate(psit, 0., grid1d);
+        dg::blas1::scal ( psit, 1/psit_tot);
+        dg::blas1::transform( psit, psit, dg::SQRT<double>());
+        map1d.emplace_back("rho_t", psit,
+            "Toroidal flux label rho_t = sqrt( psit/psit_tot) evaluated on grid1d");
         map1d.emplace_back("psip1d",    dg::evaluate( dg::cooX1d, grid1d),
-            "Flux label psi evaluated on grid1d");
+            "Poloidal flux label psi_p evaluated on grid1d");
         dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
         dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
         map1d.emplace_back("rho", rho,
diff --git a/inc/geometries/geometry_params_Xpoint.js b/inc/geometries/geometry_params_Xpoint.js
index dcc96e915..c99bc4393 100644
--- a/inc/geometries/geometry_params_Xpoint.js
+++ b/inc/geometries/geometry_params_Xpoint.js
@@ -22,6 +22,7 @@
     "inverseaspectratio" : 0.41071428571428575, //a/R_0
     "elongation"         : 1.75,
     "triangularity"      : 0.47,
+    "equilibrium"  : "solovev",
     //----------------------Miscellaneous----------------------
     "alpha"         :  0.05, //damping thickness
     "rk4eps"        :  1e-6,
-- 
GitLab


From 5cf95d903b455e2874a65709e78024fc749c4a2d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 5 Dec 2019 14:23:17 +0100
Subject: [PATCH 180/540] Add missing terms in diag

fluctuating Apar still missing though
---
 src/feltor/feltor.tex   | 10 +++++-----
 src/feltor/feltordiag.h | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 17f8b5762..8eaefb507 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1077,17 +1077,17 @@ the \ExB vorticity equation
 \subsubsection{Parallel momentum balance}
 The flux surface average over the parallel momentum equation under species summation and the LWL yields
 \begin{align}
-  \frac{\partial}{\partial t}\GA{m_iN_iU_{i} }
+  \frac{\partial}{\partial t}\GA{\mu_iN_iU_{i} }
     % \nonumber\\
-    + \frac{\partial}{\partial v} \GA{m_iN_iU_i u_E^{\;v} + \sum_s (N_sT_s + mN_sU_s^2) \widetilde{ b_{\perp}^{\;v} } }
+    + \frac{\partial}{\partial v} \GA{-\mu_iN_iU_i \partial_{\hat\eta}\phi + \sum_s (z_s\tau_sN_s + \mu_s N_sU_s^2) \widetilde{ b_{\perp}^{\;v} } }
    % \nonumber\\
-   = \sum_s\GA{-N_sT_s\npar \ln B}
+   = \sum_s\GA{-z_s\tau_s N_s\npar \ln B}
    \label{eq:parallel_momentum}
 \end{align}
 while the toroidal parallel angular momentum contribution reads
 \begin{align}\label{eq:parallel_momentum_direction}
-    \frac{\partial}{\partial t}  \GA{m_iNU_i b_\varphi}
-    + \frac{\partial}{\partial v} \GA{m_iN_iU_i b_\varphi u_E^{\;v} + \sum_s (N_sT_s + m_sN_sU_s^2) b_\varphi\widetilde{ b_{\perp}^{\;v} }}
+    \frac{\partial}{\partial t}  \GA{\mu_iNU_i b_\varphi}
+    + \frac{\partial}{\partial v} \GA{-\mu_iN_iU_i b_\varphi \partial_{\hat\eta}\phi + \sum_s (z_s\tau_s N_s + \mu_sN_sU_s^2) b_\varphi\widetilde{ b_{\perp}^{\;v} }}
    = -\GA{S_\Omega}
 \end{align}
 
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 09fbc3fdf..2eb78ec24 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -834,6 +834,42 @@ std::vector<Record> diagnostics2d_list = {
             }
         }
     },
+    /// --------------------- Parallel momentum flux terms ---------------------------//
+    {"jsparexbi_tt", "Parallel momentum radial flux by ExB velocity with ion density (Time average)", true,
+        []( DVec& result, Variables& v){
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
+
+            // parallel momentum mu_iN_iU_i
+            dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), 0., v.tmp[0]);
+
+            // Multiply everything
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
+        }
+    },
+    {"jsparbhiexbi_tt", "Parallel angular momentum radial flux by ExB velocity with ion density (Time average)", true,
+        []( DVec& result, Variables& v){
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
+
+            // parallel momentum mu_iN_iU_i
+            dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), 0., v.tmp[0]);
+
+            // Multiply everything
+            dg::blas1::pointwiseDot( 1., result, v.tmp[0],v.f.bphi(), 0., result);
+        }
+    },
+    /// --------------------- Mirror force term ---------------------------//
+    {"sparmirrore_tt", "Mirror force term with electron density (Time average)", true,
+        []( DVec& result, Variables& v){
+            dg::blas1::pointwiseDot( -v.p.mu[0], v.f.divb(), v.f.density(0), 0., result);
+        }
+    },
+    {"sparmirrori_tt", "Mirror force term with ion density (Time average)", true,
+        []( DVec& result, Variables& v){
+            dg::blas1::pointwiseDot( v.p.mu[1], v.f.divb(), v.f.density(1), 0., result);
+        }
+    },
     /// --------------------- Vorticity source terms ---------------------------//
     {"socurve_tt", "Vorticity source term electron curvature (Time average)", true,
         []( DVec& result, Variables& v) {
-- 
GitLab


From 8e1e1924a54bf9bb41321fe0c09c845ab7367b0f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 5 Dec 2019 16:15:35 +0100
Subject: [PATCH 181/540] Bring feltordiag up to date

---
 src/feltor/Makefile      |   2 +-
 src/feltor/feltor.tex    |  35 ++++----
 src/feltor/feltordiag.cu | 188 ++++++++++++++++++++++++++-------------
 src/feltor/feltordiag.h  |   2 +-
 4 files changed, 146 insertions(+), 81 deletions(-)

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index c97025b2c..a1ad5a254 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -13,7 +13,7 @@ all: feltor_hpc manufactured
 manufactured: manufactured.cu manufactured.h feltor.h implicit.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
 
-feltordiag: feltordiag.cu
+feltordiag: feltordiag.cu feltordiag.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g
 
 feltor: feltor.cu feltor.h implicit.h init.h parameters.h init_from_file.h
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 8eaefb507..093818dad 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1079,7 +1079,7 @@ The flux surface average over the parallel momentum equation under species summa
 \begin{align}
   \frac{\partial}{\partial t}\GA{\mu_iN_iU_{i} }
     % \nonumber\\
-    + \frac{\partial}{\partial v} \GA{-\mu_iN_iU_i \partial_{\hat\eta}\phi + \sum_s (z_s\tau_sN_s + \mu_s N_sU_s^2) \widetilde{ b_{\perp}^{\;v} } }
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \GA{-\mu_iN_iU_i \partial_{\hat\eta}\phi + \sum_s (z_s\tau_sN_s + \mu_s N_sU_s^2) \widetilde{ b_{\perp}^{\;v} } }
    % \nonumber\\
    = \sum_s\GA{-z_s\tau_s N_s\npar \ln B}
    \label{eq:parallel_momentum}
@@ -1087,7 +1087,7 @@ The flux surface average over the parallel momentum equation under species summa
 while the toroidal parallel angular momentum contribution reads
 \begin{align}\label{eq:parallel_momentum_direction}
     \frac{\partial}{\partial t}  \GA{\mu_iNU_i b_\varphi}
-    + \frac{\partial}{\partial v} \GA{-\mu_iN_iU_i b_\varphi \partial_{\hat\eta}\phi + \sum_s (z_s\tau_s N_s + \mu_sN_sU_s^2) b_\varphi\widetilde{ b_{\perp}^{\;v} }}
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \GA{-\mu_iN_iU_i b_\varphi \partial_{\hat\eta}\phi + \sum_s (z_s\tau_s N_s + \mu_sN_sU_s^2) b_\varphi\widetilde{ b_{\perp}^{\;v} }}
    = -\GA{S_\Omega}
 \end{align}
 
@@ -1336,8 +1336,8 @@ the value is not found,
 the program exits with an error message.
 
 \subsection{Output} \label{sec:output_file}
-Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Dataset with the same name as a dimension.
-We follow CF Conventions CF-1.7
+Output file format: netcdf-4/hdf5; A \textit{coordinate variable (Coord. Var.)} is a Dataset with the same name as a dimension.
+We follow \textbf{CF Conventions CF-1.7}
 \url{http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html}
 and write according attributes into the file.
 The command \texttt{ncdump -h output.nc} gives a full list of what a file contains.
@@ -1458,6 +1458,10 @@ and the time-averaged quantities Y $\in$
     socurvi &$z_i\tau_i N_i \mathcal K(\psi_p)$ \\
     socurvkappae &$z_e\mu_e n_eu_e^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
     socurvkappai &$z_i\mu_i N_iU_i^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
+    jsparexbi       & $N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    jsparbphiexbi   & $N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    sparmirrore & $-z_e\tau_en_e\npar \ln B$ \\
+    sparmirrori & $-z_i\tau_iN_i\npar \ln B$ \\
 \bottomrule
 \end{longtable}
 Even though this might look like a lot of things to compute the actual time spent on diagnostics is negligible if {\tt inner\_loop} parameter is greater than 1.
@@ -1488,7 +1492,7 @@ Compilation\\
 Usage \\
 \texttt{./feltordiag input.nc output.nc} \\
 
-Output file format: netcdf-4/hdf5; A coordinate variable (Coord. Var.) is a Dataset with the same name as a dimension.
+Output file format: netcdf-4/hdf5, Conventions: CF-1.7; A \textit{coordinate variable (Coord. Var.)} is a Dataset with the same name as a dimension.
 
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
@@ -1497,24 +1501,23 @@ inputfile  &     text attribute & - & verbose input file as a string (valid JSON
 geomfile   &     text attribute & - & verbose geometry input file as a string (valid JSON, no comments) \\
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
-z                & Coord. Var. & 1 (z) & $\varphi$-coordinate (computational space, compressed size: $N_z$)\\
-psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( size: $3\cdot 50$) \\
+psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( default size: $3\cdot 64$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
-xc           & Dataset & 3 (z,y,x) & Cartesian x-coordinate $x=R\sin(\varphi)$ \\
-yc           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
-zc           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
-q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} \\
-psi\_psi         & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
-rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
-psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\vn\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
-psi\_vol         & Dataset & 1 (psi) & The volume enclosed by the flux surfaces $v(\psi_p) = \int_{\psi_p} \dV $ \\
 dvdpsip          & Dataset & 1 (psi) & $\d v/\d\psi_p$ \\
+psi\_vol         & Dataset & 1 (psi) & The volume enclosed by the flux surfaces $v(\psi_p) = \int_{\psi_p} \dV $ \\
+psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\vn\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
+rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
+q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} using direct integration ( accurate but unavailable outside separatrix) \\
+psi\_psi         & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
+q-profile\_fsa   & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} computed with average integration (less accurate but defined outside separatrix) \\
+psit1d           & Dataset & 1 (psi) & Toroidal flux \\
+rho\_t           & Dataset & 1 (psi) & Toroidal flux label $\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$\\
 Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \langle Z\rangle_{\psi_{p}}$ \\
 Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\langle Z\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
 Z\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle Z\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
 Z\_ifs           & Dataset & 2 (time, psi) & Volume integrated flux surface average $\int\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the volume derived flux-surface average $\partial_v \langle Z\rangle_{\psi_p}$ \\
 Z\_ifs\_lcfs     & Dataset & 1 (time) & Volume integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the fsa evaluated $\langle j_v\rangle_{\psi_p}(0)$ \\
-Z\_ifs\_norm     & Dataset & 1 (time) & Volume integrated square flux surface average, unless Z is a current, then it is the square derivative of the flux surface average \\
+Z\_ifs\_norm     & Dataset & 1 (time) & Volume integrated square flux surface average $\sqrt{\int \d v \GA{Z}^2}$, unless Z is a current, then it is the square derivative of the flux surface average $\sqrt{\int\d v (\partial_v j^v)^2}$\\
 \bottomrule
 \end{longtable}
 where Z $\in$ \{X, Y\_tt\}
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 54595cd69..c5f111f8b 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -110,9 +110,10 @@ int main( int argc, char* argv[])
     const double psipmin = mag.psip()(R_O, Z_O);
 
 
-    std::cout << "Type X-point grid resolution (n(3), Npsi(32), Neta(640)) Must be divisible by 8\n";
-    unsigned npsi = 3, Npsi = 32, Neta = 320;//set number of psivalues (NPsi % 8 == 0)
-    std::cin >> npsi >> Npsi >> Neta;
+    //std::cout << "Type X-point grid resolution (n(3), Npsi(32), Neta(640)) Must be divisible by 8\n";
+    std::cout << "Using default X-point grid resolution (n(3), Npsi(64), Neta(640))\n";
+    unsigned npsi = 3, Npsi = 64, Neta = 640;//set number of psivalues (NPsi % 8 == 0)
+    //std::cin >> npsi >> Npsi >> Neta;
     std::cout << "You typed "<<npsi<<" x "<<Npsi<<" x "<<Neta<<"\n";
     std::cout << "Generate X-point flux-aligned grid!\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
@@ -172,6 +173,19 @@ int main( int argc, char* argv[])
     map1d.emplace_back("psi_psi",    dg::evaluate( dg::cooX1d, g1d_out),
         "Flux label psi (same as coordinate)");
 
+    dg::geo::SafetyFactorAverage qprof( g2d_out, mag);
+    dg::HVec qprofv = dg::evaluate( qprof, g1d_out);
+    map1d.emplace_back("q-profile_fsa", qprofv,
+        "q-profile (Safety factor) using average integration");
+    dg::HVec psit = dg::integrate( qprofv, g1d_out);
+    map1d.emplace_back("psit1d", psit,
+        "Toroidal flux label psi_t evaluated on grid1d");
+    double psit_tot = dg::interpolate(psit, 0., g1d_out);
+    dg::blas1::scal ( psit, 1./psit_tot);
+    dg::blas1::transform( psit, psit, dg::SQRT<double>());
+    map1d.emplace_back("rho_t", psit,
+        "Toroidal flux label rho_t = sqrt( psit/psit_tot) evaluated on grid1d");
+
 
     // interpolate from 2d grid to X-point points
     dg::IHMatrix grid2gridX2d  = dg::create::interpolation(
@@ -188,7 +202,7 @@ int main( int argc, char* argv[])
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
     int dim_ids1d[2] = {dim_ids[0], 0}; //time , psi
-    err = file::define_dimension( ncid_out, "psi", &dim_ids1d[1], g1d_out);
+    err = file::define_dimension( ncid_out, &dim_ids1d[1], g1d_out, "psi" );
     //Write long description
     std::string long_name = "Time at which 2d fields are written";
     err = nc_put_att_text( ncid_out, tvarID, "long_name", long_name.size(),
@@ -294,77 +308,125 @@ int main( int argc, char* argv[])
                 record_name[1] = 'v';
             //1. Read toroidal average
             int dataID =0;
-            err = nc_inq_varid(ncid, (record.name+"_ta2d").data(), &dataID);
-            err = nc_get_vara_double( ncid, dataID,
-                start2d, count2d, transferH2d.data());
-            //2. Compute fsa and output fsa
-            dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
-            dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
-            poloidal_average( transferH2dX, t1d, false); //average over eta
-            dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
-            dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
-            if( record_name[0] == 'j')
-                dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d );
+            bool available = true;
+            try{
+                err = nc_inq_varid(ncid, (record.name+"_ta2d").data(), &dataID);
+            } catch ( file::NC_Error error)
+            {
+                if(  i == 0)
+                {
+                    std::cerr << error.what() <<std::endl;
+                    std::cerr << "Offending variable is "<<record.name+"_ta2d\n";
+                    std::cerr << "Writing zeros ... \n";
+                }
+                available = false;
+            }
+            if( available)
+            {
+                err = nc_get_vara_double( ncid, dataID,
+                    start2d, count2d, transferH2d.data());
+                //2. Compute fsa and output fsa
+                dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
+                dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
+                poloidal_average( transferH2dX, t1d, false); //average over eta
+                dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
+                dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
+                if( record_name[0] == 'j')
+                    dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d );
+                //3. Interpolate fsa on 2d plane : <f>
+                dg::blas2::gemv(fsa2rzmatrix, fsa1d, transferH2d); //fsa on RZ grid
+            }
+            else
+            {
+                dg::blas1::scal( fsa1d, 0.);
+                dg::blas1::scal( transferH2d, 0.);
+            }
             err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_fsa"),
                 start1d, count1d, fsa1d.data());
-            //3. Interpolate fsa on 2d plane : <f>
-            dg::blas2::gemv(fsa2rzmatrix, fsa1d, transferH2d); //fsa on RZ grid
             err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fsa2d"),
                 start2d, count2d, transferH2d.data() );
-
             //4. Read 2d variable and compute fluctuations
-            err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
-            err = nc_get_vara_double( ncid, dataID, start2d, count2d,
-                t2d_mp.data());
-            if( record_name[0] == 'j')
-                dg::blas1::pointwiseDot( t2d_mp, dvdpsip2d, t2d_mp );
-            dg::blas1::axpby( 1.0, t2d_mp, -1.0, transferH2d);
-            err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
-                start2d, count2d, transferH2d.data() );
-
-            //5. flux surface integral/derivative
-            double result =0.;
-            if( record_name[0] == 'j') //j indicates a flux
+            available = true;
+            try{
+                err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
+            } catch ( file::NC_Error error)
             {
-                dg::blas2::symv( dpsi, fsa1d, t1d);
-                dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
-
-                result = dg::interpolate( fsa1d, 0., g1d_out);
+                if(  i == 0)
+                {
+                    std::cerr << error.what() <<std::endl;
+                    std::cerr << "Offending variable is "<<record.name+"_2d\n";
+                    std::cerr << "Writing zeros ... \n";
+                }
+                available = false;
             }
-            else
+            if( available)
             {
-                dg::blas1::pointwiseDot( fsa1d, dvdpsip, t1d);
-                transfer1d = dg::integrate( t1d, g1d_out);
-
-                result = dg::interpolate( transfer1d, 0., g1d_out);
-            }
-            err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
-                start1d, count1d, transfer1d.data());
-            //flux surface integral/derivative on last closed flux surface
-            err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
-                start2d, count2d, &result );
-            //6. Compute norm of time-integral terms to get relative importance
-            if( record_name[0] == 'j') //j indicates a flux
-            {
-                dg::blas2::symv( dpsi, fsa1d, t1d);
-                dg::blas1::pointwiseDivide( t1d, dvdpsip, t1d); //dvjv
-                dg::blas1::pointwiseDot( t1d, t1d, t1d);//dvjv2
-                dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);//dvjv2
-                transfer1d = dg::integrate( t1d, g1d_out);
-                result = dg::interpolate( transfer1d, 0., g1d_out);
-                result = sqrt(result);
+                err = nc_get_vara_double( ncid, dataID, start2d, count2d,
+                    t2d_mp.data());
+                if( record_name[0] == 'j')
+                    dg::blas1::pointwiseDot( t2d_mp, dvdpsip2d, t2d_mp );
+                dg::blas1::axpby( 1.0, t2d_mp, -1.0, transferH2d);
+                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
+                    start2d, count2d, transferH2d.data() );
+
+                //5. flux surface integral/derivative
+                double result =0.;
+                if( record_name[0] == 'j') //j indicates a flux
+                {
+                    dg::blas2::symv( dpsi, fsa1d, t1d);
+                    dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
+
+                    result = dg::interpolate( fsa1d, 0., g1d_out);
+                }
+                else
+                {
+                    dg::blas1::pointwiseDot( fsa1d, dvdpsip, t1d);
+                    transfer1d = dg::integrate( t1d, g1d_out);
+
+                    result = dg::interpolate( transfer1d, 0., g1d_out);
+                }
+                err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
+                    start1d, count1d, transfer1d.data());
+                //flux surface integral/derivative on last closed flux surface
+                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
+                    start2d, count2d, &result );
+                //6. Compute norm of time-integral terms to get relative importance
+                if( record_name[0] == 'j') //j indicates a flux
+                {
+                    dg::blas2::symv( dpsi, fsa1d, t1d);
+                    dg::blas1::pointwiseDivide( t1d, dvdpsip, t1d); //dvjv
+                    dg::blas1::pointwiseDot( t1d, t1d, t1d);//dvjv2
+                    dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);//dvjv2
+                    transfer1d = dg::integrate( t1d, g1d_out);
+                    result = dg::interpolate( transfer1d, 0., g1d_out);
+                    result = sqrt(result);
+                }
+                else
+                {
+                    dg::blas1::pointwiseDot( fsa1d, fsa1d, t1d);
+                    dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);
+                    transfer1d = dg::integrate( t1d, g1d_out);
+
+                    result = dg::interpolate( transfer1d, 0., g1d_out);
+                    result = sqrt(result);
+                }
+                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
+                    start2d, count2d, &result );
             }
             else
             {
-                dg::blas1::pointwiseDot( fsa1d, fsa1d, t1d);
-                dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);
-                transfer1d = dg::integrate( t1d, g1d_out);
-
-                result = dg::interpolate( transfer1d, 0., g1d_out);
-                result = sqrt(result);
+                dg::blas1::scal( transferH2d, 0.);
+                dg::blas1::scal( transfer1d, 0.);
+                double result = 0.;
+                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
+                    start2d, count2d, transferH2d.data() );
+                err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
+                    start1d, count1d, transfer1d.data());
+                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
+                    start2d, count2d, &result );
+                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
+                    start2d, count2d, &result );
             }
-            err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
-                start2d, count2d, &result );
 
         }
 
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 2eb78ec24..b1225e744 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -847,7 +847,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
         }
     },
-    {"jsparbhiexbi_tt", "Parallel angular momentum radial flux by ExB velocity with ion density (Time average)", true,
+    {"jsparbphiexbi_tt", "Parallel angular momentum radial flux by ExB velocity with ion density (Time average)", true,
         []( DVec& result, Variables& v){
             // ExB Dot GradPsi
             routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
-- 
GitLab


From 02df54091da2a98fd9cb0fb80b4118abcaafaf23 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 5 Dec 2019 17:32:04 +0100
Subject: [PATCH 182/540] Fix q-integration close to O-point

now we can have nice rho_t flux labels
---
 inc/geometries/flux.h           |  8 +++++++-
 inc/geometries/flux_t.cu        | 11 +++++++++--
 inc/geometries/geometry_diag.cu | 24 ++++++++++++++++--------
 inc/geometries/utilities.h      | 20 ++++++++++++++++++++
 inc/geometries/utilitiesX.h     | 13 +------------
 src/feltor/feltor.tex           |  5 ++---
 src/feltor/feltordiag.cu        | 25 ++++++++++++-------------
 7 files changed, 67 insertions(+), 39 deletions(-)

diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index 0cfb2a7c1..e2fdb9c92 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -30,9 +30,14 @@ struct Fpsi
 {
 
     //firstline = 0 -> conformal, firstline = 1 -> equalarc
-    Fpsi( const CylindricalFunctorsLvl1& psip, const CylindricalFunctorsLvl1& ipol, double x0, double y0, bool verbose = false):
+    Fpsi( const CylindricalFunctorsLvl2& psip, const CylindricalFunctorsLvl1& ipol, double x0, double y0, bool verbose = false):
         psip_(psip), fieldRZYT_(psip, ipol, x0, y0), fieldRZtau_(psip),m_verbose(verbose)
     {
+        //Find O-point
+        double R_O = x0, Z_O = y0;
+        dg::geo::findOpoint( psip, R_O, Z_O);
+        //define angle with respect to O-point
+        fieldRZYT_ = dg::geo::flux::FieldRZYT(psip, ipol, R_O, Z_O);
         X_init = x0, Y_init = y0;
         while( fabs( psip.dfx()(X_init, Y_init)) <= 1e-10 && fabs( psip.dfy()( X_init, Y_init)) <= 1e-10)
             X_init +=  1.;
@@ -69,6 +74,7 @@ struct Fpsi
             eps_old = eps, end_old = end; N*=2;
             dg::stepperRK( "Feagin-17-8-10",  fieldRZYT_, 0., begin, 2*M_PI, end, N);
             eps = sqrt( (end[0]-begin[0])*(end[0]-begin[0]) + (end[1]-begin[1])*(end[1]-begin[1]));
+            if(m_verbose)std::cout << "\t error "<<eps<<" with "<<N<<" steps\t";
             //Attention: if this succeeds on the first attempt end_old won't be updated
         }
         if(m_verbose)std::cout << "\t error "<<eps<<" with "<<N<<" steps\t";
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index ab06e914d..76f904776 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -57,8 +57,15 @@ int main( int argc, char* argv[])
     }
     //write parameters from file into variables
     dg::geo::solovev::Parameters gp(js);
-    {dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
-    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";}
+    {
+        dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField( gp);
+        //Find O-point
+        double R_O = gp.R_0, Z_O = 0.;
+        if( !gp.isToroidal() )
+            dg::geo::findOpoint( mag.get_psip(), R_O, Z_O);
+        const double psipmin = mag.psip()(R_O, Z_O);
+        std::cout << "O-point "<<R_O<<" "<<Z_O<<" with Psip = "<<psipmin<<std::endl;
+    }
     std::cout << "Type n(3), Nx(8), Ny(80), Nz(20)\n";
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 8467c8cb1..d6f4b31db 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -187,7 +187,7 @@ int main( int argc, char* argv[])
     //Find O-point
     double R_O = gp.R_0, Z_O = 0.;
     if( !gp.isToroidal() )
-        dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
+        dg::geo::findOpoint( mag.get_psip(), R_O, Z_O);
     const double psipmin = mag.psip()(R_O, Z_O);
 
     std::cout << "O-point "<<R_O<<" "<<Z_O<<" with Psip = "<<psipmin<<std::endl;
@@ -385,19 +385,27 @@ int main( int argc, char* argv[])
             qprofile = dg::evaluate( qprof, grid1d);
             map1d.emplace_back("q-profile", qprofile,
                 "q-profile (Safety factor) using direct integration");
+            dg::HVec psit = dg::integrate( qprofile, grid1d);
+            map1d.emplace_back("psit1d", psit,
+                "Toroidal flux label psi_t integrated  on grid1d using direct q");
+            //we need to avoid integrating >=0
+            dg::Grid1d g1d_fine(psipmin<0. ? psipmin : psipmax, psipmin<0. ? 0. : psipmin, npsi ,Npsi,dg::NEU);
+            qprofile = dg::evaluate( qprof, g1d_fine);
+            dg::HVec w1d = dg::create::weights( g1d_fine);
+            double psit_tot = dg::blas1::dot( w1d, qprofile);
+            std::cout << "psit tot "<<psit_tot<<"\n";
+            dg::blas1::scal ( psit, 1./psit_tot);
+            dg::blas1::transform( psit, psit, dg::SQRT<double>());
+            map1d.emplace_back("rho_t", psit,
+                "Toroidal flux label rho_t = sqrt( psit/psit_tot) evaluated on grid1d");
         }
         dg::geo::SafetyFactorAverage qprof( grid2d, mag);
         qprofile = dg::evaluate( qprof, grid1d);
         map1d.emplace_back("q-profile_fsa", qprofile,
             "q-profile (Safety factor) using average integration");
         dg::HVec psit = dg::integrate( qprofile, grid1d);
-        map1d.emplace_back("psit1d", psit,
-            "Toroidal flux label psi_t evaluated on grid1d");
-        double psit_tot = dg::interpolate(psit, 0., grid1d);
-        dg::blas1::scal ( psit, 1/psit_tot);
-        dg::blas1::transform( psit, psit, dg::SQRT<double>());
-        map1d.emplace_back("rho_t", psit,
-            "Toroidal flux label rho_t = sqrt( psit/psit_tot) evaluated on grid1d");
+        map1d.emplace_back("psit1d_fsa", psit,
+            "Toroidal flux label psi_t integrated on grid1d using average q");
         map1d.emplace_back("psip1d",    dg::evaluate( dg::cooX1d, grid1d),
             "Poloidal flux label psi_p evaluated on grid1d");
         dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
diff --git a/inc/geometries/utilities.h b/inc/geometries/utilities.h
index be1470414..d3db17578 100644
--- a/inc/geometries/utilities.h
+++ b/inc/geometries/utilities.h
@@ -18,6 +18,7 @@ namespace flux{
  */
 struct FieldRZYT
 {
+    //x0 and y0 sould be O-point to define angle with respect to O-point
     FieldRZYT( const CylindricalFunctorsLvl1& psip, const CylindricalFunctorsLvl1& ipol, double R0, double Z0): R_0_(R0), Z_0_(Z0), psip_(psip), ipol_(ipol){}
     void operator()(double t, const std::array<double,3>& y, std::array<double,3>& yp) const
     {
@@ -133,6 +134,7 @@ namespace ribeiro{
 
 struct FieldRZYT
 {
+    //x0 and y0 sould be O-point to define angle with respect to O-point
     FieldRZYT( const CylindricalFunctorsLvl1& psip, double R0, double Z0, const CylindricalSymmTensorLvl1& chi = CylindricalSymmTensorLvl1() ): R_0_(R0), Z_0_(Z0), psip_(psip), chi_(chi){}
     void operator()(double t, const std::array<double,3>& y, std::array<double,3>& yp) const
     {
@@ -247,6 +249,7 @@ namespace equalarc{
 
 struct FieldRZYT
 {
+    //x0 and y0 sould be O-point to define angle with respect to O-point
     FieldRZYT( const CylindricalFunctorsLvl1& psip, double R0, double Z0, const CylindricalSymmTensorLvl1& chi = CylindricalSymmTensorLvl1() ): R_0_(R0), Z_0_(Z0), psip_(psip), chi_(chi){}
     void operator()(double t, const std::array<double,3>& y, std::array<double,3>& yp) const
     {
@@ -469,6 +472,23 @@ struct MinimalCurve
     bool norm_;
     CylindricalFunctorsLvl1 psip_;
 };
+
+///@copydoc findXpoint()
+static inline void findOpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
+{
+    dg::geo::HessianRZtau hessianRZtau(  psi);
+    std::array<double, 2> X{ {0,0} }, XN(X), X_OLD(X);
+    X[0] = R_X, X[1] = Z_X;
+    double eps = 1e10, eps_old= 2e10;
+    while( (eps < eps_old || eps > 1e-7) && eps > 1e-10)
+    {
+        X_OLD = X; eps= eps_old;
+        hessianRZtau.newton_iteration( X, XN);
+        XN.swap(X);
+        eps = sqrt( (X[0]-X_OLD[0])*(X[0]-X_OLD[0]) + (X[1]-X_OLD[1])*(X[1]-X_OLD[1]));
+    }
+    R_X = X[0], Z_X = X[1];
+}
 ////////////////////////////////////////////////////////////////////////////////
 
 
diff --git a/inc/geometries/utilitiesX.h b/inc/geometries/utilitiesX.h
index e4e2c7a00..15a3df2ad 100644
--- a/inc/geometries/utilitiesX.h
+++ b/inc/geometries/utilitiesX.h
@@ -20,18 +20,7 @@ namespace geo
  */
 static inline void findXpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
 {
-    dg::geo::HessianRZtau hessianRZtau(  psi);
-    std::array<double, 2> X{ {0,0} }, XN(X), X_OLD(X);
-    X[0] = R_X, X[1] = Z_X;
-    double eps = 1e10, eps_old= 2e10;
-    while( (eps < eps_old || eps > 1e-7) && eps > 1e-10)
-    {
-        X_OLD = X; eps= eps_old;
-        hessianRZtau.newton_iteration( X, XN);
-        XN.swap(X);
-        eps = sqrt( (X[0]-X_OLD[0])*(X[0]-X_OLD[0]) + (X[1]-X_OLD[1])*(X[1]-X_OLD[1]));
-    }
-    R_X = X[0], Z_X = X[1];
+    findOpoint( psi, R_X, Z_X);
 }
 
 ///@cond
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 093818dad..b0736d6f5 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1509,9 +1509,8 @@ psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p)
 rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
 q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} using direct integration ( accurate but unavailable outside separatrix) \\
 psi\_psi         & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
-q-profile\_fsa   & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} computed with average integration (less accurate but defined outside separatrix) \\
-psit1d           & Dataset & 1 (psi) & Toroidal flux \\
-rho\_t           & Dataset & 1 (psi) & Toroidal flux label $\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$\\
+psit1d           & Dataset & 1 (psi) & Toroidal flux (integrated q-profile) $\psi_t = \int^\psi_p \d\psi_p q(\psi_p)$ \\
+rho\_t           & Dataset & 1 (psi) & Toroidal flux label $\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$ (is similar to $\rho$ in the edge but $\rho_t$ is nicer in the core domain, because equidistant $\rho_t$ make more equidistant flux-surfaces)\\
 Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \langle Z\rangle_{\psi_{p}}$ \\
 Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\langle Z\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
 Z\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle Z\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index c5f111f8b..a1910e9e1 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -167,25 +167,24 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
     map1d.emplace_back("rho", rho,
         "Alternative flux label rho = -psi/psimin + 1");
-    dg::geo::SafetyFactor qprofile( mag);
-    map1d.emplace_back("q-profile", dg::evaluate( qprofile,   g1d_out),
+    dg::geo::SafetyFactor qprof( mag);
+    dg::HVec qprofile = dg::evaluate( qprof, g1d_out);
+    map1d.emplace_back("q-profile", qprofile,
         "q-profile (Safety factor) using direct integration");
     map1d.emplace_back("psi_psi",    dg::evaluate( dg::cooX1d, g1d_out),
-        "Flux label psi (same as coordinate)");
-
-    dg::geo::SafetyFactorAverage qprof( g2d_out, mag);
-    dg::HVec qprofv = dg::evaluate( qprof, g1d_out);
-    map1d.emplace_back("q-profile_fsa", qprofv,
-        "q-profile (Safety factor) using average integration");
-    dg::HVec psit = dg::integrate( qprofv, g1d_out);
+        "Poloidal flux label psi (same as coordinate)");
+    dg::HVec psit = dg::integrate( qprofile, g1d_out);
     map1d.emplace_back("psit1d", psit,
-        "Toroidal flux label psi_t evaluated on grid1d");
-    double psit_tot = dg::interpolate(psit, 0., g1d_out);
+        "Toroidal flux label psi_t integrated using q-profile");
+    //we need to avoid integrating >=0 for total psi_t
+    dg::Grid1d g1d_fine(psipmin, 0., 3, Npsi, dg::DIR_NEU);
+    qprofile = dg::evaluate( qprof, g1d_fine);
+    dg::HVec w1d = dg::create::weights( g1d_fine);
+    double psit_tot = dg::blas1::dot( w1d, qprofile);
     dg::blas1::scal ( psit, 1./psit_tot);
     dg::blas1::transform( psit, psit, dg::SQRT<double>());
     map1d.emplace_back("rho_t", psit,
-        "Toroidal flux label rho_t = sqrt( psit/psit_tot) evaluated on grid1d");
-
+        "Toroidal flux label rho_t = sqrt( psit/psit_tot)");
 
     // interpolate from 2d grid to X-point points
     dg::IHMatrix grid2gridX2d  = dg::create::interpolation(
-- 
GitLab


From c0292a0d025a39acc1d2626803135e3441fc8985 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 20 Dec 2019 10:15:17 +0100
Subject: [PATCH 183/540] Fix: status variable in average_gpu

---
 inc/dg/backend/average_gpu.cuh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/inc/dg/backend/average_gpu.cuh b/inc/dg/backend/average_gpu.cuh
index db9704624..9ad3bc0c4 100644
--- a/inc/dg/backend/average_gpu.cuh
+++ b/inc/dg/backend/average_gpu.cuh
@@ -98,8 +98,11 @@ void average_mpi( CudaTag, unsigned nx, unsigned ny, const value_type* in0, cons
     static thrust::host_vector<value_type> h_round;
     d_accumulator.resize( ny*exblas::BIN_COUNT);
     int64_t* d_ptr = thrust::raw_pointer_cast( d_accumulator.data());
+    int status = 0;
     for( unsigned i=0; i<ny; i++)
-        exblas::exdot_gpu(nx, &in0[i*nx], &in1[i*nx], &d_ptr[i*exblas::BIN_COUNT]);
+        exblas::exdot_gpu(nx, &in0[i*nx], &in1[i*nx], &d_ptr[i*exblas::BIN_COUNT], &status);
+    if(status != 0)
+        throw dg::Error(dg::Message(_ping_)<<"MPI GPU Average failed since one of the inputs contains NaN or Inf");
     h_accumulator2 = d_accumulator;
     h_accumulator.resize( h_accumulator2.size());
     exblas::reduce_mpi_cpu( ny, &h_accumulator2[0], &h_accumulator[0], comm, comm_mod, comm_mod_reduce);
-- 
GitLab


From 6bcaaca9477265a36bdf2f81c6b8bc5ec2d193f1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 20 Dec 2019 10:16:50 +0100
Subject: [PATCH 184/540] Update geometry_diag

---
 inc/geometries/geometry_diag.cu | 30 +++++++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 5 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index d6f4b31db..8593a11d2 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -281,7 +281,7 @@ int main( int argc, char* argv[])
     double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     double volumeXGrid;
     /// -------  Elements for fsa of curvature operators ----------------
-    // This is a numerical test because the fsa of the curvature operators exactly vanishes (at least for I = cte)
+    // This is a numerical test because the fsa of the curvature operators exactly vanishes (at least for I = cte and almost for everything else)!!
     dg::geo::CylindricalVectorLvl0 bhat_ = dg::geo::createBHat( mag);
     dg::geo::CylindricalVectorLvl0 curvB_ = dg::geo::createTrueCurvatureNablaB( mag);
     dg::geo::CylindricalVectorLvl0 curvK_ = dg::geo::createTrueCurvatureKappa( mag);
@@ -307,7 +307,7 @@ int main( int argc, char* argv[])
         double fx_0 = 1./8.;
         psipmax = -fx_0/(1.-fx_0)*psipmin;
         std::cout << "psi 1 is          "<<psipmax<<"\n";
-        dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., npsi, Npsi, 360, dg::DIR, dg::NEU);
+        dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., npsi, Npsi, 640, dg::DIR, dg::NEU);
         std::cout << "DONE! Generate X-point flux-aligned grid!\n";
         dg::Average<dg::HVec > avg_eta( gX2d.grid(), dg::coo2d::y);
         std::vector<dg::HVec> coordsX = gX2d.map();
@@ -324,6 +324,8 @@ int main( int argc, char* argv[])
         dg::blas1::scal( dvdzeta, 4.*M_PI*M_PI);
         dg::Grid1d gX1d( gX2d.x0(), gX2d.x1(), npsi, Npsi, dg::NEU);
         dg::HVec X_psi_vol = dg::integrate( dvdzeta, gX1d);
+        map1d.emplace_back( "X_dvdzeta", dvdzeta,
+            "dvdzeta on X-point grid");
         map1d.emplace_back( "X_psi_vol", X_psi_vol,
             "Flux volume on X-point grid");
         dg::blas1::pointwiseDivide( X_psip1d, dvdzeta, X_psip1d);
@@ -364,6 +366,22 @@ int main( int argc, char* argv[])
             "Flux surface average of true Kappa curvature Dot Grad Psip");
         map1d.emplace_back( "X_gradPsip_fsa", X_gradPsip_fsa,
             "Flux surface average of |Grad Psip|");
+        // h02 factor
+        dg::HVec h02 = dg::pullback( Hoo(mag), gX2d), X_h02_fsa;
+        dg::blas1::pointwiseDot( volX2d, h02, h02);
+        avg_eta( h02, X_h02_fsa, false);
+        dg::blas1::scal( X_h02_fsa, 4*M_PI*M_PI); //
+        dg::blas1::pointwiseDivide( X_h02_fsa, dvdzeta, X_h02_fsa );
+        map1d.emplace_back( "X_hoo_fsa", X_h02_fsa,
+            "Flux surface average of novel h02 factor");
+        //divb
+        h02 = dg::pullback( dg::geo::Divb(mag), gX2d);
+        dg::blas1::pointwiseDot( volX2d, h02, h02);
+        avg_eta( h02, X_h02_fsa, false);
+        dg::blas1::scal( X_h02_fsa, 4*M_PI*M_PI); //
+        dg::blas1::pointwiseDivide( X_h02_fsa, dvdzeta, X_h02_fsa );
+        map1d.emplace_back( "X_divb_fsa", X_h02_fsa,
+            "Flux surface average of divb");
     }
 
     ///////////////////Compute flux average////////////////////
@@ -439,8 +457,9 @@ int main( int argc, char* argv[])
         // h02 factor
         dg::HVec h02 = dg::evaluate( Hoo(mag), grid2d);
         fsa.set_container( h02);
-        map1d.emplace_back( "hoo", dg::evaluate( fsa, grid1d),
-            "Flux surface average of novel h02 factor");
+        map1d.emplace_back( "hoo_fsa", dg::evaluate( fsa, grid1d),
+           "Flux surface average of novel h02 factor with delta function");
+
 
 
         dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, mag);
@@ -568,7 +587,8 @@ int main( int argc, char* argv[])
     err = nc_redef(ncid);
     //////////////////////////////Finalize////////////////////////////////////
     err = nc_close(ncid);
-    std::cout << "TEST ACCURACY OF CURVATURES (values must be close to 0!)\n";
+    std::cout << "FILE CLOSED AND READY TO USE NOW!\n";
+    std::cout << "Test accuracy of curvatures (values must be close to 0!)\n";
     std::array<dg::HVec, 3> bhat, curvB, curvK;
     dg::pushForward( bhat_.x(), bhat_.y(), bhat_.z(),
             bhat[0], bhat[1], bhat[2], grid3d);
-- 
GitLab


From 7741ce947ed957a3ef839630291989493a957a02 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 9 Jan 2020 16:00:54 +0100
Subject: [PATCH 185/540] Make nc_utilities accept template parameter

in order to be able to write float dimension variables
---
 inc/file/nc_utilities.h | 94 ++++++++++++++++++++++++++++-------------
 src/README.adoc         |  3 --
 2 files changed, 65 insertions(+), 32 deletions(-)

diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index ecd444559..05ddef8db 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -26,7 +26,29 @@
 */
 namespace file
 {
+///@cond
+template<class value_type>
+static inline nc_type getNCDataType(){ assert( false && "Type not supported!\n" ); return NC_DOUBLE; }
+template<>
+inline nc_type getNCDataType<double>(){ return NC_DOUBLE;}
+template<>
+inline nc_type getNCDataType<float>(){ return NC_FLOAT;}
+template<>
+inline nc_type getNCDataType<int>(){ return NC_INT;}
+template<>
+inline nc_type getNCDataType<unsigned>(){ return NC_UINT;}
+///@endcond
 
+template<class T>
+inline int define_real_time( int ncid, const char* name, int* dimID, int* tvarID)
+{
+    int retval;
+    if( (retval = nc_def_dim( ncid, name, NC_UNLIMITED, dimID)) ){ return retval;}
+    if( (retval = nc_def_var( ncid, name, getNCDataType<T>(), 1, dimID, tvarID))){return retval;}
+    std::string t = "time since start"; //needed for paraview to recognize timeaxis
+    if( (retval = nc_put_att_text(ncid, *tvarID, "units", t.size(), t.data())) ){ return retval;}
+    return retval;
+}
 /**
  * @brief Define an unlimited time dimension and variable following
   <a href="http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html">CF-conventions</a>
@@ -35,20 +57,16 @@ namespace file
  * @param ncid file ID
  * @param name Name of time variable (variable names are not standardized)
  * @param dimID time-dimension ID
- * @param tvarID time-variable ID
+ * @param tvarID time-variable ID (for a time variable of type \c NC_DOUBLE)
  *
  * @return netcdf error code if any
  */
 static inline int define_time( int ncid, const char* name, int* dimID, int* tvarID)
 {
-    int retval;
-    if( (retval = nc_def_dim( ncid, name, NC_UNLIMITED, dimID)) ){ return retval;}
-    if( (retval = nc_def_var( ncid, name, NC_DOUBLE, 1, dimID, tvarID))){return retval;}
-    std::string t = "time since start"; //needed for paraview to recognize timeaxis
-    if( (retval = nc_put_att_text(ncid, *tvarID, "units", t.size(), t.data())) ){ return retval;}
-    return retval;
+    return define_real_time<double>( ncid, name, dimID, tvarID);
 }
 
+
 /**
  * @brief Define a limited time dimension and variable following
   <a href="http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html">CF-conventions</a>
@@ -58,7 +76,7 @@ static inline int define_time( int ncid, const char* name, int* dimID, int* tvar
  * @param name Name of the time variable (usually "time")
  * @param size The number of timesteps
  * @param dimID time-dimension ID
- * @param tvarID time-variable ID
+ * @param tvarID time-variable ID (for a time variable of type \c NC_DOUBLE)
  *
  * @return netcdf error code if any
  */
@@ -81,17 +99,19 @@ static inline int define_limited_time( int ncid, const char* name, int size, int
  * @param g The 1d DG grid from which data points are generated (input)
  * @param name_dim Name of dimension (input)
  * @param axis The axis attribute (input), ("X", "Y" or "Z")
+ * @tparam T determines the datatype of the dimension variables
  *
  * @return netcdf error code if any
  */
-static inline int define_dimension( int ncid, int* dimID, const dg::Grid1d& g, std::string name_dim = "x", std::string axis = "X")
+template<class T>
+inline int define_dimension( int ncid, int* dimID, const dg::RealGrid1d<T>& g, std::string name_dim = "x", std::string axis = "X")
 {
     int retval;
     std::string long_name = name_dim+"-coordinate in Computational coordinate system";
-    thrust::host_vector<double> points = dg::create::abscissas( g);
+    thrust::host_vector<T> points = dg::create::abscissas( g);
     if( (retval = nc_def_dim( ncid, name_dim.data(), points.size(), dimID)) ) { return retval;}
     int varID;
-    if( (retval = nc_def_var( ncid, name_dim.data(), NC_DOUBLE, 1, dimID, &varID))){return retval;}
+    if( (retval = nc_def_var( ncid, name_dim.data(), getNCDataType<T>(), 1, dimID, &varID))){return retval;}
     if( (retval = nc_enddef(ncid)) ) {return retval;} //not necessary for NetCDF4 files
     if( (retval = nc_put_var_double( ncid, varID, points.data())) ){ return retval;}
     if( (retval = nc_redef(ncid))) {return retval;} //not necessary for NetCDF4 files
@@ -109,13 +129,15 @@ static inline int define_dimension( int ncid, int* dimID, const dg::Grid1d& g, s
  * @param tvarID time variable ID (unlimited)
  * @param g The 1d DG grid from which data points are generated
  * @param name_dims Names for the dimension variables
+ * @tparam T determines the datatype of the dimension variables
  *
  * @return netcdf error code if any
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::Grid1d& g, std::array<std::string,2> name_dims = {"time","x"})
+template<class T>
+inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::RealGrid1d<T>& g, std::array<std::string,2> name_dims = {"time","x"})
 {
     int retval;
-    retval = define_time( ncid, name_dims[0].data(), &dimsIDs[0], tvarID);
+    retval = define_real_time<T>( ncid, name_dims[0].data(), &dimsIDs[0], tvarID);
     if(retval)
         return retval;
     return define_dimension( ncid, &dimsIDs[1], g, name_dims[1], "X");
@@ -128,14 +150,16 @@ static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const
  * @param dimsIDs (write - only) 2D array of dimension IDs (Y,X)
  * @param g The 2d grid from which to derive the dimensions
  * @param name_dims Names for the dimension variables
+ * @tparam T determines the datatype of the dimension variables
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology2d& g, std::array<std::string,2> name_dims = {"y", "x"})
+template<class T>
+inline int define_dimensions( int ncid, int* dimsIDs, const dg::aRealTopology2d<T>& g, std::array<std::string,2> name_dims = {"y", "x"})
 {
-    dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
-    dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
+    dg::RealGrid1d<T> gx( g.x0(), g.x1(), g.n(), g.Nx());
+    dg::RealGrid1d<T> gy( g.y0(), g.y1(), g.n(), g.Ny());
     int retval;
     retval = define_dimension( ncid, &dimsIDs[0], gy, name_dims[0], "Y");
     if(retval)
@@ -151,14 +175,16 @@ static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology
  * @param tvarID (write - only) The ID of the time variable ( unlimited)
  * @param g The 2d grid from which to derive the dimensions
  * @param name_dims Names for the dimension variables ( time, Y, X)
+ * @tparam T determines the datatype of the dimension variables
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aTopology2d& g, std::array<std::string,3> name_dims = {"time", "y", "x"})
+template<class T>
+inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRealTopology2d<T>& g, std::array<std::string,3> name_dims = {"time", "y", "x"})
 {
     int retval;
-    retval = define_time( ncid, name_dims[0].data(), &dimsIDs[0], tvarID);
+    retval = define_real_time<T>( ncid, name_dims[0].data(), &dimsIDs[0], tvarID);
     if(retval)
         return retval;
     return define_dimensions( ncid, &dimsIDs[1], g, {name_dims[1], name_dims[2]});
@@ -174,11 +200,13 @@ static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const
  * @param tvarID (write - only) The ID of the time variable (limited)
  * @param g The 2d grid from which to derive the dimensions
  * @param name_dims Names for the dimension variables (time, Y, X)
+ * @tparam T determines the datatype of the dimension variables
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_limtime_xy( int ncid, int* dimsIDs, int size, int* tvarID, const dg::aTopology2d& g, std::array<std::string, 3> name_dims = {"time", "y", "x"})
+template<class T>
+inline int define_limtime_xy( int ncid, int* dimsIDs, int size, int* tvarID, const dg::aRealTopology2d<T>& g, std::array<std::string, 3> name_dims = {"time", "y", "x"})
 {
     int retval;
     retval = define_limited_time( ncid, name_dims[0].data(), size, &dimsIDs[0], tvarID);
@@ -194,15 +222,17 @@ static inline int define_limtime_xy( int ncid, int* dimsIDs, int size, int* tvar
  * @param dimsIDs (write - only) 3D array of dimension IDs (Z,Y,X)
  * @param g The grid from which to derive the dimensions
  * @param name_dims Names for the dimension variables ( Z, Y, X)
+ * @tparam T determines the datatype of the dimension variables
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology3d& g, std::array<std::string, 3> name_dims = {"z", "y", "x"})
+template<class T>
+inline int define_dimensions( int ncid, int* dimsIDs, const dg::aRealTopology3d<T>& g, std::array<std::string, 3> name_dims = {"z", "y", "x"})
 {
-    dg::Grid1d gx( g.x0(), g.x1(), g.n(), g.Nx());
-    dg::Grid1d gy( g.y0(), g.y1(), g.n(), g.Ny());
-    dg::Grid1d gz( g.z0(), g.z1(), 1, g.Nz());
+    dg::RealGrid1d<T> gx( g.x0(), g.x1(), g.n(), g.Nx());
+    dg::RealGrid1d<T> gy( g.y0(), g.y1(), g.n(), g.Ny());
+    dg::RealGrid1d<T> gz( g.z0(), g.z1(), 1, g.Nz());
     int retval;
     retval = define_dimension( ncid, &dimsIDs[0], gz, name_dims[0], "Z");
     if(retval)
@@ -222,14 +252,16 @@ static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aTopology
  * @param tvarID (write - only) The ID of the time variable ( unlimited)
  * @param g The grid from which to derive the dimensions
  * @param name_dims Names for the dimension variables ( time, Z, Y, X)
+ * @tparam T determines the datatype of the dimension variables
  *
  * @return if anything goes wrong it returns the netcdf code, else SUCCESS
  * @note File stays in define mode
  */
-static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aTopology3d& g, std::array<std::string, 4> name_dims = {"time", "z", "y", "x"})
+template<class T>
+inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRealTopology3d<T>& g, std::array<std::string, 4> name_dims = {"time", "z", "y", "x"})
 {
     int retval;
-    retval = define_time( ncid, "time", &dimsIDs[0], tvarID);
+    retval = define_real_time<T>( ncid, "time", &dimsIDs[0], tvarID);
     if(retval)
         return retval;
     return define_dimensions( ncid, &dimsIDs[1], g, {name_dims[1], name_dims[2], name_dims[3]});
@@ -239,22 +271,26 @@ static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const
 #ifdef MPI_VERSION
 
 /// Convenience function that just calls the corresponding serial version with the global grid
-static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aMPITopology2d& g, std::array<std::string,2> name_dims = {"y", "x"})
+template<class T>
+inline int define_dimensions( int ncid, int* dimsIDs, const dg::aRealMPITopology2d<T>& g, std::array<std::string,2> name_dims = {"y", "x"})
 {
     return define_dimensions( ncid, dimsIDs, g.global(), name_dims);
 }
 /// Convenience function that just calls the corresponding serial version with the global grid
-static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aMPITopology2d& g, std::array<std::string,3> name_dims = {"time", "y", "x"})
+template<class T>
+inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRealMPITopology2d<T>& g, std::array<std::string,3> name_dims = {"time", "y", "x"})
 {
     return define_dimensions( ncid, dimsIDs, tvarID, g.global(), name_dims);
 }
 /// Convenience function that just calls the corresponding serial version with the global grid
-static inline int define_dimensions( int ncid, int* dimsIDs, const dg::aMPITopology3d& g, std::array<std::string, 3> name_dims = {"z", "y", "x"})
+template<class T>
+inline int define_dimensions( int ncid, int* dimsIDs, const dg::aRealMPITopology3d<T>& g, std::array<std::string, 3> name_dims = {"z", "y", "x"})
 {
     return define_dimensions( ncid, dimsIDs, g.global(), name_dims);
 }
 /// Convenience function that just calls the corresponding serial version with the global grid
-static inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aMPITopology3d& g, std::array<std::string, 4> name_dims = {"time", "z", "y", "x"})
+template<class T>
+inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRealMPITopology3d<T>& g, std::array<std::string, 4> name_dims = {"time", "z", "y", "x"})
 {
     return define_dimensions( ncid, dimsIDs, tvarID, g.global(), name_dims);
 }
diff --git a/src/README.adoc b/src/README.adoc
index c3b65880e..4eb61cf21 100644
--- a/src/README.adoc
+++ b/src/README.adoc
@@ -12,9 +12,6 @@ with realistic flux surfaces. Used in the paper "Three discontinuous
 Galerkin schemes for the anisotropic heat conduction equation on
 non-aligned grids" Comput. Phys. Commun. 199, 29-39 (2016)
 
-feltor2d::  2d gyrofluid electrodynamic model (the phi symmetric version
-of feltor)
-
 lamb_dipole:: Programs that reproduce results for the paper "A
 conservative discontinuous Galerkin scheme for the 2D incompressible
 Navier-Stokes equations" Comput. Phys. Commun. 185, 2865-2873 (2014) (The
-- 
GitLab


From c666b8afda3f41887843eacf997aa8f7e2b1e5b5 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 9 Jan 2020 16:35:10 +0100
Subject: [PATCH 186/540] Add interpolate_in_3d file

---
 src/feltor/interpolate_in_3d.cu | 326 ++++++++++++++++++++++++++++++++
 1 file changed, 326 insertions(+)
 create mode 100644 src/feltor/interpolate_in_3d.cu

diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
new file mode 100644
index 000000000..4f7b78248
--- /dev/null
+++ b/src/feltor/interpolate_in_3d.cu
@@ -0,0 +1,326 @@
+#include <iostream>
+#include <fstream>
+#include <iomanip>
+#include <vector>
+#include <string>
+#include <functional>
+#include "json/json.h"
+
+#include "dg/algorithm.h"
+#include "dg/geometries/geometries.h"
+#include "dg/file/nc_utilities.h"
+using HVec = dg::HVec;
+using DVec = dg::DVec;
+using DMatrix = dg::DMatrix;
+using IDMatrix = dg::IDMatrix;
+using IHMatrix = dg::IHMatrix;
+using Geometry = dg::CylindricalGrid3d;
+#define MPI_OUT
+
+//convert all 3d variables of the last timestep to float and interpolate to a 3 times finer grid in phi
+//we need a time variable when available
+int main( int argc, char* argv[])
+{
+    if( argc != 3)
+    {
+        std::cerr << "Usage: "<<argv[0]<<" [input.nc] [output.nc]\n";
+        return -1;
+    }
+    std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
+
+    //------------------------open input nc file--------------------------------//
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input( length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "geomfile", &length);
+    std::string geom( length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "geomfile", &geom[0]);
+    err = nc_close(ncid);
+
+    //std::cout << "input "<<input<<std::endl;
+    //std::cout << "geome "<<geom <<std::endl;
+    Json::Value js,gs;
+    Json::CharReaderBuilder parser;
+    parser["collectComments"] = false;
+    std::string errs;
+    std::stringstream ss( input);
+    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    ss.str( geom);
+    parseFromStream( parser, ss, &gs, &errs); //read input without comments
+    const feltor::Parameters p(js);
+    const dg::geo::solovev::Parameters gp(gs);
+    p.display();
+    gp.display();
+    std::vector<std::string> names_static_input{
+        "xc", "yc", "zc", "BR", "BZ", "BP", "Nprof", "Psip", "Source"
+    }
+    std::vector<std::string> names_input{
+        "electrons", "ions", "Ue", "Ui", "potential", "induction"
+    };
+
+    //-----------------Create Netcdf output file with attributes----------//
+    int ncid_out;
+    err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
+
+    /// Set global attributes
+    std::map<std::string, std::string> att;
+    att["title"] = "Output file of feltor/src/feltor/interpolate_in_3d.cu";
+    att["Conventions"] = "CF-1.7";
+    ///Get local time and begin file history
+    auto ttt = std::time(nullptr);
+    auto tm = *std::localtime(&ttt);
+    std::ostringstream oss;
+    ///time string  + program-name + args
+    oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
+    for( int i=0; i<argc; i++) oss << " "<<argv[i];
+    att["history"] = oss.str();
+    att["comment"] = "Find more info in feltor/src/feltor.tex";
+    att["source"] = "FELTOR";
+    att["references"] = "https://github.com/feltor-dev/feltor";
+    att["inputfile"] = input;
+    att["geomfile"] = geom;
+    for( auto pair : att)
+        err = nc_put_att_text( ncid_out, NC_GLOBAL,
+            pair.first.data(), pair.second.size(), pair.second.data());
+
+    //-------------------Construct grids-------------------------------------//
+
+    const float Rmin=gp.R_0-p.boxscaleRm*gp.a;
+    const float Zmin=-p.boxscaleZm*gp.a*gp.elongation;
+    const float Rmax=gp.R_0+p.boxscaleRp*gp.a;
+    const float Zmax=p.boxscaleZp*gp.a*gp.elongation;
+
+    dg::RealCylindricalGrid3d<float> g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
+        p.n_out, p.Nx_out, p.Ny_out, 3*p.Nz_out, p.bcxN, p.bcyN, dg::PER);
+
+    // Construct weights and temporaries
+    dg::fHVec transferH3d = dg::evaluate(dg::zero,g3d_out);
+
+    // define 2d and 1d and 0d dimensions and variables
+    int dim_ids[3];
+    err = file::define_dimensions( ncid_out, dim_ids, g3d_out, {"z", "y", "x"});
+    std::map<std::string, int> id3d;
+
+    //write 1d static vectors (psi, q-profile, ...) into file
+    for( auto tp : map1d)
+    {
+        int vid;
+        err = nc_def_var( ncid_out, std::get<0>(tp).data(), NC_DOUBLE, 1,
+            &dim_ids1d[1], &vid);
+        err = nc_put_att_text( ncid_out, vid, "long_name",
+            std::get<2>(tp).size(), std::get<2>(tp).data());
+        err = nc_enddef( ncid);
+        err = nc_put_var_double( ncid_out, vid, std::get<1>(tp).data());
+        err = nc_redef(ncid_out);
+    }
+
+    for( auto& record : feltor::diagnostics2d_list)
+    {
+        std::string record_name = record.name;
+        if( record_name[0] == 'j')
+            record_name[1] = 'v';
+        std::string name = record_name + "_fluc2d";
+        long_name = record.long_name + " (Fluctuations wrt fsa on phi = 0 plane.)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid_out, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_fsa2d";
+        long_name = record.long_name + " (Flux surface average interpolated to 2d plane.)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid_out, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_fsa";
+        long_name = record.long_name + " (Flux surface average.)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        err = nc_put_att_text( ncid_out, id1d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_ifs";
+        long_name = record.long_name + " (wrt. vol integrated flux surface average)";
+        if( record_name[0] == 'j')
+            long_name = record.long_name + " (wrt. vol derivative of the flux surface average)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        err = nc_put_att_text( ncid_out, id1d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_ifs_lcfs";
+        long_name = record.long_name + " (wrt. vol integrated flux surface average evaluated on last closed flux surface)";
+        if( record_name[0] == 'j')
+            long_name = record.long_name + " (flux surface average evaluated on the last closed flux surface)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
+            &id0d[name]);
+        err = nc_put_att_text( ncid_out, id0d[name], "long_name", long_name.size(),
+            long_name.data());
+
+        name = record_name + "_ifs_norm";
+        long_name = record.long_name + " (wrt. vol integrated square flux surface average from 0 to lcfs)";
+        if( record_name[0] == 'j')
+            long_name = record.long_name + " (wrt. vol integrated square derivative of the flux surface average from 0 to lcfs)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
+            &id0d[name]);
+        err = nc_put_att_text( ncid_out, id0d[name], "long_name", long_name.size(),
+            long_name.data());
+    }
+    /////////////////////////////////////////////////////////////////////////
+    int timeID;
+    double time=0.;
+
+    size_t steps;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
+    err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
+    err = nc_inq_dimlen( ncid, timeID, &steps);
+    //steps = 3;
+    for( unsigned i=0; i<steps; i++)//timestepping
+    {
+        start2d[0] = i;
+        start1d[0] = i;
+        // read and write time
+        err = nc_get_vara_double( ncid, timeID, start2d, count2d, &time);
+        std::cout << "Timestep = " << i << "  time = " << time << std::endl;
+        err = nc_put_vara_double( ncid_out, tvarID, start2d, count2d, &time);
+        for( auto& record : feltor::diagnostics2d_list)
+        {
+            std::string record_name = record.name;
+            if( record_name[0] == 'j')
+                record_name[1] = 'v';
+            //1. Read toroidal average
+            int dataID =0;
+            bool available = true;
+            try{
+                err = nc_inq_varid(ncid, (record.name+"_ta2d").data(), &dataID);
+            } catch ( file::NC_Error error)
+            {
+                if(  i == 0)
+                {
+                    std::cerr << error.what() <<std::endl;
+                    std::cerr << "Offending variable is "<<record.name+"_ta2d\n";
+                    std::cerr << "Writing zeros ... \n";
+                }
+                available = false;
+            }
+            if( available)
+            {
+                err = nc_get_vara_double( ncid, dataID,
+                    start2d, count2d, transferH2d.data());
+                //2. Compute fsa and output fsa
+                dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
+                dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
+                poloidal_average( transferH2dX, t1d, false); //average over eta
+                dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
+                dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
+                if( record_name[0] == 'j')
+                    dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d );
+                //3. Interpolate fsa on 2d plane : <f>
+                dg::blas2::gemv(fsa2rzmatrix, fsa1d, transferH2d); //fsa on RZ grid
+            }
+            else
+            {
+                dg::blas1::scal( fsa1d, 0.);
+                dg::blas1::scal( transferH2d, 0.);
+            }
+            err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_fsa"),
+                start1d, count1d, fsa1d.data());
+            err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fsa2d"),
+                start2d, count2d, transferH2d.data() );
+            //4. Read 2d variable and compute fluctuations
+            available = true;
+            try{
+                err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
+            } catch ( file::NC_Error error)
+            {
+                if(  i == 0)
+                {
+                    std::cerr << error.what() <<std::endl;
+                    std::cerr << "Offending variable is "<<record.name+"_2d\n";
+                    std::cerr << "Writing zeros ... \n";
+                }
+                available = false;
+            }
+            if( available)
+            {
+                err = nc_get_vara_double( ncid, dataID, start2d, count2d,
+                    t2d_mp.data());
+                if( record_name[0] == 'j')
+                    dg::blas1::pointwiseDot( t2d_mp, dvdpsip2d, t2d_mp );
+                dg::blas1::axpby( 1.0, t2d_mp, -1.0, transferH2d);
+                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
+                    start2d, count2d, transferH2d.data() );
+
+                //5. flux surface integral/derivative
+                double result =0.;
+                if( record_name[0] == 'j') //j indicates a flux
+                {
+                    dg::blas2::symv( dpsi, fsa1d, t1d);
+                    dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
+
+                    result = dg::interpolate( fsa1d, 0., g1d_out);
+                }
+                else
+                {
+                    dg::blas1::pointwiseDot( fsa1d, dvdpsip, t1d);
+                    transfer1d = dg::integrate( t1d, g1d_out);
+
+                    result = dg::interpolate( transfer1d, 0., g1d_out);
+                }
+                err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
+                    start1d, count1d, transfer1d.data());
+                //flux surface integral/derivative on last closed flux surface
+                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
+                    start2d, count2d, &result );
+                //6. Compute norm of time-integral terms to get relative importance
+                if( record_name[0] == 'j') //j indicates a flux
+                {
+                    dg::blas2::symv( dpsi, fsa1d, t1d);
+                    dg::blas1::pointwiseDivide( t1d, dvdpsip, t1d); //dvjv
+                    dg::blas1::pointwiseDot( t1d, t1d, t1d);//dvjv2
+                    dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);//dvjv2
+                    transfer1d = dg::integrate( t1d, g1d_out);
+                    result = dg::interpolate( transfer1d, 0., g1d_out);
+                    result = sqrt(result);
+                }
+                else
+                {
+                    dg::blas1::pointwiseDot( fsa1d, fsa1d, t1d);
+                    dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);
+                    transfer1d = dg::integrate( t1d, g1d_out);
+
+                    result = dg::interpolate( transfer1d, 0., g1d_out);
+                    result = sqrt(result);
+                }
+                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
+                    start2d, count2d, &result );
+            }
+            else
+            {
+                dg::blas1::scal( transferH2d, 0.);
+                dg::blas1::scal( transfer1d, 0.);
+                double result = 0.;
+                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
+                    start2d, count2d, transferH2d.data() );
+                err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
+                    start1d, count1d, transfer1d.data());
+                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
+                    start2d, count2d, &result );
+                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
+                    start2d, count2d, &result );
+            }
+
+        }
+
+
+    } //end timestepping
+    err = nc_close(ncid);
+    err = nc_close(ncid_out);
+
+    return 0;
+}
-- 
GitLab


From f8c02c9beadf60e8461dd10b74c29faa432eea1b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 9 Jan 2020 23:29:46 +0100
Subject: [PATCH 187/540] Write interpolate_in_3d file

it compiles now but no idea if it does the correct thing
---
 inc/geometries/fieldaligned.h   |  12 +-
 src/feltor/Makefile             |   2 +
 src/feltor/interpolate_in_3d.cu | 254 +++++++++-----------------------
 3 files changed, 77 insertions(+), 191 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 3e2c38675..75fc27f9b 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -563,19 +563,19 @@ container Fieldaligned<G, I,container>::interpolate_from_coarse_grid( const G& g
 
     container out = dg::evaluate( dg::zero, *m_g);
     container helper = dg::evaluate( dg::zero, *m_g);
-    dg::split( out, m_f, *m_g);
+    dg::split( in, m_f, *m_g);
     dg::split( helper, m_temp, *m_g);
-    std::vector<dg::View< container>> in_split = dg::split( in, grid);
+    std::vector<dg::View< container>> out_split = dg::split( out, grid);
     for ( int i=0; i<(int)Nz_coarse; i++)
     {
         //1. copy input vector to appropriate place in output
-        dg::blas1::copy( in_split[i], m_f[i*cphi]);
-        dg::blas1::copy( in_split[i], m_temp[i*cphi]);
+        dg::blas1::copy( m_f[i], out_split[i*cphi]);
+        dg::blas1::copy( m_f[i], m_temp[i*cphi]);
         //2. Now apply plus and minus T to fill in the rest
         for( int j=1; j<(int)cphi; j++)
         {
             //!!! The value of f at the plus plane is I^- of the current plane
-            dg::blas2::symv( m_minus, m_f[i*cphi+j-1], m_f[i*cphi+j]);
+            dg::blas2::symv( m_minus, out_split[i*cphi+j-1], out_split[i*cphi+j]);
             //!!! The value of f at the minus plane is I^+ of the current plane
             dg::blas2::symv( m_plus, m_temp[(i*cphi+cphi+1-j)%Nz], m_temp[i*cphi+cphi-j]);
         }
@@ -586,7 +586,7 @@ container Fieldaligned<G, I,container>::interpolate_from_coarse_grid( const G& g
         {
             double alpha = (double)(cphi-j)/(double)cphi;
             double beta = (double)j/(double)cphi;
-            dg::blas1::axpby( alpha, m_f[i*cphi+j], beta, m_temp[i*cphi+j], m_f[i*cphi+j]);
+            dg::blas1::axpby( alpha, out_split[i*cphi+j], beta, m_temp[i*cphi+j], out_split[i*cphi+j]);
         }
     return out;
 }
diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index a1ad5a254..1d6856200 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -15,6 +15,8 @@ manufactured: manufactured.cu manufactured.h feltor.h implicit.h
 
 feltordiag: feltordiag.cu feltordiag.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g
+interpolate_in_3d: interpolate_in_3d.cu feltordiag.h
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g
 
 feltor: feltor.cu feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index 4f7b78248..6bb63a3bb 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -12,10 +12,12 @@
 using HVec = dg::HVec;
 using DVec = dg::DVec;
 using DMatrix = dg::DMatrix;
+using HMatrix = dg::HMatrix;
 using IDMatrix = dg::IDMatrix;
 using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
+#include "feltordiag.h"
 
 //convert all 3d variables of the last timestep to float and interpolate to a 3 times finer grid in phi
 //we need a time variable when available
@@ -55,12 +57,6 @@ int main( int argc, char* argv[])
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
-    std::vector<std::string> names_static_input{
-        "xc", "yc", "zc", "BR", "BZ", "BP", "Nprof", "Psip", "Source"
-    }
-    std::vector<std::string> names_input{
-        "electrons", "ions", "Ue", "Ui", "potential", "induction"
-    };
 
     //-----------------Create Netcdf output file with attributes----------//
     int ncid_out;
@@ -89,121 +85,101 @@ int main( int argc, char* argv[])
 
     //-------------------Construct grids-------------------------------------//
 
-    const float Rmin=gp.R_0-p.boxscaleRm*gp.a;
-    const float Zmin=-p.boxscaleZm*gp.a*gp.elongation;
-    const float Rmax=gp.R_0+p.boxscaleRp*gp.a;
-    const float Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    const double Rmin=gp.R_0-p.boxscaleRm*gp.a;
+    const double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
+    const double Rmax=gp.R_0+p.boxscaleRp*gp.a;
+    const double Zmax=p.boxscaleZp*gp.a*gp.elongation;
 
-    dg::RealCylindricalGrid3d<float> g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
+    dg::RealCylindricalGrid3d<double> g3d_in( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
+        p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bcxN, p.bcyN, dg::PER);
+    dg::RealCylindricalGrid3d<double> g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
         p.n_out, p.Nx_out, p.Ny_out, 3*p.Nz_out, p.bcxN, p.bcyN, dg::PER);
 
     // Construct weights and temporaries
-    dg::fHVec transferH3d = dg::evaluate(dg::zero,g3d_out);
+    dg::HVec transferH_in = dg::evaluate(dg::zero,g3d_in);
+    dg::HVec transferH_out = dg::evaluate(dg::zero,g3d_out);
+    dg::fHVec transferH_out_float = dg::construct<dg::fHVec>( transferH_out);
 
     // define 2d and 1d and 0d dimensions and variables
-    int dim_ids[3];
-    err = file::define_dimensions( ncid_out, dim_ids, g3d_out, {"z", "y", "x"});
-    std::map<std::string, int> id3d;
+    int dim_ids[3], tvarID;
+    err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g3d_out, {"time", "z", "y", "x"});
+    std::map<std::string, int> id4d;
 
-    //write 1d static vectors (psi, q-profile, ...) into file
-    for( auto tp : map1d)
-    {
-        int vid;
-        err = nc_def_var( ncid_out, std::get<0>(tp).data(), NC_DOUBLE, 1,
-            &dim_ids1d[1], &vid);
-        err = nc_put_att_text( ncid_out, vid, "long_name",
-            std::get<2>(tp).size(), std::get<2>(tp).data());
-        err = nc_enddef( ncid);
-        err = nc_put_var_double( ncid_out, vid, std::get<1>(tp).data());
-        err = nc_redef(ncid_out);
-    }
+    /////////////////////////////////////////////////////////////////////////
+    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
 
-    for( auto& record : feltor::diagnostics2d_list)
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    auto bhat = dg::geo::createBHat( mag);
+    if( p.alpha_mag > 0.)
+        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
+    dg::geo::Fieldaligned<Geometry, IHMatrix, HVec> fieldaligned(
+        bhat, g3d_out, p.bcxN, p.bcyN, dg::geo::NoLimiter(),
+        p.rk4eps, p.mx, p.my);
+
+    for( auto& record : feltor::diagnostics3d_static_list)
     {
-        std::string record_name = record.name;
-        if( record_name[0] == 'j')
-            record_name[1] = 'v';
-        std::string name = record_name + "_fluc2d";
-        long_name = record.long_name + " (Fluctuations wrt fsa on phi = 0 plane.)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid_out, id2d[name], "long_name", long_name.size(),
+        std::string name = record.name;
+        std::string long_name = record.long_name;
+        int vID;
+        err = nc_def_var( ncid_out, name.data(), NC_FLOAT, 3, dim_ids,
+            &vID);
+        err = nc_put_att_text( ncid_out, vID, "long_name", long_name.size(),
             long_name.data());
 
-        name = record_name + "_fsa2d";
-        long_name = record.long_name + " (Flux surface average interpolated to 2d plane.)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
-            &id2d[name]);
-        err = nc_put_att_text( ncid_out, id2d[name], "long_name", long_name.size(),
-            long_name.data());
+        int dataID = 0;
+        err = nc_inq_varid(ncid, record.name.data(), &dataID);
+        err = nc_get_var_double( ncid, dataID, transferH_in.data());
+        transferH_out = fieldaligned.interpolate_from_coarse_grid(
+            g3d_in, transferH_in);
+        dg::assign( transferH_out, transferH_out_float);
 
-        name = record_name + "_fsa";
-        long_name = record.long_name + " (Flux surface average.)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
-            &id1d[name]);
-        err = nc_put_att_text( ncid_out, id1d[name], "long_name", long_name.size(),
+        err = nc_enddef( ncid);
+        err = nc_put_var_float( ncid_out, vID, transferH_out_float.data());
+        err = nc_redef(ncid_out);
+    }
+    for( auto& record : feltor::diagnostics3d_list)
+    {
+        std::string name = record.name;
+        std::string long_name = record.long_name;
+        err = nc_def_var( ncid_out, name.data(), NC_FLOAT, 3, dim_ids,
+            &id4d[name]);
+        err = nc_put_att_text( ncid_out, id4d[name], "long_name", long_name.size(),
             long_name.data());
+    }
+    err = nc_enddef( ncid);
 
-        name = record_name + "_ifs";
-        long_name = record.long_name + " (wrt. vol integrated flux surface average)";
-        if( record_name[0] == 'j')
-            long_name = record.long_name + " (wrt. vol derivative of the flux surface average)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
-            &id1d[name]);
-        err = nc_put_att_text( ncid_out, id1d[name], "long_name", long_name.size(),
-            long_name.data());
 
-        name = record_name + "_ifs_lcfs";
-        long_name = record.long_name + " (wrt. vol integrated flux surface average evaluated on last closed flux surface)";
-        if( record_name[0] == 'j')
-            long_name = record.long_name + " (flux surface average evaluated on the last closed flux surface)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
-            &id0d[name]);
-        err = nc_put_att_text( ncid_out, id0d[name], "long_name", long_name.size(),
-            long_name.data());
 
-        name = record_name + "_ifs_norm";
-        long_name = record.long_name + " (wrt. vol integrated square flux surface average from 0 to lcfs)";
-        if( record_name[0] == 'j')
-            long_name = record.long_name + " (wrt. vol integrated square derivative of the flux surface average from 0 to lcfs)";
-        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 1, dim_ids,
-            &id0d[name]);
-        err = nc_put_att_text( ncid_out, id0d[name], "long_name", long_name.size(),
-            long_name.data());
-    }
-    /////////////////////////////////////////////////////////////////////////
     int timeID;
     double time=0.;
 
     size_t steps;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
     err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
     err = nc_inq_dimlen( ncid, timeID, &steps);
-    //steps = 3;
-    for( unsigned i=0; i<steps; i++)//timestepping
+    size_t count3d_in[4]  = {1, g3d_in.Nz(), g3d_in.n()*g3d_in.Ny(), g3d_in.n()*g3d_in.Nx()};
+    size_t count3d_out[4] = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(), g3d_out.n()*g3d_out.Nx()};
+    size_t start3d[4] = {0, 0, 0, 0};
+    /////////////////////////////////////////////////////////////////////////
+    for( unsigned i=0; i<steps; i+=10)//timestepping
     {
-        start2d[0] = i;
-        start1d[0] = i;
+        start3d[0] = i;
         // read and write time
-        err = nc_get_vara_double( ncid, timeID, start2d, count2d, &time);
+        err = nc_get_vara_double( ncid, timeID, start3d, count3d_in, &time);
         std::cout << "Timestep = " << i << "  time = " << time << std::endl;
-        err = nc_put_vara_double( ncid_out, tvarID, start2d, count2d, &time);
-        for( auto& record : feltor::diagnostics2d_list)
+        err = nc_put_vara_double( ncid_out, tvarID, start3d, count3d_out, &time);
+        for( auto& record : feltor::diagnostics3d_list)
         {
             std::string record_name = record.name;
-            if( record_name[0] == 'j')
-                record_name[1] = 'v';
-            //1. Read toroidal average
             int dataID =0;
             bool available = true;
             try{
-                err = nc_inq_varid(ncid, (record.name+"_ta2d").data(), &dataID);
+                err = nc_inq_varid(ncid, record.name.data(), &dataID);
             } catch ( file::NC_Error error)
             {
                 if(  i == 0)
                 {
                     std::cerr << error.what() <<std::endl;
-                    std::cerr << "Offending variable is "<<record.name+"_ta2d\n";
+                    std::cerr << "Offending variable is "<<record.name<<"\n";
                     std::cerr << "Writing zeros ... \n";
                 }
                 available = false;
@@ -211,110 +187,18 @@ int main( int argc, char* argv[])
             if( available)
             {
                 err = nc_get_vara_double( ncid, dataID,
-                    start2d, count2d, transferH2d.data());
+                    start3d, count3d_in, transferH_in.data());
                 //2. Compute fsa and output fsa
-                dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
-                dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
-                poloidal_average( transferH2dX, t1d, false); //average over eta
-                dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
-                dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
-                if( record_name[0] == 'j')
-                    dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d );
-                //3. Interpolate fsa on 2d plane : <f>
-                dg::blas2::gemv(fsa2rzmatrix, fsa1d, transferH2d); //fsa on RZ grid
-            }
-            else
-            {
-                dg::blas1::scal( fsa1d, 0.);
-                dg::blas1::scal( transferH2d, 0.);
-            }
-            err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_fsa"),
-                start1d, count1d, fsa1d.data());
-            err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fsa2d"),
-                start2d, count2d, transferH2d.data() );
-            //4. Read 2d variable and compute fluctuations
-            available = true;
-            try{
-                err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
-            } catch ( file::NC_Error error)
-            {
-                if(  i == 0)
-                {
-                    std::cerr << error.what() <<std::endl;
-                    std::cerr << "Offending variable is "<<record.name+"_2d\n";
-                    std::cerr << "Writing zeros ... \n";
-                }
-                available = false;
-            }
-            if( available)
-            {
-                err = nc_get_vara_double( ncid, dataID, start2d, count2d,
-                    t2d_mp.data());
-                if( record_name[0] == 'j')
-                    dg::blas1::pointwiseDot( t2d_mp, dvdpsip2d, t2d_mp );
-                dg::blas1::axpby( 1.0, t2d_mp, -1.0, transferH2d);
-                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
-                    start2d, count2d, transferH2d.data() );
-
-                //5. flux surface integral/derivative
-                double result =0.;
-                if( record_name[0] == 'j') //j indicates a flux
-                {
-                    dg::blas2::symv( dpsi, fsa1d, t1d);
-                    dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
-
-                    result = dg::interpolate( fsa1d, 0., g1d_out);
-                }
-                else
-                {
-                    dg::blas1::pointwiseDot( fsa1d, dvdpsip, t1d);
-                    transfer1d = dg::integrate( t1d, g1d_out);
-
-                    result = dg::interpolate( transfer1d, 0., g1d_out);
-                }
-                err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
-                    start1d, count1d, transfer1d.data());
-                //flux surface integral/derivative on last closed flux surface
-                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
-                    start2d, count2d, &result );
-                //6. Compute norm of time-integral terms to get relative importance
-                if( record_name[0] == 'j') //j indicates a flux
-                {
-                    dg::blas2::symv( dpsi, fsa1d, t1d);
-                    dg::blas1::pointwiseDivide( t1d, dvdpsip, t1d); //dvjv
-                    dg::blas1::pointwiseDot( t1d, t1d, t1d);//dvjv2
-                    dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);//dvjv2
-                    transfer1d = dg::integrate( t1d, g1d_out);
-                    result = dg::interpolate( transfer1d, 0., g1d_out);
-                    result = sqrt(result);
-                }
-                else
-                {
-                    dg::blas1::pointwiseDot( fsa1d, fsa1d, t1d);
-                    dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);
-                    transfer1d = dg::integrate( t1d, g1d_out);
-
-                    result = dg::interpolate( transfer1d, 0., g1d_out);
-                    result = sqrt(result);
-                }
-                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
-                    start2d, count2d, &result );
+                transferH_out = fieldaligned.interpolate_from_coarse_grid(
+                    g3d_in, transferH_in);
+                dg::assign( transferH_out, transferH_out_float);
             }
             else
             {
-                dg::blas1::scal( transferH2d, 0.);
-                dg::blas1::scal( transfer1d, 0.);
-                double result = 0.;
-                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
-                    start2d, count2d, transferH2d.data() );
-                err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
-                    start1d, count1d, transfer1d.data());
-                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
-                    start2d, count2d, &result );
-                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
-                    start2d, count2d, &result );
+                dg::blas1::scal( transferH_out_float, (float)0);
             }
-
+            err = nc_put_vara_float( ncid_out, id4d.at(record.name), start3d,
+                count3d_out, transferH_out_float.data());
         }
 
 
-- 
GitLab


From 8d39a38e7cb5b9ea1bf6c4dd7ac3cc633f961b29 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 10 Jan 2020 16:02:38 +0100
Subject: [PATCH 188/540] Debug interpolate

but there is a memory leak somewhere
---
 inc/file/nc_utilities.h         |  13 +++-
 inc/geometries/fieldaligned.h   |  14 ++--
 src/feltor/feltordiag.h         |  14 ++++
 src/feltor/interpolate_in_3d.cu | 124 ++++++++++++++++++++------------
 4 files changed, 112 insertions(+), 53 deletions(-)

diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index 05ddef8db..44c0c67bc 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -37,6 +37,17 @@ template<>
 inline nc_type getNCDataType<int>(){ return NC_INT;}
 template<>
 inline nc_type getNCDataType<unsigned>(){ return NC_UINT;}
+
+template<class T>
+inline int put_var_T( int ncid, int varID, T* data);
+template<>
+inline int put_var_T<float>( int ncid, int varID, float* data){
+    return nc_put_var_float( ncid, varID, data);
+}
+template<>
+inline int put_var_T<double>( int ncid, int varID, double* data){
+    return nc_put_var_double( ncid, varID, data);
+}
 ///@endcond
 
 template<class T>
@@ -113,7 +124,7 @@ inline int define_dimension( int ncid, int* dimID, const dg::RealGrid1d<T>& g, s
     int varID;
     if( (retval = nc_def_var( ncid, name_dim.data(), getNCDataType<T>(), 1, dimID, &varID))){return retval;}
     if( (retval = nc_enddef(ncid)) ) {return retval;} //not necessary for NetCDF4 files
-    if( (retval = nc_put_var_double( ncid, varID, points.data())) ){ return retval;}
+    if( (retval = put_var_T<T>( ncid, varID, points.data())) ){ return retval;}
     if( (retval = nc_redef(ncid))) {return retval;} //not necessary for NetCDF4 files
     retval = nc_put_att_text( ncid, *dimID, "axis", axis.size(), axis.data());
     retval = nc_put_att_text( ncid, *dimID, "long_name", long_name.size(), long_name.data());
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 75fc27f9b..d4af3c170 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -563,15 +563,19 @@ container Fieldaligned<G, I,container>::interpolate_from_coarse_grid( const G& g
 
     container out = dg::evaluate( dg::zero, *m_g);
     container helper = dg::evaluate( dg::zero, *m_g);
-    dg::split( in, m_f, *m_g);
     dg::split( helper, m_temp, *m_g);
-    std::vector<dg::View< container>> out_split = dg::split( out, grid);
+    std::vector<dg::View< container>> out_split = dg::split( out, *m_g);
+    std::vector<dg::View< const container>> in_split = dg::split( in, grid);
     for ( int i=0; i<(int)Nz_coarse; i++)
     {
         //1. copy input vector to appropriate place in output
-        dg::blas1::copy( m_f[i], out_split[i*cphi]);
-        dg::blas1::copy( m_f[i], m_temp[i*cphi]);
-        //2. Now apply plus and minus T to fill in the rest
+        dg::blas1::copy( in_split[i], out_split[i*cphi]);
+        dg::blas1::copy( in_split[i], m_temp[i*cphi]);
+    }
+    //Step 1 needs to finish so that m_temp contains values everywhere
+    //2. Now apply plus and minus T to fill in the rest
+    for ( int i=0; i<(int)Nz_coarse; i++)
+    {
         for( int j=1; j<(int)cphi; j++)
         {
             //!!! The value of f at the plus plane is I^- of the current plane
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index b1225e744..196f46b34 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -240,6 +240,20 @@ std::vector<Record_static> diagnostics3d_static_list = {
     },
 };
 
+std::array<std::tuple<std::string, std::string, HVec>, 3> generate_cyl2cart( Geometry& grid)
+{
+    HVec xc = dg::evaluate( dg::cooX3d, grid);
+    HVec yc = dg::evaluate( dg::cooY3d, grid);
+    HVec zc = dg::evaluate( dg::cooZ3d, grid);
+    dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
+    std::array<std::tuple<std::string, std::string, HVec>, 3> list = {{
+        { "xc", "x-coordinate in Cartesian coordinate system", xc },
+        { "yc", "y-coordinate in Cartesian coordinate system", yc },
+        { "zc", "z-coordinate in Cartesian coordinate system", zc }
+    }};
+    return list;
+}
+
 // Here are all 3d outputs we want to have
 std::vector<Record> diagnostics3d_list = {
     {"electrons", "electron density", false,
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index 6bb63a3bb..e5118a67c 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -19,8 +19,19 @@ using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #include "feltordiag.h"
 
-//convert all 3d variables of the last timestep to float and interpolate to a 3 times finer grid in phi
-//we need a time variable when available
+thrust::host_vector<float> append( const thrust::host_vector<float>& in, const dg::aRealTopology3d<double>& g)
+{
+    unsigned size2d = g.n()*g.n()*g.Nx()*g.Ny();
+    thrust::host_vector<float> out(g.size()+size2d);
+    for( unsigned i=0; i<g.size(); i++)
+        out[i] = in[i];
+    for( unsigned i=0; i<size2d; i++)
+        out[g.size()+i] = in[i];
+    return out;
+}
+//convert all 3d variables of every n-th timestep to float
+//and interpolate to a 3 times finer grid in phi
+//also periodify in 3d
 int main( int argc, char* argv[])
 {
     if( argc != 3)
@@ -32,24 +43,24 @@ int main( int argc, char* argv[])
 
     //------------------------open input nc file--------------------------------//
     file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "geomfile", &length);
+    int ncid_in;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid_in);
+    size_t length=0;
+    err = nc_inq_attlen( ncid_in, NC_GLOBAL, "inputfile", &length);
+    std::string inputfile( length, 'x');
+    err = nc_get_att_text( ncid_in, NC_GLOBAL, "inputfile", &inputfile[0]);
+    err = nc_inq_attlen( ncid_in, NC_GLOBAL, "geomfile", &length);
     std::string geom( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "geomfile", &geom[0]);
-    err = nc_close(ncid);
+    err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geom[0]);
+    err = nc_close(ncid_in);
 
-    //std::cout << "input "<<input<<std::endl;
+    //std::cout << "inputfile "<<input<<std::endl;
     //std::cout << "geome "<<geom <<std::endl;
     Json::Value js,gs;
     Json::CharReaderBuilder parser;
     parser["collectComments"] = false;
     std::string errs;
-    std::stringstream ss( input);
+    std::stringstream ss( inputfile);
     parseFromStream( parser, ss, &js, &errs); //read input without comments
     ss.str( geom);
     parseFromStream( parser, ss, &gs, &errs); //read input without comments
@@ -77,7 +88,7 @@ int main( int argc, char* argv[])
     att["comment"] = "Find more info in feltor/src/feltor.tex";
     att["source"] = "FELTOR";
     att["references"] = "https://github.com/feltor-dev/feltor";
-    att["inputfile"] = input;
+    att["inputfile"] = inputfile;
     att["geomfile"] = geom;
     for( auto pair : att)
         err = nc_put_att_text( ncid_out, NC_GLOBAL,
@@ -94,6 +105,8 @@ int main( int argc, char* argv[])
         p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bcxN, p.bcyN, dg::PER);
     dg::RealCylindricalGrid3d<double> g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
         p.n_out, p.Nx_out, p.Ny_out, 3*p.Nz_out, p.bcxN, p.bcyN, dg::PER);
+    dg::RealCylindricalGrid3d<float> g3d_out_periodic( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI+g3d_out.hz(),
+        p.n_out, p.Nx_out, p.Ny_out, 3*p.Nz_out+1, p.bcxN, p.bcyN, dg::PER);
 
     // Construct weights and temporaries
     dg::HVec transferH_in = dg::evaluate(dg::zero,g3d_in);
@@ -102,51 +115,65 @@ int main( int argc, char* argv[])
 
     // define 2d and 1d and 0d dimensions and variables
     int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g3d_out, {"time", "z", "y", "x"});
+    err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g3d_out_periodic, {"time", "z", "y", "x"});
     std::map<std::string, int> id4d;
 
     /////////////////////////////////////////////////////////////////////////
-    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
-
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    auto bhat = dg::geo::createBHat( mag);
     if( p.alpha_mag > 0.)
         mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
+    auto bhat = dg::geo::createBHat( mag);
     dg::geo::Fieldaligned<Geometry, IHMatrix, HVec> fieldaligned(
-        bhat, g3d_out, p.bcxN, p.bcyN, dg::geo::NoLimiter(),
+        bhat, g3d_out, dg::NEU, dg::NEU, dg::geo::NoLimiter(), //let's take NEU bc because N is not homogeneous
         p.rk4eps, p.mx, p.my);
+    err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
+
 
     for( auto& record : feltor::diagnostics3d_static_list)
     {
-        std::string name = record.name;
-        std::string long_name = record.long_name;
+        if( record.name != "xc" && record.name != "yc" && record.name != "zc" )
+        {
+            int vID;
+            err = nc_def_var( ncid_out, record.name.data(), NC_FLOAT, 3, &dim_ids[1],
+                &vID);
+            err = nc_put_att_text( ncid_out, vID, "long_name", record.long_name.size(),
+                record.long_name.data());
+
+            int dataID = 0;
+            err = nc_inq_varid(ncid_in, record.name.data(), &dataID);
+            err = nc_get_var_double( ncid_in, dataID, transferH_in.data());
+            transferH_out = fieldaligned.interpolate_from_coarse_grid(
+                g3d_in, transferH_in);
+            dg::assign( transferH_out, transferH_out_float);
+
+            err = nc_enddef( ncid_out);
+            err = nc_put_var_float( ncid_out, vID, append(transferH_out_float, g3d_out).data());
+            err = nc_redef(ncid_out);
+        }
+    }
+    for( auto record : feltor::generate_cyl2cart( g3d_out) )
+    {
         int vID;
-        err = nc_def_var( ncid_out, name.data(), NC_FLOAT, 3, dim_ids,
+        err = nc_def_var( ncid_out, std::get<0>(record).data(), NC_FLOAT, 3, &dim_ids[1],
             &vID);
-        err = nc_put_att_text( ncid_out, vID, "long_name", long_name.size(),
-            long_name.data());
-
-        int dataID = 0;
-        err = nc_inq_varid(ncid, record.name.data(), &dataID);
-        err = nc_get_var_double( ncid, dataID, transferH_in.data());
-        transferH_out = fieldaligned.interpolate_from_coarse_grid(
-            g3d_in, transferH_in);
-        dg::assign( transferH_out, transferH_out_float);
-
-        err = nc_enddef( ncid);
-        err = nc_put_var_float( ncid_out, vID, transferH_out_float.data());
+        err = nc_put_att_text( ncid_out, vID, "long_name", std::get<1>(record).size(),
+            std::get<1>(record).data());
+        dg::assign( std::get<2>(record), transferH_out_float);
+        err = nc_enddef( ncid_out);
+        err = nc_put_var_float( ncid_out, vID, append(transferH_out_float, g3d_out).data());
         err = nc_redef(ncid_out);
+
     }
     for( auto& record : feltor::diagnostics3d_list)
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
-        err = nc_def_var( ncid_out, name.data(), NC_FLOAT, 3, dim_ids,
+        err = nc_def_var( ncid_out, name.data(), NC_FLOAT, 4, dim_ids,
             &id4d[name]);
         err = nc_put_att_text( ncid_out, id4d[name], "long_name", long_name.size(),
             long_name.data());
     }
-    err = nc_enddef( ncid);
+    err = nc_enddef( ncid_out);
 
 
 
@@ -154,18 +181,20 @@ int main( int argc, char* argv[])
     double time=0.;
 
     size_t steps;
-    err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
-    err = nc_inq_dimlen( ncid, timeID, &steps);
+    err = nc_inq_unlimdim( ncid_in, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
+    err = nc_inq_dimlen( ncid_in, timeID, &steps);
     size_t count3d_in[4]  = {1, g3d_in.Nz(), g3d_in.n()*g3d_in.Ny(), g3d_in.n()*g3d_in.Nx()};
-    size_t count3d_out[4] = {1, g3d_out.Nz(), g3d_out.n()*g3d_out.Ny(), g3d_out.n()*g3d_out.Nx()};
+    size_t count3d_out[4] = {1, g3d_out_periodic.Nz(), g3d_out.n()*g3d_out.Ny(), g3d_out.n()*g3d_out.Nx()};
     size_t start3d[4] = {0, 0, 0, 0};
-    /////////////////////////////////////////////////////////////////////////
+    ///////////////////////////////////////////////////////////////////////
     for( unsigned i=0; i<steps; i+=10)//timestepping
     {
+        std::cout << "Timestep = "<<i<< "/"<<steps;
         start3d[0] = i;
         // read and write time
-        err = nc_get_vara_double( ncid, timeID, start3d, count3d_in, &time);
-        std::cout << "Timestep = " << i << "  time = " << time << std::endl;
+        err = nc_get_vara_double( ncid_in, timeID, start3d, count3d_in, &time);
+        std::cout << "  time = " << time << std::endl;
+        start3d[0] = i/10;
         err = nc_put_vara_double( ncid_out, tvarID, start3d, count3d_out, &time);
         for( auto& record : feltor::diagnostics3d_list)
         {
@@ -173,7 +202,7 @@ int main( int argc, char* argv[])
             int dataID =0;
             bool available = true;
             try{
-                err = nc_inq_varid(ncid, record.name.data(), &dataID);
+                err = nc_inq_varid(ncid_in, record.name.data(), &dataID);
             } catch ( file::NC_Error error)
             {
                 if(  i == 0)
@@ -186,7 +215,7 @@ int main( int argc, char* argv[])
             }
             if( available)
             {
-                err = nc_get_vara_double( ncid, dataID,
+                err = nc_get_vara_double( ncid_in, dataID,
                     start3d, count3d_in, transferH_in.data());
                 //2. Compute fsa and output fsa
                 transferH_out = fieldaligned.interpolate_from_coarse_grid(
@@ -198,13 +227,14 @@ int main( int argc, char* argv[])
                 dg::blas1::scal( transferH_out_float, (float)0);
             }
             err = nc_put_vara_float( ncid_out, id4d.at(record.name), start3d,
-                count3d_out, transferH_out_float.data());
+                count3d_out, append(transferH_out_float, g3d_out).data());
         }
 
-
     } //end timestepping
-    err = nc_close(ncid);
+    std::cout << "Hello!\n";
+    err = nc_close(ncid_in);
     err = nc_close(ncid_out);
+    std::cout << "Hello!\n";
 
     return 0;
 }
-- 
GitLab


From 9cd374fd9cc2cbd58d84734bb41363947a9df742 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 12 Jan 2020 20:28:12 +0100
Subject: [PATCH 189/540] Fix the memory leak in interpolate_in_3d

---
 src/feltor/interpolate_in_3d.cu | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index e5118a67c..3ffc2eb0a 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -113,8 +113,8 @@ int main( int argc, char* argv[])
     dg::HVec transferH_out = dg::evaluate(dg::zero,g3d_out);
     dg::fHVec transferH_out_float = dg::construct<dg::fHVec>( transferH_out);
 
-    // define 2d and 1d and 0d dimensions and variables
-    int dim_ids[3], tvarID;
+    // define 4d dimension
+    int dim_ids[4], tvarID;
     err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g3d_out_periodic, {"time", "z", "y", "x"});
     std::map<std::string, int> id4d;
 
@@ -231,10 +231,8 @@ int main( int argc, char* argv[])
         }
 
     } //end timestepping
-    std::cout << "Hello!\n";
     err = nc_close(ncid_in);
     err = nc_close(ncid_out);
-    std::cout << "Hello!\n";
 
     return 0;
 }
-- 
GitLab


From 224b37896716f6f890f7a023a94001a581ffb005 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 13 Jan 2020 18:18:26 +0100
Subject: [PATCH 190/540] Add equidistant interpolation

---
 src/feltor/interpolate_in_3d.cu | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index 3ffc2eb0a..b9aa037b1 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -105,17 +105,22 @@ int main( int argc, char* argv[])
         p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bcxN, p.bcyN, dg::PER);
     dg::RealCylindricalGrid3d<double> g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
         p.n_out, p.Nx_out, p.Ny_out, 3*p.Nz_out, p.bcxN, p.bcyN, dg::PER);
+    dg::RealCylindricalGrid3d<double> g3d_out_equidistant( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
+        1, p.n_out*p.Nx_out, p.n_out*p.Ny_out, 3*p.Nz_out, p.bcxN, p.bcyN, dg::PER);
     dg::RealCylindricalGrid3d<float> g3d_out_periodic( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI+g3d_out.hz(),
         p.n_out, p.Nx_out, p.Ny_out, 3*p.Nz_out+1, p.bcxN, p.bcyN, dg::PER);
+    dg::RealCylindricalGrid3d<float> g3d_out_periodic_equidistant( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI+g3d_out.hz(),
+        1, p.n_out*p.Nx_out, p.n_out*p.Ny_out, 3*p.Nz_out+1, p.bcxN, p.bcyN, dg::PER);
 
     // Construct weights and temporaries
     dg::HVec transferH_in = dg::evaluate(dg::zero,g3d_in);
     dg::HVec transferH_out = dg::evaluate(dg::zero,g3d_out);
-    dg::fHVec transferH_out_float = dg::construct<dg::fHVec>( transferH_out);
+    dg::HVec transferH = dg::evaluate(dg::zero,g3d_out_equidistant);
+    dg::fHVec transferH_out_float = dg::construct<dg::fHVec>( transferH);
 
     // define 4d dimension
     int dim_ids[4], tvarID;
-    err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g3d_out_periodic, {"time", "z", "y", "x"});
+    err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g3d_out_periodic_equidistant, {"time", "z", "y", "x"});
     std::map<std::string, int> id4d;
 
     /////////////////////////////////////////////////////////////////////////
@@ -127,6 +132,7 @@ int main( int argc, char* argv[])
         bhat, g3d_out, dg::NEU, dg::NEU, dg::geo::NoLimiter(), //let's take NEU bc because N is not homogeneous
         p.rk4eps, p.mx, p.my);
     err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
+    dg::IHMatrix interpolate_in_2d = dg::create::interpolation( g3d_out_equidistant, g3d_out);
 
 
     for( auto& record : feltor::diagnostics3d_static_list)
@@ -144,14 +150,15 @@ int main( int argc, char* argv[])
             err = nc_get_var_double( ncid_in, dataID, transferH_in.data());
             transferH_out = fieldaligned.interpolate_from_coarse_grid(
                 g3d_in, transferH_in);
-            dg::assign( transferH_out, transferH_out_float);
+            dg::blas2::symv( interpolate_in_2d, transferH_out, transferH);
+            dg::assign( transferH, transferH_out_float);
 
             err = nc_enddef( ncid_out);
-            err = nc_put_var_float( ncid_out, vID, append(transferH_out_float, g3d_out).data());
+            err = nc_put_var_float( ncid_out, vID, append(transferH_out_float, g3d_out_equidistant).data());
             err = nc_redef(ncid_out);
         }
     }
-    for( auto record : feltor::generate_cyl2cart( g3d_out) )
+    for( auto record : feltor::generate_cyl2cart( g3d_out_equidistant) )
     {
         int vID;
         err = nc_def_var( ncid_out, std::get<0>(record).data(), NC_FLOAT, 3, &dim_ids[1],
@@ -160,7 +167,7 @@ int main( int argc, char* argv[])
             std::get<1>(record).data());
         dg::assign( std::get<2>(record), transferH_out_float);
         err = nc_enddef( ncid_out);
-        err = nc_put_var_float( ncid_out, vID, append(transferH_out_float, g3d_out).data());
+        err = nc_put_var_float( ncid_out, vID, append(transferH_out_float, g3d_out_equidistant).data());
         err = nc_redef(ncid_out);
 
     }
@@ -184,7 +191,7 @@ int main( int argc, char* argv[])
     err = nc_inq_unlimdim( ncid_in, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
     err = nc_inq_dimlen( ncid_in, timeID, &steps);
     size_t count3d_in[4]  = {1, g3d_in.Nz(), g3d_in.n()*g3d_in.Ny(), g3d_in.n()*g3d_in.Nx()};
-    size_t count3d_out[4] = {1, g3d_out_periodic.Nz(), g3d_out.n()*g3d_out.Ny(), g3d_out.n()*g3d_out.Nx()};
+    size_t count3d_out[4] = {1, g3d_out_periodic_equidistant.Nz(), g3d_out_equidistant.n()*g3d_out_equidistant.Ny(), g3d_out_equidistant.n()*g3d_out_equidistant.Nx()};
     size_t start3d[4] = {0, 0, 0, 0};
     ///////////////////////////////////////////////////////////////////////
     for( unsigned i=0; i<steps; i+=10)//timestepping
@@ -220,14 +227,15 @@ int main( int argc, char* argv[])
                 //2. Compute fsa and output fsa
                 transferH_out = fieldaligned.interpolate_from_coarse_grid(
                     g3d_in, transferH_in);
-                dg::assign( transferH_out, transferH_out_float);
+                dg::blas2::symv( interpolate_in_2d, transferH_out, transferH);
+                dg::assign( transferH, transferH_out_float);
             }
             else
             {
                 dg::blas1::scal( transferH_out_float, (float)0);
             }
             err = nc_put_vara_float( ncid_out, id4d.at(record.name), start3d,
-                count3d_out, append(transferH_out_float, g3d_out).data());
+                count3d_out, append(transferH_out_float, g3d_out_equidistant).data());
         }
 
     } //end timestepping
-- 
GitLab


From d0c2aefffc6201fde8f09d58f2516c177dba040b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 20 Jan 2020 16:12:44 +0100
Subject: [PATCH 191/540] Remove surface average in feltor.tex

---
 src/feltor/feltor.tex           | 52 ++++++++++++---------------------
 src/feltor/interpolate_in_3d.cu | 17 ++++++-----
 2 files changed, 28 insertions(+), 41 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index b0736d6f5..9b5aca3a3 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -474,24 +474,8 @@ to numerically compute area integrals.
 
 \subsection{Flux surface average}
 
-There is two possible ways to introduce a flux-surface average.
-The first one is the straightforward average on the actual area of the
-flux-surface.
-The {\bf area average}
-of a function $f(R,Z,\varphi)$ is given by the formula
-\begin{align}\label{eq:fsa_area}
-\langle f \rangle^{area}_{\psi_{p}} :=&
-\frac{ \oint_{\psi_p  } f(R,Z,\varphi)\dA}{\oint_{\psi_p } \dA} \nonumber \\
-=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) |\vn\psi_p| \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
-{\int_\Omega |\vn\psi_p|\delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z} \nonumber\\
-=& \frac{\oint \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g g^{\zeta\zeta}}\d\eta}
-         { \oint \sqrt{g g^{\zeta\zeta}}\d\eta}
-\end{align}
-%with $\dV := R\d R\d Z\d \varphi$ %(we define the average in computational space and omit one $R$)
-and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
-in our domain $\Omega$.
 
-The second one (the {\bf volume average} after \cite{haeseleer}) defines an average on a
+The flux surface average (as a {\bf volume average} after \cite{haeseleer}) is defined as an average over a
 small volume - a shell centered around the flux-surface - defined by two neighboring flux-surfaces.
 With the help of the volume
 flux label (notice that both the volume $v$ as well as the poloidal flux $\psi_p$ have physical 
@@ -500,40 +484,42 @@ meaning while the coordinate $\zeta(\psi_p)$ is an arbitrary choice) we define
 v(\psi_p) :=& \int_{\psi_{p,\min}}^\psi \dV = \int^{\zeta(\psi_p)} \sqrt{g}\d\zeta\d\eta\d\varphi,
 \\
 \frac{\d v}{\d\psi_p} =& \int\dA |\vn\psi_p|^{-1} = 2\pi f_0\oint_{\zeta(\psi_p)} \sqrt{g}\d\eta \\
-\langle f \rangle^{vol}_\psi :=& \frac{\partial}{\partial v} \int \dV f
+\langle f \rangle_\psi :=& \frac{\partial}{\partial v} \int \dV f
  = \frac{1}{\int \dA |\vn\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\vn\psi_p|} \dA \nonumber\\
 =& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
 {\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}\nonumber\\
  =& \left(\frac{\d v}{\d\psi_p }\right)^{-1} 2\pi f_0 \oint_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
  = \frac{1}{\oint \sqrt{g}\d\eta } \oint_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
 \end{align}
-where we used the co-area formula Eq.~\eqref{eq:coarea} for the second
-identity. We immediately see that this definition differs from the first
-Eq.~\eqref{eq:fsa_area} by the weight factor $|\vn\psi_p|$ and that it is particularly easy to compute
-in a flux-aligned coordinate system. Notice however that the volume element does appear (unlike e.g. Tokam3X papers)
+where we used the co-area formula Eq.~\eqref{eq:coarea} for the second identity
+and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
+in our domain $\Omega$.
+ We immediately see that this definition is particularly easy to compute
+in a flux-aligned coordinate system. Notice however that the volume element
+does appear (unlike e.g. Tokam3X papers)
 
-Both averages fulfill the basic identities
+The flux-surface average fulfills the basic identities
 \begin{align}
 \label{eq:fsa_identities}
 \langle \mu f + \lambda g\rangle &= \mu\langle f\rangle + \lambda \langle g\rangle \\
 \langle f(\psi_p) \rangle &= f(\psi_p)
 \end{align}
 
-
-The volume average is better suited for density-like quantities
-than the area average as we can see with the following identity.
-Assume we have a quantity $X$ with $\partial_t X + \nc \vec j_X = \Lambda_X$. Then we can use the volume average to write
+The volume average is well-suited for density-like quantities
+as we can see with the following identity.
+Assume we have a quantity $X$ with $\partial_t X + \nc \vec j_X = \Lambda_X$.
+Then we can use the volume average to write
 \begin{align}
-\frac{\partial}{\partial t} \langle X \rangle^{vol} + \frac{\partial}{
-  \partial v} \langle \vec j_X\cn v\rangle^{vol}  = \langle \Lambda_X\rangle^{vol}
+\frac{\partial}{\partial t} \langle X \rangle + \frac{\partial}{
+  \partial v} \langle \vec j_X\cn v\rangle  = \langle \Lambda_X\rangle
 \label{eq:fsa_balance}
 \end{align}
 where again $v=v(\psi_p)$ is the volume flux label.
 The {\bf total flux} of a given flux density $\vec j_X$ though the
 flux surface $\psi_p = \psi_{p0}$ is given by
 \begin{align}
-\left\langle\vec j_X\cn v\right\rangle^{vol} &:= J_X=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
- \frac{\d v}{\d\psi_p} \langle \vec j_X\cn\psi_p \rangle^{vol}\\
+\left\langle\vec j_X\cn v\right\rangle &:= J_X=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
+ \frac{\d v}{\d\psi_p} \langle \vec j_X\cn\psi_p \rangle\\
  &=
    2\pi f_0 \oint_0^{2\pi} \langle \vec j_X\cn\psi_p\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
 %2\pi\int_\Omega \vec \langle \vec j\cn\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
@@ -541,8 +527,8 @@ flux surface $\psi_p = \psi_{p0}$ is given by
 \end{align}
 Once we have the flux-surface averaged equation we can easily get the volume integrated version (again with the help of the co-area formula)
 \begin{align}
-\frac{\partial}{\partial t} \int_0^{v(\psi_p)}\langle X \rangle^{vol} \d v 
-+ \langle \vec j_X\cn v\rangle^{vol}(v(\psi_p))  = \int_0^{v(\psi_p)}\langle \Lambda_X\rangle^{vol}\d v
+\frac{\partial}{\partial t} \int_0^{v(\psi_p)}\langle X \rangle \d v 
++ \langle \vec j_X\cn v\rangle(v(\psi_p))  = \int_0^{v(\psi_p)}\langle \Lambda_X\rangle\d v
 \label{eq:integral_balance}
 \end{align}
 
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index b9aa037b1..3313b8a62 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -29,9 +29,9 @@ thrust::host_vector<float> append( const thrust::host_vector<float>& in, const d
         out[g.size()+i] = in[i];
     return out;
 }
-//convert all 3d variables of every n-th timestep to float
-//and interpolate to a 3 times finer grid in phi
-//also periodify in 3d
+//convert all 3d variables of every N-th timestep to float
+//and interpolate to a FACTOR times finer grid in phi
+//also periodify in 3d and equidistant in RZ
 int main( int argc, char* argv[])
 {
     if( argc != 3)
@@ -100,17 +100,18 @@ int main( int argc, char* argv[])
     const double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
     const double Rmax=gp.R_0+p.boxscaleRp*gp.a;
     const double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    const unsigned FACTOR = 6;
 
     dg::RealCylindricalGrid3d<double> g3d_in( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
         p.n_out, p.Nx_out, p.Ny_out, p.Nz_out, p.bcxN, p.bcyN, dg::PER);
     dg::RealCylindricalGrid3d<double> g3d_out( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
-        p.n_out, p.Nx_out, p.Ny_out, 3*p.Nz_out, p.bcxN, p.bcyN, dg::PER);
+        p.n_out, p.Nx_out, p.Ny_out, FACTOR*p.Nz_out, p.bcxN, p.bcyN, dg::PER);
     dg::RealCylindricalGrid3d<double> g3d_out_equidistant( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
-        1, p.n_out*p.Nx_out, p.n_out*p.Ny_out, 3*p.Nz_out, p.bcxN, p.bcyN, dg::PER);
+        1, p.n_out*p.Nx_out, p.n_out*p.Ny_out, FACTOR*p.Nz_out, p.bcxN, p.bcyN, dg::PER);
     dg::RealCylindricalGrid3d<float> g3d_out_periodic( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI+g3d_out.hz(),
-        p.n_out, p.Nx_out, p.Ny_out, 3*p.Nz_out+1, p.bcxN, p.bcyN, dg::PER);
+        p.n_out, p.Nx_out, p.Ny_out, FACTOR*p.Nz_out+1, p.bcxN, p.bcyN, dg::PER);
     dg::RealCylindricalGrid3d<float> g3d_out_periodic_equidistant( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI+g3d_out.hz(),
-        1, p.n_out*p.Nx_out, p.n_out*p.Ny_out, 3*p.Nz_out+1, p.bcxN, p.bcyN, dg::PER);
+        1, p.n_out*p.Nx_out, p.n_out*p.Ny_out, FACTOR*p.Nz_out+1, p.bcxN, p.bcyN, dg::PER);
 
     // Construct weights and temporaries
     dg::HVec transferH_in = dg::evaluate(dg::zero,g3d_in);
@@ -130,7 +131,7 @@ int main( int argc, char* argv[])
     auto bhat = dg::geo::createBHat( mag);
     dg::geo::Fieldaligned<Geometry, IHMatrix, HVec> fieldaligned(
         bhat, g3d_out, dg::NEU, dg::NEU, dg::geo::NoLimiter(), //let's take NEU bc because N is not homogeneous
-        p.rk4eps, p.mx, p.my);
+        p.rk4eps, 5, 5);
     err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
     dg::IHMatrix interpolate_in_2d = dg::create::interpolation( g3d_out_equidistant, g3d_out);
 
-- 
GitLab


From 46bcc04a96701212ccd26ff52266d56b81b39db9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 22 Jan 2020 22:34:10 +0100
Subject: [PATCH 192/540] Make feltordiag work on multiple files

---
 src/feltor/feltor.tex    |   4 +-
 src/feltor/feltordiag.cu | 278 ++++++++++++++++++++-------------------
 2 files changed, 146 insertions(+), 136 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 9b5aca3a3..69f69e4f8 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1472,11 +1472,11 @@ the whole simulation is lost. It is safer to just merge files afterwards with fo
 \section{Diagnostics}\label{sec:diagnostics}
 \subsection{Program and files}
 We have the program \texttt{feltor/diag/feltordiag.cu}.
-This program reads a previously generated simulation file \texttt{input.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows. \\
+This program reads one or more previously generated simulation file(s) \texttt{input0.nc ... inputN.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows. \\
 Compilation\\
 \texttt{make feltordiag} \\
 Usage \\
-\texttt{./feltordiag input.nc output.nc} \\
+\texttt{./feltordiag input0.nc ... inputN.nc output.nc} \\
 
 Output file format: netcdf-4/hdf5, Conventions: CF-1.7; A \textit{coordinate variable (Coord. Var.)} is a Dataset with the same name as a dimension.
 
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index a1910e9e1..850088260 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -20,12 +20,14 @@ using Geometry = dg::CylindricalGrid3d;
 
 int main( int argc, char* argv[])
 {
-    if( argc != 3)
+    if( argc < 3)
     {
-        std::cerr << "Usage: "<<argv[0]<<" [input.nc] [output.nc]\n";
+        std::cerr << "Usage: "<<argv[0]<<" [input0.nc ... inputN.nc] [output.nc]\n";
         return -1;
     }
-    std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
+    for( int i=1; i<argc-1; i++)
+        std::cout << argv[i]<< " ";
+    std::cout << " -> "<<argv[argc-1]<<std::endl;
 
     //------------------------open input nc file--------------------------------//
     file::NC_Error_Handle err;
@@ -60,7 +62,7 @@ int main( int argc, char* argv[])
 
     //-----------------Create Netcdf output file with attributes----------//
     int ncid_out;
-    err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
+    err = nc_create(argv[argc-1],NC_NETCDF4|NC_CLOBBER, &ncid_out);
 
     /// Set global attributes
     std::map<std::string, std::string> att;
@@ -213,7 +215,6 @@ int main( int argc, char* argv[])
     std::map<std::string, int> id0d, id1d, id2d;
 
     size_t count1d[2] = {1, g1d_out.n()*g1d_out.N()};
-    size_t start1d[2] = {0, 0};
     size_t count2d[3] = {1, g2d_out.n()*g2d_out.Ny(), g2d_out.n()*g2d_out.Nx()};
     size_t start2d[3] = {0, 0, 0};
 
@@ -284,154 +285,163 @@ int main( int argc, char* argv[])
             long_name.data());
     }
     /////////////////////////////////////////////////////////////////////////
-    int timeID;
-    double time=0.;
-
-    size_t steps;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid); //open 3d file
-    err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
-    err = nc_inq_dimlen( ncid, timeID, &steps);
-    //steps = 3;
-    for( unsigned i=0; i<steps; i++)//timestepping
+    size_t counter = 0;
+    for( int j=1; j<argc-1; j++)
     {
-        start2d[0] = i;
-        start1d[0] = i;
-        // read and write time
-        err = nc_get_vara_double( ncid, timeID, start2d, count2d, &time);
-        std::cout << "Timestep = " << i << "  time = " << time << std::endl;
-        err = nc_put_vara_double( ncid_out, tvarID, start2d, count2d, &time);
-        for( auto& record : feltor::diagnostics2d_list)
+        int timeID;
+
+        size_t steps;
+        std::cout << "Opening file "<<argv[j]<<"\n";
+        err = nc_open( argv[j], NC_NOWRITE, &ncid); //open 3d file
+        err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
+        err = nc_inq_dimlen( ncid, timeID, &steps);
+        //steps = 3;
+        for( unsigned i=0; i<steps; i++)//timestepping
         {
-            std::string record_name = record.name;
-            if( record_name[0] == 'j')
-                record_name[1] = 'v';
-            //1. Read toroidal average
-            int dataID =0;
-            bool available = true;
-            try{
-                err = nc_inq_varid(ncid, (record.name+"_ta2d").data(), &dataID);
-            } catch ( file::NC_Error error)
+            if( j > 1 && i == 0)
+                continue; // else we duplicate the first timestep
+            start2d[0] = i;
+            size_t start2d_out[3] = {counter, 0,0};
+            size_t start1d_out[2] = {counter, 0};
+            // read and write time
+            double time=0.;
+            err = nc_get_vara_double( ncid, timeID, start2d, count2d, &time);
+            std::cout << counter << " Timestep = " << i <<"/"<<steps-1 << "  time = " << time << std::endl;
+            counter++;
+            err = nc_put_vara_double( ncid_out, tvarID, start2d_out, count2d, &time);
+            for( auto& record : feltor::diagnostics2d_list)
             {
-                if(  i == 0)
-                {
-                    std::cerr << error.what() <<std::endl;
-                    std::cerr << "Offending variable is "<<record.name+"_ta2d\n";
-                    std::cerr << "Writing zeros ... \n";
-                }
-                available = false;
-            }
-            if( available)
-            {
-                err = nc_get_vara_double( ncid, dataID,
-                    start2d, count2d, transferH2d.data());
-                //2. Compute fsa and output fsa
-                dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
-                dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
-                poloidal_average( transferH2dX, t1d, false); //average over eta
-                dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
-                dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
+                std::string record_name = record.name;
                 if( record_name[0] == 'j')
-                    dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d );
-                //3. Interpolate fsa on 2d plane : <f>
-                dg::blas2::gemv(fsa2rzmatrix, fsa1d, transferH2d); //fsa on RZ grid
-            }
-            else
-            {
-                dg::blas1::scal( fsa1d, 0.);
-                dg::blas1::scal( transferH2d, 0.);
-            }
-            err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_fsa"),
-                start1d, count1d, fsa1d.data());
-            err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fsa2d"),
-                start2d, count2d, transferH2d.data() );
-            //4. Read 2d variable and compute fluctuations
-            available = true;
-            try{
-                err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
-            } catch ( file::NC_Error error)
-            {
-                if(  i == 0)
+                    record_name[1] = 'v';
+                //1. Read toroidal average
+                int dataID =0;
+                bool available = true;
+                try{
+                    err = nc_inq_varid(ncid, (record.name+"_ta2d").data(), &dataID);
+                } catch ( file::NC_Error error)
                 {
-                    std::cerr << error.what() <<std::endl;
-                    std::cerr << "Offending variable is "<<record.name+"_2d\n";
-                    std::cerr << "Writing zeros ... \n";
+                    if(  i == 0)
+                    {
+                        std::cerr << error.what() <<std::endl;
+                        std::cerr << "Offending variable is "<<record.name+"_ta2d\n";
+                        std::cerr << "Writing zeros ... \n";
+                    }
+                    available = false;
                 }
-                available = false;
-            }
-            if( available)
-            {
-                err = nc_get_vara_double( ncid, dataID, start2d, count2d,
-                    t2d_mp.data());
-                if( record_name[0] == 'j')
-                    dg::blas1::pointwiseDot( t2d_mp, dvdpsip2d, t2d_mp );
-                dg::blas1::axpby( 1.0, t2d_mp, -1.0, transferH2d);
-                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
-                    start2d, count2d, transferH2d.data() );
-
-                //5. flux surface integral/derivative
-                double result =0.;
-                if( record_name[0] == 'j') //j indicates a flux
+                if( available)
                 {
-                    dg::blas2::symv( dpsi, fsa1d, t1d);
-                    dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
-
-                    result = dg::interpolate( fsa1d, 0., g1d_out);
+                    err = nc_get_vara_double( ncid, dataID,
+                        start2d, count2d, transferH2d.data());
+                    //2. Compute fsa and output fsa
+                    dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
+                    dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
+                    poloidal_average( transferH2dX, t1d, false); //average over eta
+                    dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
+                    dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
+                    if( record_name[0] == 'j')
+                        dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d );
+                    //3. Interpolate fsa on 2d plane : <f>
+                    dg::blas2::gemv(fsa2rzmatrix, fsa1d, transferH2d); //fsa on RZ grid
                 }
                 else
                 {
-                    dg::blas1::pointwiseDot( fsa1d, dvdpsip, t1d);
-                    transfer1d = dg::integrate( t1d, g1d_out);
-
-                    result = dg::interpolate( transfer1d, 0., g1d_out);
+                    dg::blas1::scal( fsa1d, 0.);
+                    dg::blas1::scal( transferH2d, 0.);
+                }
+                err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_fsa"),
+                    start1d_out, count1d, fsa1d.data());
+                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fsa2d"),
+                    start2d_out, count2d, transferH2d.data() );
+                //4. Read 2d variable and compute fluctuations
+                available = true;
+                try{
+                    err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
+                } catch ( file::NC_Error error)
+                {
+                    if(  i == 0)
+                    {
+                        std::cerr << error.what() <<std::endl;
+                        std::cerr << "Offending variable is "<<record.name+"_2d\n";
+                        std::cerr << "Writing zeros ... \n";
+                    }
+                    available = false;
                 }
-                err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
-                    start1d, count1d, transfer1d.data());
-                //flux surface integral/derivative on last closed flux surface
-                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
-                    start2d, count2d, &result );
-                //6. Compute norm of time-integral terms to get relative importance
-                if( record_name[0] == 'j') //j indicates a flux
+                if( available)
                 {
-                    dg::blas2::symv( dpsi, fsa1d, t1d);
-                    dg::blas1::pointwiseDivide( t1d, dvdpsip, t1d); //dvjv
-                    dg::blas1::pointwiseDot( t1d, t1d, t1d);//dvjv2
-                    dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);//dvjv2
-                    transfer1d = dg::integrate( t1d, g1d_out);
-                    result = dg::interpolate( transfer1d, 0., g1d_out);
-                    result = sqrt(result);
+                    err = nc_get_vara_double( ncid, dataID, start2d, count2d,
+                        t2d_mp.data());
+                    if( record_name[0] == 'j')
+                        dg::blas1::pointwiseDot( t2d_mp, dvdpsip2d, t2d_mp );
+                    dg::blas1::axpby( 1.0, t2d_mp, -1.0, transferH2d);
+                    err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
+                        start2d_out, count2d, transferH2d.data() );
+
+                    //5. flux surface integral/derivative
+                    double result =0.;
+                    if( record_name[0] == 'j') //j indicates a flux
+                    {
+                        dg::blas2::symv( dpsi, fsa1d, t1d);
+                        dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
+
+                        result = dg::interpolate( fsa1d, 0., g1d_out);
+                    }
+                    else
+                    {
+                        dg::blas1::pointwiseDot( fsa1d, dvdpsip, t1d);
+                        transfer1d = dg::integrate( t1d, g1d_out);
+
+                        result = dg::interpolate( transfer1d, 0., g1d_out);
+                    }
+                    err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
+                        start1d_out, count1d, transfer1d.data());
+                    //flux surface integral/derivative on last closed flux surface
+                    err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
+                        start2d_out, count2d, &result );
+                    //6. Compute norm of time-integral terms to get relative importance
+                    if( record_name[0] == 'j') //j indicates a flux
+                    {
+                        dg::blas2::symv( dpsi, fsa1d, t1d);
+                        dg::blas1::pointwiseDivide( t1d, dvdpsip, t1d); //dvjv
+                        dg::blas1::pointwiseDot( t1d, t1d, t1d);//dvjv2
+                        dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);//dvjv2
+                        transfer1d = dg::integrate( t1d, g1d_out);
+                        result = dg::interpolate( transfer1d, 0., g1d_out);
+                        result = sqrt(result);
+                    }
+                    else
+                    {
+                        dg::blas1::pointwiseDot( fsa1d, fsa1d, t1d);
+                        dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);
+                        transfer1d = dg::integrate( t1d, g1d_out);
+
+                        result = dg::interpolate( transfer1d, 0., g1d_out);
+                        result = sqrt(result);
+                    }
+                    err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
+                        start2d_out, count2d, &result );
                 }
                 else
                 {
-                    dg::blas1::pointwiseDot( fsa1d, fsa1d, t1d);
-                    dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);
-                    transfer1d = dg::integrate( t1d, g1d_out);
-
-                    result = dg::interpolate( transfer1d, 0., g1d_out);
-                    result = sqrt(result);
+                    dg::blas1::scal( transferH2d, 0.);
+                    dg::blas1::scal( transfer1d, 0.);
+                    double result = 0.;
+                    err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
+                        start2d_out, count2d, transferH2d.data() );
+                    err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
+                        start1d_out, count1d, transfer1d.data());
+                    err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
+                        start2d_out, count2d, &result );
+                    err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
+                        start2d_out, count2d, &result );
                 }
-                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
-                    start2d, count2d, &result );
-            }
-            else
-            {
-                dg::blas1::scal( transferH2d, 0.);
-                dg::blas1::scal( transfer1d, 0.);
-                double result = 0.;
-                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fluc2d"),
-                    start2d, count2d, transferH2d.data() );
-                err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
-                    start1d, count1d, transfer1d.data());
-                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_lcfs"),
-                    start2d, count2d, &result );
-                err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
-                    start2d, count2d, &result );
-            }
 
-        }
+            }
 
 
-    } //end timestepping
-    err = nc_close(ncid);
+        } //end timestepping
+        err = nc_close(ncid);
+    }
     err = nc_close(ncid_out);
 
     return 0;
-- 
GitLab


From cd533c1060b682fddd04ba6dd4f15878257d45a6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 12 Feb 2020 13:56:21 +0100
Subject: [PATCH 193/540] Update source term to the long wavelength limit

Also document phi component source of ExB angular momentum
---
 doc/related_pages/newcommands.tex |   9 +-
 src/feltor/feltor.h               |  11 +--
 src/feltor/feltor.tex             | 152 +++++++++++++++---------------
 3 files changed, 89 insertions(+), 83 deletions(-)

diff --git a/doc/related_pages/newcommands.tex b/doc/related_pages/newcommands.tex
index f72318e7f..ee579c58a 100644
--- a/doc/related_pages/newcommands.tex
+++ b/doc/related_pages/newcommands.tex
@@ -15,9 +15,14 @@
 \newcommand{\GKI}{\int d^6 \bm{Z} \BSP}
 \newcommand{\GKIV}{\int dv_{\|} d \mu d \theta \BSP}
 \newcommand{\BSP}{B_{\|}^*}
-\newcommand{\GA}[1]{\left\langle #1	 \right\rangle}
-
 \newcommand{\Abar}{\langle A_\parallel \rangle}
+%Averages
+\newcommand{\RA}[1]{\left \langle #1 \right \rangle} %Reynolds (flux-surface) average
+\newcommand{\RF}[1]{\widetilde{#1}} %Reynolds fluctuation
+\newcommand{\FA}[1]{\left\llbracket #1 \right\rrbracket} %Favre average
+\newcommand{\FF}[1]{\widehat{#1}} %Favre fluctuation
+\newcommand{\PA}[1]{\left \langle #1 \right\rangle_\varphi} %Phi average
+
 %Vectors
 \newcommand{\ahat}{\bm{\hat{a}}}
 \newcommand{\bhat}{\bm{\hat{b}}}
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 360baafeb..462ea2298 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -152,8 +152,7 @@ struct ComputeSource{
     DG_DEVICE
     void operator()( double& result, double tilde_n, double profne,
         double source, double omega_source) const{
-        double temp = omega_source*source*(profne - tilde_n);
-        result = temp;
+        result = omega_source*source*(profne - tilde_n);
     }
 };
 //Resistivity (consistent density dependency,
@@ -887,12 +886,12 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
                 m_profne, m_source, m_omega_source);
         else
             dg::blas1::axpby( m_omega_source, m_source, 0., m_s[0][0]);
-        //compute FLR corrections
+        //compute FLR corrections S_N = (1-0.5*mu*tau*Lap)*S_n
         dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
         dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
-        // potential part of FLR correction
-        dg::blas1::pointwiseDot( m_p.mu[1], m_s[0][1], m_binv, m_binv, 0., m_temp0);
-        m_lapperpP.multiply_sigma( -1., m_temp0, m_phi[0], 1., m_s[0][0]);
+        // potential part of FLR correction S_N += -div*(mu S_n grad*Phi/B^2)
+        dg::blas1::pointwiseDot( m_p.mu[1], m_s[0][0], m_binv, m_binv, 0., m_temp0);
+        m_lapperpP.multiply_sigma( 1., m_temp0, m_phi[0], 1., m_s[0][1]);
 
         dg::blas1::axpby( 1., m_s[0][0], 1.0, yp[0][0]);
         dg::blas1::axpby( 1., m_s[0][1], 1.0, yp[0][1]);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 69f69e4f8..8d3a065c1 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -36,7 +36,7 @@ Given a vector field $\vec B(\vec x)$ with unit vector $\bhat(\vec x) := (\vec B
 we can define various differential operations.
 %Let us further assume that $\bhat$ is perturbed by the parallel
 %vector potential $A_\parallel$ via
-%$\tilde{ \vec b }_\perp := ({\vn \times A_\parallel \bhat)}/{B}$
+%${ \vec b }_\perp := ({\vn \times A_\parallel \bhat)}/{B}$
 \rowcolors{2}{gray!25}{white}
 %\begin{longtable}{>{\RaggedRight}p{7cm}>{\RaggedRight}p{7cm}}
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
@@ -74,7 +74,7 @@ we can define various differential operations.
     $ \npar f := \bhat\cn f$ \\
     %Perturbed parallel Derivative&
     %$\bar\npar$ &
-    %$\bar\npar f := (\bhat + \tilde{\vec b }_\perp)\cn f = \npar f + A_\parallel \mathcal K_{\vn\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$ \\
+    %$\bar\npar f := (\bhat + {\vec b }_\perp)\cn f = \npar f + A_\parallel \mathcal K_{\vn\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$ \\
     Parallel Laplacian&
     $\Delta_\parallel $&
     $\Delta_\parallel f:= \nc ( \bhat\bhat\cn f )$\\
@@ -450,7 +450,7 @@ volume integral.
 
 We define the {\bf toroidal average} of a function $f(R,Z,\varphi)$ as
 \begin{align} \label{eq:phi_average}
-\langle f\rangle_\varphi(R,Z) := \frac{1}{2\pi}\oint f(R,Z,\varphi)\d \varphi
+\PA{ f}(R,Z) := \frac{1}{2\pi}\oint f(R,Z,\varphi)\d \varphi
 \end{align}
 
 In arbitrary coordinates the area integral is defined by the pull back
@@ -484,12 +484,12 @@ meaning while the coordinate $\zeta(\psi_p)$ is an arbitrary choice) we define
 v(\psi_p) :=& \int_{\psi_{p,\min}}^\psi \dV = \int^{\zeta(\psi_p)} \sqrt{g}\d\zeta\d\eta\d\varphi,
 \\
 \frac{\d v}{\d\psi_p} =& \int\dA |\vn\psi_p|^{-1} = 2\pi f_0\oint_{\zeta(\psi_p)} \sqrt{g}\d\eta \\
-\langle f \rangle_\psi :=& \frac{\partial}{\partial v} \int \dV f
+\RA{ f }_\psi :=& \frac{\partial}{\partial v} \int \dV f
  = \frac{1}{\int \dA |\vn\psi_p|^{-1} } \int_{\psi_p} \frac{f(\vec x)}{|\vn\psi_p|} \dA \nonumber\\
-=& \frac{\int_\Omega \langle f\rangle_\varphi(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
+=& \frac{\int_\Omega \PA{ f}(R,Z) \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}
 {\int_\Omega \delta(\psi_p(R,Z)-\psi_{p})H(Z-Z_X)\ R \d R \d Z}\nonumber\\
- =& \left(\frac{\d v}{\d\psi_p }\right)^{-1} 2\pi f_0 \oint_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
- = \frac{1}{\oint \sqrt{g}\d\eta } \oint_0^{2\pi} \langle f\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
+ =& \left(\frac{\d v}{\d\psi_p }\right)^{-1} 2\pi f_0 \oint_0^{2\pi} \PA{ f}(\zeta,\eta) \sqrt{g}\d\eta
+ = \frac{1}{\oint \sqrt{g}\d\eta } \oint_0^{2\pi} \PA{ f}(\zeta,\eta) \sqrt{g}\d\eta
 \end{align}
 where we used the co-area formula Eq.~\eqref{eq:coarea} for the second identity
 and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
@@ -501,8 +501,8 @@ does appear (unlike e.g. Tokam3X papers)
 The flux-surface average fulfills the basic identities
 \begin{align}
 \label{eq:fsa_identities}
-\langle \mu f + \lambda g\rangle &= \mu\langle f\rangle + \lambda \langle g\rangle \\
-\langle f(\psi_p) \rangle &= f(\psi_p)
+\RA{ \mu f + \lambda g} &= \mu\RA{ f} + \lambda \RA{ g} \\
+\RA{ f(\psi_p)} &= f(\psi_p)
 \end{align}
 
 The volume average is well-suited for density-like quantities
@@ -510,25 +510,25 @@ as we can see with the following identity.
 Assume we have a quantity $X$ with $\partial_t X + \nc \vec j_X = \Lambda_X$.
 Then we can use the volume average to write
 \begin{align}
-\frac{\partial}{\partial t} \langle X \rangle + \frac{\partial}{
-  \partial v} \langle \vec j_X\cn v\rangle  = \langle \Lambda_X\rangle
+\frac{\partial}{\partial t} \RA{X } + \frac{\partial}{
+  \partial v} \RA{ \vec j_X\cn v}  = \RA{ \Lambda_X}
 \label{eq:fsa_balance}
 \end{align}
 where again $v=v(\psi_p)$ is the volume flux label.
 The {\bf total flux} of a given flux density $\vec j_X$ though the
 flux surface $\psi_p = \psi_{p0}$ is given by
 \begin{align}
-\left\langle\vec j_X\cn v\right\rangle &:= J_X=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
- \frac{\d v}{\d\psi_p} \langle \vec j_X\cn\psi_p \rangle\\
+\RA{\vec j_X\cn v} &:= J_X=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
+ \frac{\d v}{\d\psi_p} \RA{ \vec j_X\cn\psi_p }\\
  &=
-   2\pi f_0 \oint_0^{2\pi} \langle \vec j_X\cn\psi_p\rangle_\varphi(\zeta,\eta) \sqrt{g}\d\eta
-%2\pi\int_\Omega \vec \langle \vec j\cn\psi_p\rangle_\varphi \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
+   2\pi f_0 \oint_0^{2\pi} \PA{ \vec j_X\cn\psi_p}(\zeta,\eta) \sqrt{g}\d\eta
+%2\pi\int_\Omega \vec \PA{ \vec j\cn\psi_p} \delta(\psi_p(R,Z)-\psi_{p0}) H(Z-Z_X)\ R \d R \d Z
 \label{eq:total_flux}
 \end{align}
 Once we have the flux-surface averaged equation we can easily get the volume integrated version (again with the help of the co-area formula)
 \begin{align}
-\frac{\partial}{\partial t} \int_0^{v(\psi_p)}\langle X \rangle \d v 
-+ \langle \vec j_X\cn v\rangle(v(\psi_p))  = \int_0^{v(\psi_p)}\langle \Lambda_X\rangle\d v
+\frac{\partial}{\partial t} \int_0^{v(\psi_p)}\RA{X} \d v 
++ \RA{ \vec j_X\cn v}(v(\psi_p))  = \int_0^{v(\psi_p)}\RA{ \Lambda_X}\d v
 \label{eq:integral_balance}
 \end{align}
 
@@ -569,8 +569,8 @@ construction algorithms, i.e. we use a high-order Runge-Kutta method
 and refine the stepsize until machine-precision is reached.
 
 Notice that the safety factor diverges on the last closed flux
-surface whereas the Eq.~\eqref{eq:total_flux} and \eqref{eq:fsa_area}
-remain finite due to the $\vn\psi$ factor.
+surface whereas Eq.~\eqref{eq:total_flux}
+remains finite due to the $\vn\psi$ factor.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
@@ -589,15 +589,15 @@ where $a\in\{e,i\}$ is the species label and $z$ is the charge number.
 Omitting the species label we arrive at (dividing the density equation by $\Omega_0n_0$ and the velocity equation by $\Omega_0 c_s$)
 \begin{align}
 \frac{\partial}{\partial t} N &+ \vec\nc\left( N \left(
-    \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + \tilde{\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
+    \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + {\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
 \mu N \frac{\partial}{\partial t} U &+ \mu N \left(
-    \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + \tilde{\vec b}_\perp\right)
+    \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + {\vec b}_\perp\right)
     \right)\cn U  \nonumber \\
     &+ 2\mu \nc ( NU \vec v_{\vn\times\bhat})
     -\mu NU\nc \vec v_{\vn\times\bhat}
     + \mu NU\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
-    &= -\tau \left(\bhat + \tilde{\vec b}_\perp\right)\cn N 
-    -N \left( \left(\bhat+\tilde{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right) 
+    &= -\tau \left(\bhat + {\vec b}_\perp\right)\cn N 
+    -N \left( \left(\bhat+{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right) 
     - \eta n_e^2(U_i-u_e) + \mu N\Lambda_U
 \label{}
 \end{align}
@@ -607,7 +607,7 @@ with
 \vec v_{K} := \tau \left(\vec{\mathcal K_{\vn B}} + \vec{\mathcal K_{\vn\times\bhat}}\right)=\tau\vec{\mathcal K}  ,\nonumber\\
 \vec v_C := \mu U^2\vec{\mathcal K_{\vn\times\bhat}},\quad
 \vec v_{\vn\times\bhat} := \tau\vec{\mathcal K_{\vn\times\bhat}},\quad
-\tilde{\vec b}_\perp = \frac{\vn\times A_\parallel \bhat}{B}.
+{\vec b}_\perp = \frac{\vn\times A_\parallel \bhat}{B}.
 \label{}
 \end{align}
 
@@ -828,18 +828,19 @@ or a Torpex inspired source profile
 with $a=0.0335$m, $b=0.05$m, $c=565m^{-2}$, $R_0=0.98$m and $Z_0=-0.02$m.
 
 
-For ions we use
+In order to not generate potential with the source term the
+ion source needs to fulfill $S_{n_e} = \Gamma_{1,i}S_{N_i} + \nc\left( \frac{\mu_i S_{N_i}}{B^2}\np \phi\right)$ which in the long wavelength limit can be inverted to
 \begin{align}
-    S_{N_i} = \Gamma_{1,i}^{-1} S_{n_e} = \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S_{n_e} \\
-    \delta S_{n_e} = \nc\left( \frac{\mu_i S_{N_i}}{B^2}\np \phi\right)
+    S_{N_i} = \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S_{n_e} -\nc\left( \frac{\mu_i S_{n_e}}{B^2}\np \phi\right)
   \label{eq:ion_source}
 \end{align}
-and finally add $\delta S_{n_e}$ to $S_{n_e}$ as the potential FLR correction.
-Note that the correction $\delta S_{n_e}$ is a total divergence which means
-it does not change the volume integrated "total" particle number created by the source.
-Note that Eq.~\eqref{eq:ion_source} is explicitly chosen as to avoid potential generation
-by the particle source (cf.~Section~\ref{sec:conservation}). $S_{n_e}$ needs to be smooth
+Note that the additional terms besides $S_{n_e}$ are total divergences which means
+they do not change the volume integrated "total" particle number created by the source.
+Note that $S_{n_e}$ needs to be smooth
 so that $\np^2 S_{n_e}$ is well defined.
+Also note that with our definition of $\Lambda_{n_e}$ and $\Lambda_{N_i}$ and
+the polarisation equation we have $\Lambda_{n_e} = \Gamma_{1,i}\Lambda_{N_i} + \nc\left( \frac{\mu_i \Lambda_{N_i}}{B^2}\np \phi\right)$ in the long wavelength limit (swap the operators).
+This means that diffusion does not generate potential either.
 
 %The idea for the terms $S_U$ is mainly to provide more numerical stability
 %in the corner regions of the domain, where the parallel derivative may lead
@@ -859,7 +860,7 @@ two functions for which we have no boundary conditions
     \frac{\partial}{\partial t} N =&
         - \frac{1}{B}[\psi, N]_{\perp}%\nonumber\\
         - \bar \npar \left( NU\right)
-        - NU\left(\vec \nc\bhat+\vec \nc\tilde{\vec b}_\perp\right)
+        - NU\left(\vec \nc\bhat+\vec \nc{\vec b}_\perp\right)
         - \tau \mathcal K(N) \nonumber \\&
         - N \mathcal K(\psi)
         -\mu \mathcal K_{\vn\times\bhat}(NU^2)
@@ -885,7 +886,7 @@ two functions for which we have no boundary conditions
 \end{subequations}
 together with
 $\bar\npar f = \npar f + A_\parallel \mathcal K_{\vn\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$
-and $\nc \tilde{ \vec b}_\perp = A_\parallel \vec \nc\vec{ \mathcal{ K}_{\vn\times\bhat}} - \mathcal K_{\vn B}(A_\parallel) $
+and $\nc { \vec b}_\perp = A_\parallel \vec \nc\vec{ \mathcal{ K}_{\vn\times\bhat}} - \mathcal K_{\vn B}(A_\parallel) $
 and
 \begin{subequations} \label{eq:elliptic}
   \begin{align}
@@ -909,11 +910,11 @@ The terms of the particle conservation thus read
 \begin{align} \label{eq:mass_conservation}
   n_e= & n_e,\\
   \vec j_{n_e} =& n_e\left(
-  \vec v_E + \vec v_C + \vec v_{K} +u_e\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \nonumber\\
+  \vec v_E + \vec v_C + \vec v_{K} +u_e\left(\bhat+{\vec b}_\perp\right)  \right) \nonumber\\
   =& n_e \left(\frac{\bhat\times \vn\phi}{B} 
   + \tau_e \frac{\bhat\times\vn n_e}{n_eB} 
   + \mu_e u_e^2\vec K_{\vn\times\bhat} 
-  + u_e(\bhat + \tilde{\vec b}_\perp) \right), \\
+  + u_e(\bhat + {\vec b}_\perp) \right), \\
   \Lambda_{n_e} =&
   \nu_\perp\Delta_\perp n_e + \nu_\parallel\Delta_\parallel n_e
 \\
@@ -930,7 +931,7 @@ the rotation vanishes under the divergence.
 Let us here also derive the particle flux \eqref{eq:mass_conservation} through a flux surface
 \begin{align} \label{eq:particle_flux}
  \vec j_{N}\cn v %=& N\left( \vec v_E + \vec v_C + \vec v_{\vn
- %B} + U \left(\bhat + \tilde{\vec b}_\perp\right)\right) \cn \psi_p \nonumber\\
+ %B} + U \left(\bhat + {\vec b}_\perp\right)\right) \cn \psi_p \nonumber\\
  =&
   \frac{\d v}{\d \psi_p} N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
    \mathcal K_{\vn\times\bhat}(\psi_p) + \tau  \mathcal K_{\vn B}(\psi_p) \right] \nonumber\\
@@ -957,9 +958,9 @@ with ( $z_e=-1$ and $z_i=+1$) and $\vec u_E := {\bhat\times \vn\phi}/{B}$
   +\frac{1}{2} z_i\mu_i  N_i U_i^2,\\
   \vec j_{\mathcal E} =& \sum_s z\left[
   \left(\tau \ln N + \frac{1}{2}\mu U^2 + \psi \right)N\left(
-  \vec v_E + \vec v_C + \vec v_{K} +U\left(\bhat+\tilde{\vec b}_\perp\right)  \right) \right]
+  \vec v_E + \vec v_C + \vec v_{K} +U\left(\bhat+{\vec b}_\perp\right)  \right) \right]
   \nonumber\\
-  &+ \sum_z z\left[\mu \tau NU^2\vec K_{\vn\times\bhat} + \tau NU \left(\bhat + \tilde{\vec b}_\perp\right)\right], \\
+  &+ \sum_z z\left[\mu \tau NU^2\vec K_{\vn\times\bhat} + \tau NU \left(\bhat + {\vec b}_\perp\right)\right], \\
   \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U^2 \right)
   \left(\nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N\right)  +  \mu NU\left(\nu_\perp\Delta_\perp U + \nu_\parallel\Delta_\parallel U\right) \right]
 \nonumber \\
@@ -1012,26 +1013,26 @@ where we write $\partial_{\hat\zeta} := \vn\psi_p\cn$ and $\partial_{\hat\eta} :
 \subsubsection{Vorticity equation}
 With this we write the vorticity equation (in the LWL)
 \begin{align} \label{eq:vorticity_average}
-&\frac{\partial}{\partial t} \left\langle \mu_i N_i \left(
-\frac{\partial_{\hat\zeta}\phi\,}{B^2} + \tau_i \partial_{\hat\zeta} \ln N_i\right) \right\rangle
+&\frac{\partial}{\partial t} \RA{ \mu_i N_i \left(
+\frac{\partial_{\hat\zeta}\phi\,}{B^2} + \tau_i \partial_{\hat\zeta} \ln N_i\right) }
 \nonumber\\
-&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle -\partial_{\hat\eta} \phi \left(\mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}
-+ \mu_i \tau_i N_i\partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\eta}A_\parallel\partial_{\hat\zeta} A_\parallel  + \frac{1}{2}\partial_{\hat\eta} A_\parallel \partial_{\hat\zeta} \tau_i N_i U_i \right\rangle
+&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{ -\partial_{\hat\eta} \phi \left(\mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}
++ \mu_i \tau_i N_i\partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\eta}A_\parallel\partial_{\hat\zeta} A_\parallel  + \frac{1}{2}\partial_{\hat\eta} A_\parallel \partial_{\hat\zeta} \tau_i N_i U_i }
 \nonumber\\
-&= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p \right\rangle
+&= \RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p }
 \end{align}
 where we neglect the second order derivative $\partial_{\hat\eta}\partial_{\hat\zeta}A_\parallel$ in the magnetization density term and
 where the right hand side represents the negative toroidal component of the Lorentz force $-(\vec j\times\vec B)_\varphi$.
 Notice that
 \begin{align}
-\left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p\right\rangle
-= \left\langle \frac{ \bhat\times \vn (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cn\psi_p\right\rangle
-= -\left\langle\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)\right\rangle
+\RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p}
+= \RA{ \frac{ \bhat\times \vn (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cn\psi_p}
+= -\RA{\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)}
 \end{align}
 We can interpret Eq.~\eqref{eq:vorticity_average} as the flux surface average of
 the vorticity equation
 \begin{align}
-&\partial_t \GA{\Omega} + \GA{\nc \vec j_\Omega} = \GA{S_\Omega} \\
+&\partial_t \RA{\Omega} + \RA{\nc \vec j_\Omega} = \RA{S_\Omega} \\
 \Omega &:= \mu_i N_i \left(\frac{\vn\psi_p\cn\phi}{B^2} + \tau_i \vn\psi_p \cn \ln N_i\right) \\
 \vec j_{\Omega} &:= \Omega \vec u_E
     - \left(\frac{1}{\beta} \vn\psi_p\cn A_\parallel +\frac{1}{2} \vn\psi_p\cn \tau_i N_iU_i\right)\frac{\bhat\times\vn A_\parallel}{B} \\
@@ -1041,40 +1042,41 @@ the vorticity equation
 \subsubsection{Toroidal ExB momentum equation}
 Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB velocity
 \begin{align} \label{eq:exb_average}
-&\frac{\partial}{\partial t} \left\langle \mu_i N_i \left(
-\frac{\partial_{\hat\zeta}\phi\,}{B^2}\right) \right\rangle
+&\frac{\partial}{\partial t} \RA{ \mu_i N_i
+\frac{\partial_{\hat\zeta}\phi\,}{B^2}}
 \nonumber\\
-&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \left\langle \mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}\left(-\partial_{\hat\eta} \phi - \tau_i \partial_{\hat\eta} \ln N_i \right)
- + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel + \frac{1}{2}\partial_{\hat\zeta} A_\parallel \partial_{\hat\eta} \tau_i N_i U_i\right \rangle
+&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{ \mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}\left(-\partial_{\hat\eta} \phi - \tau_i \partial_{\hat\eta} \ln N_i \right)
+ + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel + \frac{1}{2}\partial_{\hat\zeta} A_\parallel \partial_{\hat\eta} \tau_i N_i U_i}
 \nonumber\\
-&= \left\langle (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p \right\rangle
+&= \RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p } + \RA{\mu_i (S_{n_e} + \Lambda_{n_e})\frac{\partial_{\hat\zeta}\phi\,}{B^2}}
 \end{align}
 We can interpret Eq.~\eqref{eq:exb_average} as the flux surface average of
 the \ExB vorticity equation
 \begin{align}
-&\partial_t \GA{\Omega_E} + \GA{\nc \vec j_{\Omega_E}} = \GA{S_\Omega} \\
+&\partial_t \RA{\Omega_E} + \RA{\nc \vec j_{\Omega_E}} = \RA{S_\Omega} \\
 \Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \\
 \vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
     - \frac{1}{\beta} \vn\psi_p\cn A_\parallel \left(\frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
-    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p)
+    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p) \\
+    \Lambda_\Omega &:= \RA{\mu_i (S_{n_e} + \Lambda_{n_e})\frac{\vn\psi_p\cn\phi}{B^2}}
 \end{align}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Parallel momentum balance}
 The flux surface average over the parallel momentum equation under species summation and the LWL yields
 \begin{align}
-  \frac{\partial}{\partial t}\GA{\mu_iN_iU_{i} }
+  \frac{\partial}{\partial t}\RA{\mu_iN_iU_{i} }
     % \nonumber\\
-    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \GA{-\mu_iN_iU_i \partial_{\hat\eta}\phi + \sum_s (z_s\tau_sN_s + \mu_s N_sU_s^2) \widetilde{ b_{\perp}^{\;v} } }
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{-\mu_iN_iU_i \partial_{\hat\eta}\phi + \sum_s (z_s\tau_sN_s + \mu_s N_sU_s^2) b_{\perp}^{\;v}  }
    % \nonumber\\
-   = \sum_s\GA{-z_s\tau_s N_s\npar \ln B}
+   = \sum_s\RA{-z_s\tau_s N_s\npar \ln B}
    \label{eq:parallel_momentum}
 \end{align}
 while the toroidal parallel angular momentum contribution reads
 \begin{align}\label{eq:parallel_momentum_direction}
-    \frac{\partial}{\partial t}  \GA{\mu_iNU_i b_\varphi}
-    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \GA{-\mu_iN_iU_i b_\varphi \partial_{\hat\eta}\phi + \sum_s (z_s\tau_s N_s + \mu_sN_sU_s^2) b_\varphi\widetilde{ b_{\perp}^{\;v} }}
-   = -\GA{S_\Omega}
+    \frac{\partial}{\partial t}  \RA{\mu_iNU_i b_\varphi}
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{-\mu_iN_iU_i b_\varphi \partial_{\hat\eta}\phi + \sum_s (z_s\tau_s N_s + \mu_sN_sU_s^2) b_\varphi b_{\perp}^{\;v} }
+   = -\RA{S_\Omega}
 \end{align}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1357,13 +1359,13 @@ Ui               & Dataset & 4 (time, z, y, x) & ion velocity $U_i$ \\
 potential        & Dataset & 4 (time, z, y, x) & electric potential $\phi$ \\
 induction        & Dataset & 4 (time, z, y, x) & parallel vector potential $A_\parallel$ \\
 X\_2d            & Dataset & 3 (time,y,x) & Selected plane $X(\varphi=0)$ \\
-X\_ta2d          & Dataset & 3 (time,y,x) & Toroidal average $\langle X
-    \rangle_\varphi$ Eq.~\eqref{eq:phi_average} \\
+X\_ta2d          & Dataset & 3 (time,y,x) & Toroidal average $\PA{ X }$
+Eq.~\eqref{eq:phi_average} \\
 Y\_tt\_2d        & Dataset & 3 (time,y,x) & Time integrated (between two outputs) selected plane
 $\int_{t_0}^{t_1}\d t Y(\varphi=0) $
 where $t_1 - t_0 = ${\tt dt*inner\_loop*itstp} and {\tt itstp} is the number of discretization points\\
 Y\_tt\_ta2d      & Dataset & 3 (time,y,x) & Time integrated (between two outputs) toroidal average (Eq.~\eqref{eq:phi_average})
-$\int_{t_0}^{t_1}\d t \langle Y \rangle_\varphi$
+$\int_{t_0}^{t_1}\d t \PA{ Y }$
 where $t_1 - t_0 = ${\tt dt*inner\_loop*itstp} and {\tt itstp} is the number of discretization points\\
 \bottomrule
 \end{longtable}
@@ -1417,7 +1419,7 @@ and the time-averaged quantities Y $\in$
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}\\
 \midrule
     jsne &$ n_e (\vec v_E + \vec v_K + \vec v_C )\cn \psi_p$ \\
-    jsneA &$ n_e u_e \vec{\tilde b}_\perp  \cn \psi_p$ \\
+    jsneA &$ n_e u_e \vec{ b}_\perp  \cn \psi_p$ \\
     lneperp &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ \\
     lneparallel &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
     resistivity &-$\eta_\parallel n_e^2 (U_i-u_e)^2$ \\
@@ -1425,10 +1427,10 @@ and the time-averaged quantities Y $\in$
         + z_e \tau_e n_e u_e^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
     jsei &$z_i(\tau_i \ln N_i + \mu_i U_i^2/2 + \psi_i)N_i(\vec v_E^i + \vec v_C + \vec v_K)\cn \psi_p
         + z_i \tau_i N_i U_i^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
-    jseea &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec {\tilde b}_\perp\cn \psi_p
-        + z_e \tau_e n_e u_e \vec{\tilde b}_\perp \cn \psi_p $ \\
-    jseia &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec {\tilde b}_\perp\cn \psi_p
-        + z_i \tau_i N_i U_i \vec{\tilde b}_\perp \cn \psi_p $ \\
+    jseea &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec { b}_\perp\cn \psi_p
+        + z_e \tau_e n_e u_e \vec{ b}_\perp \cn \psi_p $ \\
+    jseia &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec { b}_\perp\cn \psi_p
+        + z_i \tau_i N_i U_i \vec{ b}_\perp \cn \psi_p $ \\
     leeperp &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_e \nu_\perp \Delta_\perp u_e$ \\
     leiperp &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_i \nu_\perp \Delta_\perp U_i$ \\
     leeparallel &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_e \nu_\parallel \Delta_\parallel u_e$ \\
@@ -1497,12 +1499,12 @@ q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:s
 psi\_psi         & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
 psit1d           & Dataset & 1 (psi) & Toroidal flux (integrated q-profile) $\psi_t = \int^\psi_p \d\psi_p q(\psi_p)$ \\
 rho\_t           & Dataset & 1 (psi) & Toroidal flux label $\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$ (is similar to $\rho$ in the edge but $\rho_t$ is nicer in the core domain, because equidistant $\rho_t$ make more equidistant flux-surfaces)\\
-Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \langle Z\rangle_{\psi_{p}}$ \\
-Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\langle Z\rangle_{\psi_p}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
-Z\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\langle Z\rangle_{\psi_p}$ Eq.~\eqref{eq:fsa_vol} \\
-Z\_ifs           & Dataset & 2 (time, psi) & Volume integrated flux surface average $\int\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the volume derived flux-surface average $\partial_v \langle Z\rangle_{\psi_p}$ \\
-Z\_ifs\_lcfs     & Dataset & 1 (time) & Volume integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\langle Z\rangle_{\psi_p}$ unless Z is a current, then it is the fsa evaluated $\langle j_v\rangle_{\psi_p}(0)$ \\
-Z\_ifs\_norm     & Dataset & 1 (time) & Volume integrated square flux surface average $\sqrt{\int \d v \GA{Z}^2}$, unless Z is a current, then it is the square derivative of the flux surface average $\sqrt{\int\d v (\partial_v j^v)^2}$\\
+Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \RA{ Z}(R,Z)$ \\
+Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\RA{ Z}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
+Z\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\RA{ Z}$ Eq.~\eqref{eq:fsa_vol} \\
+Z\_ifs           & Dataset & 2 (time, psi) & Volume integrated flux surface average $\int\d v\RA{ Z}$ unless Z is a current, then it is the volume derived flux-surface average $\partial_v \RA{ Z}$ \\
+Z\_ifs\_lcfs     & Dataset & 1 (time) & Volume integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\RA{ Z}$ unless Z is a current, then it is the fsa evaluated $\RA{ j_v}(0)$ \\
+Z\_ifs\_norm     & Dataset & 1 (time) & Volume integrated square flux surface average $\sqrt{\int \d v \RA{Z}^2}$, unless Z is a current, then it is the square derivative of the flux surface average $\sqrt{\int\d v (\partial_v \RA{j^v})^2}$\\
 \bottomrule
 \end{longtable}
 where Z $\in$ \{X, Y\_tt\}
-- 
GitLab


From 49161d4fdf8e994eee7e0dd5e13bd1e9e582240c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 14 Feb 2020 14:52:43 +0100
Subject: [PATCH 194/540] Rewrite vorticity equation in feltor docu

---
 inc/geometries/geometry_diag.cu |  2 +
 src/feltor/feltor.tex           | 68 ++++++++++++++-------------------
 2 files changed, 31 insertions(+), 39 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 8593a11d2..38f5f193d 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -471,6 +471,8 @@ int main( int argc, char* argv[])
         map1d.emplace_back( "psi_vol", psi_vol,
             "Flux volume with delta function");
         double volumeFVI = 2.*M_PI*fvi(psipmax);
+        double volumeSep = 2.*M_PI*fvi(0.);
+        std::cout << "VOLUME ENCLOSED BY SEPARATRIX: "<<volumeSep<<"\n";
         std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeCoarea<<" "<<volumeFVI
                   <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
         if(gp.hasXpoint()){
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 8d3a065c1..1e9faee49 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -989,76 +989,65 @@ We have the energy flux through a flux surface
 \end{align}
 
 \subsection{ Vorticity and momentum balance equation}
-In order to discuss poloidal flows let us first define orthogonal vectors $\{\vec{ \hat\zeta}, \vec{\hat\eta},\bhat\}$
+From $\vec u_E = \bhat\times \vn\phi/B$ and $\vec u_D = \tau_i \bhat\times\vn \ln N_i$
+we can derive using $\vn\psi_p = e_\varphi \times\vec B$
 \begin{align}
-\vec{\hat\zeta} := \vn\psi_p,\quad
-\vec{\hat \eta} := \frac{\bhat \times\hat \zeta}{B}
+u_{E,\varphi} &= \frac{\vn\phi\cn\psi_p}{B^2}
+&&u_{D,\varphi} = \frac{\tau_i \vn \ln N_i\cn\psi_p}{B} \\
+u_E^{v}&= \frac{\bhat\times \vn\phi}{B}\cn v
+&&u_D^{v}= \tau_i \bhat\times \vn \ln N_i\cn v
 \end{align}
-where $\vec{\hat\eta}$
-points in the counter-clockwise poloidal direction with our choice of the magnetic field direction.
-Notice that $\vec{\hat \eta}$ in general has a (small) toroidal component in addition to the dominant poloidal component.
-
-%We then have $\vn\phi = \hat \zeta\partial_{\hat\zeta}\phi + \hat\eta\partial_{\hat\eta} \phi$
-%with $\partial_{\hat\zeta}:= \hat\zeta\cn$ and $\partial_{\hat\eta}:=\hat\eta\cn$.
-Furthermore, from $\vec u_E = \bhat\times \vn\phi/B$ and $\vec u_d = \tau_i \bhat\times\vn \ln N_i$
-we can derive
-\begin{align}
-u_E^{\hat\eta} &:= \vec u_E\cdot \vec{\hat\eta} = \frac{\partial_{\hat\zeta} \phi}{B^2} \sim u_{E,\varphi}
-&&u_d^{\hat\eta} := \vec u_d\cdot \vec{\hat\eta} =  \frac{\tau_i \partial_{\hat\zeta} \ln N_i}{B} \sim u_{D,\varphi} \\
-u_E^{\hat\zeta}&:= \vec u_E\cdot \vec{\hat\zeta}  = -\partial_{\hat\eta} \phi \sim u_E^v \sim -\partial_\varphi\phi
-&&u_d^{\hat\zeta}:= \vec u_d\cdot \vec{\hat\zeta} = -B \tau_i \partial_{\hat\eta} \ln N_i \sim u_D^v
-\end{align}
-where we write $\partial_{\hat\zeta} := \vn\psi_p\cn$ and $\partial_{\hat\eta} := \vec{\hat\eta}\cn$.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Vorticity equation}
 With this we write the vorticity equation (in the LWL)
 \begin{align} \label{eq:vorticity_average}
 &\frac{\partial}{\partial t} \RA{ \mu_i N_i \left(
-\frac{\partial_{\hat\zeta}\phi\,}{B^2} + \tau_i \partial_{\hat\zeta} \ln N_i\right) }
+\frac{\vn\psi_p\cn\phi\,}{B^2} + \tau_i \vn\psi_p\cn \ln N_i\right) }
 \nonumber\\
-&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{ -\partial_{\hat\eta} \phi \left(\mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}
-+ \mu_i \tau_i N_i\partial_{\hat\zeta} \ln N_i \right) + \frac{1}{\beta} \partial_{\hat\eta}A_\parallel\partial_{\hat\zeta} A_\parallel  + \frac{1}{2}\partial_{\hat\eta} A_\parallel \partial_{\hat\zeta} \tau_i N_i U_i }
+&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{ \mu_i N_i \frac{\bhat\times\vn\phi}{B}\cn\psi_p \left(\frac{\vn\psi_p\cn\phi }{B^2}
++ \tau_i \vn\psi_p\cn \ln N_i \right) } \nonumber\\
+&- \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\frac{\bhat\times\vn A_\parallel}{B}\cn\psi_p\left(\frac{1}{\beta} \vn A_\parallel\cn\psi_p  + \frac{1}{2}\vn \tau_i N_i U_i\cn\psi_p\right) }
 \nonumber\\
 &= \RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p }
 \end{align}
-where we neglect the second order derivative $\partial_{\hat\eta}\partial_{\hat\zeta}A_\parallel$ in the magnetization density term and
+where we neglect the second order derivative $\frac{\bhat\times\vn\psi_p}{B}\cn\vn\psi_p\cn A_\parallel$ in the magnetization density term and
 where the right hand side represents the negative toroidal component of the Lorentz force $-(\vec j\times\vec B)_\varphi$.
-Notice that
+Notice that the Lorentz force is related to the diamagnetic velocity
 \begin{align}
 \RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p}
 = \RA{ \frac{ \bhat\times \vn (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cn\psi_p}
-= -\RA{\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)}
+%= -\RA{\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)}
 \end{align}
 We can interpret Eq.~\eqref{eq:vorticity_average} as the flux surface average of
 the vorticity equation
 \begin{align}
-&\partial_t \RA{\Omega} + \RA{\nc \vec j_\Omega} = \RA{S_\Omega} \\
+&\partial_t \RA{\Omega} + \RA{\nc \vec j_\Omega} = -\RA{F_{L,\varphi}} \\
 \Omega &:= \mu_i N_i \left(\frac{\vn\psi_p\cn\phi}{B^2} + \tau_i \vn\psi_p \cn \ln N_i\right) \\
 \vec j_{\Omega} &:= \Omega \vec u_E
     - \left(\frac{1}{\beta} \vn\psi_p\cn A_\parallel +\frac{1}{2} \vn\psi_p\cn \tau_i N_iU_i\right)\frac{\bhat\times\vn A_\parallel}{B} \\
-    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p)
+    F_{L,\varphi} &:=  -(z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) - (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p)
 \end{align}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Toroidal ExB momentum equation}
 Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB velocity
 \begin{align} \label{eq:exb_average}
 &\frac{\partial}{\partial t} \RA{ \mu_i N_i
-\frac{\partial_{\hat\zeta}\phi\,}{B^2}}
+\frac{\vn\psi_p\cn\phi\,}{B^2}}
+\nonumber\\
+&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{ \mu_i N_i\frac{\vn\psi_p\cn\phi }{B^2}\left(\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \tau_i \frac{\bhat\times\vn N_i}{N_i}\cn\psi_p \right)}
 \nonumber\\
-&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{ \mu_i N_i\frac{\partial_{\hat\zeta}\phi }{B^2}\left(-\partial_{\hat\eta} \phi - \tau_i \partial_{\hat\eta} \ln N_i \right)
- + \frac{1}{\beta} \partial_{\hat\zeta} A_\parallel \partial_{\hat\eta}A_\parallel + \frac{1}{2}\partial_{\hat\zeta} A_\parallel \partial_{\hat\eta} \tau_i N_i U_i}
+ &- \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\vn\psi_p\cn A_\parallel\left(\frac{1}{\beta}  \frac{\bhat\times\vn A_\parallel}{B}\cn\psi_p + \frac{1}{2} {\bhat\times\vn \tau_i N_i U_i}\cn\psi_p\right)}
 \nonumber\\
-&= \RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p } + \RA{\mu_i (S_{n_e} + \Lambda_{n_e})\frac{\partial_{\hat\zeta}\phi\,}{B^2}}
+&= \RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p } + \RA{\mu_i (S_{n_e} + \Lambda_{n_e})\frac{\vn\psi_p\cn\phi\,}{B^2}}
 \end{align}
 We can interpret Eq.~\eqref{eq:exb_average} as the flux surface average of
-the \ExB vorticity equation
+the toroidal \ExB angular momentum density equation
 \begin{align}
-&\partial_t \RA{\Omega_E} + \RA{\nc \vec j_{\Omega_E}} = \RA{S_\Omega} \\
+&\partial_t \RA{\Omega_E} + \RA{\nc \vec j_{\Omega_E}} = -\RA{F_{L,\varphi}} + \RA{\Lambda_{\Omega_E}} \\
 \Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \\
 \vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
     - \frac{1}{\beta} \vn\psi_p\cn A_\parallel \left(\frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
-    S_\Omega &:=  (z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p) \\
-    \Lambda_\Omega &:= \RA{\mu_i (S_{n_e} + \Lambda_{n_e})\frac{\vn\psi_p\cn\phi}{B^2}}
+    \Lambda_{\Omega_E} &:= \RA{\mu_i (S_{n_e} + \Lambda_{n_e})\frac{\vn\psi_p\cn\phi}{B^2}}
 \end{align}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1067,16 +1056,17 @@ The flux surface average over the parallel momentum equation under species summa
 \begin{align}
   \frac{\partial}{\partial t}\RA{\mu_iN_iU_{i} }
     % \nonumber\\
-    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{-\mu_iN_iU_i \partial_{\hat\eta}\phi + \sum_s (z_s\tau_sN_s + \mu_s N_sU_s^2) b_{\perp}^{\;v}  }
-   % \nonumber\\
-   = \sum_s\RA{-z_s\tau_s N_s\npar \ln B}
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i \frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_sN_s + \mu_s N_sU_s^2) b_{\perp}^{\;v}  }
+    \nonumber\\
+   = \sum_s\RA{-z_s\tau_s N_s\npar \ln B} + \mu_i \RA{ \left(S_{N_i} + \Lambda_{N_i}\right) U_i}
    \label{eq:parallel_momentum}
 \end{align}
 while the toroidal parallel angular momentum contribution reads
 \begin{align}\label{eq:parallel_momentum_direction}
     \frac{\partial}{\partial t}  \RA{\mu_iNU_i b_\varphi}
-    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{-\mu_iN_iU_i b_\varphi \partial_{\hat\eta}\phi + \sum_s (z_s\tau_s N_s + \mu_sN_sU_s^2) b_\varphi b_{\perp}^{\;v} }
-   = -\RA{S_\Omega}
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i b_\varphi\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_s N_s + \mu_sN_sU_s^2) b_\varphi b_{\perp}^{\;v} }
+    \nonumber\\
+   = \RA{F_{L,\varphi}} + \mu_i \RA{ \left(S_{N_i} + \Lambda_{N_i}\right) U_i b_\varphi}
 \end{align}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-- 
GitLab


From 204c1d29f52123a12be94f436c3b6b55e3b2a672 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 16 Feb 2020 00:26:35 +0100
Subject: [PATCH 195/540] Clean up vorticity equation

---
 src/feltor/feltor.tex | 73 +++++++++++--------------------------------
 1 file changed, 18 insertions(+), 55 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 1e9faee49..4bd39a189 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -989,66 +989,28 @@ We have the energy flux through a flux surface
 \end{align}
 
 \subsection{ Vorticity and momentum balance equation}
-From $\vec u_E = \bhat\times \vn\phi/B$ and $\vec u_D = \tau_i \bhat\times\vn \ln N_i$
-we can derive using $\vn\psi_p = e_\varphi \times\vec B$
-\begin{align}
-u_{E,\varphi} &= \frac{\vn\phi\cn\psi_p}{B^2}
-&&u_{D,\varphi} = \frac{\tau_i \vn \ln N_i\cn\psi_p}{B} \\
-u_E^{v}&= \frac{\bhat\times \vn\phi}{B}\cn v
-&&u_D^{v}= \tau_i \bhat\times \vn \ln N_i\cn v
-\end{align}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Vorticity equation}
-With this we write the vorticity equation (in the LWL)
-\begin{align} \label{eq:vorticity_average}
-&\frac{\partial}{\partial t} \RA{ \mu_i N_i \left(
-\frac{\vn\psi_p\cn\phi\,}{B^2} + \tau_i \vn\psi_p\cn \ln N_i\right) }
-\nonumber\\
-&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{ \mu_i N_i \frac{\bhat\times\vn\phi}{B}\cn\psi_p \left(\frac{\vn\psi_p\cn\phi }{B^2}
-+ \tau_i \vn\psi_p\cn \ln N_i \right) } \nonumber\\
-&- \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\frac{\bhat\times\vn A_\parallel}{B}\cn\psi_p\left(\frac{1}{\beta} \vn A_\parallel\cn\psi_p  + \frac{1}{2}\vn \tau_i N_i U_i\cn\psi_p\right) }
-\nonumber\\
-&= \RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p }
-\end{align}
-where we neglect the second order derivative $\frac{\bhat\times\vn\psi_p}{B}\cn\vn\psi_p\cn A_\parallel$ in the magnetization density term and
-where the right hand side represents the negative toroidal component of the Lorentz force $-(\vec j\times\vec B)_\varphi$.
-Notice that the Lorentz force is related to the diamagnetic velocity
-\begin{align}
-\RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p}
-= \RA{ \frac{ \bhat\times \vn (z_e\tau_e n_e + z_i\tau_i N_i)}{B}\cn\psi_p}
-%= -\RA{\partial_{\hat\eta} (z_e\tau_e n_e + z_i\tau_i N_i)}
-\end{align}
-We can interpret Eq.~\eqref{eq:vorticity_average} as the flux surface average of
-the vorticity equation
+We write the vorticity equation (in the LWL)
 \begin{align}
-&\partial_t \RA{\Omega} + \RA{\nc \vec j_\Omega} = -\RA{F_{L,\varphi}} \\
-\Omega &:= \mu_i N_i \left(\frac{\vn\psi_p\cn\phi}{B^2} + \tau_i \vn\psi_p \cn \ln N_i\right) \\
+&\partial_t \RA{\Omega} + \frac{\partial}{\partial v}\frac{\d v}{\d\psi_p}\RA{\vec j_\Omega\cn\psi_p} = -\RA{F_{L,\varphi}} \label{eq:vorticity_average} \\
+\Omega &:= \mu_i N_i \left(\frac{\vn\phi\cn\psi_p}{B^2} + \tau_i \vn\ln N_i\cn\psi_p\right) \\
 \vec j_{\Omega} &:= \Omega \vec u_E
-    - \left(\frac{1}{\beta} \vn\psi_p\cn A_\parallel +\frac{1}{2} \vn\psi_p\cn \tau_i N_iU_i\right)\frac{\bhat\times\vn A_\parallel}{B} \\
+    - \left(\frac{1}{\beta} \vn A_\parallel\cn\psi_p +\frac{1}{2} \vn\tau_i N_iU_i\cn\psi_p\right)\frac{\bhat\times\vn A_\parallel}{B} \\
     F_{L,\varphi} &:=  -(z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) - (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p)
 \end{align}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Toroidal ExB momentum equation}
 Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB velocity
-\begin{align} \label{eq:exb_average}
-&\frac{\partial}{\partial t} \RA{ \mu_i N_i
-\frac{\vn\psi_p\cn\phi\,}{B^2}}
-\nonumber\\
-&+ \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{ \mu_i N_i\frac{\vn\psi_p\cn\phi }{B^2}\left(\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \tau_i \frac{\bhat\times\vn N_i}{N_i}\cn\psi_p \right)}
-\nonumber\\
- &- \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\vn\psi_p\cn A_\parallel\left(\frac{1}{\beta}  \frac{\bhat\times\vn A_\parallel}{B}\cn\psi_p + \frac{1}{2} {\bhat\times\vn \tau_i N_i U_i}\cn\psi_p\right)}
-\nonumber\\
-&= \RA{ (z_e \tau_e n_e + z_i\tau_i N_i)\vec K\cn\psi_p + (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\vec K_{\vn\times\bhat}\cn\psi_p } + \RA{\mu_i (S_{n_e} + \Lambda_{n_e})\frac{\vn\psi_p\cn\phi\,}{B^2}}
-\end{align}
-We can interpret Eq.~\eqref{eq:exb_average} as the flux surface average of
-the toroidal \ExB angular momentum density equation
 \begin{align}
-&\partial_t \RA{\Omega_E} + \RA{\nc \vec j_{\Omega_E}} = -\RA{F_{L,\varphi}} + \RA{\Lambda_{\Omega_E}} \\
-\Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \\
+&\partial_t \RA{\Omega_E} + \frac{\partial}{\partial v} \frac{\d v}{\d \psi_p}\RA{ \vec j_{\Omega_E}\cn\psi_p} = -\RA{F_{L,\varphi}}+ \RA{\mathcal S_{\Omega_E}} + \RA{\Lambda_{\Omega_E}} \label{eq:exb_average} \\
+\Omega_E &:= \mu_i N_i \frac{\vn\phi\cn\psi_p}{B^2} \equiv u_{E,\varphi} \\
 \vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
-    - \frac{1}{\beta} \vn\psi_p\cn A_\parallel \left(\frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
-    \Lambda_{\Omega_E} &:= \RA{\mu_i (S_{n_e} + \Lambda_{n_e})\frac{\vn\psi_p\cn\phi}{B^2}}
+    - \frac{1}{\beta} \vn A_\parallel\cn\psi_p \left(\frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
+    \mathcal S_{\Omega_E} &:= \mu_i S_{n_e} \frac{\vn\phi\cn\psi_p}{B^2} \quad
+    \Lambda_{\Omega_E} := \mu_i \Lambda_{n_e}\frac{\vn\phi\cn\psi_p}{B^2}
 \end{align}
+where here we also monitor the source and diffusion terms.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Parallel momentum balance}
@@ -1058,15 +1020,15 @@ The flux surface average over the parallel momentum equation under species summa
     % \nonumber\\
     + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i \frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_sN_s + \mu_s N_sU_s^2) b_{\perp}^{\;v}  }
     \nonumber\\
-   = \sum_s\RA{-z_s\tau_s N_s\npar \ln B} + \mu_i \RA{ \left(S_{N_i} + \Lambda_{N_i}\right) U_i}
+   = \sum_s\RA{-z_s\tau_s N_s\npar \ln B} + \mu_i \RA{ S_{N_i} U_i}
    \label{eq:parallel_momentum}
 \end{align}
 while the toroidal parallel angular momentum contribution reads
 \begin{align}\label{eq:parallel_momentum_direction}
-    \frac{\partial}{\partial t}  \RA{\mu_iNU_i b_\varphi}
+    \frac{\partial}{\partial t}  \RA{\mu_iN_iU_i b_\varphi}
     + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i b_\varphi\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_s N_s + \mu_sN_sU_s^2) b_\varphi b_{\perp}^{\;v} }
     \nonumber\\
-   = \RA{F_{L,\varphi}} + \mu_i \RA{ \left(S_{N_i} + \Lambda_{N_i}\right) U_i b_\varphi}
+   = \RA{F_{L,\varphi}} + \mu_i \RA{ S_{N_i} U_i b_\varphi}
 \end{align}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -1359,8 +1321,8 @@ $\int_{t_0}^{t_1}\d t \PA{ Y }$
 where $t_1 - t_0 = ${\tt dt*inner\_loop*itstp} and {\tt itstp} is the number of discretization points\\
 \bottomrule
 \end{longtable}
-where the time integrals are computed with the help of Simpson's rule
-and X $\in$
+where
+X $\in$
 \begin{longtable}{llll}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} & \textbf{Name} &  \textbf{Equation}\\
@@ -1403,7 +1365,7 @@ and X $\in$
     odiai &$\mu_i \tau_i\vn\psi_p\cn N_i$ \\
 \bottomrule
 \end{longtable}
-and the time-averaged quantities Y $\in$
+and the time-averaged (the time integrals are computed with the help of Simpson's rule) quantities Y $\in$
 \begin{longtable}{ll}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}\\
@@ -1442,7 +1404,8 @@ and the time-averaged quantities Y $\in$
     sparmirrori & $-z_i\tau_iN_i\npar \ln B$ \\
 \bottomrule
 \end{longtable}
-Even though this might look like a lot of things to compute the actual time spent on diagnostics is negligible if {\tt inner\_loop} parameter is greater than 1.
+Even though this might look like a lot of things to compute the actual time spent on diagnostics is negligible if {\tt inner\_loop} parameter is greater than 1. Also
+remember that these are all two-dimensional fields, which take up much less disk-space than three-dimensional fields.
 \subsection{Restart file} \label{sec:restart_file}
 The program \texttt{feltor\_hpc.cu} has the possibility to initialize time and the fields with
 the results of a previous simulation. In particular, this feature is motivated by chain jobs on a cluster
-- 
GitLab


From eaf9980e0b5a091822225691afbc66512fd22055 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 19 Feb 2020 00:09:41 +0100
Subject: [PATCH 196/540] Restructure docu of output quantities in feltor

---
 src/feltor/feltor.tex | 239 +++++++++++++++++++++++++-----------------
 1 file changed, 140 insertions(+), 99 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 4bd39a189..8d10d7aed 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -589,12 +589,12 @@ where $a\in\{e,i\}$ is the species label and $z$ is the charge number.
 Omitting the species label we arrive at (dividing the density equation by $\Omega_0n_0$ and the velocity equation by $\Omega_0 c_s$)
 \begin{align}
 \frac{\partial}{\partial t} N &+ \vec\nc\left( N \left(
-    \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + {\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
+    \vec u_E + \vec u_K + \vec u_{C} + U\left(\bhat + {\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
 \mu N \frac{\partial}{\partial t} U &+ \mu N \left(
-    \vec v_E + \vec v_K + \vec v_{C} + U\left(\bhat + {\vec b}_\perp\right)
+    \vec u_E + \vec u_K + \vec u_{C} + U\left(\bhat + {\vec b}_\perp\right)
     \right)\cn U  \nonumber \\
-    &+ 2\mu \nc ( NU \vec v_{\vn\times\bhat})
-    -\mu NU\nc \vec v_{\vn\times\bhat}
+    &+ 2\mu \nc ( NU \vec u_{\vn\times\bhat})
+    -\mu NU\nc \vec u_{\vn\times\bhat}
     + \mu NU\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
     &= -\tau \left(\bhat + {\vec b}_\perp\right)\cn N 
     -N \left( \left(\bhat+{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right) 
@@ -603,10 +603,10 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
 \end{align}
 with
 \begin{align}
-\vec v_E := \frac{\bhat\times\vn\psi}{B},\quad
-\vec v_{K} := \tau \left(\vec{\mathcal K_{\vn B}} + \vec{\mathcal K_{\vn\times\bhat}}\right)=\tau\vec{\mathcal K}  ,\nonumber\\
-\vec v_C := \mu U^2\vec{\mathcal K_{\vn\times\bhat}},\quad
-\vec v_{\vn\times\bhat} := \tau\vec{\mathcal K_{\vn\times\bhat}},\quad
+\vec u_E := \frac{\bhat\times\vn\psi}{B},\quad
+\vec u_{K} := \tau \left(\vec{\mathcal K_{\vn B}} + \vec{\mathcal K_{\vn\times\bhat}}\right)=\tau\vec{\mathcal K}  ,\nonumber\\
+\vec u_C := \mu U^2\vec{\mathcal K_{\vn\times\bhat}},\quad
+\vec u_{\vn\times\bhat} := \tau\vec{\mathcal K_{\vn\times\bhat}},\quad
 {\vec b}_\perp = \frac{\vn\times A_\parallel \bhat}{B}.
 \label{}
 \end{align}
@@ -829,7 +829,7 @@ with $a=0.0335$m, $b=0.05$m, $c=565m^{-2}$, $R_0=0.98$m and $Z_0=-0.02$m.
 
 
 In order to not generate potential with the source term the
-ion source needs to fulfill $S_{n_e} = \Gamma_{1,i}S_{N_i} + \nc\left( \frac{\mu_i S_{N_i}}{B^2}\np \phi\right)$ which in the long wavelength limit can be inverted to
+ion source needs to fulfill $S_{n_e} = \Gamma_{1,i}S_{N_i} + \nc\left( \frac{\mu_i S_{N_i}}{B^2}\np \phi\right)$ which in the long wavelength limit can be inverted to (the long wavelength limit should be well-fulfilled for a realistic source term since the amplitude is typically quite small)
 \begin{align}
     S_{N_i} = \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S_{n_e} -\nc\left( \frac{\mu_i S_{n_e}}{B^2}\np \phi\right)
   \label{eq:ion_source}
@@ -897,7 +897,22 @@ and
     A_\parallel &= \beta\left(N_iW_i-n_e w_e\right)
   \end{align}
 \end{subequations}
-Note that the negative signs make the operators in Eq.~\eqref{eq:elliptic} positive definite.
+Note that the negative signs make the operators in Eqs.~\eqref{eq:elliptic} positive definite.
+
+In the output file we have
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} & \textbf{Name} &  \textbf{Equation}\\
+\midrule
+    electrons &$n_e$ &
+    ions &$N_i$ \\
+    Ue &$u_e$ &
+    Ui &$U_i$ \\
+    potential &$\phi$ &
+    psi &$\psi$ \\
+    induction &$A_\parallel$ & \\
+\bottomrule
+\end{longtable}
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
 The density equation directly yields the particle conservation
@@ -907,14 +922,14 @@ The density equation directly yields the particle conservation
   =  \Lambda_{n_e}+S_{n_e}
 \end{align}
 The terms of the particle conservation thus read
-\begin{align} \label{eq:mass_conservation}
+\begin{align}
   n_e= & n_e,\\
   \vec j_{n_e} =& n_e\left(
-  \vec v_E + \vec v_C + \vec v_{K} +u_e\left(\bhat+{\vec b}_\perp\right)  \right) \nonumber\\
+  \vec u_E + \vec u_C + \vec u_{K} +u_e\left(\bhat+{\vec b}_\perp\right)  \right) \nonumber\\
   =& n_e \left(\frac{\bhat\times \vn\phi}{B} 
   + \tau_e \frac{\bhat\times\vn n_e}{n_eB} 
   + \mu_e u_e^2\vec K_{\vn\times\bhat} 
-  + u_e(\bhat + {\vec b}_\perp) \right), \\
+  + u_e(\bhat + {\vec b}_\perp) \right), \label{eq:particle_flux} \\
   \Lambda_{n_e} =&
   \nu_\perp\Delta_\perp n_e + \nu_\parallel\Delta_\parallel n_e
 \\
@@ -928,17 +943,33 @@ n_e \vec K = n_e\vn\times\frac{\bhat}{B} = \vn\times n_e\frac{\bhat}{B} + \frac{
 such that we can define the diamagnetic flux in the particle flux since
 the rotation vanishes under the divergence.
 
-Let us here also derive the particle flux \eqref{eq:mass_conservation} through a flux surface
+Let us here also derive the particle flux \eqref{eq:particle_flux} through a flux surface
 \begin{align} \label{eq:particle_flux}
- \vec j_{N}\cn v %=& N\left( \vec v_E + \vec v_C + \vec v_{\vn
+ \vec j_{N}\cn v %=& N\left( \vec u_E + \vec u_C + \vec u_{\vn
  %B} + U \left(\bhat + {\vec b}_\perp\right)\right) \cn \psi_p \nonumber\\
  =&
   \frac{\d v}{\d \psi_p} N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
    \mathcal K_{\vn\times\bhat}(\psi_p) + \tau  \mathcal K_{\vn B}(\psi_p) \right] \nonumber\\
  &+ NU\frac{\d v}{\d \psi_p}\left [\left( A_\parallel \mathcal
- K_{\vn\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right] \\
+ K_{\vn\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right]
 \end{align}
 
+The relevant terms in the output file are
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} & \textbf{Name} &  \textbf{Equation}\\
+\midrule
+    electrons & $n_e$ &
+    jsne\_tt &$ n_e (\vec u_E + \vec u_K + \vec u_C )\cn \psi_p$ \\
+    jsneA\_tt &$ n_e u_e \vec{ b}_\perp  \cn \psi_p$ &
+    jsneE\_tt & $ n_e \vec u_E\cn\psi_p$ \\
+    lneperp\_tt &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ &
+    lneparallel\_tt &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
+    sne & $S_{n_e}$ & \\
+\bottomrule
+\end{longtable}
+
+
 
 \subsubsection{Energy theorem}
 The terms of the energy theorem are
@@ -958,7 +989,7 @@ with ( $z_e=-1$ and $z_i=+1$) and $\vec u_E := {\bhat\times \vn\phi}/{B}$
   +\frac{1}{2} z_i\mu_i  N_i U_i^2,\\
   \vec j_{\mathcal E} =& \sum_s z\left[
   \left(\tau \ln N + \frac{1}{2}\mu U^2 + \psi \right)N\left(
-  \vec v_E + \vec v_C + \vec v_{K} +U\left(\bhat+{\vec b}_\perp\right)  \right) \right]
+  \vec u_E + \vec u_C + \vec u_{K} +U\left(\bhat+{\vec b}_\perp\right)  \right) \right]
   \nonumber\\
   &+ \sum_z z\left[\mu \tau NU^2\vec K_{\vn\times\bhat} + \tau NU \left(\bhat + {\vec b}_\perp\right)\right], \\
   \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U^2 \right)
@@ -987,30 +1018,84 @@ We have the energy flux through a flux surface
  K_{\vn\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
 \label{eq:energy_flux}
 \end{align}
+The relevant terms in the output file are
+\begin{longtable}{ll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}\\
+\midrule
+    nelnne &$ z_e\tau_e n_e \ln n_e$ \\
+    nilnni &$ z_i\tau_i N_i \ln N_i$ \\
+    aperp2 &$ (\np A_\parallel)^2/2/\beta$ \\
+    ue2   &$z_i\mu_i N_i u_E^2 /2$ \\
+    neue2 &$ z_e\mu_e n_e u_e^2/2$ \\
+    niui2 &$ z_i\mu_i N_i U_i^2/2$ \\
+    see & $z_e(\tau_e (1+\ln n_e) + \phi + \frac{1}{2}\mu_e u_e^2) S_{n_e} $ \\
+    sei & $z_i(\tau_i (1+\ln N_i) + \psi + \frac{1}{2}\mu_i U_i^2) S_{N_i} $ \\
+    resistivity\_tt &-$\eta_\parallel n_e^2 (U_i-u_e)^2$ \\
+    jsee\_tt &$z_e(\tau_e \ln n_e + \mu_e u_e^2/2 + \phi)n_e(\vec u_E + \vec u_C + \vec u_K)\cn \psi_p
+        + z_e \tau_e n_e u_e^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
+    jsei\_tt &$z_i(\tau_i \ln N_i + \mu_i U_i^2/2 + \psi_i)N_i(\vec u_E^i + \vec u_C + \vec u_K)\cn \psi_p
+        + z_i \tau_i N_i U_i^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
+    jseea\_tt &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec { b}_\perp\cn \psi_p
+        + z_e \tau_e n_e u_e \vec{ b}_\perp \cn \psi_p $ \\
+    jseia\_tt &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec { b}_\perp\cn \psi_p
+        + z_i \tau_i N_i U_i \vec{ b}_\perp \cn \psi_p $ \\
+    leeperp\_tt &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_e \nu_\perp \Delta_\perp u_e$ \\
+    leiperp\_tt &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_i \nu_\perp \Delta_\perp U_i$ \\
+    leeparallel\_tt &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_e \nu_\parallel \Delta_\parallel u_e$ \\
+    leiparallel\_tt &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_i \nu_\parallel \Delta_\parallel U_i$ \\
+\bottomrule
+\end{longtable}
 
 \subsection{ Vorticity and momentum balance equation}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\subsubsection{Vorticity equation}
+\subsubsection{Vorticity and toroidal ExB angular momentum equation} \label{sec:vorticity_eq}
 We write the vorticity equation (in the LWL)
 \begin{align}
 &\partial_t \RA{\Omega} + \frac{\partial}{\partial v}\frac{\d v}{\d\psi_p}\RA{\vec j_\Omega\cn\psi_p} = -\RA{F_{L,\varphi}} \label{eq:vorticity_average} \\
-\Omega &:= \mu_i N_i \left(\frac{\vn\phi\cn\psi_p}{B^2} + \tau_i \vn\ln N_i\cn\psi_p\right) \\
+\Omega &:= \mu_i N_i \left(\frac{\vn\phi\cn\psi_p}{B^2} + \tau_i \vn\ln N_i\cn\psi_p\right) \equiv \mu_i N_i(u_{E,\varphi} + u_{D,\varphi}) \\
 \vec j_{\Omega} &:= \Omega \vec u_E
     - \left(\frac{1}{\beta} \vn A_\parallel\cn\psi_p +\frac{1}{2} \vn\tau_i N_iU_i\cn\psi_p\right)\frac{\bhat\times\vn A_\parallel}{B} \\
     F_{L,\varphi} &:=  -(z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) - (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p)
 \end{align}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\subsubsection{Toroidal ExB momentum equation}
 Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB velocity
 \begin{align}
 &\partial_t \RA{\Omega_E} + \frac{\partial}{\partial v} \frac{\d v}{\d \psi_p}\RA{ \vec j_{\Omega_E}\cn\psi_p} = -\RA{F_{L,\varphi}}+ \RA{\mathcal S_{\Omega_E}} + \RA{\Lambda_{\Omega_E}} \label{eq:exb_average} \\
-\Omega_E &:= \mu_i N_i \frac{\vn\phi\cn\psi_p}{B^2} \equiv u_{E,\varphi} \\
+\Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \equiv \mu_i N_i u_{E,\varphi} \\
 \vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
     - \frac{1}{\beta} \vn A_\parallel\cn\psi_p \left(\frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
     \mathcal S_{\Omega_E} &:= \mu_i S_{n_e} \frac{\vn\phi\cn\psi_p}{B^2} \quad
     \Lambda_{\Omega_E} := \mu_i \Lambda_{n_e}\frac{\vn\phi\cn\psi_p}{B^2}
 \end{align}
 where here we also monitor the source and diffusion terms.
+In the output file we have
+\begingroup
+%\setlength{\tabcolsep}{10pt} % Default value: 6pt
+\renewcommand{\arraystretch}{1.5} % Default value: 1
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}&
+\textbf{Name} &  \textbf{Equation}\\
+\midrule
+    oexbe &$\mu_i n_e \frac{\vn\psi_p\cn\phi}{B^2}$ &
+    oexbi &$\mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2}$ \\
+    odiae &$\mu_i \tau_i\vn\psi_p\cn n_e$ &
+    odiai &$\mu_i \tau_i\vn\psi_p\cn N_i$ \\
+    jsoexbi\_tt &$\mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \frac{\bhat\times\vn\phi\cn \psi_p}{B}$ &
+    jsoexbe\_tt &$\mu_i n_e \frac{\vn\psi_p\cn\phi}{B^2} \frac{\bhat\times\vn\phi\cn \psi_p}{B}$ \\
+    jsodiaiUE\_tt &$\mu_i \tau_i\vn\psi_p\cn N_i \frac{\bhat\times\vn\phi\cn \psi_p}{B}$ &
+    jsodiaeUE\_tt &$\mu_i \tau_i\vn\psi_p\cn n_e \frac{\bhat\times\vn\phi\cn \psi_p}{B}$ \\
+    jsoexbiUD\_tt &$\mu_i\tau_i \frac{\vn\psi_p\cn\phi}{B^2} \frac{\bhat\times\vn N_i\cn \psi_p}{B}$ &
+    jsoexbeUD\_tt &$\mu_i\tau_i \frac{\vn\psi_p\cn\phi}{B^2} \frac{\bhat\times\vn n_e\cn \psi_p}{B}$ \\
+    jsoapar\_tt &$ \vn\psi_p\cn A_\parallel \frac{\bhat\times\vn A_\parallel\cn \psi_p}{B\beta}$ &
+    socurve\_tt &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
+    socurvi\_tt &$z_i\tau_i N_i \mathcal K(\psi_p)$ &
+    socurvkappae\_tt &$z_e\mu_e n_eu_e^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
+    socurvkappai\_tt &$z_i\mu_i N_iU_i^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ & \\
+    sosne\_tt & $\mu_i S_{n_e} \vn\psi_p\cn\phi/B^2$ & \\
+\bottomrule
+\end{longtable}
+\endgroup
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Parallel momentum balance}
@@ -1031,6 +1116,25 @@ while the toroidal parallel angular momentum contribution reads
    = \RA{F_{L,\varphi}} + \mu_i \RA{ S_{N_i} U_i b_\varphi}
 \end{align}
 
+The relevant terms in the output file are (the Lorentz force term is described in the previous subsection \ref{sec:vorticity_eq})
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} &
+\textbf{Name} &  \textbf{Equation}\\
+\midrule
+    neue &$n_e u_e$ &
+    niui &$\mu_i N_i U_i$ \\
+    neuebphi &$n_eu_eb_\varphi$ &
+    niuibphi &$\mu_i N_iU_ib_\varphi$ \\
+    jsparexbi\_tt       & $\mu_i N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ &
+    jsparbphiexbi\_tt   & $\mu_i N_iU_ib_\varphi(\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    sparmirrore\_tt & $-z_e\tau_en_e\npar \ln B$ &
+    sparmirrori\_tt & $-z_i\tau_iN_i\npar \ln B$ \\
+    jsparexbi\_tt       & $\mu_i N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ &
+    jsparbphiexbi\_tt   & $\mu_i N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ \\
+\bottomrule
+\end{longtable}
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 
@@ -1313,99 +1417,36 @@ induction        & Dataset & 4 (time, z, y, x) & parallel vector potential $A_\p
 X\_2d            & Dataset & 3 (time,y,x) & Selected plane $X(\varphi=0)$ \\
 X\_ta2d          & Dataset & 3 (time,y,x) & Toroidal average $\PA{ X }$
 Eq.~\eqref{eq:phi_average} \\
-Y\_tt\_2d        & Dataset & 3 (time,y,x) & Time integrated (between two outputs) selected plane
+Y\_tt\_2d        & Dataset & 3 (time,y,x) & Time integrated (between two outputs, Simpson's rule) selected plane
 $\int_{t_0}^{t_1}\d t Y(\varphi=0) $
 where $t_1 - t_0 = ${\tt dt*inner\_loop*itstp} and {\tt itstp} is the number of discretization points\\
-Y\_tt\_ta2d      & Dataset & 3 (time,y,x) & Time integrated (between two outputs) toroidal average (Eq.~\eqref{eq:phi_average})
+Y\_tt\_ta2d      & Dataset & 3 (time,y,x) & Time integrated (between two outputs, Simpson's rule) toroidal average (Eq.~\eqref{eq:phi_average})
 $\int_{t_0}^{t_1}\d t \PA{ Y }$
 where $t_1 - t_0 = ${\tt dt*inner\_loop*itstp} and {\tt itstp} is the number of discretization points\\
 \bottomrule
 \end{longtable}
 where
-X $\in$
+X and Y\_tt represent the quantities described in previous sections. Additionally we output the miscellaneous quantities
 \begin{longtable}{llll}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} & \textbf{Name} &  \textbf{Equation}\\
 \midrule
-    electrons &$n_e$ &
-    ions &$N_i$ \\
-    Ue &$u_e$ &
-    Ui &$U_i$ \\
-    potential &$\phi$ &
-    psi &$\psi$ \\
-    induction &$A_\parallel$ &
-    vorticity &$-\Delta_\perp\phi$ \\
+    vorticity &$-\Delta_\perp\phi$ &
+    apar\_vorticity &$-\Delta_\perp A_\parallel$ \\
     dssue & $\npar^2 u_e$&
     dppue & $\partial_\varphi^2 u_e$\\
     dpue2 & $(\partial_\varphi u_e)^2$&
-    apar\_vorticity &$-\Delta_\perp A_\parallel$ \\
-    neue &$n_e u_e$ &
-    niui &$N_i U_i$ \\
-    neuebphi &$n_eu_eb_\varphi$ &
-    niuibphi &$N_iU_ib_\varphi$ \\
-    lperpinv &$L_\perp^{-1} := |\vec\np n_e|/n_e$ &
-    perpaligned &$(\vec\np n_e)^2/n_e$ \\
-    lparallelinv &$L_\parallel^{-1} := |\npar n_e|/n_e$ &
-    aligned &$ (\npar n_e)^2/n_e$ \\
-    ne2 & $n_e^2$ &
-    phi2 & $\phi^2$ \\
-    nephi & $n_e\phi$ &
-    sne & $S_{n_e}$ \\
-    see & $z_e(\tau_e (1+\ln n_e) + \phi + \frac{1}{2}\mu_e u_e^2) S_{n_e} $ &
-    sei & $z_i(\tau_i (1+\ln N_i) + \psi + \frac{1}{2}\mu_i U_i^2) S_{N_i} $ \\
-    nelnne &$ z_e\tau_e n_e \ln n_e$ &
-    nilnni &$ z_i\tau_i N_i \ln N_i$ \\
-    aperp2 &$ (\np A_\parallel)^2/2/\beta$ &
-    ue2   &$z_i\mu_i N_i u_E^2 /2$ \\
-    neue2 &$ z_e\mu_e n_e u_e^2/2$ &
-    niui2 &$ z_i\mu_i N_i U_i^2/2$ \\
-    oexbe &$\mu_i n_e \vn\psi_p\cn\phi/B^2$ &
-    oexbi &$\mu_i N_i \vn\psi_p\cn\phi/B^2$ \\
-    odiae &$\mu_i \tau_i\vn\psi_p\cn n_e$ &
-    odiai &$\mu_i \tau_i\vn\psi_p\cn N_i$ \\
-\bottomrule
-\end{longtable}
-and the time-averaged (the time integrals are computed with the help of Simpson's rule) quantities Y $\in$
-\begin{longtable}{ll}
-\toprule
-\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}\\
-\midrule
-    jsne &$ n_e (\vec v_E + \vec v_K + \vec v_C )\cn \psi_p$ \\
-    jsneA &$ n_e u_e \vec{ b}_\perp  \cn \psi_p$ \\
-    lneperp &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ \\
-    lneparallel &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
-    resistivity &-$\eta_\parallel n_e^2 (U_i-u_e)^2$ \\
-    jsee &$z_e(\tau_e \ln n_e + \mu_e u_e^2/2 + \phi)n_e(\vec v_E + \vec v_C + \vec v_K)\cn \psi_p
-        + z_e \tau_e n_e u_e^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
-    jsei &$z_i(\tau_i \ln N_i + \mu_i U_i^2/2 + \psi_i)N_i(\vec v_E^i + \vec v_C + \vec v_K)\cn \psi_p
-        + z_i \tau_i N_i U_i^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
-    jseea &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec { b}_\perp\cn \psi_p
-        + z_e \tau_e n_e u_e \vec{ b}_\perp \cn \psi_p $ \\
-    jseia &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec { b}_\perp\cn \psi_p
-        + z_i \tau_i N_i U_i \vec{ b}_\perp \cn \psi_p $ \\
-    leeperp &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_e \nu_\perp \Delta_\perp u_e$ \\
-    leiperp &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_i \nu_\perp \Delta_\perp U_i$ \\
-    leeparallel &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_e \nu_\parallel \Delta_\parallel u_e$ \\
-    leiparallel &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_i \nu_\parallel \Delta_\parallel U_i$ \\
-    jsoexbi &$(\mu_i N_i \vn\psi_p\cn\phi/B^2) (\bhat\times\vn\phi)\cn \psi_p/B$ \\
-    jsoexbe &$(\mu_i n_e \vn\psi_p\cn\phi/B^2) (\bhat\times\vn\phi)\cn \psi_p/B$ \\
-    jsodiaiUE &$(\mu_i \tau_i\vn\psi_p\cn N_i) (\bhat\times\vn\phi)\cn \psi_p/B$ \\
-    jsodiaeUE &$(\mu_i \tau_i\vn\psi_p\cn n_e) (\bhat\times\vn\phi)\cn \psi_p/B$ \\
-    jsoexbiUD &$(\mu_i\tau_i \vn\psi_p\cn\phi/B^2) (\bhat\times\vn N_i)\cn \psi_p/B$ \\
-    jsoexbeUD &$(\mu_i\tau_i \vn\psi_p\cn\phi/B^2) (\bhat\times\vn n_e)\cn \psi_p/B$ \\
-    jsoapar &$ \vn\psi_p\cn A_\parallel \bhat\times\vn A_\parallel\cn \psi_p/B/\beta$ \\
-    socurve &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
-    socurvi &$z_i\tau_i N_i \mathcal K(\psi_p)$ \\
-    socurvkappae &$z_e\mu_e n_eu_e^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
-    socurvkappai &$z_i\mu_i N_iU_i^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
-    jsparexbi       & $N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ \\
-    jsparbphiexbi   & $N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ \\
-    sparmirrore & $-z_e\tau_en_e\npar \ln B$ \\
-    sparmirrori & $-z_i\tau_iN_i\npar \ln B$ \\
+    lperpinv &$L_\perp^{-1} := |\vec\np n_e|/n_e$ \\
+    perpaligned &$(\vec\np n_e)^2/n_e$ &
+    lparallelinv &$L_\parallel^{-1} := |\npar n_e|/n_e$ \\
+    aligned &$ (\npar n_e)^2/n_e$ &
+    ne2 & $n_e^2$ \\
+    phi2 & $\phi^2$ &
+    nephi & $n_e\phi$ \\
 \bottomrule
 \end{longtable}
-Even though this might look like a lot of things to compute the actual time spent on diagnostics is negligible if {\tt inner\_loop} parameter is greater than 1. Also
-remember that these are all two-dimensional fields, which take up much less disk-space than three-dimensional fields.
+The computation time spent on diagnostics is negligible if {\tt inner\_loop} parameter is greater than 1. Also
+remember that the X and Y fields are all two-dimensional, which takes up much less disk-space than three-dimensional fields.
 \subsection{Restart file} \label{sec:restart_file}
 The program \texttt{feltor\_hpc.cu} has the possibility to initialize time and the fields with
 the results of a previous simulation. In particular, this feature is motivated by chain jobs on a cluster
-- 
GitLab


From 95dce09b53cf64bef15bb9cc97ec9dff74ef08df Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 19 Feb 2020 21:40:59 +0100
Subject: [PATCH 197/540] Update feltor docu with zonal flow energy

---
 doc/related_pages/newcommands.tex |  2 +-
 src/feltor/feltor.tex             | 63 +++++++++++++++++++++++--------
 2 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/doc/related_pages/newcommands.tex b/doc/related_pages/newcommands.tex
index ee579c58a..1cc99ef18 100644
--- a/doc/related_pages/newcommands.tex
+++ b/doc/related_pages/newcommands.tex
@@ -19,7 +19,7 @@
 %Averages
 \newcommand{\RA}[1]{\left \langle #1 \right \rangle} %Reynolds (flux-surface) average
 \newcommand{\RF}[1]{\widetilde{#1}} %Reynolds fluctuation
-\newcommand{\FA}[1]{\left\llbracket #1 \right\rrbracket} %Favre average
+\newcommand{\FA}[1]{\left[\left[ #1 \right]\right]} %Favre average
 \newcommand{\FF}[1]{\widehat{#1}} %Favre fluctuation
 \newcommand{\PA}[1]{\left \langle #1 \right\rangle_\varphi} %Phi average
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 8d10d7aed..6ddbc15c4 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -26,6 +26,7 @@ analyses the magnetic field geometry.
 file(s) of \texttt{feltor\_hpc.cu}.
 
 \end{abstract}
+\tableofcontents
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The magnetic field}\label{sec:magnetic}
@@ -189,7 +190,7 @@ and represent a solution to Equation~\eqref{eq:GSEdimless} as~\cite{Cerfon2010}
 \label{eq:solovev}
 \begin{align}
  \psi_p (R,Z) &= \mathcal P_{\psi} R_0 \left[ A\left( \frac{1}{2} \bar{R}^2 \ln{\bar{R}}
-   - \frac{1}{8}\bar{R}^4\right)+ \frac{1}{8}\bar{R}^4 
+   - \frac{1}{8}\bar{R}^4\right)+ \frac{1}{8}\bar{R}^4
    + \sum_{i=1}^{12} c_{i}  \bar{\psi}_{pi}\right],\\
    I(\psi_p) &= \mathcal P_I\sqrt{ - 2A\psi_p/(R_0\mathcal P_{\psi}) +1},
 \end{align}
@@ -944,7 +945,7 @@ such that we can define the diamagnetic flux in the particle flux since
 the rotation vanishes under the divergence.
 
 Let us here also derive the particle flux \eqref{eq:particle_flux} through a flux surface
-\begin{align} \label{eq:particle_flux}
+\begin{align} \label{eq:radial_particle_flux}
  \vec j_{N}\cn v %=& N\left( \vec u_E + \vec u_C + \vec u_{\vn
  %B} + U \left(\bhat + {\vec b}_\perp\right)\right) \cn \psi_p \nonumber\\
  =&
@@ -1047,16 +1048,15 @@ The relevant terms in the output file are
 \bottomrule
 \end{longtable}
 
-\subsection{ Vorticity and momentum balance equation}
-%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Vorticity and toroidal ExB angular momentum equation} \label{sec:vorticity_eq}
 We write the vorticity equation (in the LWL)
 \begin{align}
-&\partial_t \RA{\Omega} + \frac{\partial}{\partial v}\frac{\d v}{\d\psi_p}\RA{\vec j_\Omega\cn\psi_p} = -\RA{F_{L,\varphi}} \label{eq:vorticity_average} \\
-\Omega &:= \mu_i N_i \left(\frac{\vn\phi\cn\psi_p}{B^2} + \tau_i \vn\ln N_i\cn\psi_p\right) \equiv \mu_i N_i(u_{E,\varphi} + u_{D,\varphi}) \\
+    &\partial_t \RA{\Omega} + \frac{\partial}{\partial v}\frac{\d v}{\d\psi_p}\RA{\vec j_\Omega\cn\psi_p} = -\RA{F_{L,\varphi}} + \RA{\mathcal S_\Omega} \label{eq:vorticity_average} \\
+\Omega &:= \mu_i N_i \left(\frac{\vn\psi_p\cn\phi}{B^2} + \tau_i \vn\ln N_i\cn\psi_p\right) \equiv \mu_i N_i(u_{E,\varphi} + u_{D,\varphi}) \\
 \vec j_{\Omega} &:= \Omega \vec u_E
-    - \left(\frac{1}{\beta} \vn A_\parallel\cn\psi_p +\frac{1}{2} \vn\tau_i N_iU_i\cn\psi_p\right)\frac{\bhat\times\vn A_\parallel}{B} \\
-    F_{L,\varphi} &:=  -(z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) - (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p)
+    - \left(\frac{1}{\beta} \vn\psi_p \cn A_\parallel +\frac{1}{2}\tau_i \vn\psi_p\cn  (N_iU_i)\right)\frac{\bhat\times\vn A_\parallel}{B} \\
+    F_{L,\varphi} &:=  -(z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) - (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p) \\
+    \mathcal S_\Omega &:= \mu_i S_{n_e} \frac{\vn\psi_p\cn \phi}{B^2} + \mu_i\tau_i\vn\psi_p\cn S_{n_e} \label{eq:em_source}
 \end{align}
 Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB velocity
 \begin{align}
@@ -1064,14 +1064,11 @@ Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continui
 \Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \equiv \mu_i N_i u_{E,\varphi} \\
 \vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
     - \frac{1}{\beta} \vn A_\parallel\cn\psi_p \left(\frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
-    \mathcal S_{\Omega_E} &:= \mu_i S_{n_e} \frac{\vn\phi\cn\psi_p}{B^2} \quad
-    \Lambda_{\Omega_E} := \mu_i \Lambda_{n_e}\frac{\vn\phi\cn\psi_p}{B^2}
+    \mathcal S_{\Omega_E} &:= \mu_i S_{n_e} \frac{\vn\psi_p\cn\phi}{B^2} \quad
+    \Lambda_{\Omega_E} := \mu_i \Lambda_{n_e}\frac{\vn\psi_p\cn\phi}{B^2}
 \end{align}
 where here we also monitor the source and diffusion terms.
 In the output file we have
-\begingroup
-%\setlength{\tabcolsep}{10pt} % Default value: 6pt
-\renewcommand{\arraystretch}{1.5} % Default value: 1
 \begin{longtable}{llll}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation}&
@@ -1092,10 +1089,11 @@ In the output file we have
     socurvi\_tt &$z_i\tau_i N_i \mathcal K(\psi_p)$ &
     socurvkappae\_tt &$z_e\mu_e n_eu_e^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
     socurvkappai\_tt &$z_i\mu_i N_iU_i^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ & \\
-    sosne\_tt & $\mu_i S_{n_e} \vn\psi_p\cn\phi/B^2$ & \\
+    sosne\_tt & $\mu_i S_{n_e} \vn\psi_p\cn\phi/B^2$ &
+    sospi\_tt & $\mu_i \tau_i \vn\psi_p \cn S_{n_e}$\\
+    loe\_tt & $ \mu_i \Lambda_{n_e} \vn\psi_p\cn\phi/B^2$ & \\
 \bottomrule
 \end{longtable}
-\endgroup
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Parallel momentum balance}
@@ -1132,12 +1130,46 @@ The relevant terms in the output file are (the Lorentz force term is described i
     sparmirrori\_tt & $-z_i\tau_iN_i\npar \ln B$ \\
     jsparexbi\_tt       & $\mu_i N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ &
     jsparbphiexbi\_tt   & $\mu_i N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    sparsni\_tt & $\mu_i S_{N_i} U_i$ &
+    sparsnibphi\_tt & $\mu_i S_{N_i} U_ib_\varphi $ \\
 \bottomrule
 \end{longtable}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 
+\subsubsection{Zonal flow energy}
+\begin{align}
+    E_\mathrm{zonal} = \frac{1}{2}\RA{\rho_M}\FA{ \iota^{-2}\mathcal I^{\vartheta\vartheta} + 2\iota^{-1}\mathcal I^{\vartheta\varphi} + \mathcal I^{\varphi\varphi}} \FA{u_{E,\varphi}}^2
+    \equiv \frac{1}{2} \RA{\rho_M} \FA{u_{E,\varphi}}^2  \FA{\mathcal I_0}
+    \label{eq:zonal_energy}
+\end{align}
+For symmetry flux coordinates we have $g_{\vartheta\vartheta} = R^2 (\vn\psi_p)^2/I^2\iota^2$, $g_{\varphi\vartheta} =0$ and $g_{\varphi\varphi}=R^2$ and thus $\mathcal I_0 = R^{-2}( 1 + I^2/|\vn\psi_p|^2)= B^2 / |\vn\psi_p|^2$.
+\begin{align}\label{eq:perp_kinetic}
+      \frac{\partial}{\partial t}E_{\mathrm{zonal}} +\frac{\partial}{\partial v } \left(E_{\mathrm{zonal}}\FA{u^v} \right)
+  =&-\FA{\mathcal I_0}\FA{u_{E,\varphi}}\left(\frac{\partial }{\partial v}  \Theta_{\varphi}^{\; v} + \RA{(\vec j_f\times\vec B)_\varphi}\right)
+  \nonumber\\
+    &-\frac{1}{2}\FA{u_{E,\varphi}}^2\frac{\partial}{\partial v}\left(\RA{\rho_M}\FA{\FF{\mathcal I_0}\FF{u^v}}\right)
+     + \mathcal S_{\mathrm{zonal}}
+\end{align}
+where we neglected the term $\RA{n\vec u\cn \mathcal I_0}$ in the continuity equation as small in our ordering
+ and we have
+ \begin{align}
+ \mathcal S_{\mathrm{zonal}} :=& \FA{\mathcal I_0} \FA{u_{E,\varphi}} \mathcal S_{u_{E,\varphi}} + \frac{1}{2}\FA{u_{E,\varphi}}^2  \RA{mS_n\mathcal I_0}
+%  \nonumber\\
+ \label{eq:zonal_source}
+ \end{align}
+ and in the output file
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} &
+\textbf{Name} &  \textbf{Equation}\\
+\midrule
+    nei0 &$n_e \mathcal I_0$ &
+    snei0 & $S_{n_e } \mathcal I_0$ \\
+\bottomrule
+\end{longtable}
+
 \subsection{Manufactured Solution}
 In order to test the implementation we manufacture a solution to Eqs.~\eqref{eq:Egyrofluid} and \eqref{eq:elliptic} of the form
 \begin{align*}
@@ -1466,7 +1498,6 @@ the whole simulation is lost. It is safer to just merge files afterwards with fo
 \texttt{ncrcat output1.nc output2.nc output.nc}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Diagnostics}\label{sec:diagnostics}
-\subsection{Program and files}
 We have the program \texttt{feltor/diag/feltordiag.cu}.
 This program reads one or more previously generated simulation file(s) \texttt{input0.nc ... inputN.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows. \\
 Compilation\\
-- 
GitLab


From d5c7714d8e73141935384f4c949191968d917a95 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 19 Feb 2020 23:21:04 +0100
Subject: [PATCH 198/540] Added missing terms in feltor diagnostics

required a slight restructure of the Variables and an additional grid
and compute_gradS method in feltor
---
 inc/dg/multigrid.h              |   2 +-
 inc/geometries/geometry_diag.cu |  21 +---
 inc/geometries/magnetic_field.h |  15 +++
 src/feltor/feltor.cu            |   2 +-
 src/feltor/feltor.h             |   8 ++
 src/feltor/feltor.tex           |   6 +-
 src/feltor/feltor_hpc.cu        |   6 +-
 src/feltor/feltordiag.h         | 197 +++++++++++++++++++++-----------
 8 files changed, 166 insertions(+), 91 deletions(-)

diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index c4ea2a0e0..5767fadb9 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -208,7 +208,7 @@ struct MultigridCG2d
     unsigned num_stages()const{return m_stages;}
 
     ///observe the grids at all stages
-    ///@param stage must fulfill \c 0<stage<stages()
+    ///@param stage must fulfill \c 0 <= stage < stages()
     const Geometry& grid( unsigned stage) const {
         return *(m_grids[stage]);
     }
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 38f5f193d..f8bdfaad4 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -86,21 +86,6 @@ struct IPhi
     private:
     double R_0, A;
 };
-//The newly derived h02 factor
-struct Hoo : public dg::geo::aCylindricalFunctor<Hoo>
-{
-    Hoo( dg::geo::TokamakMagneticField mag): mag_(mag){}
-    double do_compute( double R, double Z) const
-    {
-        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
-        double psip2 = psipR*psipR+psipZ*psipZ;
-        if( psip2 == 0)
-            psip2 = 1e-16;
-        return (ipol*ipol + psip2)/R/R/psip2;
-    }
-    private:
-    dg::geo::TokamakMagneticField mag_;
-};
 
 int main( int argc, char* argv[])
 {
@@ -270,7 +255,7 @@ int main( int argc, char* argv[])
         {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
         {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
             M_PI, p.sigma, p.sigma, p.sigma, p.amp)},
-        { "Hoo", "The novel h02 factor", Hoo( mag) }
+        { "Hoo", "The novel h02 factor", dg::geo::Hoo( mag) }
 
     };
     dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
@@ -367,7 +352,7 @@ int main( int argc, char* argv[])
         map1d.emplace_back( "X_gradPsip_fsa", X_gradPsip_fsa,
             "Flux surface average of |Grad Psip|");
         // h02 factor
-        dg::HVec h02 = dg::pullback( Hoo(mag), gX2d), X_h02_fsa;
+        dg::HVec h02 = dg::pullback( dg::geo::Hoo(mag), gX2d), X_h02_fsa;
         dg::blas1::pointwiseDot( volX2d, h02, h02);
         avg_eta( h02, X_h02_fsa, false);
         dg::blas1::scal( X_h02_fsa, 4*M_PI*M_PI); //
@@ -455,7 +440,7 @@ int main( int argc, char* argv[])
         map1d.emplace_back( "psi_area", psi_area,
             "Flux area with delta function");
         // h02 factor
-        dg::HVec h02 = dg::evaluate( Hoo(mag), grid2d);
+        dg::HVec h02 = dg::evaluate( dg::geo::Hoo(mag), grid2d);
         fsa.set_container( h02);
         map1d.emplace_back( "hoo_fsa", dg::evaluate( fsa, grid1d),
            "Flux surface average of novel h02 factor with delta function");
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 2cabc92e4..6650eb429 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -678,6 +678,21 @@ struct GradPsip: public aCylindricalFunctor<GradPsip>
     TokamakMagneticField m_mag;
 
 };
+///@brief Inertia factor \f$ \mathcal I_0 \f$
+struct Hoo : public dg::geo::aCylindricalFunctor<Hoo>
+{
+    Hoo( dg::geo::TokamakMagneticField mag): mag_(mag){}
+    double do_compute( double R, double Z) const
+    {
+        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
+        double psip2 = psipR*psipR+psipZ*psipZ;
+        if( psip2 == 0)
+            psip2 = 1e-16;
+        return (ipol*ipol + psip2)/R/R/psip2;
+    }
+    private:
+    dg::geo::TokamakMagneticField mag_;
+};
 ///@}
 
 } //namespace geo
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index fc0e0ee56..dc928a632 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -76,7 +76,7 @@ int main( int argc, char* argv[])
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
     gradPsip[2] =  result; //zero
     feltor::Variables var = {
-        feltor, p, gradPsip, gradPsip
+        feltor, p,gp,mag, gradPsip, gradPsip
     };
 
 
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 462ea2298..32f431f71 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -199,6 +199,9 @@ struct Explicit
     }
 
     /// ///////////////////DIAGNOSTIC MEMBERS //////////////////////
+    const Geometry& grid() const {
+        return m_multigrid.grid(0);
+    }
     //potential[0]: electron potential, potential[1]: ion potential
     const Container& uE2() const {
         return m_UE2;
@@ -227,6 +230,11 @@ struct Explicit
     const std::array<Container, 3> & gradA () const {
         return m_dA;
     }
+    void compute_gradS( int i, std::array<Container,3>& gradS) const{
+        dg::blas2::symv( m_dx_N, m_s[0][i], gradS[0]);
+        dg::blas2::symv( m_dy_N, m_s[1][i], gradS[1]);
+        if(!m_p.symmetric)dg::blas2::symv( m_dz, m_s[2][i], gradS[2]);
+    }
     const Container & dsN (int i) const {
         return m_dsN[i];
     }
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 6ddbc15c4..a3d82f99b 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1091,7 +1091,7 @@ In the output file we have
     socurvkappai\_tt &$z_i\mu_i N_iU_i^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ & \\
     sosne\_tt & $\mu_i S_{n_e} \vn\psi_p\cn\phi/B^2$ &
     sospi\_tt & $\mu_i \tau_i \vn\psi_p \cn S_{n_e}$\\
-    loe\_tt & $ \mu_i \Lambda_{n_e} \vn\psi_p\cn\phi/B^2$ & \\
+    loexbe\_tt & $ \mu_i \Lambda_{n_e} \vn\psi_p\cn\phi/B^2$ & \\
 \bottomrule
 \end{longtable}
 
@@ -1436,7 +1436,7 @@ yc           & Dataset & 3 (z,y,x) & Cartesian y-coordinate $y=R\cos(\varphi)$\\
 zc           & Dataset & 3 (z,y,x) & Cartesian z-coordinate $z=Z$ \\
 Psip             & Dataset & 3 (z,y,x) & Flux function $\psi_p(R,Z)$ \\
 Nprof            & Dataset & 3 (z,y,x) & Density profile $n_\text{prof}$ used in the forcing source \\
-Source           & Dataset & 3 (z,y,x) & Source  profile $\Theta_\alpha(\rho(R,Z) - \rho_s) H(Z-Z_X)$\\
+Source           & Dataset & 3 (z,y,x) & Source profile $S_{prof}$\\
 BR               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^R$ \\
 BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^Z$ \\
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
@@ -1458,7 +1458,7 @@ where $t_1 - t_0 = ${\tt dt*inner\_loop*itstp} and {\tt itstp} is the number of
 \bottomrule
 \end{longtable}
 where
-X and Y\_tt represent the quantities described in previous sections. Additionally we output the miscellaneous quantities
+X and Y\_tt represent the quantities described in the tables in previous sections and the miscellaneous quantities
 \begin{longtable}{llll}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} & \textbf{Name} &  \textbf{Equation}\\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index f71be7e46..6b666a338 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -178,7 +178,7 @@ int main( int argc, char* argv[])
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
     gradPsip[2] =  resultD; //zero
     feltor::Variables var = {
-        feltor, p, gradPsip, gradPsip
+        feltor, p, gp, mag, gradPsip, gradPsip
     };
     // the vector ids
     std::map<std::string, int> id3d, id4d, restart_ids;
@@ -263,7 +263,7 @@ int main( int argc, char* argv[])
             "long_name", record.long_name.size(), record.long_name.data());
         MPI_OUT err = nc_enddef( ncid);
         MPI_OUT std::cout << "Computing "<<record.name<<"\n";
-        record.function( resultH, var, grid, gp, mag);
+        record.function( resultH, var, grid);
         dg::blas2::symv( projectH, resultH, transferH);
         file::put_var_double( ncid, vecID, g3d_out, transferH);
         MPI_OUT err = nc_redef(ncid);
@@ -278,7 +278,7 @@ int main( int argc, char* argv[])
             "long_name", record.long_name.size(), record.long_name.data());
         MPI_OUT err = nc_enddef( ncid);
         MPI_OUT std::cout << "Computing2d "<<record.name<<"\n";
-        record.function( resultH, var, grid, gp, mag);
+        record.function( resultH, var, grid);
         dg::blas2::symv( projectH, resultH, transferH);
         if(write2d)file::put_var_double( ncid, vecID, *g2d_out_ptr, transferH);
         MPI_OUT err = nc_redef(ncid);
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 196f46b34..73f5a1f88 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -150,6 +150,8 @@ void jacobian(
 struct Variables{
     feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
     feltor::Parameters p;
+    dg::geo::solovev::Parameters gp;
+    dg::geo::TokamakMagneticField mag;
     std::array<DVec, 3> gradPsip;
     std::array<DVec, 3> tmp;
 };
@@ -164,7 +166,7 @@ struct Record{
 struct Record_static{
     std::string name;
     std::string long_name;
-    std::function<void( HVec&, Variables&, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag)> function;
+    std::function<void( HVec&, Variables&, Geometry& grid)> function;
 };
 
 ///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -173,46 +175,46 @@ struct Record_static{
 //Here is a list of static (time-independent) 3d variables that go into the output
 std::vector<Record_static> diagnostics3d_static_list = {
     { "BR", "R-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            dg::geo::BFieldR fieldR(mag);
+        []( HVec& result, Variables& v, Geometry& grid){
+            dg::geo::BFieldR fieldR(v.mag);
             result = dg::pullback( fieldR, grid);
         }
     },
     { "BZ", "Z-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            dg::geo::BFieldZ fieldZ(mag);
+        []( HVec& result, Variables& v, Geometry& grid){
+            dg::geo::BFieldZ fieldZ(v.mag);
             result = dg::pullback( fieldZ, grid);
         }
     },
     { "BP", "Contravariant P-component of magnetic field in cylindrical coordinates",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            dg::geo::BFieldP fieldP(mag);
+        []( HVec& result, Variables& v, Geometry& grid){
+            dg::geo::BFieldP fieldP(v.mag);
             result = dg::pullback( fieldP, grid);
         }
     },
     { "Psip", "Flux-function psi",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-             result = dg::pullback( mag.psip(), grid);
+        []( HVec& result, Variables& v, Geometry& grid){
+             result = dg::pullback( v.mag.psip(), grid);
         }
     },
     { "Nprof", "Density profile (that the source may force)",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid ){
             result = dg::evaluate( dg::zero, grid);
             bool fixed_profile;
             HVec source = feltor::source_profiles.at(v.p.source_type)(
-                fixed_profile, result, grid, v.p, gp, mag);
+                fixed_profile, result, grid, v.p, v.gp, v.mag);
         }
     },
     { "Source", "Source region",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid ){
             bool fixed_profile;
             HVec profile;
             result = feltor::source_profiles.at(v.p.source_type)(
-                fixed_profile, profile, grid, v.p, gp, mag);
+                fixed_profile, profile, grid, v.p, v.gp, v.mag);
         }
     },
     { "xc", "x-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -221,7 +223,7 @@ std::vector<Record_static> diagnostics3d_static_list = {
         }
     },
     { "yc", "y-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -230,7 +232,7 @@ std::vector<Record_static> diagnostics3d_static_list = {
         }
     },
     { "zc", "z-coordinate in Cartesian coordinate system",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid ){
             HVec xc = dg::evaluate( dg::cooX3d, grid);
             HVec yc = dg::evaluate( dg::cooY3d, grid);
             HVec zc = dg::evaluate( dg::cooZ3d, grid);
@@ -289,70 +291,70 @@ std::vector<Record> diagnostics3d_list = {
 };
 
 //Here is a list of static (time-independent) 2d variables that go into the output
-//( we make 3d variables here that but only the first 2d slice is output)
+//( we make 3d variables here but only the first 2d slice is output)
 std::vector<Record_static> diagnostics2d_static_list = {
     { "Psip2d", "Flux-function psi",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            result = dg::pullback( mag.psip(), grid);
+        []( HVec& result, Variables& v, Geometry& grid ){
+            result = dg::pullback( v.mag.psip(), grid);
         }
     },
     { "Ipol", "Poloidal current",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            result = dg::pullback( mag.ipol(), grid);
+        []( HVec& result, Variables& v, Geometry& grid ){
+            result = dg::pullback( v.mag.ipol(), grid);
         }
     },
     { "Bmodule", "Magnetic field strength",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
-            result = dg::pullback( dg::geo::Bmodule(mag), grid);
+        []( HVec& result, Variables& v, Geometry& grid ){
+            result = dg::pullback( dg::geo::Bmodule(v.mag), grid);
         }
     },
     { "Divb", "The divergence of the magnetic unit vector",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid ){
             result = v.f.divb();
         }
     },
     { "InvB", "Inverse of Bmodule",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid ){
             result = v.f.binv();
         }
     },
     { "CurvatureKappaR", "R-component of the Kappa B curvature vector",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid ){
             result = v.f.curvKappa()[0];
         }
     },
     { "CurvatureKappaZ", "Z-component of the Kappa B curvature vector",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid){
             result = v.f.curvKappa()[1];
         }
     },
     { "CurvatureKappaP", "Contravariant Phi-component of the Kappa B curvature vector",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid){
             result = v.f.curvKappa()[2];
         }
     },
     { "DivCurvatureKappa", "Divergence of the Kappa B curvature vector",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid){
             result = v.f.divCurvKappa();
         }
     },
     { "CurvatureR", "R-component of the curvature vector",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid){
             result = v.f.curv()[0];
         }
     },
     { "CurvatureZ", "Z-component of the full curvature vector",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid){
             result = v.f.curv()[1];
         }
     },
     { "CurvatureP", "Contravariant Phi-component of the full curvature vector",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid){
             result = v.f.curv()[2];
         }
     },
     { "bphi", "Contravariant Phi-component of the magnetic unit vector",
-        []( HVec& result, Variables& v, Geometry& grid, const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag ){
+        []( HVec& result, Variables& v, Geometry& grid){
             result = v.f.bphi();
         }
     }
@@ -394,11 +396,17 @@ std::vector<Record> diagnostics2d_list = {
              dg::blas1::copy(v.f.induction(), result);
         }
     },
+    /// -----------------Miscellaneous additions --------------------//
     {"vorticity", "Minus Lap_perp of electric potential", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.lapMperpP(0), result);
         }
     },
+    {"apar_vorticity", "Minus Lap_perp of magnetic potential", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.lapMperpA(), result);
+        }
+    },
     {"dssue", "2nd parallel derivative of electron velocity", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.dssU(0), result);
@@ -414,35 +422,6 @@ std::vector<Record> diagnostics2d_list = {
              dg::blas1::pointwiseDot(v.f.gradU(0)[2], v.f.gradU(0)[2], result);
         }
     },
-    {"apar_vorticity", "Minus Lap_perp of magnetic potential", false,
-        []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.lapMperpA(), result);
-        }
-    },
-    {"neue", "Product of electron density and velocity", false,
-        []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot(
-                v.f.density(0), v.f.velocity(0), result);
-        }
-    },
-    {"niui", "Product of ion gyrocentre density and velocity", false,
-        []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot(
-                v.f.density(1), v.f.velocity(1), result);
-        }
-    },
-    {"neuebphi", "Product of neue and covariant phi component of magnetic field unit vector", false,
-        []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( 1.,
-                v.f.density(0), v.f.velocity(0), v.f.bphi(), 0., result);
-        }
-    },
-    {"niuibphi", "Product of NiUi and covariant phi component of magnetic field unit vector", false,
-        []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( 1.,
-                v.f.density(1), v.f.velocity(1), v.f.bphi(), 0., result);
-        }
-    },
     {"lperpinv", "Perpendicular density gradient length scale", false,
         []( DVec& result, Variables& v ) {
             const std::array<DVec, 3>& dN = v.f.gradN(0);
@@ -522,6 +501,13 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
+    {"jsneE_tt", "Radial electron particle flux: ExB contribution (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
+            dg::blas1::pointwiseDot( result, v.f.density(0), result);
+        }
+    },
     {"lneperp_tt", "Perpendicular electron diffusion (Time average)", true,
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), v.tmp[0], result);
@@ -848,7 +834,62 @@ std::vector<Record> diagnostics2d_list = {
             }
         }
     },
-    /// --------------------- Parallel momentum flux terms ---------------------------//
+    {"sosne_tt", "ExB vorticity source term with electron source", true,
+        []( DVec& result, Variables& v){
+            routines::dot( v.f.gradP(0), v.gradPsip, result);
+            dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
+            dg::blas1::pointwiseDot( v.p.mu[1], result, v.f.sources()[0][0], 0., result);
+        }
+    },
+    {"sospi_tt", "Diamagnetic vorticity source term with electron source", true,
+        []( DVec& result, Variables& v){
+            v.f.compute_gradS( 0, v.tmp);
+            routines::dot( v.tmp, v.gradPsip, result);
+            dg::blas1::scal( result, v.p.mu[1]*v.p.tau[1]);
+        }
+    },
+    {"loexbe_tt", "Vorticity dissipation term with electron Lambda", true,
+        []( DVec& result, Variables& v){
+            routines::dot( v.f.gradP(0), v.gradPsip, result);
+            dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
+
+            v.f.compute_diffusive_lapMperpN( v.f.density(0), v.tmp[0], v.tmp[1]);
+            dg::blas1::scal( v.tmp[1], -v.p.nu_perp);
+            dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(0),
+                                     0., v.tmp[2]);
+            dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., v.tmp[2]);
+            dg::blas1::axpby( 1., v.tmp[1], 1., v.tmp[2]); //Lambda_ne
+            dg::blas1::pointwiseDot( v.tmp[2], result, result);
+
+            dg::blas1::scal( result, v.p.mu[1]);
+        }
+    },
+    ///-----------------------Parallel momentum terms ------------------------//
+    {"neue", "Product of electron density and velocity", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot(
+                v.f.density(0), v.f.velocity(0), result);
+        }
+    },
+    {"niui", "Product of ion gyrocentre density and velocity", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot(
+                v.f.density(1), v.f.velocity(1), result);
+        }
+    },
+    {"neuebphi", "Product of neue and covariant phi component of magnetic field unit vector", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 1.,
+                v.f.density(0), v.f.velocity(0), v.f.bphi(), 0., result);
+        }
+    },
+    {"niuibphi", "Product of NiUi and covariant phi component of magnetic field unit vector", false,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 1.,
+                v.f.density(1), v.f.velocity(1), v.f.bphi(), 0., result);
+        }
+    },
+    /// --------------------- Parallel momentum flux terms ---------------------//
     {"jsparexbi_tt", "Parallel momentum radial flux by ExB velocity with ion density (Time average)", true,
         []( DVec& result, Variables& v){
             // ExB Dot GradPsi
@@ -858,7 +899,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), 0., v.tmp[0]);
 
             // Multiply everything
-            dg::blas1::pointwiseDot( 1., result, v.tmp[0], 0., result);
+            dg::blas1::pointwiseDot( result, v.tmp[0], result);
         }
     },
     {"jsparbphiexbi_tt", "Parallel angular momentum radial flux by ExB velocity with ion density (Time average)", true,
@@ -873,6 +914,19 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( 1., result, v.tmp[0],v.f.bphi(), 0., result);
         }
     },
+    /// --------------------- Parallel momentum source terms ---------------------//
+    {"sparsni_tt", "Parallel momentum source by density source", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( v.p.mu[1],
+                v.f.sources()[0][1], v.f.velocity(1), 0., result);
+        }
+    },
+    {"sparsnibphi_tt", "Parallel angular momentum source by density source", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( v.p.mu[1],
+                v.f.sources()[0][1], v.f.velocity(0), v.f.bphi(), 0., result);
+        }
+    },
     /// --------------------- Mirror force term ---------------------------//
     {"sparmirrore_tt", "Mirror force term with electron density (Time average)", true,
         []( DVec& result, Variables& v){
@@ -884,7 +938,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( v.p.mu[1], v.f.divb(), v.f.density(1), 0., result);
         }
     },
-    /// --------------------- Vorticity source terms ---------------------------//
+    /// --------------------- Lorentz force terms ---------------------------//
     {"socurve_tt", "Vorticity source term electron curvature (Time average)", true,
         []( DVec& result, Variables& v) {
             routines::dot( v.f.curv(), v.gradPsip, result);
@@ -911,6 +965,19 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[0], result, 0., result);
         }
     },
+    /// --------------------- Zonal flow energy terms------------------------//
+    {"nei0", "inertial factor", false,
+        []( DVec& result, Variables& v ) {
+            result = dg::pullback( dg::geo::Hoo( v.mag), v.f.grid());
+            dg::blas1::pointwiseDot( v.f.density(0), result, result);
+        }
+    },
+    {"snei0", "inertial factor source", false,
+        []( DVec& result, Variables& v ) {
+            result = dg::pullback( dg::geo::Hoo( v.mag), v.f.grid());
+            dg::blas1::pointwiseDot( v.f.sources()[0][0], result, result);
+        }
+    },
 
 
 };
-- 
GitLab


From 921f5ab5d34d713ebe9cc1e6ff3561b7c08d4a51 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 3 Apr 2020 17:55:30 +0200
Subject: [PATCH 199/540] Thoughts on normalisation of equilibrium current

and scale invariance of feltor equations
---
 src/feltor/feltor.tex | 96 ++++++++++++++++++++++++++-----------------
 1 file changed, 58 insertions(+), 38 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index a3d82f99b..7418b7936 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -143,59 +143,61 @@ In cylindrical coordinates the general axisymmetric  magnetic field can be writt
 \end{align}
 which can obviously not be manipulated to be in Clebsch form.
 Hence we are dealing with a non-flux aligned coordinate system.
-We scaled $R$, $Z$ and $R_0$ with $\rho_s$, the magnetic field with $B_0$, the flux with $\psi_{p0} = B_0\rho_s \hat R_0$
-and $I_0 = B_0 \hat R_0$ (with $\hat R_0$ being the dimensional major radius).
-Note that this normalization is in line with the one later chosen for the gyrofluid
-equations but is unnatural for the MHD type equilibrium equations through the introduction
-of $\rho_s$.
-Also note that with a typically convex function $\psi$, $I(\psi)>0$ and the previously defined coordinate system the field line winding is a {\bf left handed screw} in the positive $\vec B$-direction.
 For the sake of clarity we define the poloidal magnetic field \( \vec{B}_p = \frac{R_0}{R}\left( \frac{\partial \psi}{\partial Z}\ehat_R - \frac{\partial \psi}{\partial R}\ehat_Z\right)
 \) and the toroidal magnetic field \(\vec{B}_t =\frac{R_0I}{R} \ehat_{\varphi}\).
-%The unit vectors are denoted by \(\ehat_{R}\), \(\ehat_{Z}\), \(\ehat_{\varphi}\).
-Note that with $\psi_p = I(\psi_p) \equiv 1$ we recover a purely toroidal field.
-With a subsequent identification of $x:=R-R_0$ and the transition to
-Cartesian coordinates while straightening the field lines for large $R_0$
-we recover the familiar slab geometry.
-
-We have the equilibrium equations in toroidally symmetric, ideal MHD (
-$\vn p = \vec j\times \vec B$ and $\vn\times\vec B = \vec j$ normalized with $p_0 = B_0^2/\mu_0$, and $j_0 = B_0/\rho_s/\mu_0$ )
+Note that with a typically convex function $\psi$, $I(\psi)>0$ and the previously defined coordinate system the field line winding is a {\bf left handed screw} in the positive $\vec B$-direction.
+
+We scaled $R$, $Z$ and $R_0$ with $\rho_s = \sqrt{T_e m_i}/(eB_0)$, the
+magnetic field with $B_0$, the poloidal flux with $\psi_{p0} = B_0\rho_s \hat
+R_0$ and the poloidal equilibrium current streamfunction with $I_0 = B_0 \hat R_0$ (with $\hat R_0 =
+\rho_s R_0$ the dimensional major radius).
+We have the equilibrium equations in toroidally symmetric, ideal MHD
+$\vn p = \vec j\times \vec B$ and $\vn\times\vec B = \beta \vec j$ normalized with $p_0 = n_0 T_0$, and $j_0 = e n_0 c_S$, where we introduce $\beta = n_0 T_0 \mu_0 /B_0^2$.
+Note that this normalization is in line with the one later chosen for the gyrofluid
+equations but is unnatural for the MHD type equilibrium equations through the introduction
+of $\rho_s$ and $\beta$.
 \begin{align}
-    \vn\times \vec B &= \frac{R_0}{R}\left[ -\Delta^*\psi_p\ehat_\varphi + I_Z \ehat_R - I_R\ehat_Z \right]\equiv \vec j\\
- j_\parallel &= \vec j\cdot \bhat = \frac{\d p}{\d\psi_p} \frac{I(\psi_p)}{B} +
+    \vn\times \vec B &= \frac{R_0}{R}\left[ -\Delta^*\psi_p\ehat_\varphi + I_Z \ehat_R - I_R\ehat_Z \right]\equiv \beta \vec j\\
+ \beta j_\parallel &= \beta \vec j\cdot \bhat = \beta \frac{\d p}{\d\psi_p} \frac{I(\psi_p)}{B} +
  \frac{\d I}{\d\psi_p} B \quad \text{  Pfirsch-Schl\"uter \& Bootstrap current } \\
- \vec j_\perp &= \bhat\times\left(\vec j\times\bhat\right)=
- \frac{\bhat \times \vn p}{B} \quad\quad\quad \text{ diamagnetic current} \\
- \vec j\times\vec B &= \frac{R_0^2}{R^2}\left[ -\Delta^* \psi_p - I
-     \frac{\d I}{\d \psi_p} \right]\vn\psi_p \equiv \frac{\d p}{\d\psi_p}\vn\psi_p =\vn p
+ \beta \vec j_\perp &= \beta \bhat\times\left(\vec j\times\bhat\right)=
+ \beta \frac{\bhat \times \vn p}{B} \quad\quad\quad \text{ diamagnetic current} \\
+ \beta \vec j\times\vec B &= \frac{R_0^2}{R^2}\left[ -\Delta^* \psi_p - I
+     \frac{\d I}{\d \psi_p} \right]\vn\psi_p \equiv \beta \frac{\d p}{\d\psi_p}\vn\psi_p =\beta \vn p
 \end{align}
 from where we recover the Grad-Shafranov equation
 \begin{align}\label{eq:GSEdimless}
-    -\Delta^*_\perp  \psi_p &= \frac{R^2}{R_0^2} \frac{d p}{d  \psi_p } + I \frac{d I}{d  \psi_p } \equiv \frac{R}{R_0} j_{\hat\varphi}
+    -\Delta^*_\perp  \psi_p &= \beta \frac{R^2}{R_0^2} \frac{d p}{d  \psi_p } + I \frac{d I}{d  \psi_p } \equiv \beta \frac{R}{R_0} j_{\hat\varphi}
 \end{align}
 with $\Delta^*_\perp \psi_p = R\partial_R (R^{-1}\psi_R) + \psi_{ZZ}$.
 The Solov'ev assumptions consist of \(A/R_0 = -I \frac{d I}{d  \psi_p }\) and \((1-A)/R_0 = -\frac{d p}{d  \psi_p }\), where \(A\) is a constant~\cite{Cerfon2010,Cerfon2014}.
 By integration over \(\psi_p\) we find
 $
- p(\psi_p) = (A-1)\psi_p/R_0$,
+ p(\psi_p) = (A-1)\psi_p/R_0/\beta$,
  $I(\psi_p) = \sqrt{-2 A \psi_p/R_0 + 1}$,
  and
-    $j_{\hat\varphi} = \left[(A-1)R^2/R_0^2 - A \right]/R $.
+    $j_{\hat\varphi} = \left[(A-1)R^2/R_0^2 - A \right]/R/\beta $.
 Note that if $\psi_p$, $I(\psi)$ and $p(\psi)$ are a solution to Eq.~\eqref{eq:GSEdimless}
 then so are $\mathcal P_\psi \psi_p$ , $\mathcal P_\psi I(\psi_p)$ and $\mathcal P_\psi^2 p(\psi_p)$.
-Also note that for $A=0$ the constant current $I$ is arbitrary $\mathcal P_I$.
+Also note that for $A=0$ the constant current $I$ becomes arbitrary $\mathcal P_I$.
 
 We introduce \(\bar{R} \equiv \frac{R}{R_0}\) and \(\bar{Z} \equiv\frac{Z}{R_0}\)
-and represent a solution to Equation~\eqref{eq:GSEdimless} as~\cite{Cerfon2010}
+and thus represent a general solution to Equation~\eqref{eq:GSEdimless} as~\cite{Cerfon2010}
 \begin{subequations}
 \label{eq:solovev}
 \begin{align}
  \psi_p (R,Z) &= \mathcal P_{\psi} R_0 \left[ A\left( \frac{1}{2} \bar{R}^2 \ln{\bar{R}}
    - \frac{1}{8}\bar{R}^4\right)+ \frac{1}{8}\bar{R}^4
    + \sum_{i=1}^{12} c_{i}  \bar{\psi}_{pi}\right],\\
-   I(\psi_p) &= \mathcal P_I\sqrt{ - 2A\psi_p/(R_0\mathcal P_{\psi}) +1},
+   I(\psi_p) &= \mathcal P_I\sqrt{ - 2A\frac{\psi_p}{R_0\mathcal P_{\psi}} +1},
 \end{align}
 \end{subequations}
-with $\mathcal P_I = \pm \mathcal P_\psi$ for $A\neq 0$, $p(\psi_p) = \mathcal P_\psi (A-1)\psi_p/R_0$ and
+with $\mathcal P_\psi$ a free constant, $\mathcal P_I = \pm \mathcal P_\psi$ for $A\neq 0$ and $\mathcal P_I$ arbitrary for $A=0$ (purely toroidal equilibrium current).
+We have
+\begin{align}
+    p(\psi_p) = \mathcal P_\psi \frac{( A-1)\psi_p}{\beta R_0 } \qquad
+    j_{\hat\varphi} = \frac{\mathcal P_\psi}{\beta } \left[\frac{(A-1)R}{R_0^2} - \frac{A}{R}\right]
+\end{align}
 \rowcolors{2}{gray!25}{white}
 \begin{longtable}{>{\RaggedRight}p{7cm}>{\RaggedRight}p{7cm}}
 \toprule
@@ -227,12 +229,19 @@ $\bar{\psi}_{p11}=3 \bar{Z}\bar{R}^4 - 4\bar{Z}^3\bar{R}^2$\\
    & \\
 \bottomrule
 \end{longtable}
-The choice of the coefficients \(c_{i}\) and \(A\) determines the actual form of the magnetic field, while $R_0$ appears as an artificial scaling factor (note here that a change in $\rho_s$ changes $R_0$ but not the form or size of the dimensional equilibrium magnetic field).
+The choice of the coefficients \(c_{i}\) and \(A\) determines the actual form
+of the magnetic field.
+Eq.~\eqref{eq:solovev} can for example represent single and asymmetric double X-point configurations, force-free states,
+field reversed configurations and low and high beta tokamak equilibria.
+$R_0$ appears as an artificial scaling factor
+(note here that a change in $\rho_s$ changes $R_0$ but not the form or size of
+the dimensional equilibrium magnetic field).
 The scaling factors $\mathcal P_\psi$ and $\mathcal P_I$ are mainly introduced to maximize the flexibility e.g. to adapt the solution to experimental equilibria or to reverse the sign of the magnetic field.
-Remember that $\mathcal  P_I \neq \mathcal P_\psi$ only yields a solution for $A=0$.
 
-Eq.~\eqref{eq:solovev} allows axisymmetric equilibria with e.g. single and asymmetric double X-point configurations, force-free states,
-field reversed configurations and low and high beta tokamak equilibria. This casts this simple analytical equilibrium to the ideal choice in order to study geometric effects (e.g. inverse aspect ratio, elongation and triangularity) in magnetised plasmas.
+Since Eq.~\eqref{eq:solovev} is given analytically we can numerically evaluate $\psi_p$ and $I$
+and all their derivatives
+at arbitrary points to machine precision, which is simple to implement and fast to execute.
+This translates to an exact representation of the magnetic field and related quantities like the curvature operators in code.
 
 Note that
 \begin{align}
@@ -565,14 +574,18 @@ We can then directly integrate the safety factor as
 \frac{\d \varphi}{\d\Theta} = \frac{B^\varphi}{B^\Theta}\\
 q\equiv\frac{1}{2\pi}\oint \frac{B^\varphi}{B^\Theta} \d\Theta
 \end{align}
-We integrate this equation with the help of one of our grid
-construction algorithms, i.e. we use a high-order Runge-Kutta method
+We integrate this equation with the help of one of our ODE integrators, i.e. we use a high-order Runge-Kutta method
 and refine the stepsize until machine-precision is reached.
-
 Notice that the safety factor diverges on the last closed flux
 surface whereas Eq.~\eqref{eq:total_flux}
 remains finite due to the $\vn\psi$ factor.
 
+We find the toroidal flux $\psi_t$ by integrating the q-profile $\psi_t = \int^{\psi_p} \d\psi_p q(\psi_p)$. Since $q$ diverges, $\psi_t$, in contrast to $\psi_p$,
+is not defined outside the last closed flux-surface. We define the normalized toroidal flux label
+$\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$, which is useful because
+equidistant $\rho_t$ values tend to translate to equidistant flux-surfaces
+in configuration space.
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
 \subsection{Conservative form}
@@ -630,7 +643,8 @@ potential $A_\parallel$.
 We have the continuity equation for the electron density \(n_e\) and the ion gyro-centre
 density \(N_i\) and the momentum conservation equation for
 the parallel electron velocity \(u_e\) and the parallel ion gyro-centre velocity \(U_i\)~\cite{WiesenbergerPhD, HeldPhD}.
-\subsection{ Sign reversals of the magnetic field}\label{sec:field_reversal}
+\subsection{ Scale invariance}
+\subsubsection{Sign reversals of the magnetic field}\label{sec:field_reversal}
 If we change the direction of the magnetic field vector $\bhat$, we immediately see that all perpendicular
 drifts and $U\bhat$ change directions. On the other side, the diffusive and resistive terms remain unchanged.
 Without resistivity and diffusion a change in direction of the magnetic field thus corresponds to
@@ -639,8 +653,14 @@ a time reversal $t\rightarrow t'=-t$.
 Also note that changing the sign of the magnetic field only in the parallel derivatives $\npar \rightarrow -\npar$ does not
 have any effect. This can be seen by simply renormalizing $U'=-U$. This reverts the equations back to the original equations.
 In the code this situation can happen when you change the sign of $\bhat$ using $\mathcal P_\psi$  and $\mathcal P_I$
-and use the toroidal field approximation at the same time.
-To actually change the sign of the magnetic field you need to use the "negative toroidal field approximation".
+and use the toroidal field approximation at the same time, because $\ehat_\varphi$
+does not change sign automatically.
+To actually change the sign of the magnetic field in code you need to use the "toroidal
+negative" value for the \textit{curvmode} input parameter, which reverts $\ehat_\varphi$ in the other direction.
+\subsubsection{Scaling of density}
+If $N, U, \phi, A_\parallel$ are a solution to the model equations
+then so are $N'=\alpha N$, $U'=U$, $\phi'=\phi$ and $A_\parallel'=A_\parallel$ with the changed parameters $S_N' = \alpha S_N$, $\eta' = \eta/\alpha$ and $ \beta' = \beta/\alpha$. If $N$
+has a Dirichlet boundary condition, then $N'$ satisfies a correspondingly scaled boundary condition.
 
 
 \subsection{Diffusive terms}\label{sec:dissres}
@@ -1522,7 +1542,7 @@ psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p)
 rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
 q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} using direct integration ( accurate but unavailable outside separatrix) \\
 psi\_psi         & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
-psit1d           & Dataset & 1 (psi) & Toroidal flux (integrated q-profile) $\psi_t = \int^\psi_p \d\psi_p q(\psi_p)$ \\
+psit1d           & Dataset & 1 (psi) & Toroidal flux (integrated q-profile) $\psi_t = \int^{\psi_p} \d\psi_p q(\psi_p)$ \\
 rho\_t           & Dataset & 1 (psi) & Toroidal flux label $\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$ (is similar to $\rho$ in the edge but $\rho_t$ is nicer in the core domain, because equidistant $\rho_t$ make more equidistant flux-surfaces)\\
 Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \RA{ Z}(R,Z)$ \\
 Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\RA{ Z}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
-- 
GitLab


From fea1efe1fc44ac6a7c10ad55d60183d6100b6759 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 8 Apr 2020 00:37:34 +0200
Subject: [PATCH 200/540] Create a mirrored flux function

start by unifying mirror and shift_topologic function into a corresponds
function in grid class. Probably more tests should be written for this.
---
 inc/dg/topology/grid.h             | 123 ++++++++++++++++-------------
 inc/dg/topology/interpolation.h    |  82 ++++++-------------
 inc/dg/topology/interpolation_t.cu |   2 +-
 inc/dg/topology/projection_mpit.cu |   6 +-
 inc/geometries/fieldaligned.h      |  18 ++---
 inc/geometries/fluxfunctions.h     |  37 +++++++++
 inc/geometries/init.h              |  27 -------
 7 files changed, 136 insertions(+), 159 deletions(-)

diff --git a/inc/dg/topology/grid.h b/inc/dg/topology/grid.h
index 7a3f25bfb..6a2a25066 100644
--- a/inc/dg/topology/grid.h
+++ b/inc/dg/topology/grid.h
@@ -193,43 +193,52 @@ struct RealGrid1d
     }
 
     /**
-     * @brief Shifts a point coordinate if periodic
+     * @brief Shift any point coordinate to a corresponding grid coordinate
      *
-     * This function shifts a point coordinate to its value between x0() and x1() if bcx() returns dg::PER
-     * @param x0 arbitrary point (irrelevant for the function, it's there to be consistent with GridX1d)
-     * @param x1 end point (inout)
+     * This function shifts a point coordinate to its corresponding value
+     * between x0() and x1() according to the boundary condition: if periodic
+     * the shift is simply a modulus operation, if Dirichlet or Neumann the
+     * point is mirrored at the boundary, if Dirichlet an additional sign flag
+     * is raised and returned.
+     * @param negative swap value if there was a sign swap (happens when a point is mirrored along a Dirichlet boundary)
+     * @param x point to shift (inout) the result is guaranteed to lie inside the grid
      */
-    void shift_topologic( real_type x0, real_type& x1)const
+    void correspond_coordinates( bool& negative, real_type& x)const
     {
-        real_type deltaX;
-        if( x1 > x0_) deltaX = x1 -x0_;
-        else deltaX = x1_ - x1;
-        unsigned N = floor(deltaX/lx());
-        if( x1  > x1_ && bcx_ == dg::PER) x1 -= N*lx();
-        if( x1  < x0_ && bcx_ == dg::PER) x1 += N*lx();
+        correspond_coordinates( negative, x, bcx_);
     }
     /**
-     * @brief Shifts a point coordinate if periodic
-     *
-     * This function shifts a point coordinate to its value between x0() and x1() if \c bcx==dg::PER
-     * @param x0 arbitrary point (irrelevant for the function, it's there to be consistent with GridX1d)
-     * @param x1 end point (inout)
-     * @param bcx overrule grid internal boundary condition
+     * @copydoc correspond_coordinates(bool&,real_type&)const
+     * @param bcx boundary condition to use for shift:
      */
-    void shift_topologic( real_type x0, real_type& x1, bc bcx)const
+    void correspond_coordinates( bool& negative, real_type &x, bc bcx)const
     {
-        real_type deltaX;
-        if( x1 > x0_) deltaX = x1 -x0_;
-        else deltaX = x1_ - x1;
-        unsigned N = floor(deltaX/lx());
-        if( x1  > x1_ && bcx == dg::PER) x1 -= N*lx();
-        if( x1  < x0_ && bcx == dg::PER) x1 += N*lx();
+        if( bcx == dg::PER)
+        {
+            real_type N = floor((x-x0_)/(x1_-x0_)); // ... -2| -1| 0| 1| 2| ...
+            x = x - N*(x1_-x0_); //shift
+        }
+        //mirror along boundary as often as necessary
+        while( (x<x0_) || (x>x1_) )
+        {
+            if( x < x0_){
+                x = 2.*x0_ - x;
+                //every mirror swaps the sign if Dirichlet
+                if( bcx == dg::DIR || bcx == dg::DIR_NEU)
+                    negative = !negative;//swap sign
+            }
+            if( x > x1_){
+                x = 2.*x1_ - x;
+                if( bcx == dg::DIR || bcx == dg::NEU_DIR) //notice the different boundary NEU_DIR to the above DIR_NEU !
+                    negative = !negative; //swap sign
+            }
+        }
     }
 
     /**
      * @brief Check if the grid contains a point
      *
-     * @note Doesn't check periodicity!!
+     * @note Does not consider periodicity!!
      * @param x point to check
      *
      * @return true if x0()<=x<=x1(), false else
@@ -399,29 +408,30 @@ struct aRealTopology2d
             <<"    "<<bc2str(bcy())<<"\n";
     }
     /**
-     * @brief Shifts point coordinates if periodic
+     * @brief Shift any point coordinate to a corresponding grid coordinate
      *
-     * This function shifts point coordinates to its values inside
-     the domain if the respective boundary condition is periodic
-     * @param x0 arbitrary coordinate (irrelevant for the function, it's there to be consistent with aRealTopologyX2d)
-     * @param y0 arbitrary coordinate (irrelevant for the function, it's there to be consistent with aRealTopologyX2d)
-     * @param x1 x-coordinate to shift (inout)
-     * @param y1 y-coordinate to shift (inout)
+     * This function shifts a point coordinate to its corresponding value
+     * inside the grid domain according to the boundary condition: if periodic
+     * the shift is simply a modulus operation, if Dirichlet or Neumann the
+     * point is mirrored at the boundary, if Dirichlet an additional sign flag
+     * is raised and returned.
+     * @param negative swap value if there was a sign swap (happens when a point is mirrored along a Dirichlet boundary)
+     * @param x point (x) to shift (inout) the result is guaranteed to lie inside the grid
+     * @param y point (y) to shift (inout) the result is guaranteed to lie inside the grid
      */
-    void shift_topologic( real_type x0, real_type y0, real_type& x1, real_type& y1)const
+    void correspond_coordinates( bool& negative, real_type& x, real_type& y)const
     {
-        gx_.shift_topologic( x0,x1);
-        gy_.shift_topologic( y0,y1);
+        correspond_coordinates( negative, x, y, bcx(), bcy());
     }
     /**
-     * @copydoc shift_topologic(real_type,real_type,real_type&,real_type&)const
+     * @copydoc correspond_coordinates(bool&,real_type&,real_type&)const
      * @param bcx overrule grid internal boundary condition with this value
      * @param bcy overrule grid internal boundary condition with this value
      */
-    void shift_topologic( real_type x0, real_type y0, real_type& x1, real_type& y1, bc bcx, bc bcy)const
+    void correspond_coordinates( bool& negative, real_type& x, real_type& y, bc bcx, bc bcy)const
     {
-        gx_.shift_topologic( x0,x1,bcx);
-        gy_.shift_topologic( y0,y1,bcy);
+        gx_.correspond_coordinates( negative, x,bcx);
+        gy_.correspond_coordinates( negative, y,bcy);
     }
     /**
      * @brief Check if the grid contains a point
@@ -646,34 +656,33 @@ struct aRealTopology3d
     }
 
     /**
-     * @brief Shifts point coordinates if periodic
+     * @brief Shift any point coordinate to a corresponding grid coordinate
      *
-     * This function shifts point coordinates to its values inside
-     the domain if the respective boundary condition is periodic
-     * @param x0 arbitrary x-coordinate (irrelevant for the function, it's there to be consistent with aRealTopologyX3d)
-     * @param y0 arbitrary y-coordinate (irrelevant for the function, it's there to be consistent with aRealTopologyX3d)
-     * @param z0 arbitrary z-coordinate (irrelevant for the function, it's there to be consistent with aRealTopologyX3d)
-     * @param x1 x-coordinate to shift (inout)
-     * @param y1 y-coordinate to shift (inout)
-     * @param z1 z-coordinate to shift (inout)
+     * This function shifts a point coordinate to its corresponding value
+     * inside the grid domain according to the boundary condition: if periodic
+     * the shift is simply a modulus operation, if Dirichlet or Neumann the
+     * point is mirrored at the boundary, if Dirichlet an additional sign flag
+     * is raised and returned.
+     * @param negative swap value if there was a sign swap (happens when a point is mirrored along a Dirichlet boundary)
+     * @param x point (x) to shift (inout) the result is guaranteed to lie inside the grid
+     * @param y point (y) to shift (inout) the result is guaranteed to lie inside the grid
+     * @param z point (z) to shift (inout) the result is guaranteed to lie inside the grid
      */
-    void shift_topologic( real_type x0, real_type y0, real_type z0, real_type& x1, real_type& y1, real_type& z1)const
+    void correspond_coordinates( bool& negative, real_type& x, real_type& y, real_type& z)const
     {
-        gx_.shift_topologic( x0,x1);
-        gy_.shift_topologic( y0,y1);
-        gz_.shift_topologic( z0,z1);
+        correspond_coordinates( negative, x,y,z, bcx(), bcy(), bcz());
     }
     /**
-     * @copydoc shift_topologic(real_type,real_type,real_type,real_type&,real_type&,real_type&)const
+     * @copydoc correspond_coordinates(bool&,real_type&,real_type&,real_type&)const
      * @param bcx overrule grid internal boundary condition with this value
      * @param bcy overrule grid internal boundary condition with this value
      * @param bcz overrule grid internal boundary condition with this value
      */
-    void shift_topologic( real_type x0, real_type y0, real_type z0, real_type& x1, real_type& y1, real_type& z1, bc bcx, bc bcy, bc bcz)const
+    void correspond_coordinates( bool& negative, real_type& x, real_type& y, real_type& z, bc bcx, bc bcy, bc bcz)const
     {
-        gx_.shift_topologic( x0,x1,bcx);
-        gy_.shift_topologic( y0,y1,bcy);
-        gz_.shift_topologic( z0,z1,bcz);
+        gx_.correspond_coordinates( negative, x,bcx);
+        gy_.correspond_coordinates( negative, y,bcy);
+        gz_.correspond_coordinates( negative, z,bcz);
     }
 
     /**
diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index 4f2b2cac0..0ca4a9d0e 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -68,32 +68,6 @@ std::vector<real_type> coefficients( real_type xn, unsigned n)
     return px;
 }
 
-template<class real_type>
-void mirror( bool& negative, real_type& x, real_type x0, real_type x1, bc boundary) {
-    while( (x<x0) || (x>x1) )
-    {
-        //mirror along boundary as often as necessary
-        //every mirror swaps the sign if Dirichlet
-        if( x < x0){
-            x = 2.*x0 - x;
-            if( boundary == dg::DIR || boundary == dg::DIR_NEU)
-                negative = !negative;//swap sign
-        }
-        if( x > x1){
-            x = 2.*x1 - x;
-            if( boundary == dg::DIR || boundary == dg::NEU_DIR)
-                negative = !negative; //swap sign
-        }
-    }
-}
-template<class real_type>
-void assert_contains( real_type X, real_type x0, real_type x1, char const * point){
-    if (!(X >= x0 && X <= x1)) {
-        std::cerr << x0<<"< "<<point<<" = " << X <<" < "<<x1<<std::endl;
-    }
-    assert(X >= x0 && X <= x1);
-}
-
 }//namespace detail
 ///@endcond
 ///@addtogroup interpolation
@@ -129,10 +103,8 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
     for( unsigned i=0; i<x.size(); i++)
     {
         real_type X = x[i];
-        g.shift_topologic( X,X, bcx);
-        //mirror at boundary
         bool negative = false;
-        detail::mirror( negative, X, g.x0(), g.x1(), bcx);
+        g.correspond_coordinates( negative, X, bcx);
 
         //determine which cell (x) lies in
         real_type xnn = (X-g.x0())/g.h();
@@ -190,14 +162,8 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
     for( int i=0; i<(int)x.size(); i++)
     {
         real_type X = x[i], Y = y[i];
-        g.shift_topologic( X,Y,X,Y, bcx, bcy);
-        //mirror at boundary
-        bool negative = false;
-        detail::mirror( negative, X, g.x0(), g.x1(), bcx);
-        detail::mirror( negative, Y, g.y0(), g.y1(), bcy);
-        //assert that point is inside the grid boundaries
-        //detail::assert_contains( X, g.x0(), g.x1(), "xi");
-        //detail::assert_contains( Y, g.y0(), g.y1(), "yi");
+        bool negative=false;
+        g.correspond_coordinates( negative,X,Y, bcx, bcy);
 
         //determine which cell (x,y) lies in
         real_type xnn = (X-g.x0())/g.hx();
@@ -341,15 +307,8 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
     for( int i=0; i<(int)x.size(); i++)
     {
         real_type X = x[i], Y = y[i], Z = z[i];
-        g.shift_topologic( X,Y,Z,X,Y,Z, bcx, bcy, bcz);
         bool negative = false;
-        detail::mirror( negative, X, g.x0(), g.x1(), bcx);
-        detail::mirror( negative, Y, g.y0(), g.y1(), bcy);
-        detail::mirror( negative, Z, g.z0(), g.z1(), bcz);
-        //assert that point is inside the grid boundaries
-        //detail::assert_contains( X, g.x0(), g.x1(), "xi");
-        //detail::assert_contains( Y, g.y0(), g.y1(), "yi");
-        //detail::assert_contains( Z, g.z0(), g.z1(), "zi");
+        g.correspond_coordinates( negative,X,Y,Z, bcx, bcy, bcz);
 
         //determine which cell (x,y) lies in
         real_type xnn = (X-g.x0())/g.hx();
@@ -549,6 +508,12 @@ thrust::host_vector<real_type> forward_transform( const thrust::host_vector<real
 /**
  * @brief Interpolate a vector on a single point on a 1d Grid
  *
+ * @param sp Indicate whether the elements of the vector
+ * v are in xspace or lspace
+ *  (choose dg::xspace if you don't know what is going on here,
+ *      It is faster to interpolate in dg::lspace so consider
+ *      transforming v using dg::forward_transform( )
+ *      if you do it very many times)
  * @param v The vector to interpolate in dg::xspace
  * @param x X-coordinate of interpolation point
  * @param g The Grid on which to operate
@@ -556,20 +521,18 @@ thrust::host_vector<real_type> forward_transform( const thrust::host_vector<real
  *
  * @ingroup interpolation
  * @return interpolated point
- * @note \c g.contains(x,y) must return true
  */
 template<class real_type>
 real_type interpolate(
+    dg::space sp,
     const thrust::host_vector<real_type>& v,
     real_type x,
     const RealGrid1d<real_type>& g,
     dg::bc bcx = dg::NEU)
 {
     assert( v.size() == g.size());
-    g.shift_topologic( x,x, bcx);
-    //mirror at boundary
     bool negative = false;
-    create::detail::mirror( negative, x, g.x0(), g.x1(), bcx);
+    g.correspond_coordinates( negative, x, bcx);
 
     //determine which cell (x) lies in
 
@@ -585,13 +548,16 @@ real_type interpolate(
     }
     //evaluate 1d Legendre polynomials at (xn)...
     std::vector<real_type> px = create::detail::coefficients( xn, g.n());
-    dg::Operator<real_type> forward( g.dlt().forward());
-    std::vector<real_type> pxF(g.n(),0);
-    for( unsigned l=0; l<g.n(); l++)
+    if( sp == dg::xspace)
+    {
+        dg::Operator<real_type> forward( g.dlt().forward());
+        std::vector<real_type> pxF(g.n(),0);
+        for( unsigned l=0; l<g.n(); l++)
+            for( unsigned k=0; k<g.n(); k++)
+                pxF[l]+= px[k]*forward(k,l);
         for( unsigned k=0; k<g.n(); k++)
-            pxF[l]+= px[k]*forward(k,l);
-    for( unsigned k=0; k<g.n(); k++)
-        px[k] = pxF[k];
+            px[k] = pxF[k];
+    }
     //these are the matrix coefficients with which to multiply
     unsigned col_begin = (n)*g.n();
     //multiply x
@@ -624,7 +590,6 @@ real_type interpolate(
  *
  * @ingroup interpolation
  * @return interpolated point
- * @note \c g.contains(x,y) must return true
  */
 template<class real_type>
 real_type interpolate(
@@ -635,11 +600,8 @@ real_type interpolate(
     dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU )
 {
     assert( v.size() == g.size());
-    g.shift_topologic( x,y,x,y, bcx, bcy);
-    //mirror at boundary
     bool negative = false;
-    create::detail::mirror( negative, x, g.x0(), g.x1(), bcx);
-    create::detail::mirror( negative, y, g.y0(), g.y1(), bcy);
+    g.correspond_coordinates( negative, x,y, bcx, bcy);
 
     //determine which cell (x,y) lies in
 
diff --git a/inc/dg/topology/interpolation_t.cu b/inc/dg/topology/interpolation_t.cu
index 25584b3b8..9d6ad4b0c 100644
--- a/inc/dg/topology/interpolation_t.cu
+++ b/inc/dg/topology/interpolation_t.cu
@@ -94,7 +94,7 @@ int main()
         //create equidistant values
         x[i] = g1d.x0() + g1d.lx() + (i+0.5)*g1d.h()/(double)(g1d.n());
         //use DIR because the cooX1d is zero on the right boundary
-        double xi = dg::interpolate( xs, x[i], g1d, dg::DIR);
+        double xi = dg::interpolate( dg::xspace,xs, x[i], g1d, dg::DIR);
         if( x[i] - xi > 1e-14)
         {
             std::cerr << "X NOT EQUAL "<<i<<"\t"<<x[i]<<"  \t"<<xi<<"\n";
diff --git a/inc/dg/topology/projection_mpit.cu b/inc/dg/topology/projection_mpit.cu
index 61ab2db9e..8ea1fc553 100644
--- a/inc/dg/topology/projection_mpit.cu
+++ b/inc/dg/topology/projection_mpit.cu
@@ -64,7 +64,8 @@ int main(int argc, char* argv[])
     {
         x[i] +=shift;
         y[i] +=shift;
-        g2d.global().shift_topologic( x[i], y[i], x[i], y[i]);
+        bool negative = false;
+        g2d.global().correspond_coordinates( negative, x[i], y[i]);
     }
     dg::MIHMatrix converted_i = dg::create::interpolation( x,y,g2d);
     dg::IHMatrix  direct_i = dg::create::interpolation( x,y,g2d.global());
@@ -95,7 +96,8 @@ int main(int argc, char* argv[])
     {
         x[i] +=shift;
         y[i] +=shift;
-        g2d.global().shift_topologic( x[i], y[i], x[i], y[i]);
+        bool negative = false;
+        g2d.global().correspond_coordinates( negative, x[i], y[i]);
     }
     direct_i = dg::transpose(dg::create::interpolation( x,y,g2d.global()));
     g_temp.resize( g2d.global().size());
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index d4af3c170..f2215766a 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -46,10 +46,9 @@ namespace detail{
 
 struct DSFieldCylindrical
 {
-    DSFieldCylindrical( const dg::geo::CylindricalVectorLvl0& v, Grid2d boundary):v_(v), m_b(boundary) { }
+    DSFieldCylindrical( const dg::geo::CylindricalVectorLvl0& v):v_(v){ }
     void operator()( double t, const std::array<double,3>& y, std::array<double,3>& yp) const {
         double R = y[0], Z = y[1];
-        m_b.shift_topologic( y[0], y[1], R, Z); //shift R,Z onto domain
         double vz = v_.z()(R, Z);
         yp[0] = v_.x()(R, Z)/vz;
         yp[1] = v_.y()(R, Z)/vz;
@@ -58,12 +57,12 @@ struct DSFieldCylindrical
 
     private:
     dg::geo::CylindricalVectorLvl0 v_;
-    dg::Grid2d m_b;
 };
 
 struct DSField
 {
     //z component of v may not vanish
+    //Fields outside g are extended according to bc in g
     DSField( const dg::geo::CylindricalVectorLvl0& v, const dg::aGeometry2d& g): g_(g)
     {
         thrust::host_vector<double> v_zeta, v_eta;
@@ -77,16 +76,11 @@ struct DSField
         dsdphi_     = dg::forward_transform( v_phi, g );
     }
     //interpolate the vectors given in the constructor on the given point
-    //if point lies outside of grid boundaries zero is returned
     void operator()(double t, const std::array<double,3>& y, std::array<double,3>& yp) const
     {
-        double R = y[0], Z = y[1];
-        g_->shift_topologic( y[0], y[1], R, Z); //shift R,Z onto domain
-        {
-            yp[0] = interpolate(dg::lspace, dzetadphi_, R, Z, *g_);
-            yp[1] = interpolate(dg::lspace, detadphi_,  R, Z, *g_);
-            yp[2] = interpolate(dg::lspace, dsdphi_,    R, Z, *g_);
-        }
+        yp[0] = interpolate(dg::lspace, dzetadphi_, y[0], y[1], *g_);
+        yp[1] = interpolate(dg::lspace, detadphi_,  y[0], y[1], *g_);
+        yp[2] = interpolate(dg::lspace, dsdphi_,    y[0], y[1], *g_);
     }
     private:
     thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
@@ -117,7 +111,7 @@ void integrate_all_fieldlines2d( const dg::geo::CylindricalVectorLvl0& vec,
     //construct field on high polynomial grid, then integrate it
     dg::geo::detail::DSField field( vec, grid_field);
     //field in case of cartesian grid
-    dg::geo::detail::DSFieldCylindrical cyl_field(vec, (dg::Grid2d)grid_field);
+    dg::geo::detail::DSFieldCylindrical cyl_field(vec);
     unsigned size = grid_evaluate.size();
     for( unsigned i=0; i<size; i++)
     {
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 71e627a79..dbd3e8ff6 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -121,6 +121,43 @@ struct Constant: public aCylindricalFunctor<Constant>
     private:
     double c_;
 };
+/**
+* @brief Composition \f[ f\circ\psi = f(\psi(R,Z)) \f]
+*
+* @tparam UnaryFunctor A unary Functor with interface <tt>double (double x)</tt>
+* @ingroup profiles
+*/
+template<class UnaryFunctor>
+struct Compose : public aCylindricalFunctor<Compose<UnaryFunctor>>
+{
+    /**
+    * @brief Construct from 2d functor and forward any parameters to \c UnaryFunctor
+    *
+    * @param psi A binary functor
+    * @param ps Parameters that are forwarded to the constructor of \c UnaryFunctor
+    * @tparam FunctorParams Determined by Compiler
+    */
+    template<class ...FunctorParams>
+    Compose ( CylindricalFunctor psi, FunctorParams&& ... ps): m_psip(psi),
+        m_f(std::forward<FunctorParams>(ps)...){}
+    double do_compute( double R, double Z) const
+    {
+        return m_f(m_psip(R,Z));
+    }
+    private:
+    CylindricalFunctor m_psip;
+    UnaryFunctor m_f;
+};
+
+struct Periodify : public aCylindricalFunctor<Periodify>
+{
+    Periodify( CylindricalFunctor functor): m_f(functor){}
+    double do_compute( double R, double Z) const
+    {
+    }
+    private:
+    CylindricalFunctor m_f;
+};
 
 /**
 * @brief This struct bundles a function and its first derivatives
diff --git a/inc/geometries/init.h b/inc/geometries/init.h
index 3cbc3bf7d..722b64b65 100644
--- a/inc/geometries/init.h
+++ b/inc/geometries/init.h
@@ -12,33 +12,6 @@ namespace dg
 namespace geo
 {
 
-/**
-* @brief Composition \f[ f\circ\psi = f(\psi(R,Z)) \f]
-*
-* @tparam UnaryFunctor A unary Functor with interface <tt>double (double x)</tt>
-* @ingroup profiles
-*/
-template<class UnaryFunctor>
-struct Compose : public aCylindricalFunctor<Compose<UnaryFunctor>>
-{
-    /**
-    * @brief Construct from 2d functor and forward any parameters to \c UnaryFunctor
-    *
-    * @param psi A binary functor
-    * @param ps Parameters that are forwarded to the constructor of \c UnaryFunctor
-    * @tparam FunctorParams Determined by Compiler
-    */
-    template<class ...FunctorParams>
-    Compose ( CylindricalFunctor psi, FunctorParams&& ... ps): m_psip(psi),
-        m_f(std::forward<FunctorParams>(ps)...){}
-    double do_compute( double R, double Z) const
-    {
-        return m_f(m_psip(R,Z));
-    }
-    private:
-    CylindricalFunctor m_psip;
-    UnaryFunctor m_f;
-};
 ///@addtogroup profiles
 ///@{
 
-- 
GitLab


From 38f1087b6296f9abdecca582e3e3f2e3ad8965c8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 8 Apr 2020 12:00:09 +0200
Subject: [PATCH 201/540] Rename new function to shift and add tests

---
 inc/dg/topology/grid.h             | 85 ++++++++++++++----------------
 inc/dg/topology/grid_t.cu          | 52 ++++++++++++++++++
 inc/dg/topology/interpolation.h    | 10 ++--
 inc/dg/topology/projection_mpit.cu |  4 +-
 4 files changed, 98 insertions(+), 53 deletions(-)
 create mode 100644 inc/dg/topology/grid_t.cu

diff --git a/inc/dg/topology/grid.h b/inc/dg/topology/grid.h
index 6a2a25066..f719d7933 100644
--- a/inc/dg/topology/grid.h
+++ b/inc/dg/topology/grid.h
@@ -49,6 +49,19 @@
  * @param bcy boundary condition in y
  * @param bcz boundary condition in z
  */
+/*!@class hide_shift_doc
+ * @brief Shift any point coordinate to a corresponding grid coordinate
+ *
+ * This function shifts a point coordinate to its corresponding value
+ * inside the grid according to the boundary condition: if periodic
+ * the shift is simply a modulus operation, if Dirichlet or Neumann the
+ * point is mirrored at the boundary, if Dirichlet an additional sign flag
+ * is raised and returned. This function forms the basis for periodifying a
+ * function discretized on the grid beyond the grid boundaries.
+ * @note If periodic the right boundary point is counted to lie outside the grid and is shifted to the left boundary point.
+ * @param negative swap value if there was a sign swap (happens when a point is mirrored along a Dirichlet boundary)
+ * @param x point to shift (inout) the result is guaranteed to lie inside the grid
+ */
 
 namespace dg{
 
@@ -193,29 +206,21 @@ struct RealGrid1d
     }
 
     /**
-     * @brief Shift any point coordinate to a corresponding grid coordinate
-     *
-     * This function shifts a point coordinate to its corresponding value
-     * between x0() and x1() according to the boundary condition: if periodic
-     * the shift is simply a modulus operation, if Dirichlet or Neumann the
-     * point is mirrored at the boundary, if Dirichlet an additional sign flag
-     * is raised and returned.
-     * @param negative swap value if there was a sign swap (happens when a point is mirrored along a Dirichlet boundary)
-     * @param x point to shift (inout) the result is guaranteed to lie inside the grid
+     * @copydoc hide_shift_doc
      */
-    void correspond_coordinates( bool& negative, real_type& x)const
+    void shift( bool& negative, real_type& x)const
     {
-        correspond_coordinates( negative, x, bcx_);
+        shift( negative, x, bcx_);
     }
     /**
-     * @copydoc correspond_coordinates(bool&,real_type&)const
-     * @param bcx boundary condition to use for shift:
+     * @copydoc hide_shift_doc
+     * @param bcx overrule grid internal boundary condition with this value
      */
-    void correspond_coordinates( bool& negative, real_type &x, bc bcx)const
+    void shift( bool& negative, real_type &x, bc bcx)const
     {
         if( bcx == dg::PER)
         {
-            real_type N = floor((x-x0_)/(x1_-x0_)); // ... -2| -1| 0| 1| 2| ...
+            real_type N = floor((x-x0_)/(x1_-x0_)); // ... -2[ -1[ 0[ 1[ 2[ ...
             x = x - N*(x1_-x0_); //shift
         }
         //mirror along boundary as often as necessary
@@ -245,6 +250,7 @@ struct RealGrid1d
      */
     bool contains( real_type x)const
     {
+        //should we catch the case x1_==x && dg::PER?
         if( (x>=x0_ && x <= x1_)) return true;
         return false;
     }
@@ -408,30 +414,23 @@ struct aRealTopology2d
             <<"    "<<bc2str(bcy())<<"\n";
     }
     /**
-     * @brief Shift any point coordinate to a corresponding grid coordinate
-     *
-     * This function shifts a point coordinate to its corresponding value
-     * inside the grid domain according to the boundary condition: if periodic
-     * the shift is simply a modulus operation, if Dirichlet or Neumann the
-     * point is mirrored at the boundary, if Dirichlet an additional sign flag
-     * is raised and returned.
-     * @param negative swap value if there was a sign swap (happens when a point is mirrored along a Dirichlet boundary)
-     * @param x point (x) to shift (inout) the result is guaranteed to lie inside the grid
+     * @copydoc hide_shift_doc
      * @param y point (y) to shift (inout) the result is guaranteed to lie inside the grid
      */
-    void correspond_coordinates( bool& negative, real_type& x, real_type& y)const
+    void shift( bool& negative, real_type& x, real_type& y)const
     {
-        correspond_coordinates( negative, x, y, bcx(), bcy());
+        shift( negative, x, y, bcx(), bcy());
     }
     /**
-     * @copydoc correspond_coordinates(bool&,real_type&,real_type&)const
+     * @copydoc hide_shift_doc
+     * @param y point (y) to shift (inout) the result is guaranteed to lie inside the grid
      * @param bcx overrule grid internal boundary condition with this value
      * @param bcy overrule grid internal boundary condition with this value
      */
-    void correspond_coordinates( bool& negative, real_type& x, real_type& y, bc bcx, bc bcy)const
+    void shift( bool& negative, real_type& x, real_type& y, bc bcx, bc bcy)const
     {
-        gx_.correspond_coordinates( negative, x,bcx);
-        gy_.correspond_coordinates( negative, y,bcy);
+        gx_.shift( negative, x,bcx);
+        gy_.shift( negative, y,bcy);
     }
     /**
      * @brief Check if the grid contains a point
@@ -656,33 +655,27 @@ struct aRealTopology3d
     }
 
     /**
-     * @brief Shift any point coordinate to a corresponding grid coordinate
-     *
-     * This function shifts a point coordinate to its corresponding value
-     * inside the grid domain according to the boundary condition: if periodic
-     * the shift is simply a modulus operation, if Dirichlet or Neumann the
-     * point is mirrored at the boundary, if Dirichlet an additional sign flag
-     * is raised and returned.
-     * @param negative swap value if there was a sign swap (happens when a point is mirrored along a Dirichlet boundary)
-     * @param x point (x) to shift (inout) the result is guaranteed to lie inside the grid
+     * @copydoc hide_shift_doc
      * @param y point (y) to shift (inout) the result is guaranteed to lie inside the grid
      * @param z point (z) to shift (inout) the result is guaranteed to lie inside the grid
      */
-    void correspond_coordinates( bool& negative, real_type& x, real_type& y, real_type& z)const
+    void shift( bool& negative, real_type& x, real_type& y, real_type& z)const
     {
-        correspond_coordinates( negative, x,y,z, bcx(), bcy(), bcz());
+        shift( negative, x,y,z, bcx(), bcy(), bcz());
     }
     /**
-     * @copydoc correspond_coordinates(bool&,real_type&,real_type&,real_type&)const
+     * @copydoc hide_shift_doc
+     * @param y point (y) to shift (inout) the result is guaranteed to lie inside the grid
+     * @param z point (z) to shift (inout) the result is guaranteed to lie inside the grid
      * @param bcx overrule grid internal boundary condition with this value
      * @param bcy overrule grid internal boundary condition with this value
      * @param bcz overrule grid internal boundary condition with this value
      */
-    void correspond_coordinates( bool& negative, real_type& x, real_type& y, real_type& z, bc bcx, bc bcy, bc bcz)const
+    void shift( bool& negative, real_type& x, real_type& y, real_type& z, bc bcx, bc bcy, bc bcz)const
     {
-        gx_.correspond_coordinates( negative, x,bcx);
-        gy_.correspond_coordinates( negative, y,bcy);
-        gz_.correspond_coordinates( negative, z,bcz);
+        gx_.shift( negative, x,bcx);
+        gy_.shift( negative, y,bcy);
+        gz_.shift( negative, z,bcz);
     }
 
     /**
diff --git a/inc/dg/topology/grid_t.cu b/inc/dg/topology/grid_t.cu
new file mode 100644
index 000000000..20ee721a5
--- /dev/null
+++ b/inc/dg/topology/grid_t.cu
@@ -0,0 +1,52 @@
+#include <iostream>
+#include <iomanip>
+#include "grid.h"
+
+int main()
+{
+    dg::Grid1d g1d( 1., 1.+2.*M_PI, 3, 10, dg::PER);
+    double x = 0.-10.*M_PI;
+    dg::bc bcx[] = {dg::NEU, dg::DIR, dg::DIR_NEU, dg::NEU_DIR};
+    std::cout << "Test the shift function. The numbers should be alternatively 2 and 2*M_PI read from up to down.\n";
+    std::cout  << "PER NEU  DIR  DIR_NEU NEU_DIR\n";
+    std::cout << std::boolalpha;
+    for( int i =0; i<11; i++)
+    {
+        double x0 = x + i*2*M_PI;
+        std::cout << std::setw( 8)<< x0 << " ";
+        bool mirrored = false;
+        g1d.shift( mirrored, x0);
+        if( false == mirrored && ( x0-2.*M_PI)<1e-15 ) std::cout << "PASSED ";
+        else std::cout << "FAILED "<< mirrored<<" "<<x0<<" ";
+
+        for( int j=0; j<4; j++)
+        {
+            x0 = x + i*2*M_PI;
+            mirrored = false;
+            g1d.shift( mirrored, x0, bcx[j]);
+            std::cout << std::setw( 6)<<mirrored<<std::setw(8)<<x0 << " ";
+        }
+        std::cout <<std::endl;
+    }
+    std::cout << "Test 2d and 3d shift functions\n";
+    dg::Grid2d g2d( 1., 1.+2.*M_PI, 1., 1.+2.*M_PI, 3, 10, 10, dg::DIR, dg::DIR);
+    dg::Grid3d g3d( 1., 1.+2.*M_PI, 1., 1.+2.*M_PI, 0., 2.*M_PI, 3, 10, 10, 10, dg::DIR, dg::DIR, dg::PER);
+    double y=0;
+    for( int i=0; i<2; i++)
+    {
+        double x0 = 0;
+        double y0 = y + i*2.*M_PI;
+        std::cout << std::setw( 8)<< y0 << " ";
+        bool mirrored = false;
+        g2d.shift( mirrored, x0, y0);
+        std::cout << std::setw( 6)<<mirrored<<std::setw(8)<<x0 <<std::setw(8)<<y0 << "\n";
+        x0 = 0;
+        y0 = y + i*2.*M_PI;
+        double z0 = 2.*M_PI; //interesting
+        mirrored = false;
+        g3d.shift( mirrored, x0, y0, z0);
+        std::cout << std::setw( 15)<<mirrored<<std::setw(8)<<x0 <<std::setw(8)<<y0 <<std::setw(8)<<z0 << "\n";
+    }
+
+    return 0;
+}
diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index 0ca4a9d0e..2b0c31dd0 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -104,7 +104,7 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
     {
         real_type X = x[i];
         bool negative = false;
-        g.correspond_coordinates( negative, X, bcx);
+        g.shift( negative, X, bcx);
 
         //determine which cell (x) lies in
         real_type xnn = (X-g.x0())/g.h();
@@ -163,7 +163,7 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
     {
         real_type X = x[i], Y = y[i];
         bool negative=false;
-        g.correspond_coordinates( negative,X,Y, bcx, bcy);
+        g.shift( negative,X,Y, bcx, bcy);
 
         //determine which cell (x,y) lies in
         real_type xnn = (X-g.x0())/g.hx();
@@ -308,7 +308,7 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
     {
         real_type X = x[i], Y = y[i], Z = z[i];
         bool negative = false;
-        g.correspond_coordinates( negative,X,Y,Z, bcx, bcy, bcz);
+        g.shift( negative,X,Y,Z, bcx, bcy, bcz);
 
         //determine which cell (x,y) lies in
         real_type xnn = (X-g.x0())/g.hx();
@@ -532,7 +532,7 @@ real_type interpolate(
 {
     assert( v.size() == g.size());
     bool negative = false;
-    g.correspond_coordinates( negative, x, bcx);
+    g.shift( negative, x, bcx);
 
     //determine which cell (x) lies in
 
@@ -601,7 +601,7 @@ real_type interpolate(
 {
     assert( v.size() == g.size());
     bool negative = false;
-    g.correspond_coordinates( negative, x,y, bcx, bcy);
+    g.shift( negative, x,y, bcx, bcy);
 
     //determine which cell (x,y) lies in
 
diff --git a/inc/dg/topology/projection_mpit.cu b/inc/dg/topology/projection_mpit.cu
index 8ea1fc553..5d2703893 100644
--- a/inc/dg/topology/projection_mpit.cu
+++ b/inc/dg/topology/projection_mpit.cu
@@ -65,7 +65,7 @@ int main(int argc, char* argv[])
         x[i] +=shift;
         y[i] +=shift;
         bool negative = false;
-        g2d.global().correspond_coordinates( negative, x[i], y[i]);
+        g2d.global().shift( negative, x[i], y[i]);
     }
     dg::MIHMatrix converted_i = dg::create::interpolation( x,y,g2d);
     dg::IHMatrix  direct_i = dg::create::interpolation( x,y,g2d.global());
@@ -97,7 +97,7 @@ int main(int argc, char* argv[])
         x[i] +=shift;
         y[i] +=shift;
         bool negative = false;
-        g2d.global().correspond_coordinates( negative, x[i], y[i]);
+        g2d.global().shift( negative, x[i], y[i]);
     }
     direct_i = dg::transpose(dg::create::interpolation( x,y,g2d.global()));
     g_temp.resize( g2d.global().size());
-- 
GitLab


From cede3269e878d4ab4ff1e0a6a4529a8deb4cb31a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 21 Apr 2020 10:32:14 +0200
Subject: [PATCH 202/540] Elevate inverse function to dg namespace

and refine docu for shift function a little
---
 inc/dg/elliptic.h      | 28 ----------------------------
 inc/dg/enums.h         | 29 +++++++++++++++++++++++++++++
 inc/dg/topology/grid.h | 14 +++++++-------
 3 files changed, 36 insertions(+), 35 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 4e3e806d6..4a90a35bf 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -294,20 +294,6 @@ class Elliptic
             dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
     }
     private:
-    bc inverse( bc bound)
-    {
-        if( bound == DIR) return NEU;
-        if( bound == NEU) return DIR;
-        if( bound == DIR_NEU) return NEU_DIR;
-        if( bound == NEU_DIR) return DIR_NEU;
-        return PER;
-    }
-    direction inverse( direction dir)
-    {
-        if( dir == forward) return backward;
-        if( dir == backward) return forward;
-        return centered;
-    }
     Matrix m_leftx, m_lefty, m_rightx, m_righty, m_jumpX, m_jumpY;
     Container m_weights, m_inv_weights, m_precond, m_weights_wo_vol;
     Container m_tempx, m_tempy, m_temp;
@@ -577,20 +563,6 @@ class Elliptic3d
     }
 
     private:
-    bc inverse( bc bound)
-    {
-        if( bound == DIR) return NEU;
-        if( bound == NEU) return DIR;
-        if( bound == DIR_NEU) return NEU_DIR;
-        if( bound == NEU_DIR) return DIR_NEU;
-        return PER;
-    }
-    direction inverse( direction dir)
-    {
-        if( dir == forward) return backward;
-        if( dir == backward) return forward;
-        return centered;
-    }
     Matrix m_leftx, m_lefty, m_leftz, m_rightx, m_righty, m_rightz, m_jumpX, m_jumpY;
     Container m_weights, m_inv_weights, m_precond, m_weights_wo_vol;
     Container m_tempx, m_tempy, m_tempz, m_temp;
diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index a6d5c0a18..7371ac869 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -78,6 +78,21 @@ static inline bc str2bc( std::string s)
     throw std::runtime_error( "Boundary condition '"+s+"' not recognized!");
 }
 
+/**
+ * @brief invert boundary condition
+ *
+ * @param bound boundary condition to invert
+ * @return NEU for DIR, DIR for NEU, NEU_DIR for DIR_NEU, DIR_NEU for NEU_DIR and PER for PER
+ */
+bc inverse( bc bound)
+{
+    if( bound == DIR) return NEU;
+    if( bound == NEU) return DIR;
+    if( bound == DIR_NEU) return NEU_DIR;
+    if( bound == NEU_DIR) return DIR_NEU;
+    return PER;
+}
+
 ///@brief Switch between normalisations
 enum norm{
     normed,   //!< indicates that output is properly normalized
@@ -91,6 +106,20 @@ enum direction{
     centered //!< centered derivative
 };
 
+
+/**
+ * @brief invert direction
+ *
+ * @param dir direction to invert
+ * @return backward for forward, forward for backward, centered for centered
+ */
+direction inverse( direction dir)
+{
+    if( dir == forward) return backward;
+    if( dir == backward) return forward;
+    return centered;
+}
+
 ///@brief Space of DG coefficients
 enum space{
     lspace, //!< DG Polynomial space
diff --git a/inc/dg/topology/grid.h b/inc/dg/topology/grid.h
index f719d7933..4f45a4298 100644
--- a/inc/dg/topology/grid.h
+++ b/inc/dg/topology/grid.h
@@ -50,15 +50,15 @@
  * @param bcz boundary condition in z
  */
 /*!@class hide_shift_doc
- * @brief Shift any point coordinate to a corresponding grid coordinate
+ * @brief Shift any point coordinate to a corresponding grid coordinate according to the boundary condition
  *
- * This function shifts a point coordinate to its corresponding value
- * inside the grid according to the boundary condition: if periodic
- * the shift is simply a modulus operation, if Dirichlet or Neumann the
- * point is mirrored at the boundary, if Dirichlet an additional sign flag
- * is raised and returned. This function forms the basis for periodifying a
+ * If the given point is already inside the grid, the function does nothing, else along each dimension the following happens: check the boundary condition.
+ *If \c dg::PER, the point will be shifted topologically back onto the domain (modulo operation). Else the
+ * point will be mirrored at the closest boundary. If the boundary is a Dirichlet boundary (happens for \c dg::DIR, \c dg::DIR_NEU and \c dg::NEU_DIR; the latter two apply \c dg::DIR to the respective left or right boundary )
+ * an additional sign flag is swapped. This process is repeated until the result lies inside the grid. This function forms the basis for extending/periodifying a
  * function discretized on the grid beyond the grid boundaries.
- * @note If periodic the right boundary point is counted to lie outside the grid and is shifted to the left boundary point.
+ * @sa interpolate
+ * @note For periodic boundaries the right boundary point is considered outside the grid and is shifted to the left boundary point.
  * @param negative swap value if there was a sign swap (happens when a point is mirrored along a Dirichlet boundary)
  * @param x point to shift (inout) the result is guaranteed to lie inside the grid
  */
-- 
GitLab


From 8203668e599b2408aa5f230df4880f6c2608c119 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mwiesenb@login03.m100.cineca.it>
Date: Mon, 4 May 2020 23:15:30 +0200
Subject: [PATCH 203/540] Add Marconi M100 Makefile

and debug MPI_alltoallv
---
 config/default.mk               |  4 ++--
 config/m100.mk                  | 20 ++++++++++++++++++++
 inc/dg/backend/config.h         |  4 ++++
 inc/dg/backend/mpi_collective.h |  6 ++++--
 4 files changed, 30 insertions(+), 4 deletions(-)
 create mode 100644 config/m100.mk

diff --git a/config/default.mk b/config/default.mk
index 1bccbe376..496f54704 100644
--- a/config/default.mk
+++ b/config/default.mk
@@ -4,10 +4,10 @@ INCLUDED=1
 #compiler and compiler options
 CC=g++ #C++ compiler
 MPICC=mpic++  #mpi compiler
-CFLAGS=-Wall -std=c++11 -mavx -mfma #flags for CC
+CFLAGS=-Wall -std=c++14 -mavx -mfma #flags for CC
 NVCC=nvcc #CUDA compiler
 NVCCARCH=-arch sm_35 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
-NVCCFLAGS= -std=c++11 -Xcompiler "-Wall -mavx -mfma" #flags for NVCC
+NVCCFLAGS= -std=c++14 -Xcompiler "-Wall -mavx -mfma" #flags for NVCC
 OPT=-O2 # optimization flags for host code (it is O2 and not O3 because g++-7 up to g++-8.0 have a bug with fma in -O3, fixed in g++-8.1)
 OMPFLAG=-fopenmp #openmp flag for CC and MPICC
 
diff --git a/config/m100.mk b/config/m100.mk
new file mode 100644
index 000000000..6669a212b
--- /dev/null
+++ b/config/m100.mk
@@ -0,0 +1,20 @@
+ifeq ($(strip $(HPC_SYSTEM)),m100)
+#CC=xlc++ #C++ compiler
+#MPICC=mpixlC  #mpi compiler
+#CFLAGS=-Wall -std=c++1y -DWITHOUT_VCL -mcpu=power9 -qstrict# -mavx -mfma #flags for CC
+#OMPFLAG=-qsmp=omp
+#OPT=-O3 # optimization flags for host code
+#NVCC=nvcc #CUDA compiler
+#NVCCARCH=-arch sm_70 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
+#NVCCFLAGS= -std=c++14 -Xcompiler "-mcpu=power9 -Wall"# -mavx -mfma" #flags for NVCC
+CFLAGS=-Wall -std=c++14 -DWITHOUT_VCL -mcpu=power9 # -mavx -mfma #flags for CC
+OPT=-O3 # optimization flags for host code
+NVCC=nvcc #CUDA compiler
+NVCCARCH=-arch sm_70 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
+NVCCFLAGS= -std=c++14 -Xcompiler "-mcpu=power9 -Wall"# -mavx -mfma" #flags for NVCC
+
+INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
+LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
+LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
+#is the novel jsoncpp lib folder changed?
+endif
diff --git a/inc/dg/backend/config.h b/inc/dg/backend/config.h
index 6afedb0fb..8a693cc28 100644
--- a/inc/dg/backend/config.h
+++ b/inc/dg/backend/config.h
@@ -35,6 +35,10 @@
 #define SIMD simd
 #endif//__INTEL_COMPILER
 
+#elif defined(__ibmxl__)
+#define SIMD simd
+#pragma message( "Using IBM compiler!")
+
 #elif defined(__GNUG__)
 
 #ifndef GCC_VERSION
diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 98e4d3670..9ac1443bb 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -95,8 +95,10 @@ struct Collective
     thrust::host_vector<int> m_recvFrom, m_accR;
     dg::Buffer<thrust::host_vector<get_value_type<Vector> >> m_values, m_store;
 #else
-    Index m_sendTo,   m_accS; //accumulated send
-    Index m_recvFrom, m_accR; //accumulated recv
+//surprisingly MPI_Alltoallv wants the integers to be on the host, only
+//the data is on the device (does this question the necessity of Index?)
+    thrust::host_vector<int> m_sendTo,   m_accS; //accumulated send
+    thrust::host_vector<int> m_recvFrom, m_accR; //accumulated recv
 #endif // _DG_CUDA_UNAWARE_MPI
     MPI_Comm m_comm;
 };
-- 
GitLab


From 590ba7638260a9ea5cc259ceeb714c6cffe1a292 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 5 May 2020 21:40:33 +0200
Subject: [PATCH 204/540] Changed default C++ version to C++-14

This is the default as of gcc-6 so it's time to adapt
Let's see how far we get and then actually use C++-14
language features!
---
 config/README.md          |  4 ++--
 config/davide.mk          |  4 ++--
 config/devices/devices.mk |  4 ++--
 config/marconi.mk         | 23 +++++++++--------------
 4 files changed, 15 insertions(+), 20 deletions(-)

diff --git a/config/README.md b/config/README.md
index 2314160b0..a79c862e4 100644
--- a/config/README.md
+++ b/config/README.md
@@ -14,12 +14,12 @@ Your machine specific config file (e.g. feltor/config/your-machine.mk) should ha
 | :-------: | :--------------------------- | :----------------------------------------------------------- |
 |    CC     | g++                          | C++ compiler                                                 |
 |   MPICC   | mpic++                       | the corresponding mpi wrapper for the c++ compiler           |
-|  CFLAGS   | -std=c++11 -mavx -mfma -Wall | flags for the C++ compiler, avx and fma are recommended if the CPU supports it |
+|  CFLAGS   | -std=c++14 -mavx -mfma -Wall | flags for the C++ compiler, avx and fma are recommended if the CPU supports it |
 | MPICFLAGS |                              | flags specific to the MPI compilation                        |
    OPT    | -O3                                      | optimization flags for the **host** code (can be overwritten on the command line, CUDA kernel code is always compiled with -O3) |
 |  OMPFLAG  | -fopenmp                                 | The compiler flag activating the OpenMP support |
 |   NVCC    | nvcc                                     | CUDA compiler                            |
-| NVCCFLAGS | -std=c++11  -Xcompiler "-Wall -mavx -mfma"                             | flags for nvcc  and underlying host compiler, (minimum instruction set is sse4.1, avx and fma are recommended)                         |
+| NVCCFLAGS | -std=c++14  -Xcompiler "-Wall -mavx -mfma"                             | flags for nvcc  and underlying host compiler, (minimum instruction set is sse4.1, avx and fma are recommended)                         |
 | NVCCARCH  | -arch sm_35                              | specify the **gpu** compute capability  https://developer.nvidia.com/cuda-gpus (note: can be overwritten on the command line) |
 |                                          |                                          |     |
 |  INCLUDE  | -I$(HOME)/include                        | cusp, thrust, json, vcl and the draw libraries. The default expects to find (symbolic links to ) these libraries in your home folder |
diff --git a/config/davide.mk b/config/davide.mk
index 6480f6870..7c7746b48 100644
--- a/config/davide.mk
+++ b/config/davide.mk
@@ -1,9 +1,9 @@
 ifeq ($(strip $(HPC_SYSTEM)),davide)
-CFLAGS=-Wall -std=c++11 -DWITHOUT_VCL -mcpu=power8 # -mavx -mfma #flags for CC
+CFLAGS=-Wall -std=c++14 -DWITHOUT_VCL -mcpu=power8 # -mavx -mfma #flags for CC
 OPT=-O3 # optimization flags for host code
 NVCC=nvcc #CUDA compiler
 NVCCARCH=-arch sm_60 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
-NVCCFLAGS= -std=c++11 -Xcompiler "-mcpu=power8 -Wall"# -mavx -mfma" #flags for NVCC
+NVCCFLAGS= -std=c++14 -Xcompiler "-mcpu=power8 -Wall"# -mavx -mfma" #flags for NVCC
 INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
 LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
 LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
diff --git a/config/devices/devices.mk b/config/devices/devices.mk
index 1a57c9fe4..682c3b83f 100644
--- a/config/devices/devices.mk
+++ b/config/devices/devices.mk
@@ -1,7 +1,7 @@
 ifeq ($(strip $(device)),gpu)
 ccc_:=$(CC)
 CC = $(NVCC) --compiler-bindir $(ccc_)
-CFLAGS = $(NVCCARCH) $(NVCCFLAGS) 
+CFLAGS = $(NVCCARCH) $(NVCCFLAGS)
 CFLAGS+=-D_FORCE_INLINES # solves issue with std=c++11
 CFLAGS+=-D_MWAITXINTRIN_H_INCLUDED # solves issue with std=c++11
 ############################################
@@ -17,7 +17,7 @@ endif #device=gpu
 ifeq ($(strip $(device)),omp)
 endif #device=omp
 ifeq ($(strip $(device)),knl)
-OPT=-O3 -xMIC-AVX512 
+OPT=-O3 -xMIC-AVX512
 #OPT=-O3 -mavx512er
 endif #device=mic
 ifeq ($(strip $(device)),skl)
diff --git a/config/marconi.mk b/config/marconi.mk
index 6998f43f3..e85083490 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -2,10 +2,10 @@
 ifeq ($(strip $(HPC_SYSTEM)),marconi)
 CC=icc
 MPICC=mpiicc -mt_mpi
-OPT=-O3 -xHost  # overwritten for mic in devices.mk
+OPT=-O3 -xHost  # overwritten for skl in devices.mk
 #MPICFLAGS+= -DMPICH_IGNORE_CXX_SEEK
 OMPFLAG=-qopenmp
-CFLAGS=-Wall -std=c++11 -restrict -fp-model precise -fimf-arch-consistency=true #-mfma  #flags for CC
+CFLAGS=-Wall -std=c++14 -restrict -fp-model precise -fimf-arch-consistency=true #-mfma  #flags for CC
 
 INCLUDE += -I$(HOME)/include # cusp, thrust
 INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
@@ -20,15 +20,10 @@ endif
 #LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
 #endif
 #############################modules to load in .bashrc#######################
-#module load profile/base                         
-#module load intel/pe-xe-2017--binary             
-#module load intelmpi/2017--binary                
-#module load szip/2.1--gnu--6.1.0                 
-#module load zlib/1.2.8--gnu--6.1.0               
-#module load hdf5/1.8.17--intelmpi--2017--binary  
-#module load netcdf/4.4.1--intelmpi--2017--binary 
-
-
-###########configure mic jobs#########################
-#export KMP_AFFINITY=scatter #important
-#srun --partition=knl_fua_prod --constraint="cache" --qos=knl_qos_fuadbg --account=FUA22_FELTOR --nodes=1 --time=0:30:00 --pty /bin/bash
+#module load profile/base
+#module load intel/pe-xe-2017--binary
+#module load intelmpi/2017--binary
+#module load szip/2.1--gnu--6.1.0
+#module load zlib/1.2.8--gnu--6.1.0
+#module load hdf5/1.8.17--intelmpi--2017--binary
+#module load netcdf/4.4.1--intelmpi--2017--binary
-- 
GitLab


From 3c0daad02fdbd1f7d92e6705017753e311e021da Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 5 May 2020 21:43:36 +0200
Subject: [PATCH 205/540] Clean up references.bib file

and add 4 new references from Madsen, Held and Wiesenberger
---
 doc/related_pages/references.bib | 261 ++++++-------------------------
 1 file changed, 47 insertions(+), 214 deletions(-)

diff --git a/doc/related_pages/references.bib b/doc/related_pages/references.bib
index e7685a992..8fd055d5a 100644
--- a/doc/related_pages/references.bib
+++ b/doc/related_pages/references.bib
@@ -1,15 +1,11 @@
 % Encoding: UTF-8
 
-
 @Book{AS,
   Title                    = {{Handbook of mathematical functions}},
   Author                   = {Abramowitz, M. and Stegun, I.},
   Publisher                = {Dover Publishing Inc. New York},
   Year                     = {1972},
   Edition                  = {10th},
-
-  Owner                    = {matthias},
-  Timestamp                = {2014.03.11},
   Url                      = {http://people.math.sfu.ca/~cbm/aands/}
 }
 
@@ -18,34 +14,10 @@
   Author                   = {Cerfon, A. J. and Freidberg, J. P.},
   Journal                  = {Phys. Plasmas},
   Year                     = {2010},
-
-  Month                    = mar,
   Number                   = {3},
   Pages                    = {032502},
   Volume                   = {17},
-
-  Af                       = {Cerfon, Antoine J.EOLEOLFreidberg, Jeffrey P.},
-  C1                       = {[Cerfon, Antoine J.; Freidberg, Jeffrey P.] MIT, Plasma Sci & Fus Ctr, Cambridge, MA 02139 USA.},
   Doi                      = {10.1063/1.3328818},
-  Em                       = {acerfon@mit.edu},
-  Ga                       = {577GI},
-  J9                       = {PHYS PLASMAS},
-  Ji                       = {Phys. Plasmas},
-  La                       = {English},
-  Nr                       = {16},
-  Owner                    = {matthias},
-  Pa                       = {CIRCULATION & FULFILLMENT DIV, 2 HUNTINGTON QUADRANGLE, STE 1 N O 1,EOLEOLMELVILLE, NY 11747-4501 USA},
-  Pg                       = {9},
-  Pi                       = {MELVILLE},
-  Publisher                = {Amer Inst Physics},
-  Rp                       = {Cerfon, AJ (reprint author), MIT, Plasma Sci & Fus Ctr, 167 Albany St, Cambridge, MA 02139 USA.},
-  Sc                       = {Physics},
-  Sn                       = {1070-664X},
-  Tc                       = {23},
-  Timestamp                = {2014.10.22},
-  Ut                       = {WOS:000276210100032},
-  Wc                       = {Physics, Fluids & Plasmas},
-  Z9                       = {23}
 }
 
 @Article{Cerfon2014,
@@ -53,19 +25,10 @@
   Author                   = {Cerfon, A. J. and O'Neil, M.},
   Journal                  = {Phys. Plasmas},
   Year                     = {2014},
-
-  Month                    = jun,
   Number                   = {6},
   Pages                    = {064501},
   Volume                   = {21},
-
-  Af                       = {Cerfon, Antoine J.EOLEOLO'Neil, Michael},
   Doi                      = {10.1063/1.4881466},
-  Ei                       = {1089-7674},
-  Owner                    = {matthias},
-  Sn                       = {1070-664X},
-  Timestamp                = {2017.03.14},
-  Ut                       = {WOS:000338995300118}
 }
 
 @Article{Cockburn2001runge,
@@ -76,10 +39,7 @@
   Number                   = {3},
   Pages                    = {173--261},
   Volume                   = {16},
-
   Doi                      = {10.1023/a:1012873910884},
-  Owner                    = {matthias},
-  Timestamp                = {2013.12.04}
 }
 
 @Article{Cockburn1998,
@@ -87,22 +47,10 @@
   Author                   = {Cockburn, B. and Shu, C. W.},
   Journal                  = {Siam Journal On Numerical Analysis},
   Year                     = {1998},
-
-  Month                    = nov,
   Number                   = {6},
   Pages                    = {2440--2463},
   Volume                   = {35},
-
-  __markedentry            = {[matthias:]},
-  Af                       = {Cockburn, BEOLEOLShu, CW},
   Doi                      = {10.1137/S0036142997316712},
-  Ei                       = {1095-7170},
-  Oi                       = {Shu, Chi-Wang/0000-0001-7720-9564; Cockburn,EOLEOLBernardo/0000-0001-6085-3441},
-  Owner                    = {matthias},
-  Ri                       = {Shu, Chi-Wang/A-3216-2013; Cockburn, Bernardo/M-9617-2013},
-  Sn                       = {0036-1429},
-  Timestamp                = {2017.10.12},
-  Ut                       = {WOS:000077016000016}
 }
 
 @Book{haeseleer,
@@ -111,9 +59,6 @@
   Publisher                = {Springer-Verlag},
   Year                     = {1991},
   Series                   = {Springer Series in Computational Physics},
-
-  Owner                    = {matthias},
-  Timestamp                = {2017.10.11}
 }
 
 @Article{Einkemmer2014,
@@ -121,17 +66,10 @@
   Author                   = {Einkemmer, L. and Wiesenberger, M.},
   Journal                  = {Comput. Phys. Commun.},
   Year                     = {2014},
-
-  Month                    = nov,
   Number                   = {11},
   Pages                    = {2865-2873},
   Volume                   = {185},
-
-  Adsnote                  = {Provided by the SAO/NASA Astrophysics Data System},
   Doi                      = {10.1016/j.cpc.2014.07.007},
-  Keywords                 = {Physics - Computational Physics; Computer Science - Numerical Analysis; Mathematics - Numerical Analysis; Physics - Fluid Dynamics; Physics - Plasma Physics},
-  Owner                    = {matthias},
-  Timestamp                = {2017.10.11}
 }
 
 @Book{Frankel,
@@ -140,9 +78,6 @@
   Publisher                = {Cambridge University Press},
   Year                     = {2004},
   Edition                  = {Second},
-
-  Owner                    = {Jens Madsen},
-  Timestamp                = {2008.05.14}
 }
 
 @Article{Glasser2006,
@@ -153,31 +88,7 @@
   Number                   = {6},
   Pages                    = {481--505},
   Volume                   = {21},
-
-  Af                       = {Glasser, A. H.EOLEOLLiseikin, V. D.EOLEOLVaseva, I. A.EOLEOLLikhanova, Yu. V.},
-  C1                       = {Los Alamos Natl Lab, Los Alamos, NM 87545 USA.EOLEOLRussian Acad Sci, Siberian Branch, Inst Computat Technol, Novosibirsk 630090, Russia.},
   Doi                      = {10.1515/rnam.2006.21.6.481},
-  Ga                       = {118IP},
-  J9                       = {RUSS J NUMER ANAL M},
-  Ji                       = {Russ. J. Numer. Anal. Math. Model},
-  Keywords                 = {EQUIDISTRIBUTION},
-  La                       = {English},
-  Nr                       = {22},
-  Owner                    = {matthias},
-  Pa                       = {BRILL ACADEMIC PUBLISHERS, PO BOX 9000, 2300 PA LEIDEN, NETHERLANDS},
-  Pg                       = {25},
-  Pi                       = {LEIDEN},
-  Publisher                = {Vsp Bv},
-  Rp                       = {Glasser, AH (reprint author), Los Alamos Natl Lab, POB 1663, Los Alamos, NM 87545 USA.},
-  Sc                       = {Engineering; Mathematics},
-  Sn                       = {0927-6467},
-  Tc                       = {1},
-  Timestamp                = {2016.10.14},
-  U1                       = {0},
-  U2                       = {0},
-  Ut                       = {WOS:000242936200001},
-  Wc                       = {Engineering, Multidisciplinary; Mathematics, Applied},
-  Z9                       = {1}
 }
 
 @Article{Hariri2014,
@@ -185,41 +96,12 @@
   Author                   = {Hariri, F. and Hill, P. and Ottaviani, M. and Sarazin, Y.},
   Journal                  = {Physics of Plasmas},
   Year                     = {2014},
-
-  Month                    = aug,
   Number                   = {8},
   Pages                    = {082509},
   Volume                   = {21},
-
-  __markedentry            = {[matthias:6]},
-  Af                       = {Hariri, F.EOLEOLHill, P.EOLEOLOttaviani, M.EOLEOLSarazin, Y.},
   Doi                      = {10.1063/1.4892405},
-  Ei                       = {1089-7674},
-  Owner                    = {matthias},
-  Sn                       = {1070-664X},
-  Timestamp                = {2017.10.13},
-  Ut                       = {WOS:000342760600037}
 }
 
-@Article{Held2016,
-  Title                    = {Three discontinuous Galerkin schemes for the anisotropic heat conduction equation on non-aligned grids},
-  Author                   = {Held, M. and Wiesenberger, M. and Stegmeir, A.},
-  Journal                  = {Computer Physics Communications},
-  Year                     = {2016},
-
-  Month                    = feb,
-  Pages                    = {29--39},
-  Volume                   = {199},
-
-  Af                       = {Held, M.EOLEOLWiesenberger, M.EOLEOLStegmeir, A.},
-  Doi                      = {10.1016/j.cpc.2015.10.009},
-  Ei                       = {1879-2944},
-  Oi                       = {Wiesenberger, Matthias/0000-0002-5921-0163},
-  Owner                    = {matthias},
-  Sn                       = {0010-4655},
-  Timestamp                = {2017.10.11},
-  Ut                       = {WOS:000367113200005}
-}
 
 @Article{Kube2016,
   Title                    = {Amplitude and size scaling for interchange motions of plasma filaments},
@@ -229,12 +111,7 @@
   Number                   = {12},
   Pages                    = {122302},
   Volume                   = {23},
-
   Doi                      = {10.1063/1.4971220},
-  Eid                      = {122302},
-  Owner                    = {matthias},
-  Timestamp                = {2016.12.07},
-  Url                      = {http://scitation.aip.org/content/aip/journal/pop/23/12/10.1063/1.4971220}
 }
 
 @Book{Liseikin,
@@ -243,9 +120,6 @@
   Publisher                = {Springer-Verlag},
   Year                     = {2007},
   Edition                  = {Second},
-
-  Owner                    = {matthias},
-  Timestamp                = {2016.10.06}
 }
 
 @Article{Stegmeir2017,
@@ -255,12 +129,7 @@
   Year                     = {2017},
   Pages                    = {111 - 121},
   Volume                   = {213},
-
-  Doi                      = {http://dx.doi.org/10.1016/j.cpc.2016.12.014},
-  ISSN                     = {0010-4655},
-  Keywords                 = {Flux-coordinate independent approach},
-  Owner                    = {matthias},
-  Timestamp                = {2017.03.29}
+  Doi                      = {10.1016/j.cpc.2016.12.014},
 }
 
 @Article{Vaseva2009,
@@ -271,32 +140,26 @@
   Number                   = {1},
   Pages                    = {65--78},
   Volume                   = {24},
-
-  Af                       = {Vaseva, I. A.EOLEOLLiseikin, V. D.EOLEOLLikhanova, Yu. V.EOLEOLMorokov, Yu. N.},
-  C1                       = {[Vaseva, I. A.; Liseikin, V. D.; Likhanova, Yu. V.; Morokov, Yu. N.] Russian Acad Sci, Siberian Branch, Inst Computat Technol, Novosibirsk 630090, Russia.},
   Doi                      = {10.1515/RJNAMM.2009.006},
-  Fu                       = {Russian Foundation [06-01-08009, 07-0100336-a]; Siberian Division ofEOLEOLRussian Academy of Sciences [1.8]},
-  Fx                       = {The work was supported by the Russian Foundation for Basic ResearchEOLEOL(06-01-08009,07-0100336-a) and the Grant of complex integration projectEOLEOLSiberian Division of Russian Academy of Sciences, 2006 (1.8).},
-  Ga                       = {431FB},
-  J9                       = {RUSS J NUMER ANAL M},
-  Ji                       = {Russ. J. Numer. Anal. Math. Model},
-  La                       = {English},
-  Nr                       = {12},
-  Owner                    = {matthias},
-  Pa                       = {GENTHINER STRASSE 13, D-10785 BERLIN, GERMANY},
-  Pg                       = {14},
-  Pi                       = {BERLIN},
-  Publisher                = {Walter De Gruyter \& Co},
-  Rp                       = {Vaseva, IA (reprint author), Russian Acad Sci, Siberian Branch, Inst Computat Technol, Novosibirsk 630090, Russia.},
-  Sc                       = {Engineering; Mathematics},
-  Sn                       = {0927-6467},
-  Tc                       = {1},
-  Timestamp                = {2016.10.14},
-  U1                       = {0},
-  U2                       = {1},
-  Ut                       = {WOS:000265045600006},
-  Wc                       = {Engineering, Multidisciplinary; Mathematics, Applied},
-  Z9                       = {1}
+}
+@Article{Madsen2013,
+    author      = {Madsen, J.},
+    title       = {Full-F gyrofluid model},
+    year        = {2013},
+    journal     = {Phys. Plasmas},
+    Pages       = {072301},
+    Volume      = {20},
+    doi         = {10.1063/1.4813241},
+}
+
+@Article{Madsen2016,
+  Title                    = {Collisional transport across the magnetic field in drift-fluid models},
+  Author                   = {Madsen, J. and Naulin, V. and Nielsen, A. H. and Rasmussen, J. Juul},
+  Journal                  = {Physics of Plasmas},
+  Year                     = {2016},
+  Pages                    = {032306},
+  Volume                   = {23},
+  Doi                      = {10.1063/1.4943199},
 }
 
 @PhdThesis{WiesenbergerPhD,
@@ -304,29 +167,27 @@
   title     = {Gyrofluid computations of filament dynamics in tokamak scrape-off layers},
   school    = {University of Innsbruck},
   year      = {2014},
-  owner     = {matthias},
-  timestamp = {2017.10.12},
   url       = {http://resolver.obvsg.at/urn:nbn:at:at-ubi:1-1799},
 }
 
+@Article{Wiesenberger2014,
+  Title                    = {Radial convection of finite ion temperature, high amplitude plasma blobs},
+  Author                   = {Wiesenberger, M. and Madsen, J. and Kendl, A.},
+  Journal                  = {Phys. Plasmas},
+  Year                     = {2014},
+  Pages                    = {092301},
+  Volume                   = {21},
+  Doi                      = {10.1063/1.4894220},
+}
+
 @Article{Wiesenberger2017,
   Title                    = {Streamline integration as a method for two-dimensional elliptic grid generation},
   Author                   = {Wiesenberger, M. and Held, M. and Einkemmer, L.},
   Journal                  = {Journal of Computational Physics},
   Year                     = {2017},
-
-  Month                    = jul,
   Pages                    = {435--450},
   Volume                   = {340},
-
-  Af                       = {Wiesenberger, M.EOLEOLHeld, M.EOLEOLEinkemmer, L.},
   Doi                      = {10.1016/j.jcp.2017.03.056},
-  Ei                       = {1090-2716},
-  Oi                       = {Wiesenberger, Matthias/0000-0002-5921-0163},
-  Owner                    = {matthias},
-  Sn                       = {0021-9991},
-  Timestamp                = {2017.10.11},
-  Ut                       = {WOS:000401137900020}
 }
 
 @Article{Wiesenberger2017a,
@@ -334,54 +195,20 @@
   Author                   = {Wiesenberger, M. and Held, M. and Kube, R. and Garcia, E. O.},
   Journal                  = {Physics of Plasmas},
   Year                     = {2017},
-
-  Month                    = jun,
   Number                   = {6},
   Pages                    = {064502},
   Volume                   = {24},
-
-  __markedentry            = {[matthias:]},
-  Af                       = {Wiesenberger, M.EOLEOLHeld, M.EOLEOLKube, R.EOLEOLGarcia, O. E.},
-  C1                       = {[Wiesenberger, M.; Held, M.] Univ Innsbruck, Inst Ion Phys & Appl Phys, A-6020 Innsbruck, Austria.EOLEOL[Kube, R.; Garcia, O. E.] UiT Arctic Univ Norway, Dept Phys & Technol, N-9037 Tromso, Norway.},
-  Da                       = {2018-04-06},
   Doi                      = {10.1063/1.4985318},
-  Ei                       = {1089-7674},
-  Ga                       = {EZ3VQ},
-  J9                       = {PHYS PLASMAS},
-  Ji                       = {Phys. Plasmas},
-  Keywords                 = {EQUATORIAL SPREAD F; IRREGULARITIES; IONOSPHERE; TURBULENCE; RADAR},
-  La                       = {English},
-  Nr                       = {34},
-  Oi                       = {Garcia, Odd Erik/0000-0002-2377-8718; Wiesenberger,EOLEOLMatthias/0000-0002-5921-0163; Held, Markus/0000-0001-8171-8038},
-  Owner                    = {matthias},
-  Pa                       = {1305 WALT WHITMAN RD, STE 300, MELVILLE, NY 11747-4501 USA},
-  Pg                       = {5},
-  Pi                       = {MELVILLE},
-  Publisher                = {Amer Inst Physics},
-  Ri                       = {Garcia, Odd Erik/A-4417-2013},
-  Rp                       = {Wiesenberger, M (reprint author), Univ Innsbruck, Inst Ion Phys & Appl Phys, A-6020 Innsbruck, Austria.},
-  Sc                       = {Physics},
-  Sn                       = {1070-664X},
-  Tc                       = {0},
-  Timestamp                = {2018.04.06},
-  U1                       = {1},
-  U2                       = {2},
-  Ut                       = {WOS:000404639000103},
-  Wc                       = {Physics, Fluids & Plasmas},
-  Z9                       = {0}
 }
 
-@Article{Wiesenberger2014,
-  Title                    = {Radial convection of finite ion temperature, high amplitude plasma blobs},
-  Author                   = {Wiesenberger, M. and Madsen, J. and Kendl, A.},
-  Journal                  = {Phys. Plasmas},
-  Year                     = {2014},
-  Pages                    = {092301},
-  Volume                   = {21},
-
-  Doi                      = {10.1063/1.4894220},
-  Owner                    = {matthias},
-  Timestamp                = {2014.09.05}
+@Article{Wiesenberger2018,
+    author  = {Wiesenberger, M. and Held, M. and Einkemmer, L. and Kendl, A. },
+    title   = { Streamline integration as a method for structured grid generation in X-point geometry},
+    year    = {2018},
+    journal = {J. Comput. Phys.},
+    Pages   = {370-384},
+    Volume  = {373},
+    doi     = {10.1016/j.jcp.2018.07.007},
 }
 
 @PhdThesis{HeldPhD,
@@ -389,9 +216,15 @@
   title     = {Full-F gyro-fluid modelling of the tokamak edge and scrape-off layer},
   school    = {University of Innsbruck},
   year      = {2016},
-  owner     = {matthias},
-  timestamp = {2018.06.12},
   url       = {http://resolver.obvsg.at/urn:nbn:at:at-ubi:1-6853},
 }
 
-@Comment{jabref-meta: databaseType:bibtex;}
+@Article{Held2016,
+  Title                    = {Three discontinuous Galerkin schemes for the anisotropic heat conduction equation on non-aligned grids},
+  Author                   = {Held, M. and Wiesenberger, M. and Stegmeir, A.},
+  Journal                  = {Computer Physics Communications},
+  Year                     = {2016},
+  Pages                    = {29--39},
+  Volume                   = {199},
+  Doi                      = {10.1016/j.cpc.2015.10.009},
+}
-- 
GitLab


From 98ef750bea2a57e5113a2416dd2b721bbcaa7d51 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 5 May 2020 21:45:35 +0200
Subject: [PATCH 206/540] Change config message on Cuda-aware support

---
 inc/dg/backend/config.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/dg/backend/config.h b/inc/dg/backend/config.h
index 8a693cc28..dd4fa07ea 100644
--- a/inc/dg/backend/config.h
+++ b/inc/dg/backend/config.h
@@ -63,10 +63,10 @@
 
 #include "mpi-ext.h"
 #if defined(MPIX_CUDA_AWARE_SUPPORT) && MPIX_CUDA_AWARE_SUPPORT
-#pragma message( "Using CUDA-aware MPI support!")
+#pragma message( "CUDA-aware MPI support detected! Yay!")
 //Has cuda aware MPI support. Everything fine
 #elif defined(MPIX_CUDA_AWARE_SUPPORT) && !MPIX_CUDA_AWARE_SUPPORT
-#warning "NO CUDA aware MPI installation! Falling back to regular MPI!"
+#warning "No CUDA aware MPI installation! Falling back to regular MPI!"
 #define _DG_CUDA_UNAWARE_MPI
 #else
 #pragma message( "Cannot determine CUDA-aware MPI support! Falling back to regular MPI!")
-- 
GitLab


From 406a96f41ba17d957e60ed19d6105bc5e0a111b7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 7 May 2020 13:58:51 +0200
Subject: [PATCH 207/540] Fix documentation of exblas status

---
 inc/dg/backend/exblas/exdot_cuda.cuh | 2 ++
 inc/dg/backend/exblas/exdot_omp.h    | 2 ++
 inc/dg/backend/exblas/exdot_serial.h | 2 ++
 3 files changed, 6 insertions(+)

diff --git a/inc/dg/backend/exblas/exdot_cuda.cuh b/inc/dg/backend/exblas/exdot_cuda.cuh
index f5cdc01c3..ef7351c50 100644
--- a/inc/dg/backend/exblas/exdot_cuda.cuh
+++ b/inc/dg/backend/exblas/exdot_cuda.cuh
@@ -325,6 +325,7 @@ void ExDOTComplete(
  * @param x1_ptr first array
  * @param x2_ptr second array
  * @param d_superacc pointer to an array of 64 bit integers (the superaccumulator) in device memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
+ * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
  * @sa \c exblas::gpu::Round to convert the superaccumulator into a double precision number
 */
 template<class PointerOrValue1, class PointerOrValue2, size_t NBFPE=3>
@@ -354,6 +355,7 @@ void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
  * @param x2_ptr second array
  * @param x3_ptr third array
  * @param d_superacc pointer to an array of 64 bit integegers (the superaccumulator) in device memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
+ * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
  * @sa \c exblas::gpu::Round to convert the superaccumulator into a double precision number
  */
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3, size_t NBFPE=3>
diff --git a/inc/dg/backend/exblas/exdot_omp.h b/inc/dg/backend/exblas/exdot_omp.h
index 4972cb834..fa62f0967 100644
--- a/inc/dg/backend/exblas/exdot_omp.h
+++ b/inc/dg/backend/exblas/exdot_omp.h
@@ -254,6 +254,7 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, in
  * @param x1_ptr first array
  * @param x2_ptr second array
  * @param h_superacc pointer to an array of 64 bit integers (the superaccumulator) in host memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
+ * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
  * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
 */
 template<class PointerOrValue1, class PointerOrValue2, size_t NBFPE=8>
@@ -280,6 +281,7 @@ void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
  * @param x2_ptr second array
  * @param x3_ptr third array
  * @param h_superacc pointer to an array of 64 bit integegers (the superaccumulator) in host memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
+ * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
  * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
  */
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3, size_t NBFPE=8>
diff --git a/inc/dg/backend/exblas/exdot_serial.h b/inc/dg/backend/exblas/exdot_serial.h
index d9f626f4e..bcb3b6afb 100644
--- a/inc/dg/backend/exblas/exdot_serial.h
+++ b/inc/dg/backend/exblas/exdot_serial.h
@@ -132,6 +132,7 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c
  * @param x1_ptr first array
  * @param x2_ptr second array
  * @param h_superacc pointer to an array of 64 bit integers (the superaccumulator) in host memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
+ * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
  * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
 */
 template<class PointerOrValue1, class PointerOrValue2, size_t NBFPE=8>
@@ -161,6 +162,7 @@ void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
  * @param x2_ptr second array
  * @param x3_ptr third array
  * @param h_superacc pointer to an array of 64 bit integegers (the superaccumulator) in host memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
+ * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
  * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
  */
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3, size_t NBFPE=8>
-- 
GitLab


From c5ced05f4b39b8fa20c5af9e7e13dea9de07ab49 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 7 May 2020 14:58:28 +0200
Subject: [PATCH 208/540] Many Changes in geometries

- new function dg::compose replaces old dg::geo::Compose function
- remove geometries::init.h and replace its content by dg::compose
- re-implement findOpoint and findXpoint more concisely
- introduce Periodify and periodify(mag) functions
- adapt geometry_diag to changes
- write RhoP flux label magnetic function
- modernize separatrix_orthogonal_t.cu a bit
---
 inc/dg/functors.h                         |  55 +++++
 inc/geometries/README                     |   1 -
 inc/geometries/average.h                  |  43 ----
 inc/geometries/fieldaligned.h             |   3 +-
 inc/geometries/flux_t.cu                  |   3 +-
 inc/geometries/fluxfunctions.h            | 108 +++++---
 inc/geometries/geometries.h               |   1 -
 inc/geometries/geometry_diag.cu           | 285 ++++++++++++----------
 inc/geometries/hector_t.cu                |   1 -
 inc/geometries/init.h                     | 109 ---------
 inc/geometries/magnetic_field.h           |  67 ++++-
 inc/geometries/ribeiroX_t.cu              |   1 -
 inc/geometries/ribeiro_mpit.cu            |   3 +-
 inc/geometries/ribeiro_t.cu               |   3 +-
 inc/geometries/separatrix_orthogonal_t.cu | 100 ++++----
 inc/geometries/simple_orthogonal_t.cu     |   3 +-
 inc/geometries/solovev.h                  |   4 +-
 inc/geometries/utilities.h                |  25 --
 inc/geometries/utilitiesX.h               |  11 +-
 19 files changed, 411 insertions(+), 415 deletions(-)
 delete mode 100644 inc/geometries/init.h

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 662f82fb0..436fe6b81 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -7,6 +7,7 @@
 #endif
 #include <vector>
 #include <random>
+#include <functional>
 #include "blas1.h"
 #include "topology/grid.h"
 #include "topology/evaluation.h"
@@ -1805,6 +1806,60 @@ struct Histogram2D
     double binwidthx_,binwidthy_;
     container count_;
 };
+
+
+///@cond
+namespace detail
+{
+template<class F, class G>
+struct Compose
+{
+    Compose( F f, G g):m_f(f), m_g(g){}
+    template<class ...Xs>
+    auto operator() ( Xs&& ... xs){
+        return m_f(m_g(std::forward<Xs>(xs)...));
+    }
+    template<class ...Xs>
+    auto operator() ( Xs&& ... xs) const {
+        return m_f(m_g(std::forward<Xs>(xs)...));
+    }
+    private:
+    F m_f;
+    G m_g;
+};
+}//namespace detail
+///@endcond
+
+/**
+ * @brief Function composition \f$ f(g(x_0,x_1,...)) \f$
+ *
+ * @code{.cpp}
+ * dg::Grid2d grid2d( -1., 1., -1., 1., 3, 40, 40);
+ * //Mark everything above 2 with 1s and below with 0s
+ * dg::HVec fg = dg::evaluate( dg::compose( dg::Heaviside( 2.), dg::Gaussian( 0., 0., 2., 2., 4.)), grid2d);
+ * @endcode
+ * @tparam F Outer functor
+ * @tparam G Inner functor
+ * @param f outer functor
+ * @param g inner functor
+ * @note only works for host functions. If a version for device functions is ever needed
+ * it can be easily provided
+ *
+ * @return a function object that forwards all parameters to g and returns the
+ * return value of f, which is \f$ f(g(x_0,x_1,...)) \f$
+ */
+template <class F, class G>
+auto compose( F f, G g) {
+    return detail::Compose<F,G>( f, g);
+    //a C++-14 way of generating a generic lambda with a parameter pack. Taken from:
+    //https://stackoverflow.com/questions/19071268/function-composition-in-c-c11
+    //return [f,g](auto&&... xs){ return f(g(std::forward<decltype(xs)>(xs)...));};
+}
+///Composition of an arbitrary number of functions \f$ f_0(f_1(f_2( ... (x_0, x_1, ...)))\f$
+template <class F, typename... Fs>
+auto compose(F f, Fs... fs) {
+    return compose( f , compose(fs...));
+}
 ///@}
 } //namespace dg
 
diff --git a/inc/geometries/README b/inc/geometries/README
index 62aa3d18a..bab0c1ba6 100644
--- a/inc/geometries/README
+++ b/inc/geometries/README
@@ -32,7 +32,6 @@
     #utilities
         testfunctors.h
         fluxfunctions.h
-    init.h
     magnetic_field.h
     adaption.h
 
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index 0400a97e0..bd4a08001 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -13,49 +13,6 @@ namespace dg
 {
 namespace geo
 {
-/**
- * @brief Delta function for poloidal flux \f$ B_Z\f$
-     \f[ \delta(\psi_p(R,Z)-\psi_0) = \frac{ 1}{\sqrt{2\pi\varepsilon}} \exp\left(-\frac{(\psi_p(R,Z) - \psi_{0})^2}{2\varepsilon} \right)  \f]
-     @ingroup profiles
- */
-struct DeltaFunction
-{
-    DeltaFunction(const TokamakMagneticField& c, double epsilon,double psivalue) :
-        c_(c), epsilon_(epsilon), psivalue_(psivalue){ }
-    /**
-    * @brief Set a new \f$ \varepsilon\f$
-    *
-    * @param eps new value
-    */
-    void setepsilon(double eps ){epsilon_ = eps;}
-    /**
-    * @brief Set a new \f$ \psi_0\f$
-    *
-    * @param psi_0 new value
-    */
-    void setpsi(double psi_0 ){psivalue_ = psi_0;}
-
-    /**
-     *@brief \f[ \frac{1}{\sqrt{2\pi\varepsilon}} \exp\left(-\frac{(\psi_p(R,Z) - \psi_{0})^2}{2\varepsilon} \right)  \f]
-     */
-    double operator()( double R, double Z) const
-    {
-        double psip = c_.psip()(R,Z);
-        return 1./sqrt(2.*M_PI*epsilon_)*
-               exp(-( (psip-psivalue_)* (psip-psivalue_))/2./epsilon_);
-    }
-    /**
-     * @brief == operator()(R,Z)
-     */
-    double operator()( double R, double Z, double phi) const
-    {
-        return (*this)(R,Z);
-    }
-    private:
-    TokamakMagneticField c_;
-    double epsilon_;
-    double psivalue_;
-};
 
 /**
  * @brief Flux surface integral of the form
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index f2215766a..435d73d35 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -63,6 +63,7 @@ struct DSField
 {
     //z component of v may not vanish
     //Fields outside g are extended according to bc in g
+    //this extension is not(!) the same as extending the flux function with these bc
     DSField( const dg::geo::CylindricalVectorLvl0& v, const dg::aGeometry2d& g): g_(g)
     {
         thrust::host_vector<double> v_zeta, v_eta;
@@ -141,7 +142,7 @@ void integrate_all_fieldlines2d( const dg::geo::CylindricalVectorLvl0& vec,
     * @tparam Limiter Class that can be evaluated on a 2d grid, returns 1 if there
         is a limiter and 0 if there isn't.
         If a field line crosses the limiter in the plane \f$ \phi=0\f$ then the limiter boundary conditions apply.
-    * @param vec The vector field to integrate
+    * @param vec The vector field to integrate. Note that you can control how the boundary conditions are represented by changing vec outside the grid domain using e.g. the \c periodify function.
     * @param grid The grid on which to integrate fieldlines.
     * @param bcx This parameter is passed on to \c dg::create::interpolation(const thrust::host_vector<real_type>&,const thrust::host_vector<real_type>&,const aRealTopology2d<real_type>&,dg::bc,dg::bc) (see there for more details)
     * function and deterimens what happens when the endpoint of the fieldline integration leaves the domain boundaries of \c grid. Note that \c bcx and \c grid.bcx() have to be either both periodic or both not periodic.
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 76f904776..06e70d001 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -15,7 +15,6 @@
 #include "flux.h"
 #include "fieldaligned.h"
 #include "ds.h"
-#include "init.h"
 
 #include "dg/file/nc_utilities.h"
 
@@ -191,7 +190,7 @@ int main( int argc, char* argv[])
     std::cout << "TEST VOLUME IS:\n";
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    dg::geo::Iris iris( c.psip(), gp.psipmin, gp.psipmax);
+    auto iris = dg::compose( dg::Iris( gp.psipmin, gp.psipmax), c.psip());
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,3, 2e2, 2e2, dg::PER, dg::PER);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::HVec R  = dg::evaluate( dg::cooX2d, g2dC);
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index dbd3e8ff6..7fbf171dc 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -8,6 +8,8 @@ namespace dg
 namespace geo
 {
 
+///@addtogroup fluxfunctions
+///@{
 /*! @brief Inject both 2d and 3d \c operator() to a 2d functor
  *
  * The purpose of this class is to extend any 2d Functor to a
@@ -17,7 +19,6 @@ namespace geo
  * to implement this class).
  * @note If you want to write a functor that is both 2d and 3d directly,
  * it is easier to derive from \c aCylindricalFunctor
- * @ingroup fluxfunctions
  * @sa this class is an alternative to \c aCylindricalFunctor and
  * \c aCylindricalFunctor can be converted to this class
  */
@@ -47,7 +48,6 @@ struct RealCylindricalFunctor
 };
 
 ///Most of the times we use \c double
-/// @ingroup fluxfunctions
 using CylindricalFunctor = RealCylindricalFunctor<double>;
 
 /**
@@ -58,7 +58,6 @@ using CylindricalFunctor = RealCylindricalFunctor<double>;
 * where the 3d functor trivially redirects to the 2d version.
 * This behaviour is injected into all classes that derive from this class
 * via the Curiously Recurring Template Pattern (CRTP).
-* @ingroup fluxfunctions
 * @sa \c aCylindricalFunctor
 * @sa An alternative is \c RealCylindricalFunctor
 * @tparam Derived Interface: <tt> double do_compute(double,double) const</tt>
@@ -112,7 +111,6 @@ struct aCylindricalFunctor
 /**
  * @brief The constant functor
  * \f[ f(x,y) = c\f]
-* @ingroup fluxfunctions
  */
 struct Constant: public aCylindricalFunctor<Constant>
 {
@@ -122,40 +120,58 @@ struct Constant: public aCylindricalFunctor<Constant>
     double c_;
 };
 /**
-* @brief Composition \f[ f\circ\psi = f(\psi(R,Z)) \f]
-*
-* @tparam UnaryFunctor A unary Functor with interface <tt>double (double x)</tt>
-* @ingroup profiles
-*/
-template<class UnaryFunctor>
-struct Compose : public aCylindricalFunctor<Compose<UnaryFunctor>>
+ * @brief <tt> (double Z_X)</tt>
+ \f[ f(R,Z)= \begin{cases}
+ 1 \text{ if } Z > Z_X \\
+ 0 \text{ else }
+ \end{cases}
+ \f]
+ */
+struct ZCutter : public aCylindricalFunctor<ZCutter>
 {
-    /**
-    * @brief Construct from 2d functor and forward any parameters to \c UnaryFunctor
-    *
-    * @param psi A binary functor
-    * @param ps Parameters that are forwarded to the constructor of \c UnaryFunctor
-    * @tparam FunctorParams Determined by Compiler
-    */
-    template<class ...FunctorParams>
-    Compose ( CylindricalFunctor psi, FunctorParams&& ... ps): m_psip(psi),
-        m_f(std::forward<FunctorParams>(ps)...){}
-    double do_compute( double R, double Z) const
-    {
-        return m_f(m_psip(R,Z));
+    ZCutter(double ZX): Z_X(ZX){}
+    double do_compute(double R, double Z) const {
+        if( Z > Z_X)
+            return 1;
+        return 0;
     }
     private:
-    CylindricalFunctor m_psip;
-    UnaryFunctor m_f;
+    double Z_X;
 };
 
+/**
+ * @brief This function uses the dg::Grid2d::shift member to extend another function beyond the grid boundaries
+ */
 struct Periodify : public aCylindricalFunctor<Periodify>
 {
-    Periodify( CylindricalFunctor functor): m_f(functor){}
+    /**
+     * @brief Construct from grid
+     *
+     * @param functor the functor to periodify
+     * @param g The grid provides the shift member
+     */
+    Periodify( CylindricalFunctor functor, dg::Grid2d g): m_g( g), m_f(functor) {}
+    /**
+     * @brief provide 2d grid boundaries by hand
+     *
+     * @param functor the functor to periodify
+     * @param R0 left boundary in R
+     * @param R1 right boundary in R
+     * @param Z0 lower boundary in Z
+     * @param Z1 upper boundary in Z
+     * @param bcx boundary condition in x (determines how function is periodified)
+     * @param bcy boundary condition in y (determines how function is periodified)
+     */
+    Periodify( CylindricalFunctor functor, double R0, double R1, double Z0, double Z1, dg::bc bcx, dg::bc bcy): m_g( R0, R1, Z0, Z1, 3, 10, 10, bcx, bcy), m_f(functor) {}
     double do_compute( double R, double Z) const
     {
+        bool negative = false;
+        m_g.shift( negative, R, Z);
+        if( negative) return -m_f(R,Z);
+        return m_f( R, Z);
     }
     private:
+    dg::Grid2d m_g;
     CylindricalFunctor m_f;
 };
 
@@ -163,7 +179,6 @@ struct Periodify : public aCylindricalFunctor<Periodify>
 * @brief This struct bundles a function and its first derivatives
 *
 * @snippet hector_t.cu doxygen
-* @ingroup fluxfunctions
 */
 struct CylindricalFunctorsLvl1
 {
@@ -195,11 +210,12 @@ struct CylindricalFunctorsLvl1
     private:
     std::array<CylindricalFunctor,3> p_;
 };
+
+
 /**
 * @brief This struct bundles a function and its first and second derivatives
 *
 * @snippet hector_t.cu doxygen
-* @ingroup fluxfunctions
 */
 struct CylindricalFunctorsLvl2
 {
@@ -241,9 +257,38 @@ struct CylindricalFunctorsLvl2
     CylindricalFunctorsLvl1 f0,f1;
 };
 
+
+/**
+ * @brief This function finds critical points of psi (any point with vanishing gradient, including the X-point or O-point) via Newton iteration applied to the gradient of psi
+ *
+ * The inverse of the Hessian matrix is computed analytically
+ * @param psi \f$ \psi(R,Z)\f$, where R, Z are cylindrical coordinates
+ * @param R_X start value on input, critical point on output
+ * @param Z_X start value on input, critical point on output
+ * @ingroup misc_geo
+ */
+static inline void findOpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
+{
+    std::array<double, 2> X{ {0,0} }, XN(X), X_OLD(X);
+    X[0] = R_X, X[1] = Z_X;
+    double eps = 1e10, eps_old= 2e10;
+    while( (eps < eps_old || eps > 1e-7) && eps > 1e-10)
+    {
+        X_OLD = X; eps= eps_old;
+        double psipRZ = psi.dfxy()(X[0], X[1]);
+        double psipRR = psi.dfxx()(X[0], X[1]), psipZZ = psi.dfyy()(X[0],X[1]);
+        double psipR  = psi.dfx()(X[0], X[1]), psipZ = psi.dfy()(X[0], X[1]);
+        double Dinv = 1./(psipZZ*psipRR - psipRZ*psipRZ);
+        XN[0] = X[0] - Dinv*(psipZZ*psipR - psipRZ*psipZ);
+        XN[1] = X[1] - Dinv*(-psipRZ*psipR + psipRR*psipZ);
+        XN.swap(X);
+        eps = sqrt( (X[0]-X_OLD[0])*(X[0]-X_OLD[0]) + (X[1]-X_OLD[1])*(X[1]-X_OLD[1]));
+    }
+    R_X = X[0], Z_X = X[1];
+}
+
 /// A symmetric 2d tensor field and its divergence
 ///@snippet hector_t.cu doxygen
-///@ingroup fluxfunctions
 struct CylindricalSymmTensorLvl1
 {
     /**
@@ -295,7 +340,6 @@ struct CylindricalSymmTensorLvl1
 
 /// A vector field with three components that depend only on the first two coordinates
 ///@snippet ds_t.cu doxygen
-///@ingroup fluxfunctions
 struct CylindricalVectorLvl0
 {
     CylindricalVectorLvl0(){}
@@ -331,7 +375,6 @@ struct CylindricalVectorLvl0
  * @param g The vector field is pushed unto this grid
  * @return The tensor \c chi living on the coordinate system given by \c g
  * @tparam Geometry3d A three-dimensional geometry
- * @ingroup fluxfunctions
  */
 template<class Geometry3d>
 dg::SparseTensor<dg::get_host_vector<Geometry3d>> createAlignmentTensor(
@@ -365,7 +408,6 @@ dg::SparseTensor<dg::get_host_vector<Geometry3d>> createAlignmentTensor(
  * @param g The vector field is pushed unto this grid
  * @return The tensor \c chi living on the coordinate system given by \c g
  * @tparam Geometry3d A three-dimensional geometry
- * @ingroup fluxfunctions
  */
 template<class Geometry3d>
 dg::SparseTensor<dg::get_host_vector<Geometry3d>> createProjectionTensor(
diff --git a/inc/geometries/geometries.h b/inc/geometries/geometries.h
index e1e9e7379..9e44986fc 100644
--- a/inc/geometries/geometries.h
+++ b/inc/geometries/geometries.h
@@ -24,7 +24,6 @@
 #include "guenther.h"
 #include "toroidal.h"
 
-#include "init.h"
 #include "fluxfunctions.h"
 #include "magnetic_field.h"
 #include "adaption.h"
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index f8bdfaad4..99142bfde 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -14,7 +14,6 @@
 
 #include "solovev.h"
 //#include "taylor.h"
-#include "init.h"
 #include "magnetic_field.h"
 #include "testfunctors.h"
 #include "curvilinearX.h"
@@ -26,30 +25,31 @@ struct Parameters
     unsigned n, Nx, Ny, Nz, Npsi;
     double boxscaleRm, boxscaleRp;
     double boxscaleZm, boxscaleZp;
-    double amp, k_psi, bgprofamp, nprofileamp;
+    double amp, k_psi, nprofileamp;
     double sigma, posX, posY;
-    double rho_damping, alpha, alpha_mag, rho_source;
+    double damping_boundary, source_alpha, damping_alpha, source_boundary;
+    double profile_alpha;
     Parameters( const Json::Value& js){
         n = js.get("n",3).asUInt();
         Nx = js.get("Nx",100).asUInt();
         Ny = js.get("Ny",100).asUInt();
         Nz = js.get("Nz", 1).asUInt();
-        Npsi = js.get("Npsi", 16).asUInt();
-        boxscaleRm = js["boxscaleR"].get(0u, 1.1).asDouble();
-        boxscaleRp = js["boxscaleR"].get(1u, 1.1).asDouble();
-        boxscaleZm = js["boxscaleZ"].get(0u, 1.2).asDouble();
-        boxscaleZp = js["boxscaleZ"].get(1u, 1.1).asDouble();
+        Npsi = js.get("Npsi", 32).asUInt();
+        boxscaleRm = js["box"]["scaleR"].get(0u, 1.1).asDouble();
+        boxscaleRp = js["box"]["scaleR"].get(1u, 1.1).asDouble();
+        boxscaleZm = js["box"]["scaleZ"].get(0u, 1.2).asDouble();
+        boxscaleZp = js["box"]["scaleZ"].get(1u, 1.1).asDouble();
         amp = js.get("amplitude", 1.).asDouble();
         k_psi = js.get("k_psi", 1.).asDouble();
-        bgprofamp = js.get("bgprofamp", 1.).asDouble();
-        nprofileamp = js.get("nprofileamp", 1.).asDouble();
+        nprofileamp = js["profile"].get("amp", 1.).asDouble();
+        profile_alpha = js["profile"].get("alpha", 0.1).asDouble();
         sigma = js.get("sigma", 10).asDouble();
         posX = js.get("posX", 0.5).asDouble();
         posY = js.get("posY", 0.5).asDouble();
-        rho_damping = js.get("rho_damping", 1.2).asDouble();
-        alpha_mag = js.get("alpha_mag", 0.05).asDouble();
-        alpha = js.get("alpha", 0.05).asDouble();
-        rho_source = js.get("rho_source", 0.2).asDouble();
+        damping_boundary = js["damping"].get("boundary", 1.2).asDouble();
+        damping_alpha = js["damping"].get("alpha", 0.1).asDouble();
+        source_alpha = js["source"].get("alpha", 0.5).asDouble();
+        source_boundary = js["source"].get("boundary", 0.5).asDouble();
     }
     void display( std::ostream& os = std::cout ) const
     {
@@ -63,11 +63,12 @@ struct Parameters
             <<" boxscaleRp    = "<<boxscaleRp<<"\n"
             <<" boxscaleZm    = "<<boxscaleZm<<"\n"
             <<" boxscaleZp    = "<<boxscaleZp<<"\n"
-            <<" alpha         = "<<alpha<<"\n"
-            <<" alpha_mag     = "<<alpha_mag<<"\n"
+            <<" source bound  = "<<source_boundary<<"\n"
+            <<" source alpha  = "<<source_alpha<<"\n"
+            <<" damping bound = "<<damping_boundary<<"\n"
+            <<" damping alpha = "<<damping_alpha<<"\n"
             <<" amp           = "<<amp<<"\n"
             <<" k_psi         = "<<k_psi<<"\n"
-            <<" bgprofamp     = "<<bgprofamp<<"\n"
             <<" nprofileamp   = "<<nprofileamp<<"\n"
             <<" sigma         = "<<sigma<<"\n"
             <<" posX          = "<<posX<<"\n"
@@ -76,9 +77,9 @@ struct Parameters
     }
 };
 
-struct IPhi
+struct JPhi
 {
-    IPhi( dg::geo::solovev::Parameters gp): R_0(gp.R_0), A(gp.A){}
+    JPhi( dg::geo::solovev::Parameters gp): R_0(gp.R_0), A(gp.A){}
     double operator()(double R, double Z, double phi)const
     {
         return ((A-1.)*R - A*R_0*R_0/R)/R_0/R_0/R_0;
@@ -149,9 +150,18 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*gp.a*gp.elongation;
 
     //Test coefficients
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    if(p.alpha_mag > 0 )
-        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(), 0.), p.alpha_mag);
+    dg::geo::TokamakMagneticField mag_origin = dg::geo::createSolovevField(gp);
+    dg::geo::TokamakMagneticField mag = mag_origin;
+    if( p.damping_alpha > 0.)
+    {
+        double RO=mag.R0(), ZO=0.;
+        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+        double psipO = mag.psip()( RO, ZO);
+        double damping_psi0 = (1.-p.damping_boundary*p.damping_boundary)*psipO;
+        double damping_alpha = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
+        std::cout<< " damping "<< damping_psi0 << " "<<damping_alpha<<"\n";
+        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0+damping_alpha/2., fabs(p.damping_alpha/2.), ((psipO>0)-(psipO<0)));
+    }
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
     if( gp.hasXpoint())
@@ -161,103 +171,16 @@ int main( int argc, char* argv[])
         std::cout <<  "     R - Factor "<<(gp.R_0-R_X)/gp.triangularity/gp.a << "   Z - factor "<<-(Z_X/gp.elongation/gp.a)<<std::endl;
 
     }
-    const double R_H = gp.R_0-gp.triangularity*gp.a;
-    const double Z_H = gp.elongation*gp.a;
-    const double alpha_ = asin(gp.triangularity);
-    const double N1 = -(1.+alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.+alpha_);
-    const double N2 =  (1.-alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.-alpha_);
-    const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
-    const double psip0 = mag.psip()(gp.R_0, 0);
-    std::cout << "psip( 1, 0) "<<psip0<<"\n";
     //Find O-point
     double R_O = gp.R_0, Z_O = 0.;
     if( !gp.isToroidal() )
         dg::geo::findOpoint( mag.get_psip(), R_O, Z_O);
     const double psipmin = mag.psip()(R_O, Z_O);
+    std::cout << "O-point found at "<<R_O<<" "<<Z_O<<" with Psip "<<psipmin<<std::endl;
+    const double psip0 = mag.psip()(gp.R_0, 0);
+    std::cout << "psip( R_0, 0) = "<<psip0<<"\n";
 
-    std::cout << "O-point "<<R_O<<" "<<Z_O<<" with Psip = "<<psipmin<<std::endl;
-
-    std::cout << "TEST ACCURACY OF PSI (values must be close to 0!)\n";
-    if( gp.hasXpoint())
-        std::cout << "    Equilibrium with X-point!\n";
-    else
-        std::cout << "    NO X-point in flux function!\n";
-    std::cout << "psip( 1+e,0)           "<<mag.psip()(gp.R_0 + gp.a, 0.)<<"\n";
-    std::cout << "psip( 1-e,0)           "<<mag.psip()(gp.R_0 - gp.a, 0.)<<"\n";
-    std::cout << "psip( 1-de,ke)         "<<mag.psip()(R_H, Z_H)<<"\n";
-    if( !gp.hasXpoint())
-        std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H, Z_H)<<"\n";
-    else
-    {
-        std::cout << "psip( 1-1.1de,-1.1ke)  "<<mag.psip()(R_X, Z_X)<<"\n";
-        std::cout << "psipZ( 1+e,0)          "<<mag.psipZ()(gp.R_0 + gp.a, 0.)<<"\n";
-        std::cout << "psipZ( 1-e,0)          "<<mag.psipZ()(gp.R_0 - gp.a, 0.)<<"\n";
-        std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H,Z_H)<<"\n";
-        std::cout << "psipR( 1-1.1de,-1.1ke) "<<mag.psipR()(R_X,Z_X)<<"\n";
-        std::cout << "psipZ( 1-1.1de,-1.1ke) "<<mag.psipZ()(R_X,Z_X)<<"\n";
-    }
-    std::cout << "psipZZ( 1+e,0)         "<<mag.psipZZ()(gp.R_0+gp.a,0.)+N1*mag.psipR()(gp.R_0+gp.a,0)<<"\n";
-    std::cout << "psipZZ( 1-e,0)         "<<mag.psipZZ()(gp.R_0-gp.a,0.)+N2*mag.psipR()(gp.R_0-gp.a,0)<<"\n";
-    std::cout << "psipRR( 1-de,ke)       "<<mag.psipRR()(R_H,Z_H)+N3*mag.psipZ()(R_H,Z_H)<<"\n";
-
-
-    dg::HVec hvisual;
-    //allocate mem for visual
-    dg::HVec visual;
-    std::vector< std::tuple<std::string, std::string, std::function<double(double,double)> > > map{
-        {"Psip", "Flux function", mag.psip()},
-        {"PsipR", "Flux function derivative in R", mag.psipR()},
-        {"PsipZ", "Flux function derivative in Z", mag.psipZ()},
-        {"Ipol", "Poloidal current", mag.ipol()},
-        {"Bmodule", "Magnetic field strength", dg::geo::Bmodule(mag)},
-        {"InvB", "Inverse of Bmodule", dg::geo::InvB(mag)},
-        {"LnB", "Natural logarithm of Bmodule", dg::geo::LnB(mag)},
-        {"GradLnB", "The parallel derivative of LnB", dg::geo::GradLnB(mag)},
-        {"Divb", "The divergence of the magnetic unit vector", dg::geo::Divb(mag)},
-        {"B_R", "Derivative of Bmodule in R", dg::geo::BR(mag)},
-        {"B_Z", "Derivative of Bmodule in Z", dg::geo::BZ(mag)},
-        {"CurvatureNablaBR",  "R-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBR(mag,+1)},
-        {"CurvatureNablaBZ",  "Z-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBZ(mag,+1)},
-        {"CurvatureKappaR",   "R-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaR(mag,+1)},
-        {"CurvatureKappaZ",   "Z-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaZ(mag,+1)},
-        {"DivCurvatureKappa", "Divergence of the (toroidal) Kappa B curvature vector", dg::geo::DivCurvatureKappa(mag,+1)},
-        {"DivCurvatureNablaB","Divergence of the (toroidal) Nabla B curvature vector", dg::geo::DivCurvatureNablaB(mag,+1)},
-        {"TrueCurvatureNablaBR", "R-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBR(mag)},
-        {"TrueCurvatureNablaBZ", "Z-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBZ(mag)},
-        {"TrueCurvatureNablaBP", "Contravariant Phi-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBP(mag)},
-        {"TrueCurvatureKappaR", "R-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaR(mag)},
-        {"TrueCurvatureKappaZ", "Z-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaZ(mag)},
-        {"TrueCurvatureKappaP", "Contravariant Phi-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaP(mag)},
-        {"TrueDivCurvatureKappa", "Divergence of the (true) Kappa B curvature vector", dg::geo::TrueDivCurvatureKappa(mag)},
-        {"TrueDivCurvatureNablaB","Divergence of the (true) Nabla B curvature vector",  dg::geo::TrueDivCurvatureNablaB(mag)},
-        {"BFieldR", "R-component of the magnetic field vector", dg::geo::BFieldR(mag)},
-        {"BFieldZ", "Z-component of the magnetic field vector", dg::geo::BFieldZ(mag)},
-        {"BFieldP", "Contravariant Phi-component of the magnetic field vector", dg::geo::BFieldP(mag)},
-        {"BHatR", "R-component of the magnetic field unit vector", dg::geo::BHatR(mag)},
-        {"BHatZ", "Z-component of the magnetic field unit vector", dg::geo::BHatZ(mag)},
-        {"BHatP", "Contravariant Phi-component of the magnetic field unit vector", dg::geo::BHatP(mag)},
-        {"GradBHatR", "Parallel derivative of BHatR", dg::geo::BHatR(mag)},
-        {"GradBHatZ", "Parallel derivative of BHatZ", dg::geo::BHatZ(mag)},
-        {"GradBHatP", "Parallel derivative of BHatP", dg::geo::BHatP(mag)},
-        //////////////////////////////////
-        {"Iris", "A flux aligned Heaviside", dg::geo::Iris(mag.psip(), gp.psipmin, gp.psipmax)},
-        {"Pupil", "A flux aligned Heaviside", dg::geo::Pupil(mag.psip(), gp.psipmaxcut)},
-        {"GaussianDamping", "A flux aligned Heaviside with Gaussian damping", dg::geo::GaussianDamping(mag.psip(), gp.psipmaxcut, p.alpha)},
-        {"ZonalFlow",  "Flux aligned zonal flows", dg::geo::ZonalFlow(mag.psip(), p.amp, 0., 2.*M_PI*p.k_psi )},
-        {"PsiLimiter", "A flux aligned Heaviside", dg::geo::PsiLimiter(mag.psip(), gp.psipmaxlim)},
-        {"SourceProfile", "A source profile", dg::geo::Compose<dg::PolynomialHeaviside>(
-            dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-            p.rho_source, p.alpha, ((psip0>0)-(psip0<0)) ) },
-        {"Nprofile", "A flux aligned profile", dg::geo::Nprofile(mag.psip(), p.nprofileamp/mag.psip()(mag.R0(),0.), p.bgprofamp )},
-        {"Delta", "A flux aligned delta function", dg::geo::DeltaFunction( mag, gp.alpha*gp.alpha, psip0*0.2)},
-        {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::geo::TanhDamping(mag.psip(), -3*p.alpha, p.alpha, -1)},
-        ////
-        {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
-        {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
-            M_PI, p.sigma, p.sigma, p.sigma, p.amp)},
-        { "Hoo", "The novel h02 factor", dg::geo::Hoo( mag) }
 
-    };
     dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
     dg::DVec psipog2d   = dg::evaluate( mag.psip(), grid2d);
     std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
@@ -283,17 +206,17 @@ int main( int argc, char* argv[])
     dg::blas1::transform( gradPsip, gradPsip, dg::SQRT<double>());
     if( gp.hasXpoint())
     {
-        std::cout << "Generate X-point flux-aligned grid!\n";
+        std::cout << "Generate X-point flux-aligned grid ... \n";
         double RX = gp.R_0-1.1*gp.triangularity*gp.a;
         double ZX = -1.1*gp.elongation*gp.a;
         dg::geo::findXpoint( mag.get_psip(), RX, ZX);
         dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), RX, ZX) ;
-        dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, RX, ZX, mag.R0(), 0, 0, true);
+        dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, RX, ZX, mag.R0(), 0, 0, false);
         double fx_0 = 1./8.;
         psipmax = -fx_0/(1.-fx_0)*psipmin;
-        std::cout << "psi 1 is          "<<psipmax<<"\n";
+        //std::cout << "psi 1 is          "<<psipmax<<"\n";
         dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., npsi, Npsi, 640, dg::DIR, dg::NEU);
-        std::cout << "DONE! Generate X-point flux-aligned grid!\n";
+        std::cout << "DONE! \n";
         dg::Average<dg::HVec > avg_eta( gX2d.grid(), dg::coo2d::y);
         std::vector<dg::HVec> coordsX = gX2d.map();
         dg::SparseTensor<dg::HVec> metricX = gX2d.metric();
@@ -325,7 +248,7 @@ int main( int argc, char* argv[])
         dg::blas1::scal( X_psi_area, 4.*M_PI*M_PI);
         map1d.emplace_back( "X_psi_area", X_psi_area,
             "Flux area on X-point grid");
-        volumeXGrid = dg::interpolate( X_psi_vol, gX1d.x1(), gX1d);
+        volumeXGrid = dg::interpolate( dg::xspace, X_psi_vol, gX1d.x1(), gX1d);
 
         //Compute FSA of curvature operators
         dg::HVec X_curvNablaBGradPsip( psip_X), X_curvKappaKGradPsip( psip_X), X_gradPsip(psip_X);
@@ -371,6 +294,7 @@ int main( int argc, char* argv[])
 
     ///////////////////Compute flux average////////////////////
     dg::Grid1d grid1d( 0,1,npsi,Npsi, dg::NEU);
+    double deltapsi = 0.1;
     if( !gp.isToroidal())
     {
         std::cout << "Compute flux averages\n";
@@ -392,11 +316,11 @@ int main( int argc, char* argv[])
             map1d.emplace_back("psit1d", psit,
                 "Toroidal flux label psi_t integrated  on grid1d using direct q");
             //we need to avoid integrating >=0
-            dg::Grid1d g1d_fine(psipmin<0. ? psipmin : psipmax, psipmin<0. ? 0. : psipmin, npsi ,Npsi,dg::NEU);
+            dg::Grid1d g1d_fine(psipmin<0. ? psipmin : 0., psipmin<0. ? 0. : psipmin, npsi ,Npsi,dg::NEU);
             qprofile = dg::evaluate( qprof, g1d_fine);
             dg::HVec w1d = dg::create::weights( g1d_fine);
             double psit_tot = dg::blas1::dot( w1d, qprofile);
-            std::cout << "psit tot "<<psit_tot<<"\n";
+            //std::cout << "psit tot "<<psit_tot<<"\n";
             dg::blas1::scal ( psit, 1./psit_tot);
             dg::blas1::transform( psit, psit, dg::SQRT<double>());
             map1d.emplace_back("rho_t", psit,
@@ -415,6 +339,9 @@ int main( int argc, char* argv[])
         dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
         map1d.emplace_back("rho", rho,
             "Alternative flux label rho = -psi/psimin + 1");
+        dg::blas1::transform( rho, rho, dg::SQRT<double>());
+        map1d.emplace_back("rho_p", rho,
+            "Alternative flux label rho_p = Sqrt[-psi/psimin + 1]");
         fsa.set_container( (dg::DVec)curvNablaBGradPsip);
         map1d.emplace_back("curvNablaB_fsa",   dg::evaluate( fsa,      grid1d),
             "Flux surface average of true Nabla B curvature Dot Grad Psip with delta function");
@@ -428,6 +355,7 @@ int main( int argc, char* argv[])
         //other flux labels
         dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, mag);
         fsi.set_right( xpoint_weights);
+        deltapsi = fsi.get_deltapsi();
 
         dg::HVec areaT_psip = dg::evaluate( fsi, grid1d);
         dg::HVec w1d = dg::create::weights( grid1d);
@@ -448,7 +376,7 @@ int main( int argc, char* argv[])
 
 
         dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, mag);
-        std::cout << "Delta Rho for Flux surface integrals = "<<-fsi.get_deltapsi()/psipmin<<"\n";
+        //std::cout << "Delta Rho for Flux surface integrals = "<<-deltapsi/psipmin<<"\n";
 
         fvi.set_right( xpoint_weights);
         dg::HVec psi_vol = dg::evaluate( fvi, grid1d);
@@ -457,16 +385,17 @@ int main( int argc, char* argv[])
             "Flux volume with delta function");
         double volumeFVI = 2.*M_PI*fvi(psipmax);
         double volumeSep = 2.*M_PI*fvi(0.);
-        std::cout << "VOLUME ENCLOSED BY SEPARATRIX: "<<volumeSep<<"\n";
-        std::cout << "VOLUME TEST WITH COAREA FORMULA: "<<volumeCoarea<<" "<<volumeFVI
+        std::cout << "volume enclosed by separatrix: "<<volumeSep<<"\n";
+        std::cout << "volume test with coarea formula: "<<volumeCoarea<<" "<<volumeFVI
                   <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
         if(gp.hasXpoint()){
-            std::cout << "VOLUME TEST WITH X Grid FORMULA: "<<volumeXGrid<<" "<<volumeFVI
+            std::cout << "volume test with x grid formula: "<<volumeXGrid<<" "<<volumeFVI
                       <<" rel error = "<<fabs(volumeXGrid-volumeFVI)/volumeFVI<<"\n";
         };
     }
 
     /////////////////////////////set up netcdf/////////////////////////////////////
+    std::cout << "CREATING/OPENING FILE AND WRITING ... \n";
     file::NC_Error_Handle err;
     int ncid;
     err = nc_create( newfilename.data(), NC_NETCDF4|NC_CLOBBER, &ncid);
@@ -514,6 +443,75 @@ int main( int argc, char* argv[])
         err = nc_redef(ncid);
     }
     //write 2d vectors
+    std::vector< std::tuple<std::string, std::string, std::function<double(double,double)> > > map{
+        {"Psip", "Flux function", mag.psip()},
+        {"PsipR", "Flux function derivative in R", mag.psipR()},
+        {"PsipZ", "Flux function derivative in Z", mag.psipZ()},
+        {"PsipRR", "Flux function derivative in RR", mag.psipRR()},
+        {"PsipRZ", "Flux function derivative in RZ", mag.psipRZ()},
+        {"PsipZZ", "Flux function derivative in ZZ", mag.psipZZ()},
+        {"Ipol", "Poloidal current", mag.ipol()},
+        {"IpolR", "Poloidal current derivative in R", mag.ipolR()},
+        {"IpolZ", "Poloidal current derivative in Z", mag.ipolZ()},
+        {"Rho_p", "Normalized Poloidal flux label", dg::geo::RhoP(mag)},
+        {"Bmodule", "Magnetic field strength", dg::geo::Bmodule(mag)},
+        {"InvB", "Inverse of Bmodule", dg::geo::InvB(mag)},
+        {"LnB", "Natural logarithm of Bmodule", dg::geo::LnB(mag)},
+        {"GradLnB", "The parallel derivative of LnB", dg::geo::GradLnB(mag)},
+        {"Divb", "The divergence of the magnetic unit vector", dg::geo::Divb(mag)},
+        {"B_R", "Derivative of Bmodule in R", dg::geo::BR(mag)},
+        {"B_Z", "Derivative of Bmodule in Z", dg::geo::BZ(mag)},
+        {"CurvatureNablaBR",  "R-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBR(mag,+1)},
+        {"CurvatureNablaBZ",  "Z-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBZ(mag,+1)},
+        {"CurvatureKappaR",   "R-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaR(mag,+1)},
+        {"CurvatureKappaZ",   "Z-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaZ(mag,+1)},
+        {"DivCurvatureKappa", "Divergence of the (toroidal) Kappa B curvature vector", dg::geo::DivCurvatureKappa(mag,+1)},
+        {"DivCurvatureNablaB","Divergence of the (toroidal) Nabla B curvature vector", dg::geo::DivCurvatureNablaB(mag,+1)},
+        {"TrueCurvatureNablaBR", "R-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBR(mag)},
+        {"TrueCurvatureNablaBZ", "Z-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBZ(mag)},
+        {"TrueCurvatureNablaBP", "Contravariant Phi-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBP(mag)},
+        {"TrueCurvatureKappaR", "R-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaR(mag)},
+        {"TrueCurvatureKappaZ", "Z-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaZ(mag)},
+        {"TrueCurvatureKappaP", "Contravariant Phi-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaP(mag)},
+        {"TrueDivCurvatureKappa", "Divergence of the (true) Kappa B curvature vector", dg::geo::TrueDivCurvatureKappa(mag)},
+        {"TrueDivCurvatureNablaB","Divergence of the (true) Nabla B curvature vector",  dg::geo::TrueDivCurvatureNablaB(mag)},
+        {"BFieldR", "R-component of the magnetic field vector", dg::geo::BFieldR(mag)},
+        {"BFieldZ", "Z-component of the magnetic field vector", dg::geo::BFieldZ(mag)},
+        {"BFieldP", "Contravariant Phi-component of the magnetic field vector", dg::geo::BFieldP(mag)},
+        {"BHatR", "R-component of the magnetic field unit vector", dg::geo::BHatR(mag)},
+        {"BHatZ", "Z-component of the magnetic field unit vector", dg::geo::BHatZ(mag)},
+        {"BHatP", "Contravariant Phi-component of the magnetic field unit vector", dg::geo::BHatP(mag)},
+        {"GradBHatR", "Parallel derivative of BHatR", dg::geo::BHatR(mag)},
+        {"GradBHatZ", "Parallel derivative of BHatZ", dg::geo::BHatZ(mag)},
+        {"GradBHatP", "Parallel derivative of BHatP", dg::geo::BHatP(mag)},
+        {"GradPsip", "Module of gradient of Psip", dg::geo::GradPsip(mag)},
+        //////////////////////////////////
+        {"Iris", "A flux aligned Iris", dg::compose( dg::Iris( gp.psipmin, gp.psipmax), mag.psip())},
+        {"Pupil", "A flux aligned Pupil", dg::compose( dg::Pupil(gp.psipmaxcut), mag.psip()) },
+        {"GaussianDamping", "A flux aligned Heaviside with Gaussian damping", dg::compose( dg::GaussianDamping( gp.psipmaxcut, p.source_alpha), mag.psip()) },
+        {"ZonalFlow",  "Flux aligned Sine function", dg::compose( dg::SinX ( p.amp, 0., 2.*M_PI*p.k_psi ), mag.psip())},
+        {"PsiLimiter", "A flux aligned Heaviside", dg::compose( dg::Heaviside( gp.psipmaxlim), mag.psip() )},
+        {"SourceProfile", "A source profile", dg::compose( dg::PolynomialHeaviside(
+                    p.source_boundary-p.source_alpha/2., p.source_alpha/2., -1 ),
+                dg::geo::RhoP(mag))},
+        {"ProfileDamping", "Density profile damping", dg::compose(dg::PolynomialHeaviside(
+            1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)) },
+        {"MagneticTransition", "The region where the magnetic field is modified", dg::compose(dg::DPolynomialHeaviside(
+            p.damping_boundary+p.damping_alpha/2.,
+            p.damping_alpha/2., +1 ), dg::geo::RhoP(mag_origin))},
+        {"Nprofile", "A flux aligned profile", dg::compose( dg::LinearX( p.nprofileamp/mag.psip()(mag.R0(),0.), p.nprofileamp ), mag.psip())},
+        {"Delta", "A flux aligned Gaussian peak", dg::compose( dg::GaussianX( psipmin*0.2, deltapsi, 1./(sqrt(2.*M_PI)*deltapsi)), mag.psip())},
+        {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::compose( dg::TanhProfX( -3*p.source_alpha, p.source_alpha, -1), mag.psip())},
+        ////
+        {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
+        {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
+            M_PI, p.sigma, p.sigma, p.sigma, p.amp)},
+        { "Hoo", "The novel h02 factor", dg::geo::Hoo( mag) }
+
+    };
+    //allocate mem for visual
+    dg::HVec hvisual;
+    dg::HVec visual;
     for(auto tp : map)
     {
         int vectorID;
@@ -575,7 +573,42 @@ int main( int argc, char* argv[])
     //////////////////////////////Finalize////////////////////////////////////
     err = nc_close(ncid);
     std::cout << "FILE CLOSED AND READY TO USE NOW!\n";
-    std::cout << "Test accuracy of curvatures (values must be close to 0!)\n";
+
+    {
+        std::cout << "test accuracy of psi (values must be close to 0!)\n";
+        const double R_H = gp.R_0-gp.triangularity*gp.a;
+        const double Z_H = gp.elongation*gp.a;
+        const double alpha_ = asin(gp.triangularity);
+        const double N1 = -(1.+alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.+alpha_);
+        const double N2 =  (1.-alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.-alpha_);
+        const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
+
+        if( gp.hasXpoint())
+            std::cout << "    Equilibrium with X-point!\n";
+        else
+            std::cout << "    NO X-point in flux function!\n";
+        std::cout << "psip( 1+e,0)           "<<mag.psip()(gp.R_0 + gp.a, 0.)<<"\n";
+        std::cout << "psip( 1-e,0)           "<<mag.psip()(gp.R_0 - gp.a, 0.)<<"\n";
+        std::cout << "psip( 1-de,ke)         "<<mag.psip()(R_H, Z_H)<<"\n";
+        if( !gp.hasXpoint())
+            std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H, Z_H)<<"\n";
+        else
+        {
+            std::cout << "psip( 1-1.1de,-1.1ke)  "<<mag.psip()(R_X, Z_X)<<"\n";
+            std::cout << "psipZ( 1+e,0)          "<<mag.psipZ()(gp.R_0 + gp.a, 0.)<<"\n";
+            std::cout << "psipZ( 1-e,0)          "<<mag.psipZ()(gp.R_0 - gp.a, 0.)<<"\n";
+            std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H,Z_H)<<"\n";
+            std::cout << "psipR( 1-1.1de,-1.1ke) "<<mag.psipR()(R_X,Z_X)<<"\n";
+            std::cout << "psipZ( 1-1.1de,-1.1ke) "<<mag.psipZ()(R_X,Z_X)<<"\n";
+        }
+        std::cout << "psipZZ( 1+e,0)         "<<mag.psipZZ()(gp.R_0+gp.a,0.)+N1*mag.psipR()(gp.R_0+gp.a,0)<<"\n";
+        std::cout << "psipZZ( 1-e,0)         "<<mag.psipZZ()(gp.R_0-gp.a,0.)+N2*mag.psipR()(gp.R_0-gp.a,0)<<"\n";
+        std::cout << "psipRR( 1-de,ke)       "<<mag.psipRR()(R_H,Z_H)+N3*mag.psipZ()(R_H,Z_H)<<"\n";
+    }
+
+
+    std::cout << "Test accuracy of curvatures (values must be close to 0, but Test may be inaccurate for modified Psi_p)\n";
+    //Test NablaTimes B = B^2( curvK - curvB)
     std::array<dg::HVec, 3> bhat, curvB, curvK;
     dg::pushForward( bhat_.x(), bhat_.y(), bhat_.z(),
             bhat[0], bhat[1], bhat[2], grid3d);
@@ -592,10 +625,12 @@ int main( int argc, char* argv[])
     double error = sqrt(dg::blas1::dot( normb, normb));
     std::cout << "Error in norm b == 1 :  "<<error<<std::endl;
 
+    std::cout << "Push Forward curvatures ...\n";
     dg::pushForward( curvB_.x(), curvB_.y(), curvB_.z(),
             curvB[0], curvB[1], curvB[2], grid3d);
     dg::pushForward( curvK_.x(), curvK_.y(), curvK_.z(),
             curvK[0], curvK[1], curvK[2], grid3d);
+    std::cout << "Done!\n";
     dg::blas1::axpby( 1., curvK, -1., curvB);
     dg::HVec Bmodule = dg::pullback( dg::geo::Bmodule(mag), grid3d);
     dg::blas1::pointwiseDot( Bmodule, Bmodule, Bmodule);
@@ -606,7 +641,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDivide( gp.R_0, IR, R, 0., IR);
     dg::HVec IZ =  dg::pullback( mag.ipolZ(), grid3d);
     dg::blas1::pointwiseDivide( gp.R_0, IZ, R, 0., IZ);
-    dg::HVec IP =  dg::pullback( IPhi( gp), grid3d);
+    dg::HVec IP =  dg::pullback( JPhi( gp), grid3d);
     dg::blas1::pointwiseDivide( gp.R_0, IP, R, 0., IP);
     dg::blas1::axpby( 1., IZ, -1., curvB[0]);
     dg::blas1::axpby( 1., IR, +1., curvB[1]);
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 5cfac12f6..a2c671842 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -17,7 +17,6 @@
 #include "solovev.h"
 #include "hector.h"
 //#include "refined_conformal.h"
-#include "init.h"
 
 
 thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, const dg::Grid2d& g)
diff --git a/inc/geometries/init.h b/inc/geometries/init.h
deleted file mode 100644
index 722b64b65..000000000
--- a/inc/geometries/init.h
+++ /dev/null
@@ -1,109 +0,0 @@
-#pragma once
-//#include "solovev.h"
-#include "fluxfunctions.h"
-#include "solovev_parameters.h"
-
-/*!@file
- *
- * Initialize and Damping objects
- */
-namespace dg
-{
-namespace geo
-{
-
-///@addtogroup profiles
-///@{
-
-/**
- * @brief <tt>(CylindricalFunctor psip, double psip_min, double psip_max)</tt>
-     \f[ f(R,Z) = \begin{cases}
-        1  \text{ if } \psi_{p,min} < \psi_p(R,Z) < \psi_{p,max}\\
-        0  \text{ else}
-     \end{cases}\f]
- */
-using Iris = Compose<dg::Iris>;
-/**
- * @brief <tt> (CylindricalFunctor psip, double psipmax) </tt>
-     \f[ f(R,Z) = \begin{cases}
-        0  \text{ if } \psi_p(R,Z) > \psi_{p,max} \\
-        1  \text{ else}
-     \end{cases}\f]
- */
-using Pupil = Compose<dg::Pupil>;
-
-/**
- * @brief <tt> (CylindricalFunctor psip, double psipmax)</tt>
-     \f[ f(R,Z) = \begin{cases}
-        \psi_{p,max}  \text{ if } \psi_p(R,Z) > \psi_{p,max} \\
-        \psi_p(R,Z) \text{ else}
-     \end{cases}\f]
- */
-using PsiPupil = Compose<dg::PsiPupil>;
-
-/**
- * @brief <tt> (CylindricalFunctor psip, double psipmax) </tt>
-     \f[ f(R,Z) = \begin{cases}
-        1  \text{ if } \psi_p(R,Z) > \psi_{p,\max} \\
-        0  \text{ else}
-     \end{cases}\f]
- */
-using PsiLimiter = dg::geo::Compose<dg::Heaviside>;
-
-/**
- * @brief <tt>(CylindricalFunctor psip, double psipmax, double alpha)</tt>
- * \f[ f(R,Z) = \begin{cases}
- 1 \text{ if } \psi_p(R,Z) < \psi_{p,max}\\
- 0 \text{ if } \psi_p(R,Z) > (\psi_{p,max} + 4\alpha) \\
- \exp\left( - \frac{(\psi_p(R,Z) - \psi_{p,max})^2}{2\alpha^2}\right), \text{ else}
- \end{cases}
-   \f]
- */
-using GaussianDamping = dg::geo::Compose<dg::GaussianDamping>;
-
-/**
- * @brief <tt>(CylindricalFunctor psip, double psipmax, double alpha, int sign =1,double B = 0., double A = 1.) </tt>
- * \f[ f(R,Z) = B + 0.5 A(1+ \text{sign} \tanh((\psi_p(R,Z)-\psi_{p,max})/\alpha ) ) \f]
-
- Similar to GaussianProfDamping is zero outside given \c psipmax and
- one inside of it.
- */
-using TanhDamping = dg::geo::Compose<dg::TanhProfX>;
-
-/**
- * @brief <tt> (CylindricalFunctor psip, double A, double B)</tt>
- *\f[ f(R,Z)= B + A\psi_p(R,Z)\f]
- */
-using Nprofile = dg::geo::Compose<dg::LinearX>;
-
-/**
- * @brief <tt>(CylindricalFunctor psip, double A, double B, double k_psi)</tt>
-     \f[ f(R,Z)= B + A \sin(k_\psi \psi_p(R,Z) ) \f]
- */
-using ZonalFlow = dg::geo::Compose<dg::SinX>;
-
-/**
- * @brief <tt> (double Z_X)</tt>
- \f[ f(R,Z)= \begin{cases}
- 1 \text{ if } Z > Z_X \\
- 0 \text{ else }
- \end{cases}
- \f]
- */
-struct ZCutter : public aCylindricalFunctor<ZCutter>
-{
-    ZCutter(double ZX): Z_X(ZX){}
-    double do_compute(double R, double Z) const {
-        if( Z > Z_X)
-            return 1;
-        return 0;
-    }
-    private:
-    double Z_X;
-};
-
-
-///@}
-}//namespace functors
-}//namespace dg
-
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 6650eb429..7b71f0439 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -35,8 +35,10 @@ struct TokamakMagneticField
 {
     ///as long as the field stays empty the access functions are undefined
     TokamakMagneticField(){}
-    TokamakMagneticField( double R0, const CylindricalFunctorsLvl2& psip, const CylindricalFunctorsLvl1& ipol): R0_(R0), psip_(psip), ipol_(ipol){}
-    void set( double R0, const CylindricalFunctorsLvl2& psip, const CylindricalFunctorsLvl1& ipol)
+    TokamakMagneticField( double R0, const CylindricalFunctorsLvl2& psip, const
+            CylindricalFunctorsLvl1& ipol): R0_(R0), psip_(psip), ipol_(ipol){}
+    void set( double R0, const CylindricalFunctorsLvl2& psip, const
+            CylindricalFunctorsLvl1& ipol)
     {
         R0_=R0;
         psip_=psip;
@@ -72,6 +74,48 @@ struct TokamakMagneticField
     CylindricalFunctorsLvl1 ipol_;
 };
 
+///@cond
+CylindricalFunctorsLvl1 periodify( const CylindricalFunctorsLvl1& in, double R0, double R1, double Z0, double Z1, bc bcx, bc bcy)
+{
+    return CylindricalFunctorsLvl1(
+            Periodify( in.f(),   R0, R1, Z0, Z1, bcx, bcy),
+            Periodify( in.dfx(), R0, R1, Z0, Z1, inverse(bcx), bcy),
+            Periodify( in.dfy(), R0, R1, Z0, Z1, bcx, inverse(bcy)));
+}
+CylindricalFunctorsLvl2 periodify( const CylindricalFunctorsLvl2& in, double R0, double R1, double Z0, double Z1, bc bcx, bc bcy)
+{
+    return CylindricalFunctorsLvl2(
+            Periodify( in.f(),   R0, R1, Z0, Z1, bcx, bcy),
+            Periodify( in.dfx(), R0, R1, Z0, Z1, inverse(bcx), bcy),
+            Periodify( in.dfy(), R0, R1, Z0, Z1, bcx, inverse(bcy)),
+            Periodify( in.dfxx(), R0, R1, Z0, Z1, bcx, bcy),
+            Periodify( in.dfxy(), R0, R1, Z0, Z1, inverse(bcx), inverse(bcy)),
+            Periodify( in.dfyy(), R0, R1, Z0, Z1, bcx, bcy));
+}
+///@endcond
+/**
+ * @brief Use dg::geo::Periodify to periodify every function the magnetic field
+ *
+ * Note that derivatives are periodified with dg::inverse boundary conditions
+ * @param mag The magnetic field to periodify
+ * @param R0 left boundary in R
+ * @param R1 right boundary in R
+ * @param Z0 lower boundary in Z
+ * @param Z1 upper boundary in Z
+ * @param bcx boundary condition in x (determines how function is periodified)
+ * @param bcy boundary condition in y (determines how function is periodified)
+ * @note So far this was only tested for Neumann boundary conditions. It is uncertain if Dirichlet boundary conditions work
+ *
+ * @return new periodified magnetic field
+ */
+TokamakMagneticField periodify( const TokamakMagneticField& mag, double R0, double R1, double Z0, double Z1, dg::bc bcx, dg::bc bcy)
+{
+    //what if Dirichlet BC in the current? Won't that generate a NaN?
+    return TokamakMagneticField( mag.R0(),
+            periodify( mag.get_psip(), R0, R1, Z0, Z1, bcx, bcy),
+            //what if Dirichlet BC in the current? Won't that generate a NaN?
+            periodify( mag.get_ipol(), R0, R1, Z0, Z1, bcx, bcy));
+}
 
 ///@brief \f$   |B| = R_0\sqrt{I^2+(\nabla\psi)^2}/R   \f$
 struct Bmodule : public aCylindricalFunctor<Bmodule>
@@ -678,6 +722,25 @@ struct GradPsip: public aCylindricalFunctor<GradPsip>
     TokamakMagneticField m_mag;
 
 };
+
+///@brief \f$ \sqrt{\psi_p/ \psi_{p,\min}} \f$
+struct RhoP: public aCylindricalFunctor<RhoP>
+{
+    RhoP( const TokamakMagneticField& mag): m_mag(mag){
+        double RO = m_mag.R0(), ZO = 0;
+        findOpoint( mag.get_psip(), RO, ZO);
+        m_psipmin = m_mag.psip()(RO, ZO);
+    }
+    double do_compute( double R, double Z) const
+    {
+        return sqrt( 1.-m_mag.psip()(R,Z)/m_psipmin ) ;
+    }
+    private:
+    double m_psipmin;
+    TokamakMagneticField m_mag;
+
+};
+
 ///@brief Inertia factor \f$ \mathcal I_0 \f$
 struct Hoo : public dg::geo::aCylindricalFunctor<Hoo>
 {
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index de0adf427..1b6408760 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -14,7 +14,6 @@
 //#include "guenther.h"
 #include "curvilinearX.h"
 #include "ribeiroX.h"
-#include "init.h"
 #include "ds.h"
 
 double sine( double x) {return sin(x);}
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index be3475518..7753f470f 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -15,7 +15,6 @@
 #include "ribeiro.h"
 #include "simple_orthogonal.h"
 //#include "ds.h"
-#include "init.h"
 
 #include <netcdf_par.h>
 #include "dg/file/nc_utilities.h"
@@ -154,7 +153,7 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "TEST VOLUME IS:\n";
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    dg::geo::Iris iris(psip.f(), gp.psipmin, gp.psipmax);
+    auto iris = dg::compose( dg::Iris(gp.psipmin, gp.psipmax), psip.f());
     //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
     dg::CartesianMPIGrid2d g2dC( gp.R_0 -2.*gp.a, gp.R_0 + 2.*gp.a, -2.*gp.a, 2.*gp.a, 1, 2e3, 2e3, dg::DIR, dg::PER, g2d->communicator());
     dg::MHVec vec  = dg::evaluate( iris, g2dC);
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index b72e7be04..a138125b0 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -13,7 +13,6 @@
 #include "solovev.h"
 #include "ribeiro.h"
 //#include "ds.h"
-#include "init.h"
 
 #include "dg/file/nc_utilities.h"
 
@@ -157,7 +156,7 @@ int main( int argc, char* argv[])
     std::cout << "TEST VOLUME IS:\n";
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    dg::geo::Iris iris(psip.f(), gp.psipmin, gp.psipmax);
+    auto iris = dg::compose( dg::Iris(gp.psipmin, gp.psipmax), psip.f()) ;
     //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
 //     dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, -1.2*gp.a, 1.2*gp.a, 1, 1e3, 1e3, dg::PER, dg::PER);
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a, 2.0*gp.a, 1, 2e3, 2e3, dg::PER, dg::PER);
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index ad1ee32e9..c244f375d 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -16,18 +16,18 @@
 #include "refined_curvilinearX.h"
 #include "curvilinearX.h"
 #include "separatrix_orthogonal.h"
-#include "init.h"
 #include "testfunctors.h"
+#include "fluxfunctions.h"
 
 #include "dg/file/nc_utilities.h"
 
 
 double sine( double x) {return sin(x);}
 double cosine( double x) {return cos(x);}
-//typedef dg::FieldAligned< dg::CurvilinearGridX3d<dg::HVec> , dg::IHMatrix, dg::HVec> HFA;
 
 thrust::host_vector<double> periodify( const thrust::host_vector<double>& in, const dg::GridX3d& g)
 {
+    //in is a 2d vector, out has a second layer on top
     assert( g.Nz() == 2);
     thrust::host_vector<double> out(g.size());
     for( unsigned s=0; s<g.Nz(); s++)
@@ -81,7 +81,6 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        //std::ifstream is("geometry_params_Xpoint_taylor.js");
         std::ifstream is("geometry_params_Xpoint.js");
         is >> js;
     }
@@ -91,7 +90,13 @@ int main( int argc, char* argv[])
         is >> js;
     }
     //dg::geo::taylor::Parameters gp(js);
+    //dg::geo::TokamakMagneticField mag = dg::geo::createTaylorField(gp);
     dg::geo::solovev::Parameters gp(js);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    double R_O = gp.R_0, Z_O = 0.;
+    dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
+    const double psipmin = mag.psip()(R_O, Z_O);
+    std::cout << "Psi min "<<psipmin<<"\n";
     dg::Timer t;
     std::cout << "Type psi_0 (-15) \n";
     double psi_0 = -20;
@@ -109,22 +114,16 @@ int main( int argc, char* argv[])
     gp.display( std::cout);
     std::cout << "Constructing orthogonal grid ... \n";
     t.tic();
-    //dg::geo::TokamakMagneticField c = dg::geo::createTaylorField(gp);
-    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
-    double R_O = gp.R_0, Z_O = 0.;
-    dg::geo::findXpoint( c.get_psip(), R_O, Z_O);
-    const double psipmin = c.psip()(R_O, Z_O);
-    std::cout << "Psi min "<<psipmin<<"\n";
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
     //dg::geo::CylindricalSymmTensorLvl1 monitor_chi;
-    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( c.get_psip(), R_X, Z_X) ;
-    //dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xbump_monitor( c.get_psip(), R_X, Z_X, 100, 100) ;
+    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), R_X, Z_X) ;
+    //dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xbump_monitor( mag.get_psip(), R_X, Z_X, 100, 100) ;
     std::cout << "X-point set at "<<R_X<<" "<<Z_X<<"\n";
 
     double R0 = gp.R_0, Z0 = 0;
-    dg::geo::SeparatrixOrthogonal generator(c.get_psip(), monitor_chi, psi_0, R_X,Z_X, R0, Z0,0, true);
-    //dg::geo::SimpleOrthogonalX generator(c.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
+    dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psi_0, R_X,Z_X, R0, Z0,0, true);
+    //dg::geo::SimpleOrthogonalX generator(mag.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
     dg::EquidistXRefinement equi(add_x, add_y, 1,1);
     dg::geo::CurvilinearRefinedProductGridX3d g3d(equi, generator, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
     dg::geo::CurvilinearRefinedGridX2d g2d(equi, generator, fx_0, fy_0, n, Nx, Ny,dg::DIR, dg::NEU);
@@ -137,64 +136,59 @@ int main( int argc, char* argv[])
     g1d.display( std::cout);
     dg::HVec x_left = dg::evaluate( sine, g1d), x_right(x_left);
     dg::HVec y_left = dg::evaluate( cosine, g1d);
-    int ncid;
-    file::NC_Error_Handle err;
-    err = nc_create( "orthogonalX.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
-    int dim3d[3], dim1d[1];
-    err = file::define_dimensions(  ncid, dim3d, g3d_periodic.grid());
-    //err = file::define_dimensions(  ncid, dim3d, g2d.grid());
-    err = file::define_dimension(  ncid, dim1d, g1d, "i");
-    int coordsID[2], defID, volID, divBID, gxxID, gyyID, gxyID;
-    err = nc_def_var( ncid, "psi", NC_DOUBLE, 3, dim3d, &gxyID);
-    err = nc_def_var( ncid, "deformation", NC_DOUBLE, 3, dim3d, &defID);
-    err = nc_def_var( ncid, "volume", NC_DOUBLE, 3, dim3d, &volID);
-    err = nc_def_var( ncid, "divB", NC_DOUBLE, 3, dim3d, &divBID);
-    err = nc_def_var( ncid, "num_solution", NC_DOUBLE, 3, dim3d, &gxxID);
-    err = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 3, dim3d, &gyyID);
-    err = nc_def_var( ncid, "xc", NC_DOUBLE, 3, dim3d, &coordsID[0]);
-    err = nc_def_var( ncid, "yc", NC_DOUBLE, 3, dim3d, &coordsID[1]);
-
-    thrust::host_vector<double> psi_p = dg::pullback( c.psip(), g2d);
+    std::vector<std::tuple<std::string, dg::HVec, std::string> > map2d;
+    thrust::host_vector<double> psi_p = dg::pullback( mag.psip(), g2d);
+    map2d.emplace_back( "Psi_p", psi_p, "Poloidal flux function");
     g2d.display();
-    //err = nc_put_var_double( ncid, onesID, periodify(psi_p, g3d_periodic).data());
-    //err = nc_put_var_double( ncid, onesID, periodify(g2d.g(), g3d_periodic).data());
-    dg::HVec X( g2d.size()), Y(X); //P = dg::pullback( dg::coo3, g);
+    dg::HVec X( g2d.size()), Y(X), P = dg::evaluate( dg::zero, g2d);
     for( unsigned i=0; i<g2d.size(); i++)
     {
         X[i] = g2d.map()[0][i];
         Y[i] = g2d.map()[1][i];
     }
-    err = nc_put_var_double( ncid, coordsID[0], periodify(X, g3d_periodic).data());
-    err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g3d_periodic).data());
+    map2d.emplace_back( "xc", X, "X-coordinate Cartesian");
+    map2d.emplace_back( "yc", Y, "Y-coordinate Cartesian");
+    map2d.emplace_back( "zc", P, "Z-coordinate Cartesian");
 
     dg::HVec ones = dg::evaluate( dg::one, g2d);
     dg::HVec temp0( g2d.size()), temp1(temp0);
     dg::HVec w2d = dg::create::weights( g2d);
 
     dg::SparseTensor<dg::HVec> metric = g2d.metric();
-    dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
     dg::HVec vol = dg::tensor::volume(metric);
-    dg::blas1::transfer( vol, X);
-    err = nc_put_var_double( ncid, volID, periodify(X, g3d_periodic).data());
-    dg::blas1::transfer( g_xx, X);
-    err = nc_put_var_double( ncid, gxxID, periodify(X, g3d_periodic).data());
-    dg::blas1::transfer( g_xy, X);
-    err = nc_put_var_double( ncid, gxyID, periodify(X, g3d_periodic).data());
-    dg::blas1::transfer( g_yy, X);
-    err = nc_put_var_double( ncid, gyyID, periodify(X, g3d_periodic).data());
-
-    std::cout << "Construction successful!\n";
-
-    X = dg::pullback( dg::geo::FuncDirNeu(c, psi_0, psi_1, 480, -300, 70., 1. ), g2d);
-    err = nc_put_var_double( ncid, divBID, periodify(X, g3d_periodic).data());
-    X = dg::pullback( dg::geo::FuncDirNeu(c, psi_0, psi_1, 420, -470, 50.,1.), g2d);
-    err = nc_put_var_double( ncid, defID, periodify(X, g3d_periodic).data());
+    map2d.emplace_back( "volume", vol, "Volume element");
+    map2d.emplace_back( "g_xx", metric.value(0,0), "Metric element");
+    map2d.emplace_back( "g_xy", metric.value(0,1), "Metric element");
+    map2d.emplace_back( "g_yy", metric.value(1,1), "Metric element");
+
+    X = dg::pullback( dg::geo::FuncDirNeu(mag, psi_0, psi_1, 480, -300, 70., 1. ), g2d);
+    map2d.emplace_back( "FuncDirNeu1", X, "FuncDirNeu");
+    X = dg::pullback( dg::geo::FuncDirNeu(mag, psi_0, psi_1, 420, -470, 50.,1.), g2d);
+    map2d.emplace_back( "FuncDirNeu2", X, "FuncDirNeu");
+    std::cout << "OPEN FILE orthogonalX.nc ...\n";
+    int ncid;
+    file::NC_Error_Handle err;
+    err = nc_create( "orthogonalX.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
+    int dim3d[3];
+    err = file::define_dimensions(  ncid, dim3d, g3d_periodic.grid());
+    for(auto tp : map2d)
+    {
+        int vid;
+        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 3,
+            dim3d, &vid);
+        err = nc_put_att_text( ncid, vid, "long_name",
+            std::get<2>(tp).size(), std::get<2>(tp).data());
+        err = nc_enddef( ncid);
+        err = nc_put_var_double( ncid, vid, periodify( std::get<1>(tp), g3d_periodic).data());
+        err = nc_redef(ncid);
+    }
     err = nc_close( ncid);
+    std::cout << "FILE orthogonalX.nc CLOSED AND READY TO USE NOW!\n" <<std::endl;
 
     std::cout << "TEST VOLUME IS:\n";
     dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, Z_X, 1.2*gp.a*gp.elongation, 1, 5e2, 5e2, dg::PER, dg::PER);
     gp.psipmax = 0., gp.psipmin = psi_0;
-    dg::geo::Iris iris( c.psip(), gp.psipmin, gp.psipmax);
+    auto iris = dg::compose( dg::Iris( gp.psipmin, gp.psipmax), mag.psip());
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::HVec g2d_weights = dg::create::volume( g2dC);
     double volumeRZP = dg::blas1::dot( vec, g2d_weights);
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index d338ef07e..49ca08e5d 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -11,7 +11,6 @@
 #include "guenther.h"
 #include "solovev.h"
 #include "simple_orthogonal.h"
-#include "init.h"
 #include "testfunctors.h"
 #include "curvilinear.h"
 
@@ -165,7 +164,7 @@ int main( int argc, char* argv[])
     double volume = dg::blas1::dot( vol, ones3d);
     if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
     else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    dg::geo::Iris iris( psip.f(), gp.psipmin, gp.psipmax);
+    auto iris = dg::compose( dg::Iris(gp.psipmin, gp.psipmax), psip.f());
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a, 2.0*gp.a, 1, 2e3, 2e3, dg::PER, dg::PER);
 
     dg::HVec vec  = dg::evaluate( iris, g2dC);
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 2be586da8..3c2ab0171 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -559,8 +559,8 @@ static inline dg::geo::TokamakMagneticField createSolovevField(
  * This subsequently modifies all derivatives of psi and the poloidal
  * current.
  * @param gp Solovev parameters
- * @param psi0 above this value psi is modified to a constant psi0
- * @param alpha determines how quickly the modification acts (smaller is quicker)
+ * @param psi0 boundary value where psi is modified to a constant psi0
+ * @param alpha radius of the transition region where the modification acts (smaller is quicker)
  * @param sign determines which side of Psi to dampen (negative or positive)
  * @return A magnetic field object
  * @ingroup geom
diff --git a/inc/geometries/utilities.h b/inc/geometries/utilities.h
index d3db17578..54c55be7b 100644
--- a/inc/geometries/utilities.h
+++ b/inc/geometries/utilities.h
@@ -421,15 +421,6 @@ struct HessianRZtau
         }
 
     }
-    void newton_iteration( const std::array<double,2>& y, std::array<double,2>& yp)
-    {
-        double psipRZ = psip_.dfxy()(y[0], y[1]);
-        double psipRR = psip_.dfxx()(y[0], y[1]), psipZZ = psip_.dfyy()(y[0],y[1]);
-        double psipR = psip_.dfx()(y[0], y[1]), psipZ = psip_.dfy()(y[0], y[1]);
-        double Dinv = 1./(psipZZ*psipRR - psipRZ*psipRZ);
-        yp[0] = y[0] - Dinv*(psipZZ*psipR - psipRZ*psipZ);
-        yp[1] = y[1] - Dinv*(-psipRZ*psipR + psipRR*psipZ);
-    }
   private:
     bool norm_;
     int quad_;
@@ -473,22 +464,6 @@ struct MinimalCurve
     CylindricalFunctorsLvl1 psip_;
 };
 
-///@copydoc findXpoint()
-static inline void findOpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
-{
-    dg::geo::HessianRZtau hessianRZtau(  psi);
-    std::array<double, 2> X{ {0,0} }, XN(X), X_OLD(X);
-    X[0] = R_X, X[1] = Z_X;
-    double eps = 1e10, eps_old= 2e10;
-    while( (eps < eps_old || eps > 1e-7) && eps > 1e-10)
-    {
-        X_OLD = X; eps= eps_old;
-        hessianRZtau.newton_iteration( X, XN);
-        XN.swap(X);
-        eps = sqrt( (X[0]-X_OLD[0])*(X[0]-X_OLD[0]) + (X[1]-X_OLD[1])*(X[1]-X_OLD[1]));
-    }
-    R_X = X[0], Z_X = X[1];
-}
 ////////////////////////////////////////////////////////////////////////////////
 
 
diff --git a/inc/geometries/utilitiesX.h b/inc/geometries/utilitiesX.h
index 15a3df2ad..05887af77 100644
--- a/inc/geometries/utilitiesX.h
+++ b/inc/geometries/utilitiesX.h
@@ -8,16 +8,7 @@ namespace dg
 {
 namespace geo
 {
-/**
- * @brief This function finds the X-point (or O-point for that matter) via Newton iteration applied to the gradient of psi
- *
- * The inverse of the Hessian matrix is computed analytically
- * @param psi \f$ \psi(R,Z)\f$, where R, Z are cylindrical coordinates
- * @param R_X start value on input, X-point on output
- * @param Z_X start value on input, X-point on output
- * @ingroup misc_geo
- * @note also finds the O-point or any other point with vanishing gradient
- */
+///@copydoc findOpoint()
 static inline void findXpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
 {
     findOpoint( psi, R_X, Z_X);
-- 
GitLab


From 2c7ebf4af58705e9d3417c1c61a90bd8e95c0249 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 7 May 2020 16:18:18 +0200
Subject: [PATCH 209/540] Many changes in src/feltor

- change init.h with new compose function
- change input coordinate system for rho and alpha to rho_p
- add new and make existing parameters more recognizable in parameters.h
        profile, source, damping and FCI are now dictionaries
- output X-Grid in feltoridag
- introduce periodify as an option
- document all changes in feltor.tex
- introduce wall-damping in N and U equations
---
 src/feltor/feltor.cu               |  23 ++-
 src/feltor/feltor.h                |  29 +--
 src/feltor/feltor.tex              | 275 +++++++++++++++++------------
 src/feltor/feltor_hpc.cu           |  21 ++-
 src/feltor/feltordiag.cu           |  73 +++++---
 src/feltor/init.h                  |  64 +++----
 src/feltor/input/default.json      |  56 ++++--
 src/feltor/input/manufactured.json |  55 ++++--
 src/feltor/parameters.h            |  65 +++----
 9 files changed, 404 insertions(+), 257 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index dc928a632..c86bacb6d 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -57,8 +57,20 @@ int main( int argc, char* argv[])
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    if( p.alpha_mag > 0.)
-        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
+    //Wall damping has to be constructed before modification (!)
+    HVec damping_profile = feltor::wall_damping( grid, p, gp, mag);
+    if( p.damping_alpha > 0.)
+    {
+        double RO=mag.R0(), ZO=0.;
+        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+        double psipO = mag.psip()( RO, ZO);
+        double damping_psi0 = (1.-p.damping_boundary*p.damping_boundary)*psipO;
+        double damping_alpha = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
+        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0+damping_alpha/2.,
+                fabs(p.damping_alpha/2.), ((psipO>0)-(psipO<0)));
+    }
+    if( p.periodify)
+        mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
 
     //create RHS
     //std::cout << "Constructing RHS...\n";
@@ -96,13 +108,16 @@ int main( int argc, char* argv[])
     HVec source_profile;
     try{
         source_profile = feltor::source_profiles.at(p.source_type)(
-        fixed_profile, profile, grid, p, gp, mag);
+            fixed_profile, profile, grid, p, gp, mag);
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
         return -1;
     }
 
-    feltor.set_source( fixed_profile, dg::construct<DVec>(profile), p.omega_source, dg::construct<DVec>(source_profile));
+    feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
+        p.source_rate, dg::construct<DVec>(source_profile),
+        p.damping_rate, dg::construct<DVec>(damping_profile)
+    );
 
 
     ////////////////////////create timer and timestepper
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 32f431f71..85f92a1ee 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -324,12 +324,14 @@ struct Explicit
     }
 
     //source strength, profile - 1
-    void set_source( bool fixed_profile, Container profile, double omega_source, Container source)
+    void set_source( bool fixed_profile, Container profile, double omega_source, Container source, double omega_damping, Container damping )
     {
         m_fixed_profile = fixed_profile;
         m_profne = profile;
         m_omega_source = omega_source;
         m_source = source;
+        m_omega_damping = omega_damping;
+        m_damping = damping;
     }
     void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
   private:
@@ -360,7 +362,7 @@ struct Explicit
     std::array<Container,3> m_curv, m_curvKappa, m_b;
     Container m_divCurvKappa;
     Container m_bphi, m_binv, m_divb;
-    Container m_source, m_profne;
+    Container m_source, m_profne, m_damping;
     Container m_vol3d;
 
     Container m_apar;
@@ -385,7 +387,7 @@ struct Explicit
     dg::SparseTensor<Container> m_hh;
 
     const feltor::Parameters m_p;
-    double m_omega_source = 0.;
+    double m_omega_source = 0., m_omega_damping = 0.;
     bool m_fixed_profile = true;
     bool m_full_system = false;
 
@@ -842,17 +844,17 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
     }
 }
 
-/* y[0][0] := n_e - 1
-   y[0][1] := N_i - 1
-   y[1][0] := w_e
-   y[1][1] := W_i
-*/
 template<class Geometry, class IMatrix, class Matrix, class Container>
 void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     double t,
     const std::array<std::array<Container,2>,2>& y,
     std::array<std::array<Container,2>,2>& yp)
 {
+    /* y[0][0] := n_e - 1
+       y[0][1] := N_i - 1
+       y[1][0] := w_e
+       y[1][1] := W_i
+    */
 
     // set m_phi[0]
     compute_phi( t, y[0]);
@@ -887,13 +889,16 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
 #endif
 
     //Add source terms
-    if( m_omega_source != 0)
+    if( m_omega_source != 0 || m_omega_damping != 0)
     {
         if( m_fixed_profile )
             dg::blas1::subroutine( routines::ComputeSource(), m_s[0][0], y[0][0],
                 m_profne, m_source, m_omega_source);
         else
             dg::blas1::axpby( m_omega_source, m_source, 0., m_s[0][0]);
+        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, y[0][0], 1.,  m_s[0][0]);
+        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][0], 1.,  m_s[1][0]);
+        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][1], 1.,  m_s[1][1]);
         //compute FLR corrections S_N = (1-0.5*mu*tau*Lap)*S_n
         dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
         dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
@@ -901,10 +906,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         dg::blas1::pointwiseDot( m_p.mu[1], m_s[0][0], m_binv, m_binv, 0., m_temp0);
         m_lapperpP.multiply_sigma( 1., m_temp0, m_phi[0], 1., m_s[0][1]);
 
-        dg::blas1::axpby( 1., m_s[0][0], 1.0, yp[0][0]);
-        dg::blas1::axpby( 1., m_s[0][1], 1.0, yp[0][1]);
-        //currently we ignore m_s[1]
+        dg::blas1::axpby( 1., m_s, 1.0, yp);
     }
+
+
     if( m_full_system)
     {
 #if FELTORPERP == 1
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 7418b7936..337385fcb 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -30,12 +30,12 @@ file(s) of \texttt{feltor\_hpc.cu}.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The magnetic field}\label{sec:magnetic}
-Let us assume a three-dimensional flat space with arbitrary coordinate
+We assume a three-dimensional flat space with arbitrary coordinate
 system $\vec x :=\{x_0, x_1, x_2\}$, metric
 tensor $g_{ij}$ and volume element $\sqrt{g} := \sqrt{\det g}$.
 Given a vector field $\vec B(\vec x)$ with unit vector $\bhat(\vec x) := (\vec B/B)({\vec x})$
 we can define various differential operations.
-%Let us further assume that $\bhat$ is perturbed by the parallel
+%We further assume that $\bhat$ is perturbed by the parallel
 %vector potential $A_\parallel$ via
 %${ \vec b }_\perp := ({\vn \times A_\parallel \bhat)}/{B}$
 \rowcolors{2}{gray!25}{white}
@@ -381,53 +381,8 @@ K_{\vn\times\bhat}^\varphi &= \frac{R_0}{R^2B^2}\left(
 \label{}
 \end{align}
 
-\subsection{ Modified $\psi_p$}
-Our computational domain is a box and in particular not aligned with the
-magnetic flux surfaces. This means that particularly in the corners of
-the domain the field lines inside the domain are very short (in the
-sense that the distance between the entry point and leave point is short).
-It turns out that this behaviour is numerically disadvantageous (may
-blow up the simulation in the worst case) in the
-computation of parallel derivatives. In order to remedy this situation
-we propose to modify the flux surfaces $\psi_p$ to a constant value
-if $\psi_p$ exceeds a certain critical value. In this way the poloidal
-field component vanishes in the corners of the domain at the cost
-of introducing a strong shear layer limiting the scrape-off layer width.
-
-We define an approximation to the step function with width $\alpha$
-\begin{align}
-\Theta_\alpha(\psi) := \begin{cases}
-    0 & \text{ for } \psi < - \alpha  \\
-    \frac{1}{32 \alpha^7}  \left(16 \alpha^3-29 \alpha^2 \psi+20 \alpha \psi^2-5 \psi^3\right) (\alpha+\psi)^4
-    &\text{ for } -\alpha<\psi<+\alpha \\
-    1 & \text{ for } \psi > \alpha 
-\end{cases}
-    \approx H(\psi)
-\label{eq:approx_heaviside}
-\end{align}
-if $H(\psi)$ is the Heaviside step function.
-An integral of this function is
-\begin{align}
-\theta_\alpha(\psi) := \begin{cases}
-    0 &\text{ for } \psi < -\alpha \\
-    \frac{1}{256 \alpha^7} \left(35 \alpha^3-47 \alpha^2 \psi+25 \alpha \psi^2-5 \psi^3\right) (\alpha+\psi)^5
-     &\text{ for } -\alpha<\psi<+\alpha \\
-\psi &\text{ for } \psi > \alpha
-\end{cases}
-    \approx \psi H(\psi)
-\end{align}
-
-We now use 
-\begin{align}
--\theta_\alpha(\psi_0 - \psi)+\psi_0 \approx (\psi- \psi_0)H(\psi_0-\psi) + \psi_0
-\label{eq:modified_psip}
-\end{align}
-instead of $\psi$ for the computation of the
-magnetic field, which introduces a shear layer around $\psi_0$ where the
-fieldlines are straightened to match $\ehat_\varphi$.
-Note that $\Theta_\alpha(0) = 0.5$ and $\theta_\alpha(0) = 35\alpha/256$.
-\section{Flux surface averaging and safety factor}
-\subsection{Preliminary}
+\subsection{Flux surface averaging and safety factor}
+\subsubsection{Preliminary}
 Recall that the {\bf Dirac delta-function} has the property (in any dimension):
 \begin{align} \label{eq:dirac_delta}
 \int_V f(\vec x) \delta(h(\vec x) - h') \dV = \int_{h=h'} \frac{f(\vec x)}{|\vn h|} \dA
@@ -482,13 +437,13 @@ Notice that numerically we can integrate in flux-aligned coordinates by generati
 grid and pulling back (interpolating) the relevant fields to this grid. This is the second method
 to numerically compute area integrals.
 
-\subsection{Flux surface average}
+\subsubsection{Flux surface average}
 
 
 The flux surface average (as a {\bf volume average} after \cite{haeseleer}) is defined as an average over a
 small volume - a shell centered around the flux-surface - defined by two neighboring flux-surfaces.
 With the help of the volume
-flux label (notice that both the volume $v$ as well as the poloidal flux $\psi_p$ have physical 
+flux label (notice that both the volume $v$ as well as the poloidal flux $\psi_p$ have physical
 meaning while the coordinate $\zeta(\psi_p)$ is an arbitrary choice) we define
 \begin{align} \label{eq:fsa_vol}
 v(\psi_p) :=& \int_{\psi_{p,\min}}^\psi \dV = \int^{\zeta(\psi_p)} \sqrt{g}\d\zeta\d\eta\d\varphi,
@@ -505,8 +460,12 @@ where we used the co-area formula Eq.~\eqref{eq:coarea} for the second identity
 and we use the Heaviside function $H(Z-Z_X)$ to cut away contributions from below the X-point
 in our domain $\Omega$.
  We immediately see that this definition is particularly easy to compute
-in a flux-aligned coordinate system. Notice however that the volume element
-does appear (unlike e.g. Tokam3X papers)
+ in a flux-aligned coordinate system. Notice however that the volume element
+ does appear (unlike e.g. Tokam3X papers).
+ We use our grid construction algorithm with constant monitor metric described in Reference~\cite{Wiesenberger2018} to construct a flux-aligned grid and interpolate
+ the values of any function onto its grid points.
+ Even though this grid is unusable for simulations due to the diverging metric at the X-point the
+ evaluation of integrals works well as the singularity is integrable.
 
 The flux-surface average fulfills the basic identities
 \begin{align}
@@ -542,7 +501,7 @@ Once we have the flux-surface averaged equation we can easily get the volume int
 \label{eq:integral_balance}
 \end{align}
 
-\subsection{The safety factor}
+\subsubsection{The safety factor}
 Assume that we pick a random field line and follow it (integrate it) for exactly one
 poloidal turn. The {\bf safety factor} is defined as the ratio between
 the resulting toroidal angle ($\Delta\varphi$) to the poloidal angle ($2\pi$)
@@ -554,7 +513,7 @@ Since our magnetic field is symmetric in $\varphi$ and we used one
 full poloidal turn this definition is independent of which
 fieldline we pick on a given flux surface.
 
-%Let us define the poloidal length $s$ as the fieldline following
+%We define the poloidal length $s$ as the fieldline following
 %parameter i.e. $\vec B\cn s \equiv B_p = R_0|\vn \psi_p|/R$
 %and $\d\varphi/\d s = B^\varphi(R(s), Z(s)) / B_p(R(s),Z(s))$.
 %We can then express the safety factor as the line integral
@@ -565,12 +524,12 @@ fieldline we pick on a given flux surface.
 %where we made use of Eq.~\eqref{eq:dirac_delta} in two dimensions in the
 %last equality and thus arrive at a numerical tractable expression
 %to evaluate the safety factor.
-Let us define the geometric poloidal angle $\Theta$ as the fieldline following
+We define the geometric poloidal angle $\Theta$ as the fieldline following
 parameter i.e. $\vec B\cn\Theta = R_0(\psi_R (R-R_0) + \psi_Z Z)/r^2R$.
 We can then directly integrate the safety factor as
 \begin{align}\label{eq:safety_factor}
-\frac{\d R}{\d\Theta} = \frac{B^R}{B^\Theta}\quad 
-\frac{\d Z}{\d\Theta} = \frac{B^Z}{B^\Theta}\quad 
+\frac{\d R}{\d\Theta} = \frac{B^R}{B^\Theta}\quad
+\frac{\d Z}{\d\Theta} = \frac{B^Z}{B^\Theta}\quad
 \frac{\d \varphi}{\d\Theta} = \frac{B^\varphi}{B^\Theta}\\
 q\equiv\frac{1}{2\pi}\oint \frac{B^\varphi}{B^\Theta} \d\Theta
 \end{align}
@@ -579,13 +538,71 @@ and refine the stepsize until machine-precision is reached.
 Notice that the safety factor diverges on the last closed flux
 surface whereas Eq.~\eqref{eq:total_flux}
 remains finite due to the $\vn\psi$ factor.
-
+\subsection{Alternative flux labels}
 We find the toroidal flux $\psi_t$ by integrating the q-profile $\psi_t = \int^{\psi_p} \d\psi_p q(\psi_p)$. Since $q$ diverges, $\psi_t$, in contrast to $\psi_p$,
-is not defined outside the last closed flux-surface. We define the normalized toroidal flux label
-$\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$, which is useful because
-equidistant $\rho_t$ values tend to translate to equidistant flux-surfaces
+is not defined outside the last closed flux-surface (but has a finite value on the last closed flux surface). We now define the normalized poloidal and toroidal flux labels $\rho_p$ and $\rho_t$
+\begin{align}
+    \rho_p&:= \sqrt{1-\frac{\psi_p}{\psi_{p,\min}}} \ \leftrightarrow\ \psi_p = (1-\rho_p^2)\psi_{p,\min} \\
+    \rho_t&:= \sqrt{\frac{\psi_t}{\psi_{t,\mathrm{sep}}}},\\
+    \text{In general }\psi_{p,\min} &= \psi_p(R_O, Z_O) \neq\psi_{p}(R_0,0)
+\end{align}
+where $R_O$, $Z_O$ are the coordinates of the O-point.
+These labels are useful because
+equidistant $\rho_p$ and $\rho_t$ values tend to translate to equidistant flux-surfaces
 in configuration space.
 
+\subsection{ Modified $\psi_p$}
+Our computational domain is a box and in particular not aligned with the
+magnetic flux surfaces. This means that particularly in the corners of
+the domain the field lines inside the domain are very short (in the
+sense that the distance between the entry point and leave point is short).
+It turns out that this behaviour is numerically disadvantageous (may
+blow up the simulation in the worst case) in the
+computation of parallel derivatives. In order to remedy this situation
+we propose to modify the flux surfaces $\psi_p$ to a constant value
+if $\psi_p$ exceeds a certain critical value. In this way the poloidal
+field component vanishes in the corners of the domain at the cost
+of introducing a strong shear layer limiting the scrape-off layer width.
+
+We define an approximation to the step function with a transition layer of radius $a$
+around the origin
+\begin{align}
+\Theta_a(x) := \begin{cases}
+    0 & \text{ for } x \leq -a  \\
+    \frac{1}{32 a^7}  \left(16 a^3-29 a^2 x+20 a x^2-5 x^3\right) (a+x)^4
+    &\text{ for } -a<x\leq a \\
+    1 & \text{ for } x > a
+\end{cases}
+    \approx H(x)
+\label{eq:approx_heaviside}
+\end{align}
+where $H(x)$ is the Heaviside step function.
+An integral of this function is
+\begin{align}
+\theta_a(x) := \begin{cases}
+    0 &\text{ for } x \leq -a \\
+    \frac{1}{256 a^7} \left(35 a^3-47 a^2 x+25 a x^2-5 x^3\right) (a+x)^5
+     &\text{ for } -a<x\leq a \\
+x &\text{ for } x > a
+\end{cases}
+    \approx x H(x)
+\end{align}
+Note that $\Theta_a(0) = 0.5$ and $\theta_a(0) = 35a/256$.
+
+We now use
+\begin{align}
+    -\theta_{\alpha/2}\left(\psi_{p,b} + \frac{\alpha}{2} - \psi \right)+\psi_{p,b}+\frac{\alpha}{2} \approx (\psi- \psi_{p,b})H(\psi_{p,b}-\psi) + \psi_{p,b}
+\label{eq:modified_psip}
+\end{align}
+instead of $\psi_p$ for the computation of the
+magnetic field, which introduces a shear layer between $\psi_{p,b}$ and $\psi_{p,b}+\alpha$ where the
+fieldlines are straightened to match $\ehat_\varphi$.
+In order to simplify the setup of this region we give $\psi_{p,b}$ and $\alpha$ in terms of
+$\rho_p$ and $\alpha_p$ via $\psi_{p,b} = (1-\rho_{p,b}^2)\psi_{p,\min}$ and $\alpha = -(2\rho_{p,b} \alpha_p + \alpha_p^2)\psi_{p,\min}$. In case we change the sign
+of $\psi_p$ vie $\mathcal P_\psi$ we need to adapt Eq.~\eqref{eq:modified_psip}
+accordingly.
+
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
 \subsection{Conservative form}
@@ -610,9 +627,9 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
     &+ 2\mu \nc ( NU \vec u_{\vn\times\bhat})
     -\mu NU\nc \vec u_{\vn\times\bhat}
     + \mu NU\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
-    &= -\tau \left(\bhat + {\vec b}_\perp\right)\cn N 
-    -N \left( \left(\bhat+{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right) 
-    - \eta n_e^2(U_i-u_e) + \mu N\Lambda_U
+    &= -\tau \left(\bhat + {\vec b}_\perp\right)\cn N
+    -N \left( \left(\bhat+{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right)
+    - \eta n_e^2(U_i-u_e) + \mu N\Lambda_U + \mu N S_U
 \label{}
 \end{align}
 with
@@ -749,21 +766,14 @@ and initialize the electron density with
 consisting of a toroidally symmetric background profile $n_{\text{prof}}(R,Z)$ and a perturbation
 $\tilde n(R,Z,\varphi)$, which breaks the toroidal symmetry.
 Note that we should take care to intitialize a smooth profile with ideally well-defined $\Delta^2_\perp n_e$.
-%Let us define another approximation to the Heaviside function
-%\begin{align}
-%  \Theta(x) := \frac{1}{2}\left( 1 + \tanh\left( \frac{x-3\alpha}{ \alpha} \right) \right) \quad \Theta(x) \approx H(x)
-%  \label{eq:heaviside_profile}
-%\end{align}
-%where $H(x)$ is the actual Heaviside function and
-%$\alpha$ is a (small) width parameter.
 
-Let us define a flux-aligned density profile as
+We define a flux-aligned density profile as
 \begin{align} \label{eq:density_profile}
   n_{\text{prof}}(R,Z)=
-      n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_p(R_0,0)}\Theta_{\alpha}(-\psi_p(R, Z)-\alpha) H(Z-Z_X)
+  n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_{p,\min}}\Theta_{\alpha_p/2}\left(-\rho_p(R, Z)-\frac{\alpha_p}{2}\right) H(Z-Z_X)
 \end{align}
 The second Heaviside is multiplied only if the equilibrium $\psi_p$ has an
-X-point and avoids a profile in the private flux region. The factor $\alpha$ provides a smooth transition
+X-point and avoids a profile in the private flux region. The factor $\alpha_p$ provides a smooth transition
 zone that avoids numerical oscillations.
 
 
@@ -781,7 +791,7 @@ We initialize a blob in the R-Z plane
 \begin{align} \label{eq:initial_blob}
   \tilde n_{blob}(R,Z,0) = \triangle n \exp\left( -\frac{(R - R_0 - p_x a)^2 + (Z-p_ya)^2}{\sigma^2} \right)
 \end{align}
-Then, we use fieldline integration modulated by 
+Then, we use fieldline integration modulated by
 \begin{align}
   m_{blob}(s) = \exp\left( -\frac{s^2 }{\pi^2\sigma_z^2} \right)
 \end{align}
@@ -796,14 +806,14 @@ last closed flux surface. Notice that the core region is rather stable
 and quickly damps away fluctuations.
 Again, we transform this to all poloidal planes along the magnetic field lines and multiply the bath with
 \begin{align} \label{eq:initial_turbulent}
-\tilde n_e(R,Z,\varphi) = \tilde n_{\text{bath}}(R,Z,\varphi)\Theta_{\alpha}(-\psi_p(R, Z)-\alpha) H(Z-Z_X)
+    \tilde n_e(R,Z,\varphi) = \tilde n_{\text{bath}}(R,Z,\varphi)\Theta_{\alpha_p/2}(-\rho_p(R, Z)-\alpha_p/2) H(Z-Z_X)
 \end{align}
 \subsubsection{Zonal flows}
 We can initialize the R-Z plane with zonal flows of amplitude $A$ and
 wavelength $k_\psi$ aligned with the magnetic flux surfaces.
 \begin{align} \label{eq:initial_zonal_flow}
     \tilde n_{\text{zonal}}(R,Z) &= A \sin (2\pi k_\psi \psi_p(R,Z)) \nonumber\\
-\tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta_{\alpha}(-\psi_p(R, Z)-\alpha) H(Z-Z_X)
+\tilde n_e(R,Z,\varphi) &= \tilde n_{\text{zonal}}(R,Z)\Theta_{\alpha_p}\left(-\rho_p(R, Z)-\frac{\alpha_p}{2}\right) H(Z-Z_X)
 \end{align}
 \subsubsection{Turbulence on Gaussian profile}
 Instead of the flux-aligned profile we can also choose a toroidally symmetric Gaussian profile
@@ -812,7 +822,7 @@ Instead of the flux-aligned profile we can also choose a toroidally symmetric Ga
 \end{align}
 on top of which we can add the turbulent bath $\tilde n_{\text{bath}}$ and finally dampen it by
 \begin{align}\label{eq:turbulence_on_gaussian}
-n_e(R,Z,\varphi,0) = (n_{prof}(R,Z) + \tilde n_{\text{bath}})\Theta_\alpha( 1- \sqrt{(R-R_0)^2 + Z^2}/a)
+    n_e(R,Z,\varphi,0) = (n_{prof}(R,Z) + \tilde n_{\text{bath}})\Theta_{\alpha_p/2}\left( 1- \sqrt{(R-R_0)^2 + Z^2}/a\right)
 \end{align}
 
 \subsection{Sinks and sources} \label{sec:sources}
@@ -822,20 +832,18 @@ core of our domain, where our model does not apply.
 We thus define a particle sink/source for electrons as
 \begin{align} \label{eq:electron_source}
   S_{n_e}(R,Z,\varphi, t) &= \omega_s \begin{cases}
-    (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta_\alpha( \rho(R,Z) - \rho_s) H(Z-Z_X) \quad \text{ forced}\\
+      (n_{prof}(R,Z) - n_e(R,Z,\varphi, t))\Theta_{\alpha_p/2}\left( \rho_{p,s} - \frac{\alpha_p}{2} - \rho_p(R,Z) \right ) H(Z-Z_X) \quad \text{ forced}\\
     S_{prof}(R,Z)\quad \text{ influx}
-    \end{cases} \\
-    \rho(R,Z) &:= \frac{\psi_{p,\min}- \psi_p(R,Z) }{\psi_{p,\min}}, \quad
-    \psi_p(R,Z):= (1-\rho(R,Z))\psi_{p,\min},\\ \text{In general }\psi_{p,\min} &= \psi_p(R_O, Z_O) \neq\psi_{p}(R_0,0)
+    \end{cases}
 \end{align}
-with $0 < \rho_{s}<1$
-where $\omega_s$ is the source strength parameter and $R_O$, $Z_O$ are the coordinates of the O-point.
+where $\omega_s$ is the source strength parameter. The shift of $\Theta$ is chosen
+such that the source vanishes exactly outside $\psi_{p,s}$.
 The forced source will result in exponential adaption of the core
 density profile of the form $n_e \propto n_{prof}+(n_{prof}-n_{e,0})e^{-\omega_st}$.
 
 We can choose the constant influx
 \begin{align} \label{eq:electron_source_influx}
-  S_{prof}(R,Z) &= \Theta_\alpha( \rho(R,Z) - \rho_s) H(Z-Z_X)
+    S_{prof}(R,Z) &= \Theta_{\alpha_p/2}\left( \rho_{p,s} - \frac{\alpha_p}{2} - \rho_p(R,Z) \right) H(Z-Z_X)
 \end{align}
 or a Torpex inspired source profile
 \begin{align} \label{eq:electron_source_torpex}
@@ -863,15 +871,19 @@ Also note that with our definition of $\Lambda_{n_e}$ and $\Lambda_{N_i}$ and
 the polarisation equation we have $\Lambda_{n_e} = \Gamma_{1,i}\Lambda_{N_i} + \nc\left( \frac{\mu_i \Lambda_{N_i}}{B^2}\np \phi\right)$ in the long wavelength limit (swap the operators).
 This means that diffusion does not generate potential either.
 
-%The idea for the terms $S_U$ is mainly to provide more numerical stability
-%in the corner regions of the domain, where the parallel derivative may lead
-%to unfavourable numerical instabilities.
-%For both electrons and ions we choose
-%\begin{align} \label{eq:velocity_source}
-%  S_{U}(R,Z,\varphi, t) := -\omega_d U \Theta( \rho(R,Z) - \rho_d)
-%\end{align}
-%with $\rho_d > 1$.
-Currently we do not implement any source terms for the velocity $U$.
+Now, our idea is to dampen the density and velocity in the region defined by the
+magnetic field straightening.
+The idea for the terms $S_U$ is mainly to provide more numerical stability
+in the corner regions of the domain, where the parallel derivative may lead
+to unfavourable numerical instabilities.
+For both electrons and ions we choose
+\begin{subequations} \label{eq:velocity_source}
+\begin{align}
+    S^d_{n_e}(R,Z,\varphi,t) &:= -\omega_d (n_e-1)\Theta_{\alpha_p/2}\left(\rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2}\right)\\
+    S^d_{N_i}(R,Z,\varphi,t) &= \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S^d_{n_e} -\nc\left( \frac{\mu_i S^d_{n_e}}{B^2}\np \phi\right)\\
+S^d_U(R,Z,\varphi, t)& := -\omega_d U \Theta_{\alpha_p/2}\left(  \rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2} \right)
+\end{align}
+\end{subequations}
 
 \subsection{Implemented form}
 The form that we implement avoids derivatives on the product of
@@ -899,7 +911,8 @@ two functions for which we have no boundary conditions
         -2\tau U\mathcal K_{\vn\times\bhat}(\ln N)
         - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
         + \nu_\perp\Delta_\perp U
-        + \nu_\parallel \Delta_\parallel U,
+        + \nu_\parallel \Delta_\parallel U
+        + S_U,
         \label{eq:EgyrofluidU} \\
         W&:= \left( U + \frac{A_\parallel}{\mu}\right)
     \end{align}
@@ -964,7 +977,7 @@ n_e \vec K = n_e\vn\times\frac{\bhat}{B} = \vn\times n_e\frac{\bhat}{B} + \frac{
 such that we can define the diamagnetic flux in the particle flux since
 the rotation vanishes under the divergence.
 
-Let us here also derive the particle flux \eqref{eq:particle_flux} through a flux surface
+We here also derive the particle flux \eqref{eq:particle_flux} through a flux surface
 \begin{align} \label{eq:radial_particle_flux}
  \vec j_{N}\cn v %=& N\left( \vec u_E + \vec u_C + \vec u_{\vn
  %B} + U \left(\bhat + {\vec b}_\perp\right)\right) \cn \psi_p \nonumber\\
@@ -1311,10 +1324,14 @@ eps\_gamma  & float & 1e-6  & - & Tolerance for $\Gamma_1$
 stages      & integer & 3 & 3 & number of stages in multigrid, $2^{\text{stages-1}}$
 has to evenly divide both $N_x$ and $N_y$
 \\
-refineDS     & integer[2] & [10,10] & [10,10] & refinement factor in FCI approach in R- and Z-direction.
+FCI & dict & & & Parameters for Flux coordinate independent approach
+\\
+\qquad refine     & integer[2] & [1,1] & [1,1] & refinement factor in FCI approach in R- and Z-direction.
 We use [1,1], higher values take more time.
 \\
-rk4eps     & float & 1e-6 & 1e-6 & Accuracy of fieldline integrator in FCI. The default is reasonable.
+\qquad rk4eps     & float & 1e-6 & 1e-6 & Accuracy of fieldline integrator in FCI. The default is reasonable.
+\\
+\qquad periodify & bool & true & true & Indicate if flux function is periodified beyond grid boundaries such that the contours are perpendicular to the boundaries. This is not entirely consistent but works better for small toroidal resolution
 \\
 mu         & float & -0.000272121& - & $\mu_e =-m_e/m_i$.
     One of $\left\{ -0.000544617, -0.000272121, -0.000181372 \right\}$
@@ -1354,7 +1371,7 @@ symmetric & bool & false & false & If true, initialize all quantities symmetric
 in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used
 to construct the parallel derivatives and then overwritten to $N_z\equiv 1$.
 \\
-bc & dict & & & Dictionary of boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
+bc & dict & & & Boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
 \qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y
 for $n_e$ and $N_i$, DIR (density 1 on boundary) means both convective and
     diffusive outflow while NEU (gradient 0) means no outflow by diffusion
@@ -1365,8 +1382,9 @@ better\\
 \qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for
 $\phi$ and $\psi$, DIR means that the $v_{E,\perp}=0$ on the boundary (i.e. no
 outflow by \ExB drift), NEU can can have a detrimental effect on timestep \\
-    boxscaleR  & float[2] & [1.1,1.1]     & [1.05,1.05] & $[\varepsilon_{R-}, \varepsilon_{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
-    boxscaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon_{Z-}, \varepsilon_{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box}
+box & dict & & & Bounding box \\
+    \qquad scaleR  & float[2] & [1.1,1.1]     & [1.05,1.05] & $[\varepsilon_{R-}, \varepsilon_{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
+    \qquad scaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon_{Z-}, \varepsilon_{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box}
 \\
 initne    & string & "turbulence"     & "blob"  & initial condition for the
 perturbation $\tilde n$ in \eqref{eq:initial_ne}. "zonal" (Eq.~\eqref{eq:initial_zonal_flow}),
@@ -1385,26 +1403,35 @@ posX       & float &0.3    & - & Gaussian R-position in units of $a$\\
 posY       & float &0.0    & - & Gaussian Z-position in units of $a$ \\
 sigma\_z    & float &0.25   & - & toroidal variance in units of $R_0$ of the fieldline-following initialization \\
 k\_psi     & float &0    & - & zonal mode wave number (only for "zonal" initial condition)  \\
-nprofileamp& float &4   & - & Profile peak amplitude $N_{peak}$ in
+profile & Dict & & & Density profile \\
+\qquad amp& float &4   & - & Profile amplitude $\triangle n_{peak}$ in
 Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}
 \\
-alpha       & float & 0.2 & - & Width $\alpha$ of the Heaviside
-Eq.~\eqref{eq:approx_heaviside} in the density and source profiles (should be
-small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes)
+\qquad alpha  & float & 0.2 & - & Transition width $\alpha_p$ in the Heaviside
+at the separatrix
 \\
-source      & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}
+source & dict & & & Density source  \\
+\qquad rate & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}
 \\
-source\_type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
+\qquad type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
 "influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx},
 "torpex": Torpex inspired source profile Eq.~\eqref{eq:electron_source_torpex},
 "gaussian": Gaussian shaped source profile - uses \texttt{posX}, \texttt{posY} and \texttt{sigma},
     See the file {\tt init.h} to add your own custom source.
 \\
-rho\_source & float & 0.2  & 0.2 & Source region boundary $0<\rho_{s}<1$ in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
-%damping     & float & 0    & 0   & Friction coefficient $\omega_d$ in Eq.~\eqref{eq:velocity_source} \\
-alpha\_mag   & float & 0.05 & - & Width $\alpha$ of the Heaviside in the modified $\psi_p$ function \eqref{eq:modified_psip}. If zero, then we do not modify the magnetic field and rho\_damping is ignored.\\
-rho\_damping& float & 0.2  & 1.2 & Modification region boundary $\psi_0 = (1-\rho_d)\psi_{p,\min}$ in Eq.~\eqref{eq:modified_psip}.
+\qquad boundary & float & 0.2  & 0.2 & Source region boundary $\rho_{p,b}$: yields in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
+\qquad alpha  & float & 0.2 & - & Transition width $\alpha_p$ in the Heaviside
+in the density Eq.~\eqref{eq:density_profile} (with $rho_{p,b}=0$ and source profiles Eq.~\eqref{eq:electron_source} (should be
+small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes)
+\\
+damping & dict & & & magnetic and density damping region \\
+\qquad rate & float & 0    & 0   & Friction coefficient $\omega_d$ in density and velocity damping Eq.~\eqref{eq:velocity_source} \\
+\qquad boundary & float & 0.2  & 1.2 & Modification region boundary $\rho_{p,b}$: yields $\psi_0 = (1-\rho_{p,b}^2)\psi_{p,\min}$ in Eq.~\eqref{eq:modified_psip}.
 \\
+\qquad alpha   & float & 0.05 & - & Transition width $\alpha_p$: yields
+$\alpha=-2\rho_{p,b}\alpha_p+\alpha_p^2)\psi_{p,\min}$ for the Heaviside in the modified
+$\psi_p$ function \eqref{eq:modified_psip}. If zero, then we do not modify the
+magnetic field and damping is ignored.\\
 \bottomrule
 \end{longtable}
 The default value is taken if the value name is not found in the input file. If there is no default and
@@ -1518,8 +1545,8 @@ the whole simulation is lost. It is safer to just merge files afterwards with fo
 \texttt{ncrcat output1.nc output2.nc output.nc}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Diagnostics}\label{sec:diagnostics}
-We have the program \texttt{feltor/diag/feltordiag.cu}.
-This program reads one or more previously generated simulation file(s) \texttt{input0.nc ... inputN.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows. \\
+\texttt{feltor/diag/feltordiag.cu}
+ reads one or more previously generated simulation file(s) \texttt{input0.nc ... inputN.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows. \\
 Compilation\\
 \texttt{make feltordiag} \\
 Usage \\
@@ -1535,14 +1562,18 @@ geomfile   &     text attribute & - & verbose geometry input file as a string (v
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( default size: $3\cdot 64$) \\
+eta              & Coord. var. & 1 (eta) & $\eta$-coordinate of the X-point grid ( size: $3\cdot 640$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
+xcc              & Dataset & 2 (eta,psi) & Cartesian x-coordinate $x(\psi,\eta)$ \\
+ycc              & Dataset & 2 (eta,psi) & Cartesian y-coordinate $y(\psi,\eta)$ \\
 dvdpsip          & Dataset & 1 (psi) & $\d v/\d\psi_p$ \\
 psi\_vol         & Dataset & 1 (psi) & The volume enclosed by the flux surfaces $v(\psi_p) = \int_{\psi_p} \dV $ \\
 psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\vn\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
-rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= (\psi_{p,\min} - \psi_p)/\psi_{p,\min}$ \\
 q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} using direct integration ( accurate but unavailable outside separatrix) \\
 psi\_psi         & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
 psit1d           & Dataset & 1 (psi) & Toroidal flux (integrated q-profile) $\psi_t = \int^{\psi_p} \d\psi_p q(\psi_p)$ \\
+rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= 1 - \psi_p/\psi_{p,\min}$ \\
+rho\_p           & Dataset & 1 (psi) & poloidal flux label $\rho_p:= \sqrt{1 - \psi_p/\psi_{p,\min}}$ \\
 rho\_t           & Dataset & 1 (psi) & Toroidal flux label $\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$ (is similar to $\rho$ in the edge but $\rho_t$ is nicer in the core domain, because equidistant $\rho_t$ make more equidistant flux-surfaces)\\
 Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \RA{ Z}(R,Z)$ \\
 Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\RA{ Z}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
@@ -1556,9 +1587,19 @@ where Z $\in$ \{X, Y\_tt\}
 Note that feltoridag converts all $jsX$ quantities into $jvX$
 by multiplying $\d v/\d \psi_p$
 in the sense that $\vec j\cn v  = \vec j \cn \psi_p \d v/\d\psi_p$.
+The parameters used for the X-point flux-aligned grid construction are $f_x = 1/8$, $f_y = 0$, $n_\psi = 3$, $N_\zeta = 64$ and $N_\eta = 640$ and the constant monitor metric.
 
-
-
+We also have a useful geometry diagnostic program:
+\texttt{feltor/inc/geometries/geometry\_diag.cu} reads either a previously
+generated simulation file \texttt{input.nc} or the input json files
+\texttt{input.json} and \texttt{geometry.json} and writes an output file \texttt{diag\_geometry.nc} as\\
+Compilation\\
+\texttt{make geometry\_diag device=omp} \\
+Usage \\
+\texttt{./geometry\_diag input.json geometry.json diag\_geometry.nc} \\
+The program performs a series of tests on the correct implementation of geoemetric
+quantities and outputs a host of static 1d, 2d and 3d geometric quantities.
+The output file is for example useful in connection with the ``Group Datasets'' filter in paraview, which merges Datasets from different files into one using shallow copy only.
 
 %..................................................................
 \bibliography{../../doc/related_pages/references}
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 6b666a338..2defd38da 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -149,8 +149,20 @@ int main( int argc, char* argv[])
 #endif
 
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    if( p.alpha_mag > 0.)
-        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
+    //Wall damping has to be constructed before modification (!)
+    HVec damping_profile = feltor::wall_damping( grid, p, gp, mag);
+    if( p.damping_alpha > 0.)
+    {
+        double RO=mag.R0(), ZO=0.;
+        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+        double psipO = mag.psip()( RO, ZO);
+        double damping_psi0 = (1.-p.damping_boundary*p.damping_boundary)*psipO;
+        double damping_alpha = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
+        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0+damping_alpha/2.,
+                fabs(p.damping_alpha/2.), ((psipO>0)-(psipO<0)));
+    }
+    if( p.periodify)
+        mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
 
     //create RHS
     //MPI_OUT std::cout << "Constructing RHS...\n";
@@ -212,7 +224,10 @@ int main( int argc, char* argv[])
         return -1;
     }
 
-    feltor.set_source( fixed_profile, dg::construct<DVec>(profile), p.omega_source, dg::construct<DVec>(source_profile));
+    feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
+        p.source_rate, dg::construct<DVec>(source_profile),
+        p.damping_rate, dg::construct<DVec>(damping_profile)
+    );
     }
 
     /// //////////////////////////set up netcdf/////////////////////////////////////
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 850088260..07c4ff14b 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -96,8 +96,16 @@ int main( int argc, char* argv[])
         p.n_out, p.Nx_out, p.Ny_out, p.bcxN, p.bcyN);
 
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    const double psip0 = mag.psip()(gp.R_0, 0);
-    mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*psip0, p.alpha_mag);
+    double RO=mag.R0(), ZO=0.;
+    dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+    const double psipO = mag.psip()( RO, ZO);
+    if( p.damping_alpha > 0.)
+    {
+        double damping_psi0 = (1.-p.damping_boundary*p.damping_boundary)*psipO;
+        double damping_alpha = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
+        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0+damping_alpha/2.,
+                fabs(p.damping_alpha/2.), ((psipO>0)-(psipO<0)));
+    }
     dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
     // Construct weights and temporaries
 
@@ -106,10 +114,6 @@ int main( int argc, char* argv[])
 
 
     ///--------------- Construct X-point grid ---------------------//
-    //Find O-point
-    double R_O = gp.R_0, Z_O = 0.;
-    dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
-    const double psipmin = mag.psip()(R_O, Z_O);
 
 
     //std::cout << "Type X-point grid resolution (n(3), Npsi(32), Neta(640)) Must be divisible by 8\n";
@@ -122,17 +126,18 @@ int main( int argc, char* argv[])
     double Z_X = -1.1*gp.elongation*gp.a;
     dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
     dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), R_X, Z_X) ;
-    dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, R_X, Z_X, mag.R0(), 0, 0, true);
+    dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipO, R_X, Z_X, mag.R0(), 0, 0, false);
     double fx_0 = 1./8.;
     double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     std::cout << "psi max is            "<<psipmax<<"\n";
-    psipmax = -fx_0/(1.-fx_0)*psipmin;
+    psipmax = -fx_0/(1.-fx_0)*psipO;
     std::cout << "psi max in g1d_out is "<<psipmax<<"\n";
     dg::geo::CurvilinearGridX2d gridX2d( generator, fx_0, 0., npsi, Npsi, Neta, dg::DIR_NEU, dg::NEU);
+    std::cout << "psi max in gridX2d is "<<gridX2d.x1()<<"\n";
     std::cout << "DONE!\n";
     //Create 1d grid
-    dg::Grid1d g1d_out(psipmin, psipmax, 3, Npsi, dg::DIR_NEU); //inner value is always 0
-    const double f0 = ( gridX2d.x1() - gridX2d.x0() ) / ( psipmax - psipmin );
+    dg::Grid1d g1d_out(psipO, psipmax, 3, Npsi, dg::DIR_NEU); //inner value is always 0
+    const double f0 = ( gridX2d.x1() - gridX2d.x0() ) / ( psipmax - psipO );
     dg::HVec t1d = dg::evaluate( dg::zero, g1d_out), fsa1d( t1d);
     dg::HVec transfer1d = dg::evaluate(dg::zero,g1d_out);
 
@@ -166,9 +171,12 @@ int main( int argc, char* argv[])
         "Flux area evaluated with X-point grid");
 
     dg::HVec rho = dg::evaluate( dg::cooX1d, g1d_out);
-    dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
+    dg::blas1::axpby( -1./psipO, rho, +1., 1., rho); //transform psi to rho
     map1d.emplace_back("rho", rho,
-        "Alternative flux label rho = -psi/psimin + 1");
+        "Alternative flux label rho = 1-psi/psimin");
+    dg::blas1::transform( rho, rho, dg::SQRT<double>());
+    map1d.emplace_back("rho_p", rho,
+        "Alternative flux label rho_p = sqrt(1-psi/psimin)");
     dg::geo::SafetyFactor qprof( mag);
     dg::HVec qprofile = dg::evaluate( qprof, g1d_out);
     map1d.emplace_back("q-profile", qprofile,
@@ -179,7 +187,7 @@ int main( int argc, char* argv[])
     map1d.emplace_back("psit1d", psit,
         "Toroidal flux label psi_t integrated using q-profile");
     //we need to avoid integrating >=0 for total psi_t
-    dg::Grid1d g1d_fine(psipmin, 0., 3, Npsi, dg::DIR_NEU);
+    dg::Grid1d g1d_fine(psipO<0. ? psipO : 0., psipO<0. ? 0. : psipO, 3 ,Npsi,dg::DIR_NEU);
     qprofile = dg::evaluate( qprof, g1d_fine);
     dg::HVec w1d = dg::create::weights( g1d_fine);
     double psit_tot = dg::blas1::dot( w1d, qprofile);
@@ -199,18 +207,37 @@ int main( int argc, char* argv[])
     dg::blas2::symv( fsa2rzmatrix, dvdpsip, dvdpsip2d);
     dg::HMatrix dpsi = dg::create::dx( g1d_out, dg::DIR_NEU);
 
+    //define eta, psi
+    int dim_idsX[2] = {0,0};
+    err = file::define_dimensions( ncid_out, dim_idsX, gridX2d.grid(), {"eta", "psi"} );
+    std::string long_name = "Flux surface label";
+    err = nc_put_att_text( ncid_out, dim_idsX[0], "long_name",
+        long_name.size(), long_name.data());
+    long_name = "Flux angle";
+    err = nc_put_att_text( ncid_out, dim_idsX[1], "long_name",
+        long_name.size(), long_name.data());
+    int xccID, yccID;
+    err = nc_def_var( ncid_out, "xcc", NC_DOUBLE, 2, dim_idsX, &xccID);
+    err = nc_def_var( ncid_out, "ycc", NC_DOUBLE, 2, dim_idsX, &yccID);
+    long_name="Cartesian x-coordinate";
+    err = nc_put_att_text( ncid_out, xccID, "long_name",
+        long_name.size(), long_name.data());
+    long_name="Cartesian y-coordinate";
+    err = nc_put_att_text( ncid_out, yccID, "long_name",
+        long_name.size(), long_name.data());
+    err = nc_enddef( ncid);
+    err = nc_put_var_double( ncid_out, xccID, gridX2d.map()[0].data());
+    err = nc_put_var_double( ncid_out, yccID, gridX2d.map()[1].data());
+    err = nc_redef(ncid_out);
+
     // define 2d and 1d and 0d dimensions and variables
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
-    int dim_ids1d[2] = {dim_ids[0], 0}; //time , psi
-    err = file::define_dimension( ncid_out, &dim_ids1d[1], g1d_out, "psi" );
     //Write long description
-    std::string long_name = "Time at which 2d fields are written";
+    long_name = "Time at which 2d fields are written";
     err = nc_put_att_text( ncid_out, tvarID, "long_name", long_name.size(),
             long_name.data());
-    long_name = "Flux surface label";
-    err = nc_put_att_text( ncid_out, dim_ids1d[1], "long_name",
-        long_name.size(), long_name.data());
+    int dim_ids1d[2] = {dim_ids[0], dim_idsX[1]}; //time,  psi
 
     std::map<std::string, int> id0d, id1d, id2d;
 
@@ -384,14 +411,14 @@ int main( int argc, char* argv[])
                         dg::blas2::symv( dpsi, fsa1d, t1d);
                         dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
 
-                        result = dg::interpolate( fsa1d, 0., g1d_out);
+                        result = dg::interpolate( dg::xspace, fsa1d, 0., g1d_out);
                     }
                     else
                     {
                         dg::blas1::pointwiseDot( fsa1d, dvdpsip, t1d);
                         transfer1d = dg::integrate( t1d, g1d_out);
 
-                        result = dg::interpolate( transfer1d, 0., g1d_out);
+                        result = dg::interpolate( dg::xspace, transfer1d, 0., g1d_out);
                     }
                     err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
                         start1d_out, count1d, transfer1d.data());
@@ -406,7 +433,7 @@ int main( int argc, char* argv[])
                         dg::blas1::pointwiseDot( t1d, t1d, t1d);//dvjv2
                         dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);//dvjv2
                         transfer1d = dg::integrate( t1d, g1d_out);
-                        result = dg::interpolate( transfer1d, 0., g1d_out);
+                        result = dg::interpolate( dg::xspace, transfer1d, 0., g1d_out);
                         result = sqrt(result);
                     }
                     else
@@ -415,7 +442,7 @@ int main( int argc, char* argv[])
                         dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);
                         transfer1d = dg::integrate( t1d, g1d_out);
 
-                        result = dg::interpolate( transfer1d, 0., g1d_out);
+                        result = dg::interpolate( dg::xspace, transfer1d, 0., g1d_out);
                         result = sqrt(result);
                     }
                     err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 64a8be795..aff82d192 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -3,9 +3,11 @@
 
 namespace feltor
 {
+
 namespace detail
 {
 
+
 struct TorpexSource
 {
     TorpexSource( double R0, double Z0, double a, double b, double c):m_R0(R0), m_Z0(Z0), m_a(a), m_b(b), m_c(c){}
@@ -24,11 +26,11 @@ struct TorpexSource
     double m_R0, m_Z0, m_a, m_b, m_c;
 };
 
-struct Radius
+struct Radius : public dg::geo::aCylindricalFunctor<Radius>
 {
     Radius ( double R0, double Z0): m_R0(R0), m_Z0(Z0) {}
     DG_DEVICE
-    double operator()( double R, double Z) const{
+    double do_compute( double R, double Z) const{
         return sqrt( (R-m_R0)*(R-m_R0) + (Z-m_Z0)*(Z-m_Z0));
     }
     private:
@@ -38,9 +40,9 @@ HVec circular_damping( const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
-    HVec circular = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-        Radius( mag.R0(), 0.),
-        gp.a, gp.a*p.alpha, -1), grid);
+    HVec circular = dg::pullback( dg::compose(
+                dg::PolynomialHeaviside( gp.a, gp.a*p.profile_alpha/2., -1),
+                Radius( mag.R0(), 0.)), grid);
     return circular;
 }
 
@@ -60,34 +62,24 @@ HVec xpoint_damping(const Geometry& grid,
     }
     return xpoint_damping;
 }
-HVec damping_damping(const Geometry& grid,
-    const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
-{
-    HVec damping_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-        //first change coordinate from psi to (psi_0 - psip)/psi_0
-        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift
-        p.rho_damping, p.alpha, +1), grid);
-    return damping_damping;
-}
 HVec profile_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
-    double psip0 = mag.psip()( mag.R0(), 0);
-    HVec profile_damping = dg::pullback( dg::geo::Compose<dg::PolynomialHeaviside>(
-        mag.psip(), -p.alpha, p.alpha, ((psip0>0)-(psip0<0))), grid); //sign operator!!
+    HVec profile_damping = dg::pullback( dg::compose(dg::PolynomialHeaviside(
+        1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)), grid);
     dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), profile_damping, profile_damping);
     return profile_damping;
 }
 HVec profile(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag ){
-    double psip0 = mag.psip()( mag.R0(), 0);
+    double RO=mag.R0(), ZO=0.;
+    dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+    double psipO = mag.psip()( RO, ZO);
     //First the profile and the source (on the host since we want to output those)
-    HVec profile = dg::pullback( dg::geo::Compose<dg::LinearX>( mag.psip(),
-        p.nprofamp/psip0, 0.), grid);
+    HVec profile = dg::pullback( dg::compose(dg::LinearX(
+        p.nprofamp/psipO, 0.), mag.psip()), grid);
     dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), profile, profile);
     return profile;
 }
@@ -95,13 +87,12 @@ HVec source_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
-    double psip0 = mag.psip()( mag.R0(), 0);
-    HVec source_damping = dg::pullback(dg::geo::Compose<dg::PolynomialHeaviside>(
-        //first change coordinate from psi to (psi_0 - psip)/psi_0
-        dg::geo::Compose<dg::LinearX>( mag.psip(), -1./mag.psip()(mag.R0(), 0.),1.),
-        //then shift
-        p.rho_source, p.alpha, ((psip0>0)-(psip0<0)) ), grid);
-    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), source_damping, source_damping);
+    HVec source_damping = dg::pullback(
+        dg::compose(dg::PolynomialHeaviside(
+            p.source_boundary-p.source_alpha/2.,
+        p.source_alpha/2., -1 ), dg::geo::RhoP(mag)), grid);
+    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag),
+           source_damping, source_damping);
     return source_damping;
 }
 
@@ -128,6 +119,17 @@ void init_ni(
 };
 }//namespace detail
 
+//for wall shadow
+HVec wall_damping(const Geometry& grid,
+    const feltor::Parameters& p,
+    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+{
+    HVec wall_damping = dg::pullback(dg::compose( dg::PolynomialHeaviside(
+        p.damping_boundary+p.damping_alpha/2., p.damping_alpha/2., +1),
+                dg::geo::RhoP(mag)), grid);
+    return wall_damping;
+}
+
 /* The purpose of this file is to provide an interface for custom initial conditions and
  * source profiles.  Just add your own to the relevant map below.
  */
@@ -231,8 +233,8 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
-            dg::geo::ZonalFlow init0(mag.psip(), p.amp, 0., p.k_psi);
-            ntilde = dg::pullback( init0, grid);
+            dg::SinX sinX( p.amp, 0., p.k_psi);
+            ntilde = dg::pullback( dg::compose( sinX, mag.psip()), grid);
             dg::blas1::pointwiseDot( detail::profile_damping(grid,p,gp,mag), ntilde, ntilde);
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
             detail::init_ni( y0, f,grid,p,gp,mag);
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index c0e86a0c2..69ed1e765 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -5,8 +5,12 @@
     "Nz" : 32,
     "dt" : 1e-2,
     "compression" : [2,2],
-    "refineDS" : [1,1],
-    "rk4eps" : 1e-6,
+    "FCI":
+    {
+        "refine" : [1,1],
+        "rk4eps" : 1e-6,
+        "periodify": true
+    },
     "inner_loop" : 2,
     "itstp"  : 2,
     "maxout" : 5,
@@ -23,29 +27,43 @@
     "perp_diff"   : "hyperviscous",
     "nu_parallel" : 10,
     "resistivity" : 1e-4,
-    "amplitude" : 0.2,
-    "sigma"     : 2.0,
-    "posX"      : 0.6,
-    "posY"      : 0,
-    "sigma_z"   : 0.5,
-    "k_psi"     : 0,
-    "nprofileamp" : 4,
+    "curvmode"   : "toroidal",
+    "symmetric"  : false,
     "bc" :
     {
         "density" : ["DIR", "DIR"],
         "velocity": ["NEU", "NEU"],
         "potential":["DIR", "DIR"]
     },
-    "boxscaleR" :  [1.15,1.2],
-    "boxscaleZ" :  [1.2,1.15],
+    "box":
+    {
+        "scaleR" : [1.15,1.2],
+        "scaleZ" : [1.2, 1.15]
+    },
     "initne"     : "turbulence",
     "initphi"    : "zero",
-    "curvmode"   : "toroidal",
-    "symmetric"  : false,
-    "alpha_mag"  : 0.05,
-    "alpha"      : 0.2,
-    "source_type": "influx",
-    "source"     : 0,
-    "rho_source" : 0.2,
-    "rho_damping" : 1.2
+    "amplitude" : 0.2,
+    "sigma"     : 2.0,
+    "posX"      : 0.6,
+    "posY"      : 0,
+    "sigma_z"   : 0.5,
+    "k_psi"     : 0,
+    "profile":
+    {
+        "amp": 0,
+        "alpha": 0.1
+    },
+    "source" :
+    {
+        "rate": 1e-2,
+        "type": "influx",
+        "boundary": 0.5,
+        "alpha": 0.2
+    },
+    "damping":
+    {
+        "rate": 1e-2,
+        "boundary": 1.1,
+        "alpha": 0.1
+    }
 }
diff --git a/src/feltor/input/manufactured.json b/src/feltor/input/manufactured.json
index b76e031bc..cf1f77919 100644
--- a/src/feltor/input/manufactured.json
+++ b/src/feltor/input/manufactured.json
@@ -5,8 +5,12 @@
     "Nz" : 32,
     "dt" : 1e-4,
     "compression" : [2,2],
-    "refineDS" : [1,1],
-    "rk4eps" : 1e-6,
+    "FCI":
+    {
+        "refine" : [1,1],
+        "rk4eps" : 1e-6,
+        "periodify": false
+    },
     "inner_loop" : 2,
     "itstp"  : 2,
     "maxout" : 5,
@@ -23,28 +27,43 @@
     "perp_diff"   : "viscous",
     "nu_parallel" : 0,
     "resistivity" : 0,
-    "amplitude" : 0.01,
-    "sigma"     : 2.0,
-    "posX"      : 0.6,
-    "posY"      : 0,
-    "sigma_z"   : 0.25,
-    "k_psi"     : 0,
-    "nprofileamp" : 4,
-    "bgprofamp"   : 1,
+    "curvmode"   : "true",
+    "symmetric"  : false,
     "bc" :
     {
         "density" : ["DIR", "DIR"],
         "velocity": ["DIR", "DIR"],
         "potential":["DIR", "DIR"]
     },
-    "boxscaleR" :  [1.15,1.2],
-    "boxscaleZ" :  [1.2,1.15],
+    "box":
+    {
+        "scaleR" : [1.15,1.2],
+        "scaleZ" : [1.2, 1.15]
+    },
     "initne"     : "turbulence",
     "initphi"    : "zero",
-    "curvmode"   : "true",
-    "symmetric"  : false,
-    "alpha"      : 0.05,
-    "source"     : 0,
-    "rho_source" : 0.2,
-    "rho_damping" : 1.2
+    "amplitude" : 0.01,
+    "sigma"     : 2.0,
+    "posX"      : 0.6,
+    "posY"      : 0,
+    "sigma_z"   : 0.25,
+    "k_psi"     : 0,
+    "profile":
+    {
+        "amp": 4,
+        "alpha": 0.1
+    },
+    "source" :
+    {
+        "rate": 0,
+        "type": "influx",
+        "boundary": 0.5,
+        "alpha": 0.2
+    },
+    "damping":
+    {
+        "rate": 1e-2,
+        "boundary": 1.1,
+        "alpha": 0.1
+    }
 }
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index bd48fbc66..19d28e0b0 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -28,11 +28,9 @@ struct Parameters
 
     std::array<double,2> mu; // mu[0] = mu_e, m[1] = mu_i
     std::array<double,2> tau; // tau[0] = -1, tau[1] = tau_i
-    double alpha_mag, alpha, beta;
-    double rho_source, rho_damping;
 
     double nu_perp, nu_parallel;
-    double eta;
+    double eta, beta;
 
     double amp;
     double sigma;
@@ -40,7 +38,9 @@ struct Parameters
     double sigma_z;
     double k_psi;
 
-    double omega_source, omega_damping;
+    double source_rate, damping_rate;
+    double damping_alpha, source_alpha, profile_alpha;
+    double source_boundary, damping_boundary;
     double nprofamp;
     double boxscaleRm, boxscaleRp;
     double boxscaleZm, boxscaleZp;
@@ -48,7 +48,7 @@ struct Parameters
     enum dg::bc bcxN, bcyN, bcxU, bcyU, bcxP, bcyP;
     std::string initne, initphi, curvmode, perp_diff;
     std::string source_type;
-    bool symmetric;
+    bool symmetric, periodify;
     Parameters() = default;
     Parameters( const Json::Value& js) {
         n       = js["n"].asUInt();
@@ -70,9 +70,10 @@ struct Parameters
 
         eps_gamma   = js["eps_gamma"].asDouble();
         stages      = js.get( "stages", 3).asUInt();
-        mx          = js["refineDS"].get( 0u, 10).asUInt();
-        my          = js["refineDS"].get( 1u, 10).asUInt();
-        rk4eps      = js.get( "rk4eps", 1e-6).asDouble();
+        mx          = js["FCI"]["refine"].get( 0u, 1).asUInt();
+        my          = js["FCI"]["refine"].get( 1u, 1).asUInt();
+        rk4eps      = js["FCI"].get( "rk4eps", 1e-6).asDouble();
+        periodify   = js["FCI"].get( "periodify", true).asBool();
 
         mu[0]       = js["mu"].asDouble();
         mu[1]       = +1.;
@@ -84,21 +85,25 @@ struct Parameters
         nu_parallel = js["nu_parallel"].asDouble();
         eta         = js["resistivity"].asDouble();
 
-        amp         = js["amplitude"].asDouble();
+        initne      = js.get( "initne", "blob").asString();
+        initphi     = js.get( "initphi", "zero").asString();
+        amp         = js["amp"].asDouble();
         sigma       = js["sigma"].asDouble();
         posX        = js["posX"].asDouble();
         posY        = js["posY"].asDouble();
         sigma_z     = js["sigma_z"].asDouble();
         k_psi       = js["k_psi"].asDouble();
 
-        nprofamp  = js["nprofileamp"].asDouble();
-        omega_source  = js.get("source", 0.).asDouble();
-        source_type  = js.get("source_type", "profile").asString();
-        omega_damping = js.get("damping", 0.).asDouble();
-        alpha_mag    = js.get("alpha_mag", 0.05).asDouble();
-        alpha        = js.get("alpha", 0.2).asDouble();
-        rho_source   = js.get("rho_source", 0.2).asDouble();
-        rho_damping  = js.get("rho_damping", 1.2).asDouble();
+        nprofamp   = js["profile"]["amp"].asDouble();
+        profile_alpha = js["profile"]["alpha"].asDouble();
+
+        source_rate     = js["source"].get("rate", 0.).asDouble();
+        source_type     = js["source"].get("type", "profile").asString();
+        source_boundary = js["source"].get("boundary", 0.2).asDouble();
+        source_alpha    = js["source"].get("alpha", 0.2).asDouble();
+        damping_rate = js["damping"].get("rate", 0.).asDouble();
+        damping_alpha= js["damping"].get("alpha", 0.05).asDouble();
+        damping_boundary = js["damping"].get("boundary", 1.2).asDouble();
 
         bcxN = dg::str2bc(js["bc"]["density"][0].asString());
         bcyN = dg::str2bc(js["bc"]["density"][1].asString());
@@ -107,13 +112,11 @@ struct Parameters
         bcxP = dg::str2bc(js["bc"]["potential"][0].asString());
         bcyP = dg::str2bc(js["bc"]["potential"][1].asString());
 
-        boxscaleRm  = js["boxscaleR"].get(0u,1.05).asDouble();
-        boxscaleRp  = js["boxscaleR"].get(1u,1.05).asDouble();
-        boxscaleZm  = js["boxscaleZ"].get(0u,1.05).asDouble();
-        boxscaleZp  = js["boxscaleZ"].get(1u,1.05).asDouble();
+        boxscaleRm  = js["box"]["scaleR"].get(0u,1.05).asDouble();
+        boxscaleRp  = js["box"]["scaleR"].get(1u,1.05).asDouble();
+        boxscaleZm  = js["box"]["scaleZ"].get(0u,1.05).asDouble();
+        boxscaleZp  = js["box"]["scaleZ"].get(1u,1.05).asDouble();
 
-        initne      = js.get( "initne", "blob").asString();
-        initphi     = js.get( "initphi", "zero").asString();
         curvmode    = js.get( "curvmode", "toroidal").asString();
         symmetric   = js.get( "symmetric", false).asBool();
     }
@@ -141,12 +144,13 @@ struct Parameters
             <<"    init n_e:     "<<initne<<"\n"
             <<"    init Phi:     "<<initphi<<"\n";
         os << "Profile parameters are: \n"
-            <<"     omega_source:                 "<<omega_source<<"\n"
-            <<"     rho_source:                   "<<rho_source<<"\n"
-            <<"     omega_damping:                "<<omega_damping<<"\n"
-            <<"     rho_damping:                  "<<rho_damping<<"\n"
-            <<"     alpha_mag:                    "<<alpha_mag<<"\n"
-            <<"     alpha:                        "<<alpha<<"\n"
+            <<"     source_rate:                  "<<source_rate<<"\n"
+            <<"     source_boundary:              "<<source_boundary<<"\n"
+            <<"     source_alpha:                 "<<source_alpha<<"\n"
+            <<"     source_type:                  "<<source_type<<"\n"
+            <<"     damping_rate:                 "<<damping_rate<<"\n"
+            <<"     damping_boundary:             "<<damping_boundary<<"\n"
+            <<"     damping_alpha:                "<<damping_alpha<<"\n"
             <<"     density profile amplitude:    "<<nprofamp<<"\n"
             <<"     boxscale R+:                  "<<boxscaleRp<<"\n"
             <<"     boxscale R-:                  "<<boxscaleRm<<"\n"
@@ -164,7 +168,8 @@ struct Parameters
             <<"     Accuracy Time  CG:    "<<eps_time<<"\n"
             <<"     Accuracy Time Stepper "<<rtol<<"\n"
             <<"     Accuracy Fieldline    "<<rk4eps<<"\n"
-            <<"     Refined DS            "<<mx<<" "<<my<<"\n";
+            <<"     Periodify FCI         "<<std::boolalpha<< periodify<<"\n"
+            <<"     Refined FCI           "<<mx<<" "<<my<<"\n";
         os << "Output parameters are: \n"
             <<"     n_out  =                 "<<n_out<<"\n"
             <<"     Nx_out =                 "<<Nx_out<<"\n"
-- 
GitLab


From 10ce044997657892f9d73fa52379ea72c1927e1b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 7 May 2020 21:47:54 +0200
Subject: [PATCH 210/540] Cleanup and document

---
 inc/dg/Makefile       |   2 +-
 inc/dg/cg2d_t.cu      |   8 +-
 inc/dg/chebyshev.h    |  15 +--
 inc/dg/eve.h          |  85 +++++++++++------
 inc/dg/lgmres.h       |   2 +-
 inc/dg/multigrid.h    | 216 +++++++++++++++++++++---------------------
 inc/dg/multigrid_b.cu |  64 ++++++-------
 7 files changed, 210 insertions(+), 182 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 6d90c889e..058deb17a 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -17,7 +17,7 @@ all: $(CPPFILES:%.cpp=%) $(CUFILES:%.cu=%)
 	$(CC) $(OPT) $(INCLUDE) -DDG_DEBUG $(CFLAGS) $< -o $@  -g
 
 %_b: %_b.cu
-	$(CC) $(OPT) $(CFLAGS) -DDG_BENCHMARK $< -o $@ $(INCLUDE) $(LIBS) -g
+	$(CC) $(OPT) $(CFLAGS) -DDG_BENCHMARK $< -o $@ $(INCLUDE) -g
 
 %_mpit: %_mpit.cu
 	$(MPICC) $(OPT) $(INCLUDE) -DDG_DEBUG $(MPICFLAGS) $< -o $@ -g
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index b6b72a47e..d06d5c9b3 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -21,7 +21,8 @@ void solve( std::string solver, Matrix& A, Container& x, const Container& b, con
     unsigned n = grid.n(), Nx = grid.Nx(), Ny = grid.Ny();
     if( "cheby" == solver)
     {
-        dg::ChebyshevIteration<Container> cheby( x);
+        std::cout <<" CHEBYSHEV SOLVER:\n";
+        dg::Chebyshev<Container> cheby( x);
         double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
         double hxhy = lx*ly/(n*n*Nx*Ny);
         lmin *= hxhy, lmax *= hxhy; //we multiplied the matrix by w2d
@@ -33,12 +34,14 @@ void solve( std::string solver, Matrix& A, Container& x, const Container& b, con
     }
     if( "bicgstabl" == solver)
     {
+        std::cout <<" BICGSTABl SOLVER:\n";
         dg::BICGSTABl<Container> bicg( x, 100, 2);
         unsigned num_iter = bicg.solve( A, x, b, A.precond(), A.inv_weights(), 1e-6);
         std::cout << "After "<<num_iter<<" BICGSTABl iterations we have:\n";
     }
     if( "lgmres" == solver)
     {
+        std::cout <<" LGMRES SOLVER:\n";
         dg::LGMRES<Container> lgmres( x, 30, 4, 10000);
         unsigned num_iter = lgmres.solve( A, x, b, A.precond(), A.inv_weights(), 1e-6);
         std::cout << "After "<<num_iter<<" LGMRES iterations we have:\n";
@@ -60,6 +63,7 @@ int main()
     unsigned max_iter = n*n*Nx*Ny;
     const dg::HVec& copyable_vector = x;
 
+    std::cout <<" PCG SOLVER:\n";
 //! [doxygen]
     // create volume and inverse volume on previously defined grid
     const dg::HVec w2d = dg::create::weights( grid);
@@ -100,7 +104,7 @@ int main()
     res.d = sqrt(dg::blas2::dot(w2d , error));
     std::cout << "L2 Norm of Error is           " << res.d<<"\t"<<res.i << std::endl;
     res.d = sqrt(dg::blas2::dot( w2d, resi));
-    std::cout << "L2 Norm of Residuum is        " << res.d<<"\t"<<res.i << std::endl;
+    std::cout << "L2 Norm of Residuum is        " << res.d<<"\t"<<res.i << std::endl<<std::endl;
     //Fehler der Integration des Sinus ist vernachlässigbar (vgl. evaluation_t)
 
     std::vector<std::string> solvers{ "cheby", "bicgstabl", "lgmres"};
diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index a306ddd47..32f313919 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -5,11 +5,13 @@
 
 #include "blas.h"
 
+/*!@file
+ * Chebyshev solver
+ */
 
 namespace dg
 {
-//
-//implement the classical 3-term recursion with explicit residual
+
 /**
 * @brief Three-term recursion of the Chebyshev iteration for solving
 * \f[ Ax=b\f]
@@ -38,7 +40,7 @@ class Chebyshev
     Chebyshev(){}
     ///@copydoc construct()
     Chebyshev( const ContainerType& copyable):
-        m_ax(copyable), m_xm1(m_ax), m_b( m_ax){}
+        m_ax(copyable), m_xm1(m_ax){}
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_ax;}
@@ -47,10 +49,9 @@ class Chebyshev
      * @brief Allocate memory for the pcg method
      *
      * @param copyable A ContainerType must be copy-constructible from this
-     * @param max_iterations Maximum number of iterations to be used
      */
     void construct( const ContainerType& copyable) {
-        m_xm1 = m_ax = m_b = copyable;
+        m_xm1 = m_ax = copyable;
     }
     /**
      * @brief Solve the system A*x = b using Chebyshev iteration
@@ -60,7 +61,7 @@ class Chebyshev
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
      * @param min_ev the minimum Eigenvalue
-     * @param max_ev the maximum Eigenvalue
+     * @param max_ev the maximum Eigenvalue (must be larger than \c min_ev)
      * @param num_iter the number of iterations k (equals the number of times A is applied)
      *
      * @copydoc hide_matrix
@@ -93,7 +94,7 @@ class Chebyshev
         }
     }
   private:
-    ContainerType m_ax, m_xm1, m_b;
+    ContainerType m_ax, m_xm1;
 };
 
 } //namespace dg
diff --git a/inc/dg/eve.h b/inc/dg/eve.h
index 0b222d6ad..0f855519e 100644
--- a/inc/dg/eve.h
+++ b/inc/dg/eve.h
@@ -1,8 +1,3 @@
-/* EVE adds an estimator for the largest Eigenvalue
-   to not-yet-preconditioned CG.
-
-                           */
-
 #ifndef _DG_EVE_
 #define _DG_EVE_
 
@@ -10,38 +5,69 @@
 #include "blas.h"
 #include "functors.h"
 
+/*! @file
+ * EVE adds an estimator for the largest Eigenvalue
+ *  to not-yet-preconditioned CG.
+ *  @author Eduard Reiter and Matthias Wiesenberger
+ */
+
+
 namespace dg
 {
 
-//MW: please document
-/* EVE (EigenValueEstimator) estimate largest EV using CG */
-template< class Vector>
+/*! @brief (EigenValueEstimator) estimate largest Eigenvalue using conjugate gradient method
+* @copydoc hide_ContainerType
+ * @ingroup invert
+*/
+template< class ContainerType>
 class EVE
 {
-public:
-    using value_type  = get_value_type<Vector>;
+  public:
+    using container_type = ContainerType;
+    using value_type = get_value_type<ContainerType>; //!< value type of the ContainerType class
+    ///@brief Allocate nothing, Call \c construct method before usage
     EVE() {}
-    EVE( const Vector& copyable, unsigned max_iter):r( copyable), p( r), ap( r), max_iter( max_iter) {}
-    void set_max( unsigned new_max)
-    {   max_iter = new_max;
-    }
-    unsigned get_max() const
-    {   return max_iter;
+    ///@copydoc construct()
+    EVE( const ContainerType& copyable, unsigned max_iter = 100):r( copyable), p( r), ap( r), m_max_iter( max_iter) {}
+    /**
+     * @brief Allocate memory for the pcg method
+     *
+     * @param copyable A ContainerType must be copy-constructible from this
+     * @param max_iter Maximum number of iterations to be used
+     */
+    void construct( const ContainerType& copyable, unsigned max_iter = 100) {
+        ap = p = r = copyable;
+        m_max_iter = max_iter;
     }
-    void construct( const Vector& copyable, unsigned max_iterations = 100)
-    {   ap = p = r = copyable;
-        max_iter = max_iterations;
+    /// Set maximum number of iterations
+    void set_max( unsigned new_max) {
+        m_max_iter = new_max;
     }
-    template< class Matrix>
-    unsigned operator()( Matrix& A, Vector& x, const Vector& b, value_type& ev_max, value_type eps_ev=1e-16);
-private:
-    Vector r, p, ap;
-    unsigned max_iter;
+    /// Get maximum number of iterations
+    unsigned get_max() const {   return m_max_iter; }
+    /**
+     * @brief Unpreconditioned CG to estimate maximum Eigenvalue
+     *
+     * @param A A symmetric, positive definit matrix
+     * @param x Contains an initial value on input and the solution on output.
+     * @param b The right hand side vector. x and b may be the same vector.
+     * @param ev_max (output) maximum Eigenvalue on output
+     * @param eps_ev The desired accuracy of the largest Eigenvalue
+     *
+     * @return Number of iterations used to achieve desired precision or max_iterations
+     * @copydoc hide_matrix
+     */
+    template< class MatrixType>
+    unsigned operator()( MatrixType& A, ContainerType& x, const ContainerType& b, value_type& ev_max, value_type eps_ev=1e-16);
+  private:
+    ContainerType r, p, ap;
+    unsigned m_max_iter;
 };
 
-template< class Vector>
-template< class Matrix>
-unsigned EVE< Vector>::operator()( Matrix& A, Vector& x, const Vector&
+///@cond
+template< class ContainerType>
+template< class MatrixType>
+unsigned EVE< ContainerType>::operator()( MatrixType& A, ContainerType& x, const ContainerType&
 b, value_type& ev_max, value_type eps_ev)
 {
     blas2::symv( A, x, r);
@@ -53,7 +79,7 @@ b, value_type& ev_max, value_type eps_ev)
     value_type evdash, gamma = 0., lambda, omega, beta = 0.;
     value_type ev_est = 0.;
     ev_max = 0.;
-    for( unsigned i=1; i<max_iter; i++)
+    for( unsigned i=1; i<m_max_iter; i++)
     {
         lambda = delta*alpha_inv;       // EVE!
         blas2::symv( A, p, ap);
@@ -77,8 +103,9 @@ b, value_type& ev_max, value_type eps_ev)
         nrm2r_old=nrm2r_new;
         ev_est = ev_max;
     }
-    return max_iter;
+    return m_max_iter;
 };
+///@endcond
 
 } //namespace dg
 #endif //_DG_EVE_
diff --git a/inc/dg/lgmres.h b/inc/dg/lgmres.h
index ff4dc2b99..f0533642d 100644
--- a/inc/dg/lgmres.h
+++ b/inc/dg/lgmres.h
@@ -291,7 +291,7 @@ unsigned LGMRES< ContainerType>::solve( Matrix& A, ContainerType0& x, const Cont
         dg::blas1::axpby(1.,b,-1.,residual);
         normres = sqrt(dg::blas2::dot(S,residual));
         dg::blas2::symv(P,residual,residual);
-        value_type rho = sqrt(dg::blas1::dot(residual,residual));
+        //value_type rho = sqrt(dg::blas1::dot(residual,residual));
         totalRestarts += 1;
     }
     return totalRestarts*krylovDimension;
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 4085b5adf..3d7cc0c95 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -15,44 +15,9 @@
 #include "topology/mpi_projection.h"
 #endif
 
-#include "dg/file/nc_utilities.h"
-size_t start=0;
-int ncid =0;
-int xID=0, bID=0, rID=0, tvarID=0;
-file::NC_Error_Handle err;
-
 namespace dg
 {
 
-template<class Container, class Matrix, class Geometry>
-void file_output( const std::vector<Container>& x,const std::vector<Container>& b, const std::vector<Container>& r, std::vector<Container>& out, unsigned p, std::vector<Matrix>& inter, Geometry& grid )
-{
-    size_t count = 1;
-    double time = start;
-    err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
-    dg::HVec data = out[0];
-
-    dg::blas1::copy( x[p], out[p]);
-    for( int i=(int)p; i>0; i--)
-        dg::blas2::gemv( inter[i-1], out[i], out[i-1]);
-    data = out[0];
-    file::put_vara_double( ncid, xID, start, grid, data);
-
-    dg::blas1::copy( b[p], out[p]);
-    for( int i=(int)p; i>0; i--)
-        dg::blas2::gemv( inter[i-1], out[i], out[i-1]);
-    data = out[0];
-    file::put_vara_double( ncid, bID, start, grid, data);
-
-    dg::blas1::copy( r[p], out[p]);
-    for( int i=(int)p; i>0; i--)
-        dg::blas2::gemv( inter[i-1], out[i], out[i-1]);
-    data = out[0];
-    file::put_vara_double( ncid, rID, start, grid, data);
-
-    start++;
-}
-
 /**
 * @brief Solves the Equation \f[ \frac{1}{W} \hat O \phi = \rho \f]
 *
@@ -82,21 +47,23 @@ struct MultigridCG2d
     using matrix_type = Matrix;
     using container_type = Container;
     using value_type = get_value_type<Container>;
+    ///@brief Allocate nothing, Call \c construct method before usage
     MultigridCG2d(){}
+    ///@copydoc construct()
+    template<class ...Params>
+    MultigridCG2d( const Geometry& grid, const unsigned stages, Params&& ... ps)
+    {
+        construct( grid, stages, std::forward<Params>(ps)...);
+    }
     /**
      * @brief Construct the grids and the interpolation/projection operators
      *
      * @param grid the original grid (Nx() and Ny() must be evenly divisable by pow(2, stages-1)
      * @param stages number of grids in total (The second grid contains half the points of the original grids,
      *   The third grid contains half of the second grid ...). Must be > 1
-     *   @param ps parameters necessary for \c dg::construct to construct a \c Container from a \c dg::HVec
+     * @param ps parameters necessary for \c dg::construct to construct a \c Container from a \c dg::HVec
     */
     template<class ...Params>
-    MultigridCG2d( const Geometry& grid, const unsigned stages, Params&& ... ps)
-    {
-        construct( grid, stages, std::forward<Params>(ps)...);
-    }
-    template<class ...Params>
     void construct( const Geometry& grid, const unsigned stages, Params&& ... ps)
     {
         m_stages = stages;
@@ -142,8 +109,56 @@ struct MultigridCG2d
         }
     }
 
+
+    /**
+    * @brief Project vector to all involved grids
+    * @param src the input vector (may alias first element of out)
+    * @param out the input vector projected to all grids ( index 0 contains a copy of src, 1 is the projetion to the first coarse grid, 2 is the next coarser grid, ...)
+    * @note \c out is not resized
+    */
+    template<class ContainerType0>
+    void project( const ContainerType0& src, std::vector<ContainerType0>& out)
+    {
+        dg::blas1::copy( src, out[0]);
+        for( unsigned u=0; u<m_stages-1; u++)
+            dg::blas2::gemv( m_project[u], out[u], out[u+1]);
+    }
+
+    /**
+    * @brief Project vector to all involved grids (allocate memory version)
+    * @param src the input vector
+    * @return the input vector projected to all grids ( index 0 contains a copy of src, 1 is the projetion to the first coarse grid, 2 is the next coarser grid, ...)
+    */
+    template<class ContainerType0>
+    std::vector<ContainerType0> project( const ContainerType0& src)
+    {
+        //use the fact that m_x has the correct sizes from the constructor
+        std::vector<Container> out( m_x);
+        project( src, out);
+        return out;
+
+    }
+    ///@return number of stages (same as \c num_stages)
+    unsigned stages()const{return m_stages;}
+    ///@return number of stages (same as \c stages)
+    unsigned num_stages()const{return m_stages;}
+
+    ///return the grid at given stage
+    ///@param stage must fulfill \c 0 <= stage < stages()
+    const Geometry& grid( unsigned stage) const {
+        return *(m_grids[stage]);
+    }
+
+
+    ///After a call to a solution method returns the maximum number of iterations allowed at stage  0
+    ///(if the solution method returns this number, failure is indicated)
+    unsigned max_iter() const{return m_cg[0].get_max();}
+
+    ///@brief Return an object of same size as the object used for construction on the finest grid
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const Container& copyable() const {return m_x[0];}
     /**
-     * @brief Nested iterations
+     * @brief Nested iterations (USE THIS ONE!)
      *
      * - Compute residual with given initial guess.
      * - Project residual down to the coarsest grid.
@@ -216,57 +231,31 @@ struct MultigridCG2d
         return number;
     }
 
-    /**
-    * @brief Project vector to all involved grids
-    * @param src the input vector (may alias first element of out)
-    * @param out the input vector projected to all grids ( index 0 contains a copy of src, 1 is the projetion to the first coarse grid, 2 is the next coarser grid, ...)
-    * @note \c out is not resized
-    */
-    template<class ContainerType0>
-    void project( const ContainerType0& src, std::vector<ContainerType0>& out)
-    {
-        dg::blas1::copy( src, out[0]);
-        for( unsigned u=0; u<m_stages-1; u++)
-            dg::blas2::gemv( m_project[u], out[u], out[u+1]);
-    }
 
     /**
-    * @brief Project vector to all involved grids (allocate memory version)
-    * @param src the input vector
-    * @return the input vector projected to all grids ( index 0 contains a copy of src, 1 is the projetion to the first coarse grid, 2 is the next coarser grid, ...)
+     * @brief Full multigrid cycles (experimental, use at own risk)
+     *
+     * - Compute residual with given initial guess.
+     * - If error larger than tolerance, do a full multigrid cycle with Chebeyshev iterations as smoother
+     * - repeat
+     * @note The preconditioner for the CG solver is taken from the \c precond() method in the \c SymmetricOp class
+     * @copydoc hide_symmetric_op
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     * @param op Index 0 is the \c SymmetricOp on the original grid, 1 on the half grid, 2 on the quarter grid, ...
+     * @param x (read/write) contains initial guess on input and the solution on output
+     * @param b The right hand side (will be multiplied by \c weights)
+     * @param ev The estimate of the largest Eivenvalue for each stage
+     * @param nu_pre number of pre-smoothing steps (make it >10)
+     * @param nu_post number of post-smoothing steps (make it >10)
+     * @param gamma The shape of the multigrid ( 1 is usually ok)
+     * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$
+     * @attention This method is rather unreliable, it only converges if the
+     * parameters are chosen correctly ( there need to be enough smooting steps
+     * for instance, and a large jump  factor in the Elliptic class also seems
+     * to help) and otherwise just iterates to infinity
     */
-    template<class ContainerType0>
-    std::vector<ContainerType0> project( const ContainerType0& src)
-    {
-        //use the fact that m_x has the correct sizes from the constructor
-        std::vector<Container> out( m_x);
-        project( src, out);
-        return out;
-
-    }
-    ///@return number of stages (same as \c num_stages)
-    unsigned stages()const{return m_stages;}
-    ///@return number of stages (same as \c stages)
-    unsigned num_stages()const{return m_stages;}
-
-    ///observe the grids at all stages
-    ///@param stage must fulfill \c 0 <= stage < stages()
-    const Geometry& grid( unsigned stage) const {
-        return *(m_grids[stage]);
-    }
-
-
-    ///After a call to a solution method returns the maximum number of iterations allowed at stage  0
-    ///(if the solution method returns this number, failure is indicated)
-    unsigned max_iter() const{return m_cg[0].get_max();}
-
-    ///@brief Return an object of same size as the object used for construction on the finest grid
-    ///@return A copyable object; what it contains is undefined, its size is important
-    const Container& copyable() const {return m_x[0];}
-
-
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
-    void solve( std::vector<SymmetricOp>& op,
+    void fmg_solve( std::vector<SymmetricOp>& op,
     ContainerType0& x, const ContainerType1& b, std::vector<double> ev, unsigned nu_pre, unsigned
     nu_post, unsigned gamma, double eps)
     {
@@ -286,7 +275,7 @@ struct MultigridCG2d
         dg::blas1::axpby( -1., m_r[0], 1., m_b[0]);
         dg::blas1::copy( 0., m_x[0]);
         value_type error = sqrt( blas2::dot(op[0].inv_weights(),m_b[0]) );
-        std::cout<< "# Relative Residual error is  "<<error/(nrmb+1)<<"\n";
+        //std::cout<< "# Relative Residual error is  "<<error/(nrmb+1)<<"\n";
 
         while ( error >  eps*(nrmb + 1))
         {
@@ -303,9 +292,28 @@ struct MultigridCG2d
             dg::blas1::axpby( -1., m_r[0], 1., m_b[0]);
             dg::blas1::copy( 0., m_x[0]);
             error = sqrt( blas2::dot(op[0].inv_weights(),m_b[0]) );
-            std::cout<< "# Relative Residual error is  "<<error/(nrmb+1)<<"\n";
+            //std::cout<< "# Relative Residual error is  "<<error/(nrmb+1)<<"\n";
         }
     }
+
+    /**
+     * @brief A conjugate gradient with a full multigrid cycle as preconditioner (experimental, use at own risk)
+     *
+     * @copydoc hide_symmetric_op
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     * @param op Index 0 is the \c SymmetricOp on the original grid, 1 on the half grid, 2 on the quarter grid, ...
+     * @param x (read/write) contains initial guess on input and the solution on output
+     * @param b The right hand side (will be multiplied by \c weights)
+     * @param ev The estimate of the largest Eivenvalue for each stage
+     * @param nu_pre number of pre-smoothing steps (make it >10)
+     * @param nu_post number of post-smoothing steps (make it >10)
+     * @param gamma The shape of the multigrid ( 1 is usually ok)
+     * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$
+     * @attention This method is rather unreliable, it only converges if the
+     * parameters are chosen correctly ( there need to be enough smooting steps
+     * for instance, and a large jump  factor in the Elliptic class also seems
+     * to help) and otherwise just iterates to infinity
+    */
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
     void pcg_solve( std::vector<SymmetricOp>& op,
     ContainerType0& x, const ContainerType1& b, std::vector<double> ev, unsigned nu_pre, unsigned
@@ -330,7 +338,6 @@ struct MultigridCG2d
 
         dg::blas1::copy( 0, m_x[0]);
         dg::blas1::copy( m_cgr, m_b[0]);
-        //multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
         full_multigrid( op,m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
         dg::blas1::copy( m_x[0], m_p);
 
@@ -344,13 +351,12 @@ struct MultigridCG2d
             blas1::axpby( alpha, m_p, 1., x);
             blas1::axpby( -alpha, m_x[0], 1., m_cgr);
             value_type error = sqrt( blas2::dot(op[0].inv_weights(), m_cgr))/(nrmb+1);
-            std::cout << "\t\t\tError at "<<i<<" is "<<error<<"\n";
+            //std::cout << "\t\t\tError at "<<i<<" is "<<error<<"\n";
             if( error < eps)
                 return;
-        dg::blas1::copy( 0, m_x[0]);
-        dg::blas1::copy( m_cgr, m_b[0]);
-        //multigrid_cycle( op, m_x, m_b, ev, nu_pre, nu_post, gamma, 0, eps);
-        full_multigrid( op,m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
+            dg::blas1::copy( 0, m_x[0]);
+            dg::blas1::copy( m_cgr, m_b[0]);
+            full_multigrid( op,m_x, m_b, ev, nu_pre, nu_post, gamma, 1, eps);
 
             nrmzr_new = blas1::dot( m_x[0], m_cgr);
             blas1::axpby(1., m_x[0], nrmzr_new/nrmzr_old, m_p );
@@ -358,7 +364,7 @@ struct MultigridCG2d
         }
 
     }
-
+  private:
     template<class SymmetricOp>
     void multigrid_cycle( std::vector<SymmetricOp>& op,
     std::vector<Container>& x, std::vector<Container>& b,
@@ -387,14 +393,12 @@ struct MultigridCG2d
 #endif //DG_BENCHMARK
 
         //std::vector<Container> out( x);
-        //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
 
         m_cheby[p].solve( op[p], x[p], b[p], 1e-2*ev[p], 1.1*ev[p], nu1);
         //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1, op[p].inv_weights());
         // 2. Residuum
         dg::blas2::symv( op[p], x[p], m_r[p]);
         dg::blas1::axpby( 1., b[p], -1., m_r[p]);
-        //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
         //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum after  "<<norm_res<<"\n";
         // 3. Coarsen
@@ -403,7 +407,6 @@ struct MultigridCG2d
         dg::blas1::scal( x[p+1], 0.);
         if( p+1 == m_stages-1)
         {
-            //file_output( x, b, m_r, out, p+1, m_inter, *m_grids[0] );
 #ifdef DG_BENCHMARK
             t.tic();
 #endif //DG_BENCHMARK
@@ -416,9 +419,8 @@ struct MultigridCG2d
             MPI_Comm_rank(MPI_COMM_WORLD, &rank);
             if(rank==0)
 #endif //MPI
-            std::cout << "# Multigrid stage: " << p+1 << ", iter: " << number << ", took "<<t.diff()<<"s\n";
+            //std::cout << "# Multigrid stage: " << p+1 << ", iter: " << number << ", took "<<t.diff()<<"s\n";
 #endif //DG_BENCHMARK
-            //file_output( x, b, m_r, out, p+1, m_inter, *m_grids[0] );
             //dg::blas2::symv( op[p+1], x[p+1], m_r[p+1]);
             //dg::blas1::axpby( 1., b[p+1], -1., m_r[p+1]);
             //double norm_res = sqrt(dg::blas1::dot( m_r[p+1], m_r[p+1]));
@@ -438,10 +440,8 @@ struct MultigridCG2d
         //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
         // 6. Post-Smooth nu2 times
-        //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
         m_cheby[p].solve( op[p], x[p], b[p], 1e-2*ev[p], 1.1*ev[p], nu2);
         //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2, op[p].inv_weights());
-        //file_output( x, b, m_r, out, p, m_inter, *m_grids[0] );
         //dg::blas2::symv( op[p], x[p], m_r[p]);
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
         //double norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
@@ -462,7 +462,6 @@ struct MultigridCG2d
         //std::vector<Container> out( x);
         //begins on coarsest level and cycles through to highest
         unsigned s = m_stages-1;
-        //file_output( x, b, m_r, out, s, m_inter, *m_grids[0] );
 #ifdef DG_BENCHMARK
         dg::Timer t;
         t.tic();
@@ -476,10 +475,9 @@ struct MultigridCG2d
         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
         if(rank==0)
 #endif //MPI
-        std::cout << "# Multigrid stage: " << s << ", iter: " << number << ", took "<<t.diff()<<"s\n";
+        //std::cout << "# Multigrid stage: " << s << ", iter: " << number << ", took "<<t.diff()<<"s\n";
 #endif //DG_BENCHMARK
 
-        //file_output( x, b, m_r, out, s, m_inter, *m_grids[0] );
 		for( int p=m_stages-2; p>=0; p--)
         {
             dg::blas2::gemv( m_inter[p], x[p+1],  x[p]);
@@ -487,8 +485,6 @@ struct MultigridCG2d
                 multigrid_cycle( op, x, b, ev, nu1, nu2, gamma, p, eps);
         }
     }
-
-  private:
     unsigned m_stages;
     std::vector< dg::ClonePtr< Geometry> > m_grids;
     std::vector< MultiMatrix<Matrix, Container> >  m_inter;
diff --git a/inc/dg/multigrid_b.cu b/inc/dg/multigrid_b.cu
index 58c79edfe..86b407879 100644
--- a/inc/dg/multigrid_b.cu
+++ b/inc/dg/multigrid_b.cu
@@ -88,23 +88,27 @@ int main()
         hxhy*=4;
     }
     std::cout << "\n\n";
-    //////////////////////////////setup and write netcdf//////////////////
-    err = nc_create( "multigrid.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
-    int dim3d[3];
-    err = file::define_dimensions(  ncid, dim3d, &tvarID, grid, {"step", "y", "x"} );
-
-    err = nc_def_var( ncid, "x_num", NC_DOUBLE, 3, dim3d, &xID);
-    err = nc_def_var( ncid, "b_num", NC_DOUBLE, 3, dim3d, &bID);
-    err = nc_def_var( ncid, "r_num", NC_DOUBLE, 3, dim3d, &rID);
     ////////////////////////////////////////////////////
-    std::cout << "Type nu1 (20), nu2 (20) gamma (1) max_iter (1)\n";
+    std::cout << "Type nu1 (20), nu2 (20) gamma (1) \n";
     unsigned nu1, nu2, gamma;
-    int max_iter = 3;
-    std::cin >> nu1 >> nu2 >> gamma >> max_iter;
-    x = dg::evaluate( initial, grid);
+    std::cin >> nu1 >> nu2 >> gamma;
     dg::Timer t;
-    //for( int i=0; i<max_iter; i++)
+    std::cout << "MULTIGRID NESTED ITERATIONS SOLVE:\n";
+    x = dg::evaluate( initial, grid);
+    t.tic();
+    multigrid.direct_solve(multi_pol, x, b, eps);
+    t.toc();
+    const double norm = dg::blas2::dot( w2d, solution);
+    dg::DVec error( solution);
+    dg::blas1::axpby( 1.,x,-1., solution, error);
+    double err = dg::blas2::dot( w2d, error);
+    err = sqrt( err/norm);
+    std::cout << " Error of nested iterations "<<err<<"\n";
+    std::cout << "Took "<<t.diff()<<"s\n\n";
+    ////////////////////////////////////////////////////
     {
+        std::cout << "MULTIGRID PCG SOLVE:\n";
+        x = dg::evaluate( initial, grid);
         t.tic();
         multigrid.pcg_solve(multi_pol, x, b, multi_ev, nu1, nu2, gamma, eps);
         t.toc();
@@ -117,25 +121,21 @@ int main()
         //std::cout << " At iteration "<<i<<"\n";
         std::cout << " Error of Multigrid iterations "<<err<<"\n\n";
     }
-    ////////////////////////////////////////////////////
-
-    x = dg::evaluate( initial, grid);
-    t.tic();
-    multigrid.direct_solve(multi_pol, x, b, eps);
-    t.toc();
-    const double norm = dg::blas2::dot( w2d, solution);
-    dg::DVec error( solution);
-    dg::blas1::axpby( 1.,x,-1., solution, error);
-    double err = dg::blas2::dot( w2d, error);
-    err = sqrt( err/norm);
-    std::cout << " Error of nested iterations "<<err<<"\n";
-    std::cout << "Took "<<t.diff()<<"s\n\n";
-    /////////////////////////////////////////////////////
-    int varID;
-    err = nc_def_var( ncid, "x_ana", NC_DOUBLE, 2, &dim3d[1], &varID);
-    dg::HVec data = solution;
-    file::put_var_double( ncid, varID, grid, data);
-    err = nc_close( ncid);
+    {
+        std::cout << "MULTIGRID FMG SOLVE:\n";
+        x = dg::evaluate( initial, grid);
+        t.tic();
+        multigrid.fmg_solve(multi_pol, x, b, multi_ev, nu1, nu2, gamma, eps);
+        t.toc();
+        std::cout << "Took "<<t.diff()<<"s\n";
+        const double norm = dg::blas2::dot( w2d, solution);
+        dg::DVec error( solution);
+        dg::blas1::axpby( 1.,x,-1., solution, error);
+        double err = dg::blas2::dot( w2d, error);
+        err = sqrt( err/norm);
+        //std::cout << " At iteration "<<i<<"\n";
+        std::cout << " Error of Multigrid iterations "<<err<<"\n\n";
+    }
 
 
     return 0;
-- 
GitLab


From ff5b17ed71d86a060a30c020a6ea6fce2e0205fa Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 7 May 2020 23:29:05 +0200
Subject: [PATCH 211/540] Fix bug in feltor damping region

---
 inc/geometries/geometry_diag.cu |  8 ++++----
 src/feltor/feltor.cu            | 10 +++++-----
 src/feltor/feltor.h             |  6 ++++--
 src/feltor/feltor_hpc.cu        |  8 ++++----
 4 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 99142bfde..eec5da046 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -157,10 +157,10 @@ int main( int argc, char* argv[])
         double RO=mag.R0(), ZO=0.;
         dg::geo::findOpoint( mag.get_psip(), RO, ZO);
         double psipO = mag.psip()( RO, ZO);
-        double damping_psi0 = (1.-p.damping_boundary*p.damping_boundary)*psipO;
-        double damping_alpha = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        std::cout<< " damping "<< damping_psi0 << " "<<damping_alpha<<"\n";
-        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0+damping_alpha/2., fabs(p.damping_alpha/2.), ((psipO>0)-(psipO<0)));
+        double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
+        double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
+        std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
+        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2., fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
     }
     double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
     double Z_X = -1.1*gp.elongation*gp.a;
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index c86bacb6d..02de12e45 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -64,10 +64,10 @@ int main( int argc, char* argv[])
         double RO=mag.R0(), ZO=0.;
         dg::geo::findOpoint( mag.get_psip(), RO, ZO);
         double psipO = mag.psip()( RO, ZO);
-        double damping_psi0 = (1.-p.damping_boundary*p.damping_boundary)*psipO;
-        double damping_alpha = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0+damping_alpha/2.,
-                fabs(p.damping_alpha/2.), ((psipO>0)-(psipO<0)));
+        double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
+        double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
+        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2.,
+                fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
     }
     if( p.periodify)
         mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
@@ -184,7 +184,7 @@ int main( int argc, char* argv[])
             else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
             {
                 dg::assign( *pair.second, hvisual);
-                dg::blas1::axpby( 1., hvisual, -1., profile, hvisual);
+                //dg::blas1::axpby( 1., hvisual, -1., profile, hvisual);
             }
             else
                 dg::assign( *pair.second, hvisual);
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 85f92a1ee..e84258075 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -896,9 +896,11 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
                 m_profne, m_source, m_omega_source);
         else
             dg::blas1::axpby( m_omega_source, m_source, 0., m_s[0][0]);
+        // -w_d ~n
         dg::blas1::pointwiseDot( -m_omega_damping, m_damping, y[0][0], 1.,  m_s[0][0]);
-        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][0], 1.,  m_s[1][0]);
-        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][1], 1.,  m_s[1][1]);
+        // - w_d U
+        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][0], 0.,  m_s[1][0]);
+        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][1], 0.,  m_s[1][1]);
         //compute FLR corrections S_N = (1-0.5*mu*tau*Lap)*S_n
         dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
         dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 2defd38da..228aa506d 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -156,10 +156,10 @@ int main( int argc, char* argv[])
         double RO=mag.R0(), ZO=0.;
         dg::geo::findOpoint( mag.get_psip(), RO, ZO);
         double psipO = mag.psip()( RO, ZO);
-        double damping_psi0 = (1.-p.damping_boundary*p.damping_boundary)*psipO;
-        double damping_alpha = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0+damping_alpha/2.,
-                fabs(p.damping_alpha/2.), ((psipO>0)-(psipO<0)));
+        double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
+        double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
+        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2.,
+                fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
     }
     if( p.periodify)
         mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
-- 
GitLab


From d27cc2ec31919027c853ce7b0c51b728f3dedc0c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 11 May 2020 00:06:46 +0200
Subject: [PATCH 212/540] New RZP2X functions; Refactor geometry_diag

- moved tests out of geometry_diag into new file
- geometry_diag now outputs 2d as 3d variables as well
        (next step is to output fsa as well)
- make dg::extend_line available in Documentation
- new RZP2X functions used in geometry_diag and feltor
- findXpoint moved to file fluxfunctions.h
- mention Cylindrical as physical coords in Docu
---
 inc/dg/Doxyfile                    |  34 ++---
 inc/dg/backend/average_dispatch.h  |  71 +++++----
 inc/dg/topology/base_geometry.h    |   1 +
 inc/dg/topology/functions.h        |   9 ++
 inc/dg/topology/transform.h        |   6 +-
 inc/geometries/fluxfunctions.h     |   6 +
 inc/geometries/geometry_diag.cu    | 226 ++++++++++-------------------
 inc/geometries/magnetic_field_t.cu | 133 +++++++++++++++++
 inc/geometries/utilitiesX.h        |   5 -
 src/feltor/feltor.h                |  10 --
 src/feltor/feltor.tex              |   9 +-
 src/feltor/feltordiag.h            |  25 +---
 12 files changed, 287 insertions(+), 248 deletions(-)
 create mode 100644 inc/geometries/magnetic_field_t.cu

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index e1becbc2d..9da97f0b9 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -791,28 +791,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = ./backend/exceptions.h \
-                         ./backend/memory.h \
-                         ./backend/view.h \
-                         ./backend/mpi_collective.h \
-                         ./backend/mpi_communicator.h \
-                         ./backend/mpi_init.h \
-                         ./backend/mpi_matrix.h \
-                         ./backend/mpi_vector.h \
-                         ./backend/sparseblockmat.h \
-                         ./backend/sparseblockmat.cuh \
-                         ./backend/timer.h \
-                         ./backend/typedefs.h \
-                         ./backend/transpose.h \
-                         ./backend/vector_categories.h \
-                         ./backend/matrix_categories.h \
-                         ./backend/scalar_categories.h \
-                         ./backend/execution_policy.h \
-                         ./backend/tensor_traits.h \
-                         ./backend/tensor_traits_scalar.h \
-                         ./backend/tensor_traits_cusp.h \
-                         ./backend/tensor_traits_std.h \
-                         ./backend/tensor_traits_thrust.h \
+INPUT                  = ./backend  \
                          ./topology \
                          .
 
@@ -858,6 +837,14 @@ RECURSIVE              = NO
 # run.
 
 EXCLUDE                = topology/ell_interpolation.cuh \
+                         backend/average_cpu.h \
+                         backend/average_gpu.cuh \
+                         backend/average_omp.h \
+                         backend/config.h \
+                         backend/predicate.h \
+                         backend/sparseblockmat_gpu_kernels.cuh \
+                         backend/sparseblockmat_omp_kernels.h \
+                         backend/exblas/ \
                          geometries/ \
                          file/
 
@@ -875,8 +862,7 @@ EXCLUDE_SYMLINKS       = NO
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       = */*_blas.cuh \
-                         */*_blas.h
+EXCLUDE_PATTERNS       = */blas*
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the
diff --git a/inc/dg/backend/average_dispatch.h b/inc/dg/backend/average_dispatch.h
index f72b2e7a4..990f44b02 100644
--- a/inc/dg/backend/average_dispatch.h
+++ b/inc/dg/backend/average_dispatch.h
@@ -8,77 +8,84 @@
 #endif
 
 namespace dg{
+///@addtogroup misc
+///@{
 
 /*!@brief Transpose vector
 
- * @copydoc hide_container
+ * @copydoc hide_ContainerType
  * @param nx number of columns in input vector (size of contiguous chunks) /rows in output vector
  * @param ny number of rows in input vector /columns in output vector
  * @param in input
  * @param out output (may not alias in)
 */
-template<class container>
-void transpose( unsigned nx, unsigned ny, const container& in, container& out)
+template<class ContainerType>
+void transpose( unsigned nx, unsigned ny, const ContainerType& in, ContainerType& out)
 {
     assert(&in != &out);
-    const get_value_type<container>* in_ptr = thrust::raw_pointer_cast( in.data());
-    get_value_type<container>* out_ptr = thrust::raw_pointer_cast( out.data());
-    return transpose_dispatch( get_execution_policy<container>(), nx, ny, in_ptr, out_ptr);
+    const get_value_type<ContainerType>* in_ptr = thrust::raw_pointer_cast( in.data());
+    get_value_type<ContainerType>* out_ptr = thrust::raw_pointer_cast( out.data());
+    return transpose_dispatch( get_execution_policy<ContainerType>(), nx, ny, in_ptr, out_ptr);
 }
 
-/*!@brief Copy a line  into rows of output vector
+/*!@brief Copy a line into rows of output vector
 
- * @copydoc hide_container
+  Just append the line of size \c nx until the output vector size of \c nx*ny is reached: \c out[i*nx+j] = in[j]
+ * @copydoc hide_ContainerType
  * @param nx size of the input vector/ number of columns (size of contiguous chunks) in output vector
  * @param ny number of rows in output vector
- * @param in input (size nx)
- * @param out output (may not alias in)
+ * @param in input (size \c nx)
+ * @param out output (size \c nx*ny; may not alias in)
 */
-template<class container>
-void extend_line( unsigned nx, unsigned ny, const container& in, container& out)
+template<class ContainerType>
+void extend_line( unsigned nx, unsigned ny, const ContainerType& in, ContainerType& out)
 {
     assert(&in != &out);
-    const get_value_type<container>* in_ptr = thrust::raw_pointer_cast( in.data());
-    get_value_type<container>* out_ptr = thrust::raw_pointer_cast( out.data());
-    return extend_line( get_execution_policy<container>(), nx, ny, in_ptr, out_ptr);
+    const get_value_type<ContainerType>* in_ptr = thrust::raw_pointer_cast( in.data());
+    get_value_type<ContainerType>* out_ptr = thrust::raw_pointer_cast( out.data());
+    return extend_line( get_execution_policy<ContainerType>(), nx, ny, in_ptr, out_ptr);
 }
-/*!@brief Copy a line  into columns of output vector
+/*!@brief Copy a line into columns of output vector
 
- * @copydoc hide_container
+  \c out[i*nx+j] = in[i]
+ * @copydoc hide_ContainerType
  * @param nx number of columns (size of contiguous chunks) in output vector
  * @param ny size of input vector, number of rows in output vector
- * @param in input (size ny)
- * @param out output (may not alias in)
+ * @param in input (size \c ny)
+ * @param out output (size \c nx*ny may not alias in)
 */
-template<class container>
-void extend_column( unsigned nx, unsigned ny, const container& in, container& out)
+template<class ContainerType>
+void extend_column( unsigned nx, unsigned ny, const ContainerType& in, ContainerType& out)
 {
     assert(&in != &out);
-    const get_value_type<container>* in_ptr = thrust::raw_pointer_cast( in.data());
-    get_value_type<container>* out_ptr = thrust::raw_pointer_cast( out.data());
-    return extend_column( get_execution_policy<container>(), nx, ny, in_ptr, out_ptr);
+    const get_value_type<ContainerType>* in_ptr = thrust::raw_pointer_cast( in.data());
+    get_value_type<ContainerType>* out_ptr = thrust::raw_pointer_cast( out.data());
+    return extend_column( get_execution_policy<ContainerType>(), nx, ny, in_ptr, out_ptr);
 }
+///@}
 
-template<class container>
-void average( unsigned nx, unsigned ny, const container& in0, const container& in1, container& out)
+///@cond
+template<class ContainerType>
+void average( unsigned nx, unsigned ny, const ContainerType& in0, const ContainerType& in1, ContainerType& out)
 {
-    static_assert( std::is_same<get_value_type<container>, double>::value, "We only support double precision dot products at the moment!");
+    static_assert( std::is_same<get_value_type<ContainerType>, double>::value, "We only support double precision dot products at the moment!");
     const double* in0_ptr = thrust::raw_pointer_cast( in0.data());
     const double* in1_ptr = thrust::raw_pointer_cast( in1.data());
           double* out_ptr = thrust::raw_pointer_cast( out.data());
-    average( get_execution_policy<container>(), nx, ny, in0_ptr, in1_ptr, out_ptr);
+    average( get_execution_policy<ContainerType>(), nx, ny, in0_ptr, in1_ptr, out_ptr);
 }
 
 #ifdef MPI_VERSION
-template<class container>
-void mpi_average( unsigned nx, unsigned ny, const container& in0, const container& in1, container& out, MPI_Comm comm, MPI_Comm comm_mod, MPI_Comm comm_mod_reduce)
+template<class ContainerType>
+void mpi_average( unsigned nx, unsigned ny, const ContainerType& in0, const ContainerType& in1, ContainerType& out, MPI_Comm comm, MPI_Comm comm_mod, MPI_Comm comm_mod_reduce)
 {
-    static_assert( std::is_same<get_value_type<container>, double>::value, "We only support double precision dot products at the moment!");
+    static_assert( std::is_same<get_value_type<ContainerType>, double>::value, "We only support double precision dot products at the moment!");
     const double* in0_ptr = thrust::raw_pointer_cast( in0.data());
     const double* in1_ptr = thrust::raw_pointer_cast( in1.data());
           double* out_ptr = thrust::raw_pointer_cast( out.data());
-    average_mpi( get_execution_policy<container>(), nx, ny, in0_ptr, in1_ptr, out_ptr, comm, comm_mod, comm_mod_reduce);
+    average_mpi( get_execution_policy<ContainerType>(), nx, ny, in0_ptr, in1_ptr, out_ptr, comm, comm_mod, comm_mod_reduce);
 }
 #endif //MPI_VERSION
+///@endcond
 
 }//namespace dg
diff --git a/inc/dg/topology/base_geometry.h b/inc/dg/topology/base_geometry.h
index 14d1bcab7..4c5db43a6 100644
--- a/inc/dg/topology/base_geometry.h
+++ b/inc/dg/topology/base_geometry.h
@@ -243,6 +243,7 @@ struct RealCartesianGrid3d: public dg::aRealProductGeometry3d<real_type>
 
 /**
  * @brief three-dimensional Grid with Cylindrical metric
+ * @note \c map() returns the identity, i.e. Cylindrical coordinates count as physical coordinates. Evaluate the \c dg::cooRZP2X() functions to transform to a Cartesian coordinate system.
  */
 template<class real_type>
 struct RealCylindricalGrid3d: public dg::aRealProductGeometry3d<real_type>
diff --git a/inc/dg/topology/functions.h b/inc/dg/topology/functions.h
index 96a1bf057..330007061 100644
--- a/inc/dg/topology/functions.h
+++ b/inc/dg/topology/functions.h
@@ -47,6 +47,15 @@ DG_DEVICE static inline double cooY2d( double x, double y) {return y;}
 DG_DEVICE static inline double cooY3d( double x, double y, double z) {return y;}
 ///@brief \f[ f(x,y,z) = z\f]
 DG_DEVICE static inline double cooZ3d( double x, double y, double z) {return z;}
+
+
+///@brief \f[ R\sin(\varphi)\f]
+DG_DEVICE static inline double cooRZP2X( double R, double Z, double P){ return R*sin(P);}
+///@brief \f[ R\cos(\varphi)\f]
+DG_DEVICE static inline double cooRZP2Y( double R, double Z, double P){ return R*cos(P);}
+///@brief \f[ Z\f]
+DG_DEVICE static inline double cooRZP2Z( double R, double Z, double P){ return Z;}
+
 ///@brief \f[ f(x) = 1\f]
 DG_DEVICE static inline float one( float x) {return 1;}
 
diff --git a/inc/dg/topology/transform.h b/inc/dg/topology/transform.h
index 1b2d7ec5b..508271754 100644
--- a/inc/dg/topology/transform.h
+++ b/inc/dg/topology/transform.h
@@ -12,12 +12,14 @@ namespace dg
  *
  * Pullback is equivalent to the following:
  *
- * -# generate the list of physical space coordinates (e.g. in 2d \f$ x_i = x(\zeta_i, \eta_i),\ y_i = y(\zeta_i, \eta_i)\f$ for all \c i)
+ * -# generate the list of physical space coordinates (e.g. in 2d \f$ x_i = x(\zeta_i, \eta_i),\ y_i = y(\zeta_i, \eta_i)\f$ for all \c i) using the map member of the grid e.g. aRealGeometry2d::map()
  * -#  evaluate the given function or functor at these coordinates and store the result in the output vector (e.g. in 2d  \f$ v_i = f(x_i,y_i)\f$ for all \c i)
  *.
+
+ @note the grid defines what its physical coordinates are, i.e. it could be either Cartesian or Cylindrical coordinates
  * @tparam Functor The binary (for 2d grids) or ternary (for 3d grids) function or functor with signature: <tt>real_type ( real_type x, real_type y) ; real_type ( real_type x, real_type y, real_type z) </tt>
  * @param f The function defined in physical coordinates
- * @param g a two- or three dimensional Geometry
+ * @param g a two- or three dimensional Geometry (\c g.map() is used to evaluate \c f)
  * @note Template deduction for the Functor will fail if you overload functions with different
  dimensionality (e.g. real_type sine( real_type x) and real_type sine(real_type x, real_type y) )
  * You will want to rename those uniquely
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 7fbf171dc..695d059fb 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -287,6 +287,12 @@ static inline void findOpoint( const CylindricalFunctorsLvl2& psi, double& R_X,
     R_X = X[0], Z_X = X[1];
 }
 
+///@copydoc findOpoint()
+static inline void findXpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
+{
+    findOpoint( psi, R_X, Z_X);
+}
+
 /// A symmetric 2d tensor field and its divergence
 ///@snippet hector_t.cu doxygen
 struct CylindricalSymmTensorLvl1
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index eec5da046..80d5aee95 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -20,6 +20,9 @@
 #include "separatrix_orthogonal.h"
 #include "average.h"
 
+
+// - write magnetic functions into file
+// - test performance and accuracy of Flux surface averages and integrals
 struct Parameters
 {
     unsigned n, Nx, Ny, Nz, Npsi;
@@ -77,16 +80,6 @@ struct Parameters
     }
 };
 
-struct JPhi
-{
-    JPhi( dg::geo::solovev::Parameters gp): R_0(gp.R_0), A(gp.A){}
-    double operator()(double R, double Z, double phi)const
-    {
-        return ((A-1.)*R - A*R_0*R_0/R)/R_0/R_0/R_0;
-    }
-    private:
-    double R_0, A;
-};
 
 int main( int argc, char* argv[])
 {
@@ -162,15 +155,6 @@ int main( int argc, char* argv[])
         std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
         mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2., fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
     }
-    double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
-    double Z_X = -1.1*gp.elongation*gp.a;
-    if( gp.hasXpoint())
-    {
-        dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
-        std::cout <<  "X-point found at "<<R_X << " "<<Z_X<<" with Psip "<<mag.psip()(R_X, Z_X)<<"\n";
-        std::cout <<  "     R - Factor "<<(gp.R_0-R_X)/gp.triangularity/gp.a << "   Z - factor "<<-(Z_X/gp.elongation/gp.a)<<std::endl;
-
-    }
     //Find O-point
     double R_O = gp.R_0, Z_O = 0.;
     if( !gp.isToroidal() )
@@ -282,6 +266,20 @@ int main( int argc, char* argv[])
         dg::blas1::pointwiseDivide( X_h02_fsa, dvdzeta, X_h02_fsa );
         map1d.emplace_back( "X_hoo_fsa", X_h02_fsa,
             "Flux surface average of novel h02 factor");
+        // total source term
+        h02 = dg::pullback( dg::compose( dg::PolynomialHeaviside(
+                    p.source_boundary-p.source_alpha/2., p.source_alpha/2., -1 ),
+                dg::geo::RhoP(mag)), gX2d);
+        dg::blas1::pointwiseDot( volX2d, h02, h02);
+        avg_eta( h02, X_h02_fsa, false);
+        dg::blas1::scal( X_h02_fsa, 4*M_PI*M_PI); //
+        dg::blas1::pointwiseDivide( X_h02_fsa, dvdzeta, X_h02_fsa );
+        map1d.emplace_back( "X_sne_fsa", X_h02_fsa,
+            "Flux surface average over source term");
+        dg::blas1::pointwiseDot( X_h02_fsa, dvdzeta, X_h02_fsa );
+        X_h02_fsa = dg::integrate( X_h02_fsa, gX1d);
+        map1d.emplace_back( "X_sne_ifs", X_h02_fsa,
+            "Flux surface integral over source term");
         //divb
         h02 = dg::pullback( dg::geo::Divb(mag), gX2d);
         dg::blas1::pointwiseDot( volX2d, h02, h02);
@@ -300,7 +298,13 @@ int main( int argc, char* argv[])
         std::cout << "Compute flux averages\n";
         dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
         if( gp.hasXpoint() )
-            dg::blas1::pointwiseDot( xpoint_weights , dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
+        {
+            double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
+            double Z_X = -1.1*gp.elongation*gp.a;
+            dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
+            dg::blas1::pointwiseDot( xpoint_weights,
+                    dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
+        }
         dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, mag, psipog2d, xpoint_weights);
         grid1d = dg::Grid1d (psipmin<psipmax ? psipmin : psipmax, psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::NEU);
         map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
@@ -427,10 +431,13 @@ int main( int argc, char* argv[])
     err = nc_put_att_text( ncid, dim1d_ids[0], "long_name",
         psi_long_name.size(), psi_long_name.data());
     dg::CylindricalGrid3d grid3d(Rmin,Rmax,Zmin,Zmax, 0, 2.*M_PI, n,Nx,Ny,Nz);
-    err = file::define_dimensions( ncid, &dim3d_ids[0], grid3d);
+    dg::RealCylindricalGrid3d<float> fgrid3d(Rmin,Rmax,Zmin,Zmax, 0, 2.*M_PI, n,Nx,Ny,Nz);
+
+    err = file::define_dimensions( ncid, &dim3d_ids[0], fgrid3d);
     dim2d_ids[0] = dim3d_ids[1], dim2d_ids[1] = dim3d_ids[2];
 
     //write 1d vectors
+    std::cout << "WRTING 1D FIELDS ... \n";
     for( auto tp : map1d)
     {
         int vid;
@@ -443,7 +450,7 @@ int main( int argc, char* argv[])
         err = nc_redef(ncid);
     }
     //write 2d vectors
-    std::vector< std::tuple<std::string, std::string, std::function<double(double,double)> > > map{
+    std::vector< std::tuple<std::string, std::string, dg::geo::CylindricalFunctor >> map2d{
         {"Psip", "Flux function", mag.psip()},
         {"PsipR", "Flux function derivative in R", mag.psipR()},
         {"PsipZ", "Flux function derivative in Z", mag.psipZ()},
@@ -510,147 +517,64 @@ int main( int argc, char* argv[])
 
     };
     //allocate mem for visual
-    dg::HVec hvisual;
-    dg::HVec visual;
-    for(auto tp : map)
+    dg::HVec hvisual = dg::evaluate( dg::zero, grid2d);
+    dg::HVec hvisual3d = dg::evaluate( dg::zero, grid3d);
+    dg::fHVec fvisual, fvisual3d;
+    std::cout << "WRTING 2D/3D CYLINDRICAL FIELDS ... \n";
+    for(auto tp : map2d)
     {
-        int vectorID;
-        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 2,
+        int vectorID, vectorID3d;
+        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_FLOAT, 2,
             &dim2d_ids[0], &vectorID);
+        err = nc_def_var( ncid, (std::get<0>(tp)+"3d").data(), NC_FLOAT, 3,
+            &dim3d_ids[0], &vectorID3d);
         err = nc_put_att_text( ncid, vectorID, "long_name",
             std::get<1>(tp).size(), std::get<1>(tp).data());
+        err = nc_put_att_text( ncid, vectorID3d, "long_name",
+            std::get<1>(tp).size(), std::get<1>(tp).data());
+        std::string coordinates = "zc yc xc";
+        err = nc_put_att_text( ncid, vectorID3d, "coordinates", coordinates.size(), coordinates.data());
         err = nc_enddef( ncid);
         hvisual = dg::evaluate( std::get<2>(tp), grid2d);
-        err = nc_put_var_double( ncid, vectorID, hvisual.data());
+        dg::extend_line( grid2d.size(), grid3d.Nz(), hvisual, hvisual3d);
+        dg::assign( hvisual, fvisual);
+        dg::assign( hvisual3d, fvisual3d);
+        err = nc_put_var_float( ncid, vectorID, fvisual.data());
+        err = nc_put_var_float( ncid, vectorID3d, fvisual3d.data());
         err = nc_redef(ncid);
-
     }
+    std::cout << "WRTING 3D FIELDS ... \n";
     //compute & write 3d vectors
-    dg::HVec vecR = dg::evaluate( dg::geo::BFieldR(mag), grid3d);
-    dg::HVec vecZ = dg::evaluate( dg::geo::BFieldZ(mag), grid3d);
-    dg::HVec vecP = dg::evaluate( dg::geo::BFieldP(mag), grid3d);
-    std::string vec_long[3] = {"R-component of the magnetic field vector (3d version of BFieldR)",
-        "Z-component of the magnetic field vector (3d version of BFieldZ)",
-        "Contravariant Phi-component of the magnetic field vector (3d version of BFieldP)"};
-    int vecID[3];
-    err = nc_def_var( ncid, "BR", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[0]);
-    err = nc_def_var( ncid, "BZ", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[1]);
-    err = nc_def_var( ncid, "BP", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[2]);
-    for(int i=0; i<3; i++)
-    {
-        err = nc_put_att_text( ncid, vecID[i], "long_name", vec_long[i].size(), vec_long[i].data());
-        std::string coordinates = "zc yc xc";
-        err = nc_put_att_text( ncid, vecID[i], "coordinates", coordinates.size(), coordinates.data());
-    }
-    err = nc_enddef( ncid);
-    err = nc_put_var_double( ncid, vecID[0], vecR.data());
-    err = nc_put_var_double( ncid, vecID[1], vecZ.data());
-    err = nc_put_var_double( ncid, vecID[2], vecP.data());
-    err = nc_redef(ncid);
-    vecR = dg::evaluate( dg::cooX3d, grid3d);
-    vecZ = dg::evaluate( dg::cooZ3d, grid3d);
-    vecP = dg::evaluate( dg::cooY3d, grid3d);
-    for( unsigned i=0; i<vecR.size(); i++)
-    {
-        double xc = vecR[i]*sin(vecZ[i]);
-        double yc = vecR[i]*cos(vecZ[i]);
-        vecR[i] = xc;
-        vecZ[i] = yc;
-    }
-    vec_long[0] = "x-coordinate in Cartesian coordinate system",
-    vec_long[1] = "y-coordinate in Cartesian coordinate system",
-    vec_long[2] = "z-coordinate in Cartesian coordinate system";
-    err = nc_def_var( ncid, "xc", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[0]);
-    err = nc_def_var( ncid, "yc", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[1]);
-    err = nc_def_var( ncid, "zc", NC_DOUBLE, 3, &dim3d_ids[0], &vecID[2]);
-    for(int i=0; i<3; i++)
-        err = nc_put_att_text( ncid, vecID[i], "long_name", vec_long[i].size(), vec_long[i].data());
-    err = nc_enddef( ncid);
-    err = nc_put_var_double( ncid, vecID[0], vecR.data());
-    err = nc_put_var_double( ncid, vecID[1], vecZ.data());
-    err = nc_put_var_double( ncid, vecID[2], vecP.data());
-    err = nc_redef(ncid);
-    //////////////////////////////Finalize////////////////////////////////////
-    err = nc_close(ncid);
-    std::cout << "FILE CLOSED AND READY TO USE NOW!\n";
-
+    std::vector< std::tuple<std::string, std::string, std::function< double(double,double,double)> > > map3d{
+        {"BR", "R-component of the magnetic field vector (3d version of BFieldR)",
+            dg::geo::BFieldR(mag)},
+        {"BZ", "Z-component of the magnetic field vector (3d version of BFieldZ)",
+            dg::geo::BFieldZ(mag)},
+        {"BP", "Contravariant Phi-component of the magnetic field vector (3d version of BFieldP)",
+            dg::geo::BFieldP(mag)},
+        {"xc", "x-coordinate in Cartesian coordinate system", dg::cooRZP2X},
+        {"yc", "y-coordinate in Cartesian coordinate system", dg::cooRZP2Y},
+        {"zc", "z-coordinate in Cartesian coordinate system", dg::cooRZP2Z}
+    };
+    for( auto tp : map3d)
     {
-        std::cout << "test accuracy of psi (values must be close to 0!)\n";
-        const double R_H = gp.R_0-gp.triangularity*gp.a;
-        const double Z_H = gp.elongation*gp.a;
-        const double alpha_ = asin(gp.triangularity);
-        const double N1 = -(1.+alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.+alpha_);
-        const double N2 =  (1.-alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.-alpha_);
-        const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
-
-        if( gp.hasXpoint())
-            std::cout << "    Equilibrium with X-point!\n";
-        else
-            std::cout << "    NO X-point in flux function!\n";
-        std::cout << "psip( 1+e,0)           "<<mag.psip()(gp.R_0 + gp.a, 0.)<<"\n";
-        std::cout << "psip( 1-e,0)           "<<mag.psip()(gp.R_0 - gp.a, 0.)<<"\n";
-        std::cout << "psip( 1-de,ke)         "<<mag.psip()(R_H, Z_H)<<"\n";
-        if( !gp.hasXpoint())
-            std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H, Z_H)<<"\n";
-        else
+        int vectorID;
+        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_FLOAT, 3,
+            &dim3d_ids[0], &vectorID);
+        err = nc_put_att_text( ncid, vectorID, "long_name",
+            std::get<1>(tp).size(), std::get<1>(tp).data());
+        if( std::get<1>(tp) != "xc" && std::get<1>(tp) != "yc" &&std::get<1>(tp) != "zc")
         {
-            std::cout << "psip( 1-1.1de,-1.1ke)  "<<mag.psip()(R_X, Z_X)<<"\n";
-            std::cout << "psipZ( 1+e,0)          "<<mag.psipZ()(gp.R_0 + gp.a, 0.)<<"\n";
-            std::cout << "psipZ( 1-e,0)          "<<mag.psipZ()(gp.R_0 - gp.a, 0.)<<"\n";
-            std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H,Z_H)<<"\n";
-            std::cout << "psipR( 1-1.1de,-1.1ke) "<<mag.psipR()(R_X,Z_X)<<"\n";
-            std::cout << "psipZ( 1-1.1de,-1.1ke) "<<mag.psipZ()(R_X,Z_X)<<"\n";
+            std::string coordinates = "zc yc xc";
+            err = nc_put_att_text( ncid, vectorID, "coordinates", coordinates.size(), coordinates.data());
         }
-        std::cout << "psipZZ( 1+e,0)         "<<mag.psipZZ()(gp.R_0+gp.a,0.)+N1*mag.psipR()(gp.R_0+gp.a,0)<<"\n";
-        std::cout << "psipZZ( 1-e,0)         "<<mag.psipZZ()(gp.R_0-gp.a,0.)+N2*mag.psipR()(gp.R_0-gp.a,0)<<"\n";
-        std::cout << "psipRR( 1-de,ke)       "<<mag.psipRR()(R_H,Z_H)+N3*mag.psipZ()(R_H,Z_H)<<"\n";
-    }
-
-
-    std::cout << "Test accuracy of curvatures (values must be close to 0, but Test may be inaccurate for modified Psi_p)\n";
-    //Test NablaTimes B = B^2( curvK - curvB)
-    std::array<dg::HVec, 3> bhat, curvB, curvK;
-    dg::pushForward( bhat_.x(), bhat_.y(), bhat_.z(),
-            bhat[0], bhat[1], bhat[2], grid3d);
-    std::array<dg::HVec, 3> bhat_covariant(bhat);
-    dg::tensor::inv_multiply3d( grid3d.metric(), bhat[0], bhat[1], bhat[2],
-            bhat_covariant[0], bhat_covariant[1], bhat_covariant[2]);
-    dg::HVec normb( bhat[0]), one3d = dg::evaluate( dg::one, grid3d);
-    dg::blas1::pointwiseDot( 1., bhat[0], bhat_covariant[0],
-                             1., bhat[1], bhat_covariant[1],
-                             0., normb);
-    dg::blas1::pointwiseDot( 1., bhat[2], bhat_covariant[2],
-                             1., normb);
-    dg::blas1::axpby( 1., one3d, -1, normb);
-    double error = sqrt(dg::blas1::dot( normb, normb));
-    std::cout << "Error in norm b == 1 :  "<<error<<std::endl;
-
-    std::cout << "Push Forward curvatures ...\n";
-    dg::pushForward( curvB_.x(), curvB_.y(), curvB_.z(),
-            curvB[0], curvB[1], curvB[2], grid3d);
-    dg::pushForward( curvK_.x(), curvK_.y(), curvK_.z(),
-            curvK[0], curvK[1], curvK[2], grid3d);
-    std::cout << "Done!\n";
-    dg::blas1::axpby( 1., curvK, -1., curvB);
-    dg::HVec Bmodule = dg::pullback( dg::geo::Bmodule(mag), grid3d);
-    dg::blas1::pointwiseDot( Bmodule, Bmodule, Bmodule);
-    for( int i=0; i<3; i++)
-        dg::blas1::pointwiseDot( Bmodule, curvB[i], curvB[i]);
-    dg::HVec R = dg::pullback( dg::cooX3d, grid3d);
-    dg::HVec IR =  dg::pullback( mag.ipolR(), grid3d);
-    dg::blas1::pointwiseDivide( gp.R_0, IR, R, 0., IR);
-    dg::HVec IZ =  dg::pullback( mag.ipolZ(), grid3d);
-    dg::blas1::pointwiseDivide( gp.R_0, IZ, R, 0., IZ);
-    dg::HVec IP =  dg::pullback( JPhi( gp), grid3d);
-    dg::blas1::pointwiseDivide( gp.R_0, IP, R, 0., IP);
-    dg::blas1::axpby( 1., IZ, -1., curvB[0]);
-    dg::blas1::axpby( 1., IR, +1., curvB[1]);
-    dg::blas1::axpby( 1., IP, -1., curvB[2]);
-    for( int i=0; i<3; i++)
-    {
-        error = sqrt(dg::blas1::dot( curvB[i], curvB[i] ) );
-        std::cout << "Error in curv "<<i<<" :   "<<error<<"\n";
+        err = nc_enddef( ncid);
+        hvisual3d = dg::evaluate( std::get<2>(tp), grid3d);
+        dg::assign( hvisual3d, fvisual3d);
+        err = nc_put_var_float( ncid, vectorID, fvisual3d.data());
+        err = nc_redef(ncid);
     }
-
+    //////////////////////////////Finalize////////////////////////////////////
+    err = nc_close(ncid);
     return 0;
 }
diff --git a/inc/geometries/magnetic_field_t.cu b/inc/geometries/magnetic_field_t.cu
new file mode 100644
index 000000000..2b229c663
--- /dev/null
+++ b/inc/geometries/magnetic_field_t.cu
@@ -0,0 +1,133 @@
+#include <iostream>
+#include <iomanip>
+
+#include "json/json.h"
+
+#include "dg/algorithm.h"
+#include "dg/file/nc_utilities.h"
+
+#include "solovev.h"
+//#include "taylor.h"
+#include "magnetic_field.h"
+
+struct JPhi
+{
+    JPhi( dg::geo::solovev::Parameters gp): R_0(gp.R_0), A(gp.A){}
+    double operator()(double R, double Z, double phi)const
+    {
+        return ((A-1.)*R - A*R_0*R_0/R)/R_0/R_0/R_0;
+    }
+    private:
+    double R_0, A;
+};
+
+int main( int argc, char* argv[])
+{
+    Json::Value js;
+    if( argc==1)
+    {
+        std::ifstream is("geometry_params_Xpoint.js");
+        is >> js;
+    }
+    else
+    {
+        std::ifstream is(argv[1]);
+        is >> js;
+    }
+
+    dg::geo::solovev::Parameters gp(js);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
+    double Z_X = -1.1*gp.elongation*gp.a;
+    if( gp.hasXpoint())
+    {
+        dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
+        std::cout <<  "X-point found at "<<R_X << " "<<Z_X<<" with Psip "<<mag.psip()(R_X, Z_X)<<"\n";
+        std::cout <<  "     R - Factor "<<(gp.R_0-R_X)/gp.triangularity/gp.a << "   Z - factor "<<-(Z_X/gp.elongation/gp.a)<<std::endl;
+
+    }
+
+    std::cout << "test accuracy of psi (values must be close to 0!)\n";
+    const double R_H = gp.R_0-gp.triangularity*gp.a;
+    const double Z_H = gp.elongation*gp.a;
+    const double alpha_ = asin(gp.triangularity);
+    const double N1 = -(1.+alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.+alpha_);
+    const double N2 =  (1.-alpha_)/(gp.a*gp.elongation*gp.elongation)*(1.-alpha_);
+    const double N3 = -gp.elongation/(gp.a*cos(alpha_)*cos(alpha_));
+
+    if( gp.hasXpoint())
+        std::cout << "    Equilibrium with X-point!\n";
+    else
+        std::cout << "    NO X-point in flux function!\n";
+    std::cout << "psip( 1+e,0)           "<<mag.psip()(gp.R_0 + gp.a, 0.)<<"\n";
+    std::cout << "psip( 1-e,0)           "<<mag.psip()(gp.R_0 - gp.a, 0.)<<"\n";
+    std::cout << "psip( 1-de,ke)         "<<mag.psip()(R_H, Z_H)<<"\n";
+    if( !gp.hasXpoint())
+        std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H, Z_H)<<"\n";
+    else
+    {
+        std::cout << "psip( 1-1.1de,-1.1ke)  "<<mag.psip()(R_X, Z_X)<<"\n";
+        std::cout << "psipZ( 1+e,0)          "<<mag.psipZ()(gp.R_0 + gp.a, 0.)<<"\n";
+        std::cout << "psipZ( 1-e,0)          "<<mag.psipZ()(gp.R_0 - gp.a, 0.)<<"\n";
+        std::cout << "psipR( 1-de,ke)        "<<mag.psipR()(R_H,Z_H)<<"\n";
+        std::cout << "psipR( 1-1.1de,-1.1ke) "<<mag.psipR()(R_X,Z_X)<<"\n";
+        std::cout << "psipZ( 1-1.1de,-1.1ke) "<<mag.psipZ()(R_X,Z_X)<<"\n";
+    }
+    std::cout << "psipZZ( 1+e,0)         "<<mag.psipZZ()(gp.R_0+gp.a,0.)+N1*mag.psipR()(gp.R_0+gp.a,0)<<"\n";
+    std::cout << "psipZZ( 1-e,0)         "<<mag.psipZZ()(gp.R_0-gp.a,0.)+N2*mag.psipR()(gp.R_0-gp.a,0)<<"\n";
+    std::cout << "psipRR( 1-de,ke)       "<<mag.psipRR()(R_H,Z_H)+N3*mag.psipZ()(R_H,Z_H)<<"\n";
+
+    std::cout << "Test accuracy of curvatures (values must be close to 0)\n";
+    dg::geo::CylindricalVectorLvl0 bhat_ = dg::geo::createBHat( mag);
+    dg::geo::CylindricalVectorLvl0 curvB_ = dg::geo::createTrueCurvatureNablaB( mag);
+    dg::geo::CylindricalVectorLvl0 curvK_ = dg::geo::createTrueCurvatureKappa( mag);
+    //Test NablaTimes B = B^2( curvK - curvB)
+    std::array<dg::HVec, 3> bhat, curvB, curvK;
+    double Rmin=gp.R_0-gp.a;
+    double Zmin=-gp.a*gp.elongation;
+    double Rmax=gp.R_0+gp.a;
+    double Zmax=gp.a*gp.elongation;
+    dg::CylindricalGrid3d grid3d(Rmin,Rmax,Zmin,Zmax, 0, 2.*M_PI, 3,100,100,32);
+    dg::pushForward( bhat_.x(), bhat_.y(), bhat_.z(),
+            bhat[0], bhat[1], bhat[2], grid3d);
+    std::array<dg::HVec, 3> bhat_covariant(bhat);
+    dg::tensor::inv_multiply3d( grid3d.metric(), bhat[0], bhat[1], bhat[2],
+            bhat_covariant[0], bhat_covariant[1], bhat_covariant[2]);
+    dg::HVec normb( bhat[0]), one3d = dg::evaluate( dg::one, grid3d);
+    dg::blas1::pointwiseDot( 1., bhat[0], bhat_covariant[0],
+                             1., bhat[1], bhat_covariant[1],
+                             0., normb);
+    dg::blas1::pointwiseDot( 1., bhat[2], bhat_covariant[2],
+                             1., normb);
+    dg::blas1::axpby( 1., one3d, -1, normb);
+    double error = sqrt(dg::blas1::dot( normb, normb));
+    std::cout << "Error in norm b == 1 :  "<<error<<std::endl;
+
+    std::cout << "Push Forward curvatures ...\n";
+    dg::pushForward( curvB_.x(), curvB_.y(), curvB_.z(),
+            curvB[0], curvB[1], curvB[2], grid3d);
+    dg::pushForward( curvK_.x(), curvK_.y(), curvK_.z(),
+            curvK[0], curvK[1], curvK[2], grid3d);
+    std::cout << "Done!\n";
+    dg::blas1::axpby( 1., curvK, -1., curvB);
+    dg::HVec Bmodule = dg::pullback( dg::geo::Bmodule(mag), grid3d);
+    dg::blas1::pointwiseDot( Bmodule, Bmodule, Bmodule);
+    for( int i=0; i<3; i++)
+        dg::blas1::pointwiseDot( Bmodule, curvB[i], curvB[i]);
+    dg::HVec R = dg::pullback( dg::cooX3d, grid3d);
+    dg::HVec IR =  dg::pullback( mag.ipolR(), grid3d);
+    dg::blas1::pointwiseDivide( gp.R_0, IR, R, 0., IR);
+    dg::HVec IZ =  dg::pullback( mag.ipolZ(), grid3d);
+    dg::blas1::pointwiseDivide( gp.R_0, IZ, R, 0., IZ);
+    dg::HVec IP =  dg::pullback( JPhi( gp), grid3d);
+    dg::blas1::pointwiseDivide( gp.R_0, IP, R, 0., IP);
+    dg::blas1::axpby( 1., IZ, -1., curvB[0]);
+    dg::blas1::axpby( 1., IR, +1., curvB[1]);
+    dg::blas1::axpby( 1., IP, -1., curvB[2]);
+    for( int i=0; i<3; i++)
+    {
+        error = sqrt(dg::blas1::dot( curvB[i], curvB[i] ) );
+        std::cout << "Error in curv "<<i<<" :   "<<error<<"\n";
+    }
+}
+
diff --git a/inc/geometries/utilitiesX.h b/inc/geometries/utilitiesX.h
index 05887af77..a3ddf8827 100644
--- a/inc/geometries/utilitiesX.h
+++ b/inc/geometries/utilitiesX.h
@@ -8,11 +8,6 @@ namespace dg
 {
 namespace geo
 {
-///@copydoc findOpoint()
-static inline void findXpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
-{
-    findOpoint( psi, R_X, Z_X);
-}
 
 ///@cond
 namespace detail{
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index e84258075..f990409a4 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -15,16 +15,6 @@ namespace feltor
 {
 
 namespace routines{
-struct Cylindrical2Cartesian{
-    void operator()( double R, double Z, double P, double& x, double& y, double& z)
-    {
-        //inplace possible
-        double xx = R*sin(P);
-        double yy = R*cos(P);
-        x = xx, y = yy, z = Z;
-    }
-};
-
 struct ComputePerpDrifts{
     ComputePerpDrifts( double mu, double tau):
         m_mu(mu), m_tau(tau){}
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 337385fcb..7ef043614 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -484,7 +484,7 @@ Then we can use the volume average to write
 \label{eq:fsa_balance}
 \end{align}
 where again $v=v(\psi_p)$ is the volume flux label.
-The {\bf total flux} of a given flux density $\vec j_X$ though the
+The {\bf total flux} of a given flux density $\vec j_X$ through the
 flux surface $\psi_p = \psi_{p0}$ is given by
 \begin{align}
 \RA{\vec j_X\cn v} &:= J_X=\oint_{\psi_p=\psi_{p0}} \vec j_X\cdot \vec{\dA} =
@@ -1410,8 +1410,8 @@ Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}
 \qquad alpha  & float & 0.2 & - & Transition width $\alpha_p$ in the Heaviside
 at the separatrix
 \\
-source & dict & & & Density source  \\
-\qquad rate & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}
+source & dict & & & Density source, cf. the output \texttt{sne\_ifs} in \texttt{feltordiag} (or \texttt{geometry\_diag}) to see how much mass the source with the parameters below generates.  \\
+\qquad rate & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}.
 \\
 \qquad type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
 "influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx},
@@ -1597,8 +1597,7 @@ Compilation\\
 \texttt{make geometry\_diag device=omp} \\
 Usage \\
 \texttt{./geometry\_diag input.json geometry.json diag\_geometry.nc} \\
-The program performs a series of tests on the correct implementation of geoemetric
-quantities and outputs a host of static 1d, 2d and 3d geometric quantities.
+The program outputs a host of static 1d, 2d and 3d geometric quantities.
 The output file is for example useful in connection with the ``Group Datasets'' filter in paraview, which merges Datasets from different files into one using shallow copy only.
 
 %..................................................................
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 73f5a1f88..18b267174 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -215,39 +215,26 @@ std::vector<Record_static> diagnostics3d_static_list = {
     },
     { "xc", "x-coordinate in Cartesian coordinate system",
         []( HVec& result, Variables& v, Geometry& grid ){
-            HVec xc = dg::evaluate( dg::cooX3d, grid);
-            HVec yc = dg::evaluate( dg::cooY3d, grid);
-            HVec zc = dg::evaluate( dg::cooZ3d, grid);
-            dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
-            result = xc;
+            return dg::evaluate( dg::cooRZP2X, grid);
         }
     },
     { "yc", "y-coordinate in Cartesian coordinate system",
         []( HVec& result, Variables& v, Geometry& grid ){
-            HVec xc = dg::evaluate( dg::cooX3d, grid);
-            HVec yc = dg::evaluate( dg::cooY3d, grid);
-            HVec zc = dg::evaluate( dg::cooZ3d, grid);
-            dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
-            result = yc;
+            return dg::evaluate( dg::cooRZP2Y, grid);
         }
     },
     { "zc", "z-coordinate in Cartesian coordinate system",
         []( HVec& result, Variables& v, Geometry& grid ){
-            HVec xc = dg::evaluate( dg::cooX3d, grid);
-            HVec yc = dg::evaluate( dg::cooY3d, grid);
-            HVec zc = dg::evaluate( dg::cooZ3d, grid);
-            dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
-            result = zc;
+            return dg::evaluate( dg::cooRZP2Z, grid);
         }
     },
 };
 
 std::array<std::tuple<std::string, std::string, HVec>, 3> generate_cyl2cart( Geometry& grid)
 {
-    HVec xc = dg::evaluate( dg::cooX3d, grid);
-    HVec yc = dg::evaluate( dg::cooY3d, grid);
-    HVec zc = dg::evaluate( dg::cooZ3d, grid);
-    dg::blas1::subroutine( feltor::routines::Cylindrical2Cartesian(), xc, yc, zc, xc, yc, zc);
+    HVec xc = dg::evaluate( dg::cooRZP2X, grid);
+    HVec yc = dg::evaluate( dg::cooRZP2Y, grid);
+    HVec zc = dg::evaluate( dg::cooRZP2Z, grid);
     std::array<std::tuple<std::string, std::string, HVec>, 3> list = {{
         { "xc", "x-coordinate in Cartesian coordinate system", xc },
         { "yc", "y-coordinate in Cartesian coordinate system", yc },
-- 
GitLab


From 5baac32c01db20d4ef776cbf8a88d0e9133a3be4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 11 May 2020 18:19:08 +0200
Subject: [PATCH 213/540] Overall loop in geometry_diag

- create ScalarProd and SquareNorm cylindrical Functors
- loop over fsa in geometry_diag
- move Delta FSA into new file average_t (to separate test from output)
- move X-point interpolation test to separatrix_orthogonal_t
---
 inc/geometries/average.h                  |  28 +-
 inc/geometries/average_t.cu               | 168 +++++++++++
 inc/geometries/fluxfunctions.h            |  33 ++
 inc/geometries/geometry_diag.cu           | 350 +++++++---------------
 inc/geometries/magnetic_field.h           |  24 +-
 inc/geometries/separatrix_orthogonal_t.cu |  16 +-
 src/feltor/feltor.tex                     |   2 +-
 7 files changed, 351 insertions(+), 270 deletions(-)
 create mode 100644 inc/geometries/average_t.cu

diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index bd4a08001..cf4bd10c9 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -29,16 +29,16 @@ struct FluxSurfaceIntegral
      * @brief Construct from a grid and a magnetic field
      * f and g are default initialized to 1
      * @param g2d grid
-     * @param c contains psip, psipR and psipZ
+     * @param mag contains psip, psipR and psipZ
      * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c 0.5*h*GradPsi*width_factor)
      */
-    FluxSurfaceIntegral(const dg::Grid2d& g2d, const TokamakMagneticField& c, double width_factor = 1.):
+    FluxSurfaceIntegral(const dg::Grid2d& g2d, const TokamakMagneticField& mag, double width_factor = 1.):
             m_f(dg::evaluate(dg::one, g2d)), m_g(m_f), m_delta(m_f),
-            m_psi( dg::evaluate( c.psip(), g2d)),
+            m_psi( dg::evaluate( mag.psip(), g2d)),
             m_w2d ( dg::create::weights( g2d))
     {
-        thrust::host_vector<double> psipR  = dg::evaluate( c.psipR(), g2d);
-        thrust::host_vector<double> psipZ  = dg::evaluate( c.psipZ(), g2d);
+        thrust::host_vector<double> psipR  = dg::evaluate( mag.psipR(), g2d);
+        thrust::host_vector<double> psipZ  = dg::evaluate( mag.psipZ(), g2d);
         double psipRmax = dg::blas1::reduce( psipR, 0., dg::AbsMax<double>()  );
         double psipZmax = dg::blas1::reduce( psipZ, 0., dg::AbsMax<double>()  );
         double deltapsi = 0.5*(psipZmax*g2d.hy() +psipRmax*g2d.hx())/g2d.n();
@@ -96,12 +96,12 @@ struct FluxVolumeIntegral
      * @brief Construct from a grid and a magnetic field
      * f and g are default initialized to 1
      * @param g2d grid
-     * @param c contains psip
+     * @param mag contains psip
      */
     template<class Geometry2d>
-    FluxVolumeIntegral(const Geometry2d& g2d, const TokamakMagneticField& c):
+    FluxVolumeIntegral(const Geometry2d& g2d, const TokamakMagneticField& mag):
         m_f(dg::evaluate(dg::one, g2d)), m_g(m_f), m_heavi(m_f),
-        m_psi( dg::pullback( c.psip(), g2d)),
+        m_psi( dg::pullback( mag.psip(), g2d)),
         m_w2d ( dg::create::volume( g2d))
     {
     }
@@ -213,13 +213,13 @@ struct SafetyFactorAverage
      /**
      * @brief Construct from a field and a grid
      * @param g2d 2d grid
-     * @param c contains psip, psipR and psipZ and Ipol
+     * @param mag contains psip, psipR and psipZ and Ipol
      * @param width_factor can be used to tune the width of the numerical delta function (\c width = \c h*GradPsi*width_factor)
      */
-    SafetyFactorAverage(const dg::Grid2d& g2d, const TokamakMagneticField& c, double width_factor = 1.) :
-        m_fsi( g2d, c, width_factor)
+    SafetyFactorAverage(const dg::Grid2d& g2d, const TokamakMagneticField& mag, double width_factor = 1.) :
+        m_fsi( g2d, mag, width_factor)
     {
-        thrust::host_vector<double> alpha = dg::evaluate( c.ipol(), g2d);
+        thrust::host_vector<double> alpha = dg::evaluate( mag.ipol(), g2d);
         thrust::host_vector<double> R = dg::evaluate( dg::cooX2d, g2d);
         dg::blas1::pointwiseDivide( alpha, R, alpha);
         m_fsi.set_left( alpha);
@@ -254,8 +254,8 @@ struct SafetyFactorAverage
  */
 struct SafetyFactor
 {
-    SafetyFactor( const TokamakMagneticField& c):
-        m_fpsi( c.get_psip(), c.get_ipol(), c.R0(), 0.,false){}
+    SafetyFactor( const TokamakMagneticField& mag):
+        m_fpsi( mag.get_psip(), mag.get_ipol(), mag.R0(), 0.,false){}
 
     /**
      * @brief Calculate q(psip0)
diff --git a/inc/geometries/average_t.cu b/inc/geometries/average_t.cu
new file mode 100644
index 000000000..20b65e968
--- /dev/null
+++ b/inc/geometries/average_t.cu
@@ -0,0 +1,168 @@
+#include <iostream>
+#include <iomanip>
+
+#include "json/json.h"
+
+#include "dg/algorithm.h"
+#include "dg/file/nc_utilities.h"
+
+#include "solovev.h"
+#include "average.h"
+//#include "taylor.h"
+#include "magnetic_field.h"
+int main( int argc, char* argv[])
+{
+    Json::Value js;
+    if( argc==1)
+    {
+        std::ifstream is("geometry_params_Xpoint.js");
+        is >> js;
+    }
+    else
+    {
+        std::ifstream is(argv[1]);
+        is >> js;
+    }
+
+    const dg::geo::solovev::Parameters gp(js);
+    const dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    double R_O = gp.R_0, Z_O = 0.;
+    if( !gp.isToroidal() )
+        dg::geo::findOpoint( mag.get_psip(), R_O, Z_O);
+    const double psipmin = mag.psip()(R_O, Z_O);
+    double Rmin=gp.R_0-gp.a;
+    double Zmin=-gp.a*gp.elongation;
+    double Rmax=gp.R_0+gp.a;
+    double Zmax=gp.a*gp.elongation;
+    dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, 3,200,200);
+    std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
+    dg::HVec psipog2d = dg::evaluate( mag.psip(), grid2d);
+    // This is a numerical test because the fsa of the curvature operators exactly vanishes (at least for I = cte and almost for everything else)!!
+    dg::geo::CylindricalVectorLvl0 gradPsipF = dg::geo::createGradPsip(mag);
+    dg::HVec curvNablaBGradPsip = dg::evaluate( dg::geo::ScalarProduct(
+                dg::geo::createTrueCurvatureNablaB(mag),
+                gradPsipF), grid2d);
+    dg::HVec curvKappaKGradPsip = dg::evaluate( dg::geo::ScalarProduct(
+                dg::geo::createTrueCurvatureKappa(mag),
+                gradPsipF), grid2d);
+    dg::HVec gradPsip = dg::evaluate( dg::geo::SquareNorm(gradPsipF, gradPsipF), grid2d);
+    ///////////////////Compute flux average////////////////////
+    unsigned npsi = 3, Npsi = 64;//set number of psivalues (NPsi % 8 == 0)
+    dg::Grid1d grid1d( 0,1,npsi,Npsi, dg::NEU);
+    double deltapsi = 0.1;
+    double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
+    if( !gp.isToroidal())
+    {
+        std::cout << "Compute flux averages\n";
+        dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
+        if( gp.hasXpoint() )
+        {
+            double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
+            double Z_X = -1.1*gp.elongation*gp.a;
+            dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
+            dg::blas1::pointwiseDot( xpoint_weights,
+                    dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
+        }
+        dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, mag, psipog2d, xpoint_weights);
+        grid1d = dg::Grid1d (psipmin<psipmax ? psipmin : psipmax, psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::NEU);
+        map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
+            "Flux surface average of psi with delta function");
+        dg::HVec qprofile;
+        if( gp.equilibrium == "solovev")
+        {
+            dg::geo::SafetyFactor qprof( mag);
+            qprofile = dg::evaluate( qprof, grid1d);
+            map1d.emplace_back("q-profile", qprofile,
+                "q-profile (Safety factor) using direct integration");
+            dg::HVec psit = dg::integrate( qprofile, grid1d);
+            map1d.emplace_back("psit1d", psit,
+                "Toroidal flux label psi_t integrated  on grid1d using direct q");
+            //we need to avoid integrating >=0
+            dg::Grid1d g1d_fine(psipmin<0. ? psipmin : 0., psipmin<0. ? 0. : psipmin, npsi ,Npsi,dg::NEU);
+            qprofile = dg::evaluate( qprof, g1d_fine);
+            dg::HVec w1d = dg::create::weights( g1d_fine);
+            double psit_tot = dg::blas1::dot( w1d, qprofile);
+            //std::cout << "psit tot "<<psit_tot<<"\n";
+            dg::blas1::scal ( psit, 1./psit_tot);
+            dg::blas1::transform( psit, psit, dg::SQRT<double>());
+            map1d.emplace_back("rho_t", psit,
+                "Toroidal flux label rho_t = sqrt( psit/psit_tot) evaluated on grid1d");
+        }
+        dg::geo::SafetyFactorAverage qprof( grid2d, mag);
+        qprofile = dg::evaluate( qprof, grid1d);
+        map1d.emplace_back("q-profile_fsa", qprofile,
+            "q-profile (Safety factor) using average integration");
+        dg::HVec psit = dg::integrate( qprofile, grid1d);
+        map1d.emplace_back("psit1d_fsa", psit,
+            "Toroidal flux label psi_t integrated on grid1d using average q");
+        map1d.emplace_back("psip1d",    dg::evaluate( dg::cooX1d, grid1d),
+            "Poloidal flux label psi_p evaluated on grid1d");
+        dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
+        dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
+        map1d.emplace_back("rho", rho,
+            "Alternative flux label rho = -psi/psimin + 1");
+        dg::blas1::transform( rho, rho, dg::SQRT<double>());
+        map1d.emplace_back("rho_p", rho,
+            "Alternative flux label rho_p = Sqrt[-psi/psimin + 1]");
+        fsa.set_container( (dg::DVec)curvNablaBGradPsip);
+        map1d.emplace_back("curvNablaB_fsa",   dg::evaluate( fsa,      grid1d),
+            "Flux surface average of true Nabla B curvature Dot Grad Psip with delta function");
+        fsa.set_container( (dg::DVec)curvKappaKGradPsip);
+        map1d.emplace_back("curvKappaK_fsa",   dg::evaluate( fsa,      grid1d),
+            "Flux surface average of true Kappa curvature Dot Grad Psip with delta function");
+        fsa.set_container( (dg::DVec)gradPsip);
+        map1d.emplace_back("gradPsip_fsa",   dg::evaluate( fsa,      grid1d),
+            "Flux surface average of |Grad Psip| with delta function");
+
+        //other flux labels
+        dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, mag);
+        fsi.set_right( xpoint_weights);
+        deltapsi = fsi.get_deltapsi();
+
+        dg::HVec areaT_psip = dg::evaluate( fsi, grid1d);
+        dg::HVec w1d = dg::create::weights( grid1d);
+        double volumeCoarea = 2.*M_PI*dg::blas1::dot( areaT_psip, w1d);
+
+        //area
+        fsi.set_left( gradPsip);
+        dg::HVec psi_area = dg::evaluate( fsi, grid1d);
+        dg::blas1::scal(psi_area, 2.*M_PI);
+        map1d.emplace_back( "psi_area", psi_area,
+            "Flux area with delta function");
+
+        dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, mag);
+        std::cout << "Delta Rho for Flux surface integrals = "<<-deltapsi/psipmin<<"\n";
+
+        fvi.set_right( xpoint_weights);
+        dg::HVec psi_vol = dg::evaluate( fvi, grid1d);
+        dg::blas1::scal(psi_vol, 2.*M_PI);
+        map1d.emplace_back( "psi_vol", psi_vol,
+            "Flux volume with delta function");
+        double volumeFVI = 2.*M_PI*fvi(psipmax);
+        double volumeSep = 2.*M_PI*fvi(0.);
+        std::cout << "volume enclosed by separatrix: "<<volumeSep<<"\n";
+        std::cout << "volume test with coarea formula: "<<volumeCoarea<<" "<<volumeFVI
+                  <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
+    }
+    ///////////Write file
+    int ncid;
+    file::NC_Error_Handle err;
+    err = nc_create( "average.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
+    int dim1d;
+    err = file::define_dimension(  ncid, &dim1d, grid1d, "psi");
+    for(auto tp : map1d)
+    {
+        int vid;
+        err = nc_def_var( ncid, std::get<0>(tp).data(), NC_DOUBLE, 1,
+            &dim1d, &vid);
+        err = nc_put_att_text( ncid, vid, "long_name",
+            std::get<2>(tp).size(), std::get<2>(tp).data());
+        err = nc_enddef( ncid);
+        err = nc_put_var_double( ncid, vid, std::get<1>(tp).data() );
+        err = nc_redef(ncid);
+    }
+    err = nc_close( ncid);
+    std::cout << "FILE average.nc CLOSED AND READY TO USE NOW!\n" <<std::endl;
+
+    return 0;
+}
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 695d059fb..6ad95edc2 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -371,6 +371,39 @@ struct CylindricalVectorLvl0
     std::array<CylindricalFunctor,3> p_;
 };
 
+/**
+ * @brief Return scalar product of two vector fields \f$ v_0w_0 + v_1w_1 + v_2w_2\f$
+ */
+struct ScalarProduct : public aCylindricalFunctor<ScalarProduct>
+{
+    ScalarProduct( CylindricalVectorLvl0 v, CylindricalVectorLvl0 w) : m_v(v), m_w(w){}
+    double do_compute( double R, double Z) const
+    {
+        return m_v.x()(R,Z)*m_w.x()(R,Z)
+             + m_v.y()(R,Z)*m_w.y()(R,Z)
+             + m_v.z()(R,Z)*m_w.z()(R,Z);
+    }
+  private:
+    CylindricalVectorLvl0 m_v, m_w;
+};
+
+/**
+ * @brief Return norm of scalar product of two vector fields \f$ \sqrt{v_0w_0 + v_1w_1 + v_2w_2}\f$
+ *
+ * short for \c dg::compose( sqrt, ScalarProduct( v,w))
+ */
+struct SquareNorm : public aCylindricalFunctor<SquareNorm>
+{
+    SquareNorm( CylindricalVectorLvl0 v, CylindricalVectorLvl0 w) : m_s(v, w){}
+    double do_compute( double R, double Z) const
+    {
+        return sqrt(m_s(R,Z));
+    }
+  private:
+    ScalarProduct m_s;
+};
+
+
 /*!@brief \f[ \chi^{ij} = b^ib^j\f]
  *
  * Creates the two times contravariant tensor that,
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 80d5aee95..0579d575f 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -22,7 +22,7 @@
 
 
 // - write magnetic functions into file
-// - test performance and accuracy of Flux surface averages and integrals
+// - compute Flux - surface averages and write into file
 struct Parameters
 {
     unsigned n, Nx, Ny, Nz, Npsi;
@@ -80,7 +80,6 @@ struct Parameters
     }
 };
 
-
 int main( int argc, char* argv[])
 {
     if( !(argc == 4 || argc == 3))
@@ -171,23 +170,79 @@ int main( int argc, char* argv[])
     ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
     unsigned npsi = 3, Npsi = p.Npsi;//set number of psivalues (NPsi % 8 == 0)
     double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
-    double volumeXGrid;
-    /// -------  Elements for fsa of curvature operators ----------------
-    // This is a numerical test because the fsa of the curvature operators exactly vanishes (at least for I = cte and almost for everything else)!!
-    dg::geo::CylindricalVectorLvl0 bhat_ = dg::geo::createBHat( mag);
-    dg::geo::CylindricalVectorLvl0 curvB_ = dg::geo::createTrueCurvatureNablaB( mag);
-    dg::geo::CylindricalVectorLvl0 curvK_ = dg::geo::createTrueCurvatureKappa( mag);
-    dg::HVec curvNablaBR = dg::evaluate( curvB_.x(), grid2d);
-    dg::HVec curvNablaBZ = dg::evaluate( curvB_.y(), grid2d);
-    dg::HVec curvKappaKR = dg::evaluate( curvK_.x(), grid2d);
-    dg::HVec curvKappaKZ = dg::evaluate( curvK_.y(), grid2d);
-    dg::HVec gradPsipR = dg::evaluate( mag.psipR(), grid2d);
-    dg::HVec gradPsipZ = dg::evaluate( mag.psipZ(), grid2d);
-    dg::HVec curvNablaBGradPsip(curvNablaBR), curvKappaKGradPsip( curvNablaBR), gradPsip(curvNablaBR);
-    dg::blas1::pointwiseDot( 1., curvNablaBR, gradPsipR, 1., curvNablaBZ, gradPsipZ, 0., curvNablaBGradPsip);
-    dg::blas1::pointwiseDot( 1., curvKappaKR, gradPsipR, 1., curvKappaKZ, gradPsipZ, 0., curvKappaKGradPsip);
-    dg::blas1::pointwiseDot( 1., gradPsipR, gradPsipR, 1., gradPsipZ, gradPsipZ, 0., gradPsip);
-    dg::blas1::transform( gradPsip, gradPsip, dg::SQRT<double>());
+    //Generate list of functions to evaluate
+    std::vector< std::tuple<std::string, std::string, dg::geo::CylindricalFunctor >> map{
+        {"Psip", "Flux function", mag.psip()},
+        {"PsipR", "Flux function derivative in R", mag.psipR()},
+        {"PsipZ", "Flux function derivative in Z", mag.psipZ()},
+        {"PsipRR", "Flux function derivative in RR", mag.psipRR()},
+        {"PsipRZ", "Flux function derivative in RZ", mag.psipRZ()},
+        {"PsipZZ", "Flux function derivative in ZZ", mag.psipZZ()},
+        {"Ipol", "Poloidal current", mag.ipol()},
+        {"IpolR", "Poloidal current derivative in R", mag.ipolR()},
+        {"IpolZ", "Poloidal current derivative in Z", mag.ipolZ()},
+        {"Rho_p", "Normalized Poloidal flux label", dg::geo::RhoP(mag)},
+        {"Bmodule", "Magnetic field strength", dg::geo::Bmodule(mag)},
+        {"InvB", "Inverse of Bmodule", dg::geo::InvB(mag)},
+        {"LnB", "Natural logarithm of Bmodule", dg::geo::LnB(mag)},
+        {"GradLnB", "The parallel derivative of LnB", dg::geo::GradLnB(mag)},
+        {"Divb", "The divergence of the magnetic unit vector", dg::geo::Divb(mag)},
+        {"B_R", "Derivative of Bmodule in R", dg::geo::BR(mag)},
+        {"B_Z", "Derivative of Bmodule in Z", dg::geo::BZ(mag)},
+        {"CurvatureNablaBR",  "R-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBR(mag,+1)},
+        {"CurvatureNablaBZ",  "Z-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBZ(mag,+1)},
+        {"CurvatureKappaR",   "R-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaR(mag,+1)},
+        {"CurvatureKappaZ",   "Z-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaZ(mag,+1)},
+        {"DivCurvatureKappa", "Divergence of the (toroidal) Kappa B curvature vector", dg::geo::DivCurvatureKappa(mag,+1)},
+        {"DivCurvatureNablaB","Divergence of the (toroidal) Nabla B curvature vector", dg::geo::DivCurvatureNablaB(mag,+1)},
+        {"TrueCurvatureNablaBR", "R-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBR(mag)},
+        {"TrueCurvatureNablaBZ", "Z-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBZ(mag)},
+        {"TrueCurvatureNablaBP", "Contravariant Phi-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBP(mag)},
+        {"TrueCurvatureKappaR", "R-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaR(mag)},
+        {"TrueCurvatureKappaZ", "Z-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaZ(mag)},
+        {"TrueCurvatureKappaP", "Contravariant Phi-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaP(mag)},
+        {"TrueDivCurvatureKappa", "Divergence of the (true) Kappa B curvature vector", dg::geo::TrueDivCurvatureKappa(mag)},
+        {"TrueDivCurvatureNablaB","Divergence of the (true) Nabla B curvature vector",  dg::geo::TrueDivCurvatureNablaB(mag)},
+        {"BFieldR", "R-component of the magnetic field vector", dg::geo::BFieldR(mag)},
+        {"BFieldZ", "Z-component of the magnetic field vector", dg::geo::BFieldZ(mag)},
+        {"BFieldP", "Contravariant Phi-component of the magnetic field vector", dg::geo::BFieldP(mag)},
+        {"BHatR", "R-component of the magnetic field unit vector", dg::geo::BHatR(mag)},
+        {"BHatZ", "Z-component of the magnetic field unit vector", dg::geo::BHatZ(mag)},
+        {"BHatP", "Contravariant Phi-component of the magnetic field unit vector", dg::geo::BHatP(mag)},
+        {"GradBHatR", "Parallel derivative of BHatR", dg::geo::BHatR(mag)},
+        {"GradBHatZ", "Parallel derivative of BHatZ", dg::geo::BHatZ(mag)},
+        {"GradBHatP", "Parallel derivative of BHatP", dg::geo::BHatP(mag)},
+        {"NormGradPsip", "Norm of gradient of Psip", dg::geo::SquareNorm( dg::geo::createGradPsip(mag), dg::geo::createGradPsip(mag))},
+        {"CurvatureNablaBGradPsip", "(Toroidal) Nabla B curvature dot the gradient of Psip", dg::geo::ScalarProduct( dg::geo::createCurvatureNablaB(mag, +1), dg::geo::createGradPsip(mag))},
+        {"CurvatureKappaGradPsip", "(Toroidal) Kappa curvature dot the gradient of Psip", dg::geo::ScalarProduct( dg::geo::createCurvatureKappa(mag, +1), dg::geo::createGradPsip(mag))},
+        {"TrueCurvatureNablaBGradPsip", "True Nabla B curvature dot the gradient of Psip", dg::geo::ScalarProduct( dg::geo::createTrueCurvatureNablaB(mag), dg::geo::createGradPsip(mag))},
+        {"TrueCurvatureKappaGradPsip", "True Kappa curvature dot the gradient of Psip", dg::geo::ScalarProduct( dg::geo::createTrueCurvatureKappa(mag), dg::geo::createGradPsip(mag))},
+        //////////////////////////////////
+        {"Iris", "A flux aligned Iris", dg::compose( dg::Iris( gp.psipmin, gp.psipmax), mag.psip())},
+        {"Pupil", "A flux aligned Pupil", dg::compose( dg::Pupil(gp.psipmaxcut), mag.psip()) },
+        {"GaussianDamping", "A flux aligned Heaviside with Gaussian damping", dg::compose( dg::GaussianDamping( gp.psipmaxcut, p.source_alpha), mag.psip()) },
+        {"ZonalFlow",  "Flux aligned Sine function", dg::compose( dg::SinX ( p.amp, 0., 2.*M_PI*p.k_psi ), mag.psip())},
+        {"PsiLimiter", "A flux aligned Heaviside", dg::compose( dg::Heaviside( gp.psipmaxlim), mag.psip() )},
+        {"SourceProfile", "A source profile", dg::compose( dg::PolynomialHeaviside(
+                    p.source_boundary-p.source_alpha/2., p.source_alpha/2., -1 ),
+                dg::geo::RhoP(mag))},
+        {"ProfileDamping", "Density profile damping", dg::compose(dg::PolynomialHeaviside(
+            1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)) },
+        {"MagneticTransition", "The region where the magnetic field is modified", dg::compose(dg::DPolynomialHeaviside(
+            p.damping_boundary+p.damping_alpha/2.,
+            p.damping_alpha/2., +1 ), dg::geo::RhoP(mag_origin))},
+        {"Nprofile", "A flux aligned profile", dg::compose( dg::LinearX( p.nprofileamp/mag.psip()(mag.R0(),0.), p.nprofileamp ), mag.psip())},
+        {"Delta", "A flux aligned Gaussian peak", dg::compose( dg::GaussianX( psipmin*0.2, 0.1, 1./(sqrt(2.*M_PI)*0.1)), mag.psip())},
+        {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::compose( dg::TanhProfX( -3*p.source_alpha, p.source_alpha, -1), mag.psip())},
+        ////
+        {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
+        {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
+            M_PI, p.sigma, p.sigma, p.sigma, p.amp)},
+        { "Hoo", "The novel h02 factor", dg::geo::Hoo( mag) }
+
+    };
+
+    /// -------  Elements for fsa on X-point grid ----------------
     if( gp.hasXpoint())
     {
         std::cout << "Generate X-point flux-aligned grid ... \n";
@@ -206,23 +261,15 @@ int main( int argc, char* argv[])
         dg::SparseTensor<dg::HVec> metricX = gX2d.metric();
         dg::HVec volX2d = dg::tensor::volume2d( metricX);
         dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
-        dg::IHMatrix grid2gX2d = dg::create::interpolation( coordsX[0], coordsX[1], grid2d);
-        dg::HVec psip_X = dg::evaluate( dg::one, gX2d), dvdzeta, X_psip1d;
-        dg::blas2::symv( grid2gX2d, (dg::HVec)psipog2d, psip_X);
-        dg::blas1::pointwiseDot( volX2d, psip_X, psip_X);
-        avg_eta( psip_X, X_psip1d, false);
-        dg::blas1::scal( X_psip1d, 4.*M_PI*M_PI);
+        dg::HVec dvdzeta;
         avg_eta( volX2d, dvdzeta, false);
         dg::blas1::scal( dvdzeta, 4.*M_PI*M_PI);
         dg::Grid1d gX1d( gX2d.x0(), gX2d.x1(), npsi, Npsi, dg::NEU);
         dg::HVec X_psi_vol = dg::integrate( dvdzeta, gX1d);
-        map1d.emplace_back( "X_dvdzeta", dvdzeta,
+        map1d.emplace_back( "dvdzeta", dvdzeta,
             "dvdzeta on X-point grid");
-        map1d.emplace_back( "X_psi_vol", X_psi_vol,
+        map1d.emplace_back( "psi_vol", X_psi_vol,
             "Flux volume on X-point grid");
-        dg::blas1::pointwiseDivide( X_psip1d, dvdzeta, X_psip1d);
-        map1d.emplace_back( "X_psip_fsa", X_psip1d,
-            "Flux surface average of psi on X-point grid");
 
         //NOTE: VOLUME is WITHIN cells while AREA is ON gridpoints
         dg::HVec gradZetaX = metricX.value(0,0), X_psi_area;
@@ -230,97 +277,52 @@ int main( int argc, char* argv[])
         dg::blas1::pointwiseDot( volX2d, gradZetaX, gradZetaX); //R\sqrt{g}|\nabla\zeta|
         avg_eta( gradZetaX, X_psi_area, false);
         dg::blas1::scal( X_psi_area, 4.*M_PI*M_PI);
-        map1d.emplace_back( "X_psi_area", X_psi_area,
+        map1d.emplace_back( "psi_area", X_psi_area,
             "Flux area on X-point grid");
-        volumeXGrid = dg::interpolate( dg::xspace, X_psi_vol, gX1d.x1(), gX1d);
+        std::cout << "Total volume within separatrix is "<< dg::interpolate( dg::xspace, X_psi_vol, 0., gX1d)<<std::endl;
 
-        //Compute FSA of curvature operators
-        dg::HVec X_curvNablaBGradPsip( psip_X), X_curvKappaKGradPsip( psip_X), X_gradPsip(psip_X);
-        dg::HVec X_curvNablaB_fsa, X_curvKappaK_fsa, X_gradPsip_fsa;
-        dg::blas2::symv( grid2gX2d, curvNablaBGradPsip, X_curvNablaBGradPsip);
-        dg::blas2::symv( grid2gX2d, curvKappaKGradPsip, X_curvKappaKGradPsip);
-        dg::blas2::symv( grid2gX2d, gradPsip, X_gradPsip);
-        dg::blas1::pointwiseDot( volX2d, X_curvNablaBGradPsip, X_curvNablaBGradPsip);
-        dg::blas1::pointwiseDot( volX2d, X_curvKappaKGradPsip, X_curvKappaKGradPsip);
-        dg::blas1::pointwiseDot( volX2d, X_gradPsip, X_gradPsip);
-        avg_eta( X_curvNablaBGradPsip, X_curvNablaB_fsa, false);
-        avg_eta( X_curvKappaKGradPsip, X_curvKappaK_fsa, false);
-        avg_eta( X_gradPsip, X_gradPsip_fsa, false);
-        dg::blas1::scal( X_curvNablaB_fsa, 4*M_PI*M_PI); //
-        dg::blas1::scal( X_curvKappaK_fsa, 4*M_PI*M_PI); //
-        dg::blas1::scal( X_gradPsip_fsa, 4*M_PI*M_PI); //
-        dg::blas1::pointwiseDivide( X_curvNablaB_fsa, dvdzeta, X_curvNablaB_fsa );
-        dg::blas1::pointwiseDivide( X_curvKappaK_fsa, dvdzeta, X_curvKappaK_fsa );
-        dg::blas1::pointwiseDivide( X_gradPsip_fsa, dvdzeta, X_gradPsip_fsa );
-        map1d.emplace_back( "X_curvNablaB_fsa", X_curvNablaB_fsa,
-            "Flux surface average of true Nabla B curvature Dot Grad Psip");
-        map1d.emplace_back( "X_curvKappaK_fsa", X_curvKappaK_fsa,
-            "Flux surface average of true Kappa curvature Dot Grad Psip");
-        map1d.emplace_back( "X_gradPsip_fsa", X_gradPsip_fsa,
-            "Flux surface average of |Grad Psip|");
-        // h02 factor
-        dg::HVec h02 = dg::pullback( dg::geo::Hoo(mag), gX2d), X_h02_fsa;
-        dg::blas1::pointwiseDot( volX2d, h02, h02);
-        avg_eta( h02, X_h02_fsa, false);
-        dg::blas1::scal( X_h02_fsa, 4*M_PI*M_PI); //
-        dg::blas1::pointwiseDivide( X_h02_fsa, dvdzeta, X_h02_fsa );
-        map1d.emplace_back( "X_hoo_fsa", X_h02_fsa,
-            "Flux surface average of novel h02 factor");
-        // total source term
-        h02 = dg::pullback( dg::compose( dg::PolynomialHeaviside(
-                    p.source_boundary-p.source_alpha/2., p.source_alpha/2., -1 ),
-                dg::geo::RhoP(mag)), gX2d);
-        dg::blas1::pointwiseDot( volX2d, h02, h02);
-        avg_eta( h02, X_h02_fsa, false);
-        dg::blas1::scal( X_h02_fsa, 4*M_PI*M_PI); //
-        dg::blas1::pointwiseDivide( X_h02_fsa, dvdzeta, X_h02_fsa );
-        map1d.emplace_back( "X_sne_fsa", X_h02_fsa,
-            "Flux surface average over source term");
-        dg::blas1::pointwiseDot( X_h02_fsa, dvdzeta, X_h02_fsa );
-        X_h02_fsa = dg::integrate( X_h02_fsa, gX1d);
-        map1d.emplace_back( "X_sne_ifs", X_h02_fsa,
-            "Flux surface integral over source term");
-        //divb
-        h02 = dg::pullback( dg::geo::Divb(mag), gX2d);
-        dg::blas1::pointwiseDot( volX2d, h02, h02);
-        avg_eta( h02, X_h02_fsa, false);
-        dg::blas1::scal( X_h02_fsa, 4*M_PI*M_PI); //
-        dg::blas1::pointwiseDivide( X_h02_fsa, dvdzeta, X_h02_fsa );
-        map1d.emplace_back( "X_divb_fsa", X_h02_fsa,
-            "Flux surface average of divb");
-    }
+        //Compute FSA of cylindrical functions
+        dg::HVec transferH, transferH1d;
+        for( auto tp : map)
+        {
+            transferH = dg::pullback( std::get<2>(tp), gX2d);
+            dg::blas1::pointwiseDot( volX2d, transferH, transferH);
+            avg_eta( transferH, transferH1d, false);
+            dg::blas1::scal( transferH1d, 4*M_PI*M_PI); //
+            dg::blas1::pointwiseDivide( transferH1d, dvdzeta, transferH1d );
+            map1d.emplace_back( std::get<0>(tp)+"_fsa", transferH1d,
+                std::get<1>(tp)+" (Flux surface average)");
+            dg::blas1::pointwiseDot( transferH1d, dvdzeta, transferH1d );
+            transferH1d = dg::integrate( transferH1d, gX1d);
+            map1d.emplace_back( std::get<0>(tp)+"_ifs", transferH1d,
+                std::get<1>(tp)+" (Flux surface integral)");
 
-    ///////////////////Compute flux average////////////////////
-    dg::Grid1d grid1d( 0,1,npsi,Npsi, dg::NEU);
-    double deltapsi = 0.1;
+        }
+    }
+    /// --------- More flux labels --------------------------------
+    dg::Grid1d grid1d(psipmin<psipmax ? psipmin : psipmax,
+            psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::NEU);
     if( !gp.isToroidal())
     {
-        std::cout << "Compute flux averages\n";
-        dg::HVec xpoint_weights = dg::evaluate( dg::cooX2d, grid2d);
-        if( gp.hasXpoint() )
-        {
-            double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
-            double Z_X = -1.1*gp.elongation*gp.a;
-            dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
-            dg::blas1::pointwiseDot( xpoint_weights,
-                    dg::evaluate( dg::geo::ZCutter(Z_X), grid2d), xpoint_weights);
-        }
-        dg::geo::FluxSurfaceAverage<dg::DVec>  fsa( grid2d, mag, psipog2d, xpoint_weights);
-        grid1d = dg::Grid1d (psipmin<psipmax ? psipmin : psipmax, psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::NEU);
-        map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
-            "Flux surface average of psi with delta function");
-        dg::HVec qprofile;
+        dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
+        dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
+        map1d.emplace_back("rho", rho,
+            "Alternative flux label rho = -psi/psimin + 1");
+        dg::blas1::transform( rho, rho, dg::SQRT<double>());
+        map1d.emplace_back("rho_p", rho,
+            "Alternative flux label rho_p = Sqrt[-psi/psimin + 1]");
         if( gp.equilibrium == "solovev")
         {
             dg::geo::SafetyFactor qprof( mag);
-            qprofile = dg::evaluate( qprof, grid1d);
+            dg::HVec qprofile = dg::evaluate( qprof, grid1d);
             map1d.emplace_back("q-profile", qprofile,
                 "q-profile (Safety factor) using direct integration");
             dg::HVec psit = dg::integrate( qprofile, grid1d);
             map1d.emplace_back("psit1d", psit,
                 "Toroidal flux label psi_t integrated  on grid1d using direct q");
             //we need to avoid integrating >=0
-            dg::Grid1d g1d_fine(psipmin<0. ? psipmin : 0., psipmin<0. ? 0. : psipmin, npsi ,Npsi,dg::NEU);
+            dg::Grid1d g1d_fine(psipmin<0. ? psipmin : 0.,
+                    psipmin<0. ? 0. : psipmin, npsi, Npsi,dg::NEU);
             qprofile = dg::evaluate( qprof, g1d_fine);
             dg::HVec w1d = dg::create::weights( g1d_fine);
             double psit_tot = dg::blas1::dot( w1d, qprofile);
@@ -330,72 +332,6 @@ int main( int argc, char* argv[])
             map1d.emplace_back("rho_t", psit,
                 "Toroidal flux label rho_t = sqrt( psit/psit_tot) evaluated on grid1d");
         }
-        dg::geo::SafetyFactorAverage qprof( grid2d, mag);
-        qprofile = dg::evaluate( qprof, grid1d);
-        map1d.emplace_back("q-profile_fsa", qprofile,
-            "q-profile (Safety factor) using average integration");
-        dg::HVec psit = dg::integrate( qprofile, grid1d);
-        map1d.emplace_back("psit1d_fsa", psit,
-            "Toroidal flux label psi_t integrated on grid1d using average q");
-        map1d.emplace_back("psip1d",    dg::evaluate( dg::cooX1d, grid1d),
-            "Poloidal flux label psi_p evaluated on grid1d");
-        dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
-        dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
-        map1d.emplace_back("rho", rho,
-            "Alternative flux label rho = -psi/psimin + 1");
-        dg::blas1::transform( rho, rho, dg::SQRT<double>());
-        map1d.emplace_back("rho_p", rho,
-            "Alternative flux label rho_p = Sqrt[-psi/psimin + 1]");
-        fsa.set_container( (dg::DVec)curvNablaBGradPsip);
-        map1d.emplace_back("curvNablaB_fsa",   dg::evaluate( fsa,      grid1d),
-            "Flux surface average of true Nabla B curvature Dot Grad Psip with delta function");
-        fsa.set_container( (dg::DVec)curvKappaKGradPsip);
-        map1d.emplace_back("curvKappaK_fsa",   dg::evaluate( fsa,      grid1d),
-            "Flux surface average of true Kappa curvature Dot Grad Psip with delta function");
-        fsa.set_container( (dg::DVec)gradPsip);
-        map1d.emplace_back("gradPsip_fsa",   dg::evaluate( fsa,      grid1d),
-            "Flux surface average of |Grad Psip| with delta function");
-
-        //other flux labels
-        dg::geo::FluxSurfaceIntegral<dg::HVec> fsi( grid2d, mag);
-        fsi.set_right( xpoint_weights);
-        deltapsi = fsi.get_deltapsi();
-
-        dg::HVec areaT_psip = dg::evaluate( fsi, grid1d);
-        dg::HVec w1d = dg::create::weights( grid1d);
-        double volumeCoarea = 2.*M_PI*dg::blas1::dot( areaT_psip, w1d);
-
-        //area
-        fsi.set_left( dg::evaluate( dg::geo::GradPsip(mag), grid2d));
-        dg::HVec psi_area = dg::evaluate( fsi, grid1d);
-        dg::blas1::scal(psi_area, 2.*M_PI);
-        map1d.emplace_back( "psi_area", psi_area,
-            "Flux area with delta function");
-        // h02 factor
-        dg::HVec h02 = dg::evaluate( dg::geo::Hoo(mag), grid2d);
-        fsa.set_container( h02);
-        map1d.emplace_back( "hoo_fsa", dg::evaluate( fsa, grid1d),
-           "Flux surface average of novel h02 factor with delta function");
-
-
-
-        dg::geo::FluxVolumeIntegral<dg::HVec> fvi( (dg::CartesianGrid2d)grid2d, mag);
-        //std::cout << "Delta Rho for Flux surface integrals = "<<-deltapsi/psipmin<<"\n";
-
-        fvi.set_right( xpoint_weights);
-        dg::HVec psi_vol = dg::evaluate( fvi, grid1d);
-        dg::blas1::scal(psi_vol, 2.*M_PI);
-        map1d.emplace_back( "psi_vol", psi_vol,
-            "Flux volume with delta function");
-        double volumeFVI = 2.*M_PI*fvi(psipmax);
-        double volumeSep = 2.*M_PI*fvi(0.);
-        std::cout << "volume enclosed by separatrix: "<<volumeSep<<"\n";
-        std::cout << "volume test with coarea formula: "<<volumeCoarea<<" "<<volumeFVI
-                  <<" rel error = "<<fabs(volumeCoarea-volumeFVI)/volumeFVI<<"\n";
-        if(gp.hasXpoint()){
-            std::cout << "volume test with x grid formula: "<<volumeXGrid<<" "<<volumeFVI
-                      <<" rel error = "<<fabs(volumeXGrid-volumeFVI)/volumeFVI<<"\n";
-        };
     }
 
     /////////////////////////////set up netcdf/////////////////////////////////////
@@ -450,78 +386,12 @@ int main( int argc, char* argv[])
         err = nc_redef(ncid);
     }
     //write 2d vectors
-    std::vector< std::tuple<std::string, std::string, dg::geo::CylindricalFunctor >> map2d{
-        {"Psip", "Flux function", mag.psip()},
-        {"PsipR", "Flux function derivative in R", mag.psipR()},
-        {"PsipZ", "Flux function derivative in Z", mag.psipZ()},
-        {"PsipRR", "Flux function derivative in RR", mag.psipRR()},
-        {"PsipRZ", "Flux function derivative in RZ", mag.psipRZ()},
-        {"PsipZZ", "Flux function derivative in ZZ", mag.psipZZ()},
-        {"Ipol", "Poloidal current", mag.ipol()},
-        {"IpolR", "Poloidal current derivative in R", mag.ipolR()},
-        {"IpolZ", "Poloidal current derivative in Z", mag.ipolZ()},
-        {"Rho_p", "Normalized Poloidal flux label", dg::geo::RhoP(mag)},
-        {"Bmodule", "Magnetic field strength", dg::geo::Bmodule(mag)},
-        {"InvB", "Inverse of Bmodule", dg::geo::InvB(mag)},
-        {"LnB", "Natural logarithm of Bmodule", dg::geo::LnB(mag)},
-        {"GradLnB", "The parallel derivative of LnB", dg::geo::GradLnB(mag)},
-        {"Divb", "The divergence of the magnetic unit vector", dg::geo::Divb(mag)},
-        {"B_R", "Derivative of Bmodule in R", dg::geo::BR(mag)},
-        {"B_Z", "Derivative of Bmodule in Z", dg::geo::BZ(mag)},
-        {"CurvatureNablaBR",  "R-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBR(mag,+1)},
-        {"CurvatureNablaBZ",  "Z-component of the (toroidal) Nabla B curvature vector", dg::geo::CurvatureNablaBZ(mag,+1)},
-        {"CurvatureKappaR",   "R-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaR(mag,+1)},
-        {"CurvatureKappaZ",   "Z-component of the (toroidal) Kappa B curvature vector", dg::geo::CurvatureKappaZ(mag,+1)},
-        {"DivCurvatureKappa", "Divergence of the (toroidal) Kappa B curvature vector", dg::geo::DivCurvatureKappa(mag,+1)},
-        {"DivCurvatureNablaB","Divergence of the (toroidal) Nabla B curvature vector", dg::geo::DivCurvatureNablaB(mag,+1)},
-        {"TrueCurvatureNablaBR", "R-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBR(mag)},
-        {"TrueCurvatureNablaBZ", "Z-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBZ(mag)},
-        {"TrueCurvatureNablaBP", "Contravariant Phi-component of the (true) Nabla B curvature vector", dg::geo::TrueCurvatureNablaBP(mag)},
-        {"TrueCurvatureKappaR", "R-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaR(mag)},
-        {"TrueCurvatureKappaZ", "Z-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaZ(mag)},
-        {"TrueCurvatureKappaP", "Contravariant Phi-component of the (true) Kappa B curvature vector", dg::geo::TrueCurvatureKappaP(mag)},
-        {"TrueDivCurvatureKappa", "Divergence of the (true) Kappa B curvature vector", dg::geo::TrueDivCurvatureKappa(mag)},
-        {"TrueDivCurvatureNablaB","Divergence of the (true) Nabla B curvature vector",  dg::geo::TrueDivCurvatureNablaB(mag)},
-        {"BFieldR", "R-component of the magnetic field vector", dg::geo::BFieldR(mag)},
-        {"BFieldZ", "Z-component of the magnetic field vector", dg::geo::BFieldZ(mag)},
-        {"BFieldP", "Contravariant Phi-component of the magnetic field vector", dg::geo::BFieldP(mag)},
-        {"BHatR", "R-component of the magnetic field unit vector", dg::geo::BHatR(mag)},
-        {"BHatZ", "Z-component of the magnetic field unit vector", dg::geo::BHatZ(mag)},
-        {"BHatP", "Contravariant Phi-component of the magnetic field unit vector", dg::geo::BHatP(mag)},
-        {"GradBHatR", "Parallel derivative of BHatR", dg::geo::BHatR(mag)},
-        {"GradBHatZ", "Parallel derivative of BHatZ", dg::geo::BHatZ(mag)},
-        {"GradBHatP", "Parallel derivative of BHatP", dg::geo::BHatP(mag)},
-        {"GradPsip", "Module of gradient of Psip", dg::geo::GradPsip(mag)},
-        //////////////////////////////////
-        {"Iris", "A flux aligned Iris", dg::compose( dg::Iris( gp.psipmin, gp.psipmax), mag.psip())},
-        {"Pupil", "A flux aligned Pupil", dg::compose( dg::Pupil(gp.psipmaxcut), mag.psip()) },
-        {"GaussianDamping", "A flux aligned Heaviside with Gaussian damping", dg::compose( dg::GaussianDamping( gp.psipmaxcut, p.source_alpha), mag.psip()) },
-        {"ZonalFlow",  "Flux aligned Sine function", dg::compose( dg::SinX ( p.amp, 0., 2.*M_PI*p.k_psi ), mag.psip())},
-        {"PsiLimiter", "A flux aligned Heaviside", dg::compose( dg::Heaviside( gp.psipmaxlim), mag.psip() )},
-        {"SourceProfile", "A source profile", dg::compose( dg::PolynomialHeaviside(
-                    p.source_boundary-p.source_alpha/2., p.source_alpha/2., -1 ),
-                dg::geo::RhoP(mag))},
-        {"ProfileDamping", "Density profile damping", dg::compose(dg::PolynomialHeaviside(
-            1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)) },
-        {"MagneticTransition", "The region where the magnetic field is modified", dg::compose(dg::DPolynomialHeaviside(
-            p.damping_boundary+p.damping_alpha/2.,
-            p.damping_alpha/2., +1 ), dg::geo::RhoP(mag_origin))},
-        {"Nprofile", "A flux aligned profile", dg::compose( dg::LinearX( p.nprofileamp/mag.psip()(mag.R0(),0.), p.nprofileamp ), mag.psip())},
-        {"Delta", "A flux aligned Gaussian peak", dg::compose( dg::GaussianX( psipmin*0.2, deltapsi, 1./(sqrt(2.*M_PI)*deltapsi)), mag.psip())},
-        {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::compose( dg::TanhProfX( -3*p.source_alpha, p.source_alpha, -1), mag.psip())},
-        ////
-        {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
-        {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
-            M_PI, p.sigma, p.sigma, p.sigma, p.amp)},
-        { "Hoo", "The novel h02 factor", dg::geo::Hoo( mag) }
-
-    };
     //allocate mem for visual
     dg::HVec hvisual = dg::evaluate( dg::zero, grid2d);
     dg::HVec hvisual3d = dg::evaluate( dg::zero, grid3d);
     dg::fHVec fvisual, fvisual3d;
     std::cout << "WRTING 2D/3D CYLINDRICAL FIELDS ... \n";
-    for(auto tp : map2d)
+    for(auto tp : map)
     {
         int vectorID, vectorID3d;
         err = nc_def_var( ncid, std::get<0>(tp).data(), NC_FLOAT, 2,
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 7b71f0439..3d36215a6 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -651,6 +651,16 @@ inline CylindricalVectorLvl0 createTrueCurvatureKappa( const TokamakMagneticFiel
 inline CylindricalVectorLvl0 createTrueCurvatureNablaB( const TokamakMagneticField& mag){
     return CylindricalVectorLvl0( TrueCurvatureNablaBR(mag), TrueCurvatureNablaBZ(mag), TrueCurvatureNablaBP(mag));
 }
+/**
+ * @brief Gradient Psip vector field (PsipR, PsipZ, 0)
+ *
+ * @param mag the tokamak magnetic field
+ * @return the tuple PsipR, PsipZ, 0 constructed from mag
+ * @note The contravariant components in cylindrical coordinates
+ */
+inline CylindricalVectorLvl0 createGradPsip( const TokamakMagneticField& mag){
+    return CylindricalVectorLvl0( mag.psipR(), mag.psipZ(),Constant(0));
+}
 
 //Necessary to analytically compute Laplacians:
 ///@brief \f$ \nabla_\parallel b^R \f$
@@ -709,20 +719,6 @@ struct GradBHatP: public aCylindricalFunctor<GradBHatP>
     TokamakMagneticField mag_;
 };
 
-///@brief \f$ |\nabla\psi_p| \f$
-struct GradPsip: public aCylindricalFunctor<GradPsip>
-{
-    GradPsip( const TokamakMagneticField& mag): m_mag(mag){}
-    double do_compute( double R, double Z) const
-    {
-        double psipR = m_mag.psipR()(R,Z), psipZ = m_mag.psipZ()(R,Z);
-        return sqrt(psipR*psipR +psipZ*psipZ);
-    }
-    private:
-    TokamakMagneticField m_mag;
-
-};
-
 ///@brief \f$ \sqrt{\psi_p/ \psi_{p,\min}} \f$
 struct RhoP: public aCylindricalFunctor<RhoP>
 {
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index c244f375d..a7e344b74 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -137,8 +137,22 @@ int main( int argc, char* argv[])
     dg::HVec x_left = dg::evaluate( sine, g1d), x_right(x_left);
     dg::HVec y_left = dg::evaluate( cosine, g1d);
     std::vector<std::tuple<std::string, dg::HVec, std::string> > map2d;
-    thrust::host_vector<double> psi_p = dg::pullback( mag.psip(), g2d);
+    dg::HVec psi_p = dg::pullback( mag.psip(), g2d);
     map2d.emplace_back( "Psi_p", psi_p, "Poloidal flux function");
+    //test if interpolation onto grid works
+    dg::HVec psip_X(psi_p);
+    {
+        double Rmin=gp.R_0-1.2*gp.a;
+        double Zmin=-1.2*gp.a*gp.elongation;
+        double Rmax=gp.R_0+1.2*gp.a;
+        double Zmax=1.2*gp.a*gp.elongation;
+        dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, 3,200,200);
+        std::vector<dg::HVec> coordsX = g2d.map();
+        dg::IHMatrix grid2gX2d = dg::create::interpolation( coordsX[0], coordsX[1], grid2d);
+        dg::HVec psipog2d   = dg::evaluate( mag.psip(), grid2d);
+        dg::blas2::symv( grid2gX2d, psipog2d, psip_X);
+    }
+    map2d.emplace_back( "Psi_p_interpolated", psip_X, "Poloidal flux function");
     g2d.display();
     dg::HVec X( g2d.size()), Y(X), P = dg::evaluate( dg::zero, g2d);
     for( unsigned i=0; i<g2d.size(); i++)
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 7ef043614..958a6cf12 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1410,7 +1410,7 @@ Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}
 \qquad alpha  & float & 0.2 & - & Transition width $\alpha_p$ in the Heaviside
 at the separatrix
 \\
-source & dict & & & Density source, cf. the output \texttt{sne\_ifs} in \texttt{feltordiag} (or \texttt{geometry\_diag}) to see how much mass the source with the parameters below generates.  \\
+source & dict & & & Density source, cf. the output \texttt{sne\_ifs} in \texttt{feltordiag} (or \texttt{SourceProfile\_ifs} in \texttt{geometry\_diag}) to see how much mass the source with the parameters below generates.  \\
 \qquad rate & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}.
 \\
 \qquad type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
-- 
GitLab


From 7c3f0695086f7bf5cc103006805beea65969dcb5 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 11 May 2020 18:45:52 +0200
Subject: [PATCH 214/540] Adapt geometry_diag to use dvdpsip like feltordiag

---
 inc/geometries/geometry_diag.cu | 46 ++++++++++++++++-----------------
 1 file changed, 23 insertions(+), 23 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 0579d575f..6e4071719 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -144,24 +144,22 @@ int main( int argc, char* argv[])
     //Test coefficients
     dg::geo::TokamakMagneticField mag_origin = dg::geo::createSolovevField(gp);
     dg::geo::TokamakMagneticField mag = mag_origin;
+    //Find O-point
+    double RO = gp.R_0, ZO = 0.;
+    if( !gp.isToroidal() )
+        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+    const double psipO = mag.psip()( RO, ZO);
+    const double psipmin = mag.psip()(RO, ZO);
+    std::cout << "O-point found at "<<RO<<" "<<ZO<<" with Psip "<<psipmin<<std::endl;
+    const double psip0 = mag.psip()(gp.R_0, 0);
+    std::cout << "psip( R_0, 0) = "<<psip0<<"\n";
     if( p.damping_alpha > 0.)
     {
-        double RO=mag.R0(), ZO=0.;
-        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
-        double psipO = mag.psip()( RO, ZO);
         double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
         double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
         std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
         mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2., fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
     }
-    //Find O-point
-    double R_O = gp.R_0, Z_O = 0.;
-    if( !gp.isToroidal() )
-        dg::geo::findOpoint( mag.get_psip(), R_O, Z_O);
-    const double psipmin = mag.psip()(R_O, Z_O);
-    std::cout << "O-point found at "<<R_O<<" "<<Z_O<<" with Psip "<<psipmin<<std::endl;
-    const double psip0 = mag.psip()(gp.R_0, 0);
-    std::cout << "psip( R_0, 0) = "<<psip0<<"\n";
 
 
     dg::Grid2d grid2d(Rmin,Rmax,Zmin,Zmax, n,Nx,Ny);
@@ -169,7 +167,6 @@ int main( int argc, char* argv[])
     std::vector<std::tuple<std::string, dg::HVec, std::string> > map1d;
     ///////////TEST CURVILINEAR GRID TO COMPUTE FSA QUANTITIES
     unsigned npsi = 3, Npsi = p.Npsi;//set number of psivalues (NPsi % 8 == 0)
-    double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     //Generate list of functions to evaluate
     std::vector< std::tuple<std::string, std::string, dg::geo::CylindricalFunctor >> map{
         {"Psip", "Flux function", mag.psip()},
@@ -243,6 +240,7 @@ int main( int argc, char* argv[])
     };
 
     /// -------  Elements for fsa on X-point grid ----------------
+    double psipmax = dg::blas1::reduce( psipog2d, 0., thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     if( gp.hasXpoint())
     {
         std::cout << "Generate X-point flux-aligned grid ... \n";
@@ -261,13 +259,15 @@ int main( int argc, char* argv[])
         dg::SparseTensor<dg::HVec> metricX = gX2d.metric();
         dg::HVec volX2d = dg::tensor::volume2d( metricX);
         dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
-        dg::HVec dvdzeta;
-        avg_eta( volX2d, dvdzeta, false);
-        dg::blas1::scal( dvdzeta, 4.*M_PI*M_PI);
-        dg::Grid1d gX1d( gX2d.x0(), gX2d.x1(), npsi, Npsi, dg::NEU);
-        dg::HVec X_psi_vol = dg::integrate( dvdzeta, gX1d);
-        map1d.emplace_back( "dvdzeta", dvdzeta,
-            "dvdzeta on X-point grid");
+        const double f0 = (gX2d.x1()-gX2d.x0())/ ( psipmax - psipO);
+        dg::HVec dvdpsip;
+        avg_eta( volX2d, dvdpsip, false);
+        dg::blas1::scal( dvdpsip, 4.*M_PI*M_PI*f0);
+        dg::Grid1d gX1d(psipmin<psipmax ? psipmin : psipmax,
+            psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::DIR_NEU); //inner value is always zero
+        dg::HVec X_psi_vol = dg::integrate( dvdpsip, gX1d);
+        map1d.emplace_back( "dvdpsip", dvdpsip,
+            "Derivative of flux volume with respect to flux label psi");
         map1d.emplace_back( "psi_vol", X_psi_vol,
             "Flux volume on X-point grid");
 
@@ -288,11 +288,11 @@ int main( int argc, char* argv[])
             transferH = dg::pullback( std::get<2>(tp), gX2d);
             dg::blas1::pointwiseDot( volX2d, transferH, transferH);
             avg_eta( transferH, transferH1d, false);
-            dg::blas1::scal( transferH1d, 4*M_PI*M_PI); //
-            dg::blas1::pointwiseDivide( transferH1d, dvdzeta, transferH1d );
+            dg::blas1::scal( transferH1d, 4*M_PI*M_PI*f0); //
+            dg::blas1::pointwiseDivide( transferH1d, dvdpsip, transferH1d );
             map1d.emplace_back( std::get<0>(tp)+"_fsa", transferH1d,
                 std::get<1>(tp)+" (Flux surface average)");
-            dg::blas1::pointwiseDot( transferH1d, dvdzeta, transferH1d );
+            dg::blas1::pointwiseDot( transferH1d, dvdpsip, transferH1d );
             transferH1d = dg::integrate( transferH1d, gX1d);
             map1d.emplace_back( std::get<0>(tp)+"_ifs", transferH1d,
                 std::get<1>(tp)+" (Flux surface integral)");
@@ -301,7 +301,7 @@ int main( int argc, char* argv[])
     }
     /// --------- More flux labels --------------------------------
     dg::Grid1d grid1d(psipmin<psipmax ? psipmin : psipmax,
-            psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::NEU);
+            psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::DIR_NEU); //inner value is always zero
     if( !gp.isToroidal())
     {
         dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
-- 
GitLab


From fc7d36f61fee74bfd36161d8fa51bbf3c677425e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 11 May 2020 19:14:09 +0200
Subject: [PATCH 215/540] Fix MPI bug in multigrid.h

---
 inc/dg/multigrid.h | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 3d7cc0c95..1b6aac040 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -388,9 +388,9 @@ struct MultigridCG2d
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
         //double norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
-#ifdef DG_BENCHMARK
-        Timer t;
-#endif //DG_BENCHMARK
+//#ifdef DG_BENCHMARK
+//        Timer t;
+//#endif //DG_BENCHMARK
 
         //std::vector<Container> out( x);
 
@@ -407,20 +407,20 @@ struct MultigridCG2d
         dg::blas1::scal( x[p+1], 0.);
         if( p+1 == m_stages-1)
         {
-#ifdef DG_BENCHMARK
-            t.tic();
-#endif //DG_BENCHMARK
+//#ifdef DG_BENCHMARK
+//            t.tic();
+//#endif //DG_BENCHMARK
             int number = m_cg[p+1]( op[p+1], x[p+1], b[p+1], op[p+1].precond(),
                 op[p+1].inv_weights(), eps/2.);
-#ifdef DG_BENCHMARK
-            t.toc();
-#ifdef MPI_VERSION
-            int rank;
-            MPI_Comm_rank(MPI_COMM_WORLD, &rank);
-            if(rank==0)
-#endif //MPI
+//#ifdef DG_BENCHMARK
+//            t.toc();
+//#ifdef MPI_VERSION
+//            int rank;
+//            MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+//            if(rank==0)
+//#endif //MPI
             //std::cout << "# Multigrid stage: " << p+1 << ", iter: " << number << ", took "<<t.diff()<<"s\n";
-#endif //DG_BENCHMARK
+//#endif //DG_BENCHMARK
             //dg::blas2::symv( op[p+1], x[p+1], m_r[p+1]);
             //dg::blas1::axpby( 1., b[p+1], -1., m_r[p+1]);
             //double norm_res = sqrt(dg::blas1::dot( m_r[p+1], m_r[p+1]));
-- 
GitLab


From 0eaeec275003c5e4c6538dbca82de75e8ea741b8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 11 May 2020 21:54:32 +0200
Subject: [PATCH 216/540] Reduce parameter requirements in init_from_file

---
 src/feltor/init_from_file.h | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index d71f53e94..f4bcbbba8 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -29,14 +29,18 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     parser["collectComments"] = false;
     std::string errs;
     parseFromStream( parser, is, &jsIN, &errs); //read input without comments
-    const feltor::Parameters pIN(  jsIN);
+    unsigned  pINn       = jsIN["n"].asUInt();
+    unsigned  pINNx      = jsIN["Nx"].asUInt();
+    unsigned  pINNy      = jsIN["Ny"].asUInt();
+    unsigned  pINNz      = jsIN["Nz"].asUInt();
+    double    pINdt      = jsIN["dt"].asDouble();
+    bool      pINsymmetric   = jsIN.get( "symmetric", false).asBool();
     MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
     MPI_OUT std::cout << " file parameters:" << std::endl;
-    MPI_OUT pIN.display( std::cout);
 
     // Now read in last timestep
     Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
-        pIN.n, pIN.Nx, pIN.Ny, pIN.symmetric ? 1 : pIN.Nz, pIN.bcxN, pIN.bcyN, dg::PER
+        pINn, pINNx, pINNy, pINsymmetric ? 1 : pINNz, dg::DIR, dg::DIR, dg::PER
         #ifdef FELTOR_MPI
         , grid.communicator()
         #endif //FELTOR_MPI
-- 
GitLab


From 3d0053e236d91316bc82a9cfa6c0781ef280a219 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 12 May 2020 11:06:49 +0200
Subject: [PATCH 217/540] remove unnecessary pINdt in feltordiag

---
 src/feltor/init_from_file.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index f4bcbbba8..58b66d676 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -33,7 +33,6 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     unsigned  pINNx      = jsIN["Nx"].asUInt();
     unsigned  pINNy      = jsIN["Ny"].asUInt();
     unsigned  pINNz      = jsIN["Nz"].asUInt();
-    double    pINdt      = jsIN["dt"].asDouble();
     bool      pINsymmetric   = jsIN.get( "symmetric", false).asBool();
     MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
     MPI_OUT std::cout << " file parameters:" << std::endl;
-- 
GitLab


From 122a69326f3870bb267508014d26b34cdfc6f709 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 12 May 2020 11:39:37 +0200
Subject: [PATCH 218/540] Fix Documentation bug

---
 inc/dg/Doxyfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 9da97f0b9..3be47544b 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -862,7 +862,7 @@ EXCLUDE_SYMLINKS       = NO
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories for example use the pattern */test/*
 
-EXCLUDE_PATTERNS       = */blas*
+EXCLUDE_PATTERNS       = backend/blas*
 
 # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
 # (namespaces, classes, functions, etc.) that should be excluded from the
-- 
GitLab


From 32c2ef00a7525206d87c1295515f5d7109bbfe8d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 12 May 2020 18:52:57 +0200
Subject: [PATCH 219/540] More resilience agains nan errors

- ISNAN functor for blas1::reduce check construct
- asserts in Gaussian and PolynomialHeaviside functors
- parameter checks in feltor::init.h
---
 inc/dg/blas1.h           |  21 ++--
 inc/dg/blas2.h           |   3 +-
 inc/dg/functors.h        | 200 +++++++++++++++++++++------------------
 src/feltor/feltor.cu     |   3 +-
 src/feltor/feltor.tex    |  14 +--
 src/feltor/feltor_hpc.cu |   3 +-
 src/feltor/init.h        |  29 +++++-
 src/feltor/parameters.h  |   6 +-
 8 files changed, 165 insertions(+), 114 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 91749e05a..a6352d302 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -50,21 +50,22 @@ namespace blas1
  * This routine computes \f[ x^T y = \sum_{i=0}^{N-1} x_i y_i \f]
  * @copydoc hide_iterations
  *
- * @note Our implementation guarantees binary reproducible results.
- * The sum is computed with infinite precision and the result is rounded
- * to the nearest double precision number.
- * This is possible with the help of an adapted version of the \c ::exblas library.
-
 For example
 @code
 dg::DVec two( 100,2), three(100,3);
 double result = dg::blas1::dot( two, three); // result = 600 (100*(2*3))
 @endcode
+ * @attention if one of the input vectors contains \c NaN then the behaviour is undefined and the function may throw
+ * @note Our implementation guarantees binary reproducible results.
+ * The sum is computed with infinite precision and the result is rounded
+ * to the nearest double precision number.
+ * This is possible with the help of an adapted version of the \c ::exblas library.
+
  * @param x Left Container
  * @param y Right Container may alias x
  * @return Scalar product as defined above
  * @note This routine is always executed synchronously due to the
-        implicit memcpy of the result. With mpi the result is broadcasted to all processes. Also note that the behaviour is undefined when one of the containers contains \c nan
+        implicit memcpy of the result. With mpi the result is broadcasted to all processes.
  * @copydoc hide_ContainerType
  */
 template< class ContainerType1, class ContainerType2>
@@ -86,8 +87,11 @@ inline get_value_type<ContainerType1> dot( const ContainerType1& x, const Contai
 
 For example
 @code
-dg::DVec x( 100,2);
-double result = dg::blas1::reduce( x, 0., thrust::plus<double>()); // result = 200 ( 0 + 2 + 2 + ... + 2 ))
+//Check if a vector contains NaN
+thrust::device_vector<double> x( 100);
+thrust::device_vector<bool> boolvec ( 100, false);
+dg::blas1::transform( x, boolvec, dg::ISNAN<double>());
+bool hasnan = dg::blas1::reduce( boolvec, false, thrust::logical_or<bool>());
 @endcode
  * @param x Left Container
  * @param init initial value of the reduction
@@ -101,6 +105,7 @@ double result = dg::blas1::reduce( x, 0., thrust::plus<double>()); // result = 2
 template< class ContainerType, class BinaryOp>
 inline get_value_type<ContainerType> reduce( const ContainerType& x, get_value_type<ContainerType> init, BinaryOp op )
 {
+    //init must indeed have the same type as the values of Container since op must be associative
     return dg::blas1::detail::doReduce( dg::get_tensor_category<ContainerType>(), x, init, op);
 }
 
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index 38c8ce8b4..9734ccc84 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -59,12 +59,13 @@ inline std::vector<int64_t> doDot_superacc( const ContainerType1& x, const Matri
  * This routine computes the scalar product defined by the symmetric positive definite
  * matrix M \f[ x^T M y = \sum_{i,j=0}^{N-1} x_i M_{ij} y_j \f]
  *
+ * @copydoc hide_code_evaluate2d
+ * @attention if one of the input vectors contains \c NaN then the behaviour is undefined and the function may throw
  * @note Our implementation guarantees binary reproducible results up to and excluding the last mantissa bit of the result.
  * Furthermore, the sum is computed with infinite precision and the result is then rounded
  * to the nearest double precision number. Although the products are not computed with
  * infinite precision, the order of multiplication is guaranteed.
  * This is possible with the help of an adapted version of the \c ::exblas library.
- * @copydoc hide_code_evaluate2d
  *
  * @param x Left input
  * @param m The diagonal Matrix.
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 436fe6b81..ae03412c0 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -82,12 +82,15 @@ struct Gaussian
      *
      * @param x0 x-center-coordinate
      * @param y0 y-center-coordinate
-     * @param sigma_x x - variance
-     * @param sigma_y y - variance
+     * @param sigma_x x - variance (must be !=0)
+     * @param sigma_y y - variance (must be !=0)
      * @param amp Amplitude
      */
     Gaussian( double x0, double y0, double sigma_x, double sigma_y, double amp)
-        : x00(x0), y00(y0), sigma_x(sigma_x), sigma_y(sigma_y), amplitude(amp){}
+        : m_x0(x0), m_y0(y0), m_sigma_x(sigma_x), m_sigma_y(sigma_y), m_amp(amp){
+            assert( m_sigma_x != 0  &&  "sigma_x must not be 0 in Gaussian");
+            assert( m_sigma_y != 0  &&  "sigma_y must not be 0 in Gaussian");
+    }
     /**
      * @brief Return the value of the Gaussian
      *
@@ -102,9 +105,9 @@ struct Gaussian
     DG_DEVICE
     double operator()(double x, double y) const
     {
-        return  amplitude*
-                   exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x +
-                          (y-y00)*(y-y00)/2./sigma_y/sigma_y) );
+        return  m_amp*
+                   exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x +
+                          (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
     /**
      * @brief Return the value of the Gaussian
@@ -123,7 +126,7 @@ struct Gaussian
         return  this->operator()(x,y);
     }
   private:
-    double  x00, y00, sigma_x, sigma_y, amplitude;
+    double  m_x0, m_y0, m_sigma_x, m_sigma_y, m_amp;
 
 };
 
@@ -143,11 +146,14 @@ struct Cauchy
      *
      * @param x0 x-center-coordinate
      * @param y0 y-center-coordinate
-     * @param sigma_x radius in x
-     * @param sigma_y radius in y
+     * @param sigma_x radius in x (must be !=0)
+     * @param sigma_y radius in y (must be !=0)
      * @param amp Amplitude
      */
-    Cauchy( double x0, double y0, double sigma_x, double sigma_y, double amp): x0_(x0), y0_(y0), sigmaX_(sigma_x), sigmaY_(sigma_y), amp_(amp){}
+    Cauchy( double x0, double y0, double sigma_x, double sigma_y, double amp): x0_(x0), y0_(y0), sigmaX_(sigma_x), sigmaY_(sigma_y), amp_(amp){
+        assert( sigma_x != 0  &&  "sigma_x must be !=0 in Cauchy");
+        assert( sigma_y != 0  &&  "sigma_y must be !=0 in Cauchy");
+    }
     DG_DEVICE
     double operator()(double x, double y )const{
         double xbar = (x-x0_)/sigmaX_;
@@ -217,13 +223,17 @@ struct Gaussian3d
      * @param x0 x-center-coordinate
      * @param y0 y-center-coordinate
      * @param z0 z-center-coordinate
-     * @param sigma_x x - variance
-     * @param sigma_y y - variance
-     * @param sigma_z z - variance
+     * @param sigma_x x - variance (must be !=0)
+     * @param sigma_y y - variance (must be !=0)
+     * @param sigma_z z - variance (must be !=0)
      * @param amp Amplitude
      */
     Gaussian3d( double x0, double y0, double z0, double sigma_x, double sigma_y, double sigma_z, double amp)
-        : x00(x0), y00(y0), z00(z0), sigma_x(sigma_x), sigma_y(sigma_y), sigma_z(sigma_z), amplitude(amp){}
+        : m_x0(x0), m_y0(y0), m_z0(z0), m_sigma_x(sigma_x), m_sigma_y(sigma_y), m_sigma_z(sigma_z), m_amp(amp){
+            assert( m_sigma_x != 0  &&  "sigma_x must be !=0 in Gaussian3d");
+            assert( m_sigma_y != 0  &&  "sigma_y must be !=0 in Gaussian3d");
+            assert( m_sigma_z != 0  &&  "sigma_z must be !=0 in Gaussian3d");
+    }
     /**
      * @brief Return a 2d Gaussian
      *
@@ -238,9 +248,9 @@ struct Gaussian3d
     DG_DEVICE
     double operator()(double x, double y) const
     {
-        return  amplitude*
-                   exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x +
-                          (y-y00)*(y-y00)/2./sigma_y/sigma_y) );
+        return  m_amp*
+                   exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x +
+                          (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
     /**
      * @brief Return the value of the Gaussian
@@ -257,13 +267,13 @@ struct Gaussian3d
     DG_DEVICE
     double operator()(double x, double y, double z) const
     {
-        return  amplitude*
-                exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x +
-                       (z-z00)*(z-z00)/2./sigma_z/sigma_z +
-                       (y-y00)*(y-y00)/2./sigma_y/sigma_y) );
+        return  m_amp*
+                exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x +
+                       (z-m_z0)*(z-m_z0)/2./m_sigma_z/m_sigma_z +
+                       (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
   private:
-    double  x00, y00, z00, sigma_x, sigma_y, sigma_z, amplitude;
+    double  m_x0, m_y0, m_z0, m_sigma_x, m_sigma_y, m_sigma_z, m_amp;
 
 };
 /**
@@ -278,15 +288,17 @@ struct GaussianX
      * @brief A Gaussian in x
      *
      * @param x0 x-center-coordinate
-     * @param sigma_x x - variance
+     * @param sigma_x x - variance (must be !=0)
      * @param amp Amplitude
      */
     GaussianX( double x0, double sigma_x, double amp)
-        :x00(x0), sigma_x(sigma_x), amplitude(amp){}
+        :m_x0(x0), m_sigma_x(sigma_x), m_amp(amp){
+            assert( m_sigma_x != 0  &&  "sigma_x must be !=0 in GaussianX");
+    }
     DG_DEVICE
     double operator()(double x) const
     {
-        return  amplitude* exp( -((x-x00)*(x-x00)/2./sigma_x/sigma_x ));
+        return  m_amp* exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x ));
     }
     DG_DEVICE
     double operator()(double x, double y) const
@@ -299,7 +311,7 @@ struct GaussianX
         return this->operator()(x);
     }
   private:
-    double  x00, sigma_x, amplitude;
+    double  m_x0, m_sigma_x, m_amp;
 
 };
 /**
@@ -314,11 +326,13 @@ struct GaussianY
      * @brief Functor returning a gaussian
      *
      * @param y0 y-center-coordinate
-     * @param sigma_y y - variance
+     * @param sigma_y y - variance (must be !=0)
      * @param amp Amplitude
      */
     GaussianY( double y0, double sigma_y, double amp)
-        : y00(y0), sigma_y(sigma_y), amplitude(amp){}
+        : m_y0(y0), m_sigma_y(sigma_y), m_amp(amp){
+            assert( m_sigma_y != 0  &&  "sigma_x must be !=0 in GaussianY");
+    }
     /**
      * @brief Return the value of the gaussian
      *
@@ -333,10 +347,10 @@ struct GaussianY
     DG_DEVICE
     double operator()(double x, double y) const
     {
-        return  amplitude*exp( -((y-y00)*(y-y00)/2./sigma_y/sigma_y) );
+        return  m_amp*exp( -((y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
   private:
-    double  y00, sigma_y, amplitude;
+    double  m_y0, m_sigma_y, m_amp;
 
 };
 /**
@@ -351,11 +365,13 @@ struct GaussianZ
      * @brief Functor returning a gaussian
      *
      * @param z0 z-center-coordinate
-     * @param sigma_z z - variance
+     * @param sigma_z z - variance (must be !=0)
      * @param amp Amplitude
      */
     GaussianZ( double z0, double sigma_z, double amp)
-        : z00(z0), sigma_z(sigma_z), amplitude(amp){}
+        : m_z0(z0), m_sigma_z(sigma_z), m_amp(amp){
+            assert( m_sigma_z != 0  &&  "sigma_z must be !=0 in GaussianZ");
+    }
     /**
      * @brief Return the value of the gaussian
      *
@@ -369,7 +385,7 @@ struct GaussianZ
     DG_DEVICE
     double operator()( double z) const
     {
-        return  amplitude*exp( -((z-z00)*(z-z00)/2./sigma_z/sigma_z) );
+        return  m_amp*exp( -((z-m_z0)*(z-m_z0)/2./m_sigma_z/m_sigma_z) );
     }
     /**
      * @brief Return the value of the gaussian
@@ -386,10 +402,10 @@ struct GaussianZ
     DG_DEVICE
     double operator()(double x, double y, double z) const
     {
-        return  amplitude*exp( -((z-z00)*(z-z00)/2./sigma_z/sigma_z) );
+        return  m_amp*exp( -((z-m_z0)*(z-m_z0)/2./m_sigma_z/m_sigma_z) );
     }
   private:
-    double  z00, sigma_z, amplitude;
+    double  m_z0, m_sigma_z, m_amp;
 
 };
 /**
@@ -401,10 +417,12 @@ struct IslandXY
     /**
      * @brief Construct Island
      *
-     * @param lambda amplitude
+     * @param lambda amplitude (must be != 0)
      * @param eps y-amplitude
      */
-     IslandXY( double lambda, double eps):lambda_(lambda), eps_(eps){}
+     IslandXY( double lambda, double eps):lambda_(lambda), eps_(eps){
+         assert( lambda != 0 && "Lambda parameter in IslandXY must not be zero!");
+     }
     /**
      * @brief Return profile
      *
@@ -574,15 +592,15 @@ struct InvCoshXsq
      * @param amp amplitude
      * @param kx  kx
      */
-    InvCoshXsq( double amp, double kx):amp_(amp), kx_(kx){}
+    InvCoshXsq( double amp, double kx):m_amp(amp), m_kx(kx){}
     DG_DEVICE
-    double operator()( double x)const{ return amp_/cosh(x*kx_)/cosh(x*kx_);}
+    double operator()( double x)const{ return m_amp/cosh(x*m_kx)/cosh(x*m_kx);}
     DG_DEVICE
     double operator()( double x, double y)const{ return this->operator()(x);}
     DG_DEVICE
     double operator()( double x, double y, double z)const{ return this->operator()(x);}
   private:
-    double amp_,kx_;
+    double m_amp, m_kx;
 };
 /**
  * @brief Sin prof in x-direction
@@ -597,15 +615,15 @@ struct SinProfX
      * @param bamp backgroundamp B
      * @param kx  kx
      */
-    SinProfX( double amp, double bamp, double kx):amp_(amp), bamp_(bamp),kx_(kx){}
+    SinProfX( double amp, double bamp, double kx):m_amp(amp), m_bamp(bamp),m_kx(kx){}
     DG_DEVICE
-    double operator()( double x)const{ return bamp_+amp_*(1.-sin(x*kx_));}
+    double operator()( double x)const{ return m_bamp+m_amp*(1.-sin(x*m_kx));}
     DG_DEVICE
     double operator()( double x, double y)const{ return this->operator()(x);}
     DG_DEVICE
     double operator()( double x, double y, double z)const{ return this->operator()(x);}
   private:
-    double amp_,bamp_,kx_;
+    double m_amp, m_bamp, m_kx;
 };
 /**
  * @brief Exp prof in x-direction
@@ -618,17 +636,19 @@ struct ExpProfX
      *
      * @param amp amplitude B
      * @param bamp backgroundamp A (choose zero for constant gradient length
-     * @param ln  ln
+     * @param ln  ln (must be !=0)
      */
-    ExpProfX( double amp, double bamp, double ln):amp_(amp), bamp_(bamp),ln_(ln){}
+    ExpProfX( double amp, double bamp, double ln):m_amp(amp),m_bamp(bamp),m_ln(ln){
+        assert( ln!=0 && "ln parameter must be != 0 in ExpProfX!");
+    }
     DG_DEVICE
-    double operator()( double x)const{ return bamp_+amp_*exp(-x/ln_);}
+    double operator()( double x)const{ return m_bamp+m_amp*exp(-x/m_ln);}
     DG_DEVICE
     double operator()( double x, double y)const{ return this->operator()(x);}
     DG_DEVICE
     double operator()( double x, double y, double z)const{ return this->operator()(x);}
   private:
-    double amp_,bamp_,ln_;
+    double m_amp, m_bamp, m_ln;
 };
 /**
  * @brief A linear function in x-direction
@@ -797,7 +817,9 @@ struct Heaviside
 struct GaussianDamping
 {
     GaussianDamping( double psimax, double alpha):
-        m_psimax(psimax), m_alpha(alpha) { }
+        m_psimax(psimax), m_alpha(alpha) {
+            assert( alpha!= 0 && "Damping width in GaussianDamping must not be zero");
+        }
     DG_DEVICE
     double operator()(double psi)const
     {
@@ -817,7 +839,7 @@ struct TanhProfX {
      * @brief Construct with xb, width and sign
      *
      * @param xb boundary value
-     * @param width damping width \c alpha
+     * @param width damping width \c alpha (must be !=0)
      * @param sign sign of the Tanh, defines the damping direction
      * @param bgamp background amplitude \c B
      * @param profamp profile amplitude \c A
@@ -825,7 +847,9 @@ struct TanhProfX {
      */
     TanhProfX(double xb, double width, int sign =1,double bgamp = 0.,
         double profamp = 1.) :
-        xb_(xb),w_(width), s_(sign),bga_(bgamp),profa_(profamp)  {}
+        xb_(xb),w_(width), s_(sign),bga_(bgamp),profa_(profamp)  {
+            assert( width != 0&& "Width in TanhProfX must not be zero!");
+    }
     DG_DEVICE
     double operator() (double x)const
     {
@@ -860,12 +884,14 @@ struct PolynomialHeaviside {
      * @brief Construct with xb, width and sign
      *
      * @param xb boundary value
-     * @param a transition width
+     * @param a transition width (must be != 0)
      * @param sign either +1 (original Heaviside) or -1 (the function is mirrored at the \c x=xb axis: f(2xb-x))
      * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
      */
     PolynomialHeaviside(double xb, double a, int sign = +1) :
-        x0(xb), a(a), m_s(sign){}
+        x0(xb), a(a), m_s(sign){
+            assert( a!=0 && "PolynomialHeaviside width must not be zero");
+    }
     DG_DEVICE
     double operator() (double x)const
     {
@@ -901,11 +927,13 @@ struct IPolynomialHeaviside {
      * @brief Construct with xb, width and sign
      *
      * @param xb boundary value
-     * @param a transition width
+     * @param a transition width (must be != 0)
      * @param sign either +1 (original) or -1 (the function is point mirrored at \c x=xb: 2*xb-f(2xb-x))
      */
     IPolynomialHeaviside(double xb, double a, int sign = +1) :
-        x0(xb), a(a), m_s(sign){}
+        x0(xb), a(a), m_s(sign){
+            assert( a!=0 && "IPolynomialHeaviside width must not be zero");
+        }
     DG_DEVICE
     double operator() (double x)const
     {
@@ -944,13 +972,15 @@ struct DPolynomialHeaviside {
      * @brief Construct with xb, width and sign
      *
      * @param xb boundary value
-     * @param a transition width
+     * @param a transition width ( must be !=0)
      * @param sign either +1 (original) or -1 (the function is mirrored at \c x=x0)
      * (since this function is symmetric this parameter is ignored, it's there to be
      * consistent with PolynomialHeaviside)
      */
     DPolynomialHeaviside(double xb, double a, int sign = +1) :
-        x0(xb), a(a){}
+        x0(xb), a(a){
+            assert( a!=0 && "DPolynomialHeaviside width must not be zero");
+    }
     DG_DEVICE
     double operator() (double x)const
     {
@@ -977,7 +1007,7 @@ struct EXP
      * @param amp Amplitude
      * @param lambda coefficient
      */
-    EXP( T amp = 1., T lambda = 1.): amp_(amp), lambda_(lambda){}
+    EXP( T amp = 1., T lambda = 1.): m_amp(amp), m_lambda(lambda){}
     /**
      * @brief return exponential
      *
@@ -988,10 +1018,10 @@ struct EXP
     DG_DEVICE
     T operator() ( T x) const
     {
-        return amp_*exp(lambda_*x);
+        return m_amp*exp(m_lambda*x);
     }
   private:
-    T amp_, lambda_;
+    T m_amp, m_lambda;
 };
 /**
  * @brief natural logarithm
@@ -1218,7 +1248,6 @@ struct POSVALUE
 /**
  * @brief Return a constant
  * \f[ f(x) = c\f]
- *
  */
 struct CONSTANT
 {
@@ -1226,48 +1255,22 @@ struct CONSTANT
      * @brief Construct with a value
      *
      * @param cte the constant value
-     *
      */
-    CONSTANT( double cte): value_(cte){}
+    CONSTANT( double cte): m_value(cte){}
 
-    /**
-     * @brief constant
-     *
-     * @param x
-     *
-     * @return
-     */
     DG_DEVICE
-    double operator()(double x)const{return value_;}
-    /**
-     * @brief constant
-     *
-     * @param x
-     * @param y
-     *
-     * @return
-     */
+    double operator()(double x)const{return m_value;}
     DG_DEVICE
-    double operator()(double x, double y)const{return value_;}
-    /**
-     * @brief constant
-     *
-     * @param x
-     * @param y
-     * @param z
-     *
-     * @return
-     */
+    double operator()(double x, double y)const{return m_value;}
     DG_DEVICE
-    double operator()(double x, double y, double z)const{return value_;}
+    double operator()(double x, double y, double z)const{return m_value;}
     private:
-    double value_;
+    double m_value;
 };
 
 /**
  * @brief Return one
  * \f[ f(x) = 1\f]
- *
  */
 struct ONE
 {
@@ -1281,7 +1284,6 @@ struct ONE
 /**
  * @brief Return zero
  * \f[ f(x) = 0\f]
- *
  */
 struct ZERO
 {
@@ -1808,6 +1810,20 @@ struct Histogram2D
 };
 
 
+/**
+ * @brief Check for NaN
+ * \f[ f(x) = std::isnan(x) \f]
+ */
+template <class T>
+struct ISNAN
+{
+#ifdef __CUDACC__
+    DG_DEVICE bool operator()(T x){return isnan(x);}
+#else
+    bool operator()( T x){ return std::isnan(x);}
+#endif
+};
+
 ///@cond
 namespace detail
 {
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 02de12e45..9c9d2eb9d 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -58,9 +58,10 @@ int main( int argc, char* argv[])
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     //Wall damping has to be constructed before modification (!)
-    HVec damping_profile = feltor::wall_damping( grid, p, gp, mag);
+    HVec damping_profile = dg::evaluate( dg::zero, grid);
     if( p.damping_alpha > 0.)
     {
+        damping_profile = feltor::wall_damping( grid, p, gp, mag);
         double RO=mag.R0(), ZO=0.;
         dg::geo::findOpoint( mag.get_psip(), RO, ZO);
         double psipO = mag.psip()( RO, ZO);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 958a6cf12..6e28c842e 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1401,14 +1401,14 @@ amplitude  & float &0.01   & - & amplitude $A$ of initial perturbation (blob, tu
 sigma      & float &2      & - & Gaussian variance in units of $\rho_s$ \\
 posX       & float &0.3    & - & Gaussian R-position in units of $a$\\
 posY       & float &0.0    & - & Gaussian Z-position in units of $a$ \\
-sigma\_z    & float &0.25   & - & toroidal variance in units of $R_0$ of the fieldline-following initialization \\
+sigma\_z    & float &0.25   & - & toroidal variance in units of $\pi$ of the fieldline-following initialization \\
 k\_psi     & float &0    & - & zonal mode wave number (only for "zonal" initial condition)  \\
 profile & Dict & & & Density profile \\
-\qquad amp& float &4   & - & Profile amplitude $\triangle n_{peak}$ in
+\qquad amp& float &4   & 0 & Profile amplitude $\triangle n_{peak}$ in
 Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}
 \\
-\qquad alpha  & float & 0.2 & - & Transition width $\alpha_p$ in the Heaviside
-at the separatrix
+\qquad alpha  & float & 0.2 & 0.2 & Transition width $\alpha_p$ in the Heaviside
+at the separatrix (must not be zero - even if amp is zero - it is also used for the perturbation)
 \\
 source & dict & & & Density source, cf. the output \texttt{sne\_ifs} in \texttt{feltordiag} (or \texttt{SourceProfile\_ifs} in \texttt{geometry\_diag}) to see how much mass the source with the parameters below generates.  \\
 \qquad rate & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}.
@@ -1420,15 +1420,15 @@ source & dict & & & Density source, cf. the output \texttt{sne\_ifs} in \texttt{
     See the file {\tt init.h} to add your own custom source.
 \\
 \qquad boundary & float & 0.2  & 0.2 & Source region boundary $\rho_{p,b}$: yields in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
-\qquad alpha  & float & 0.2 & - & Transition width $\alpha_p$ in the Heaviside
+\qquad alpha  & float & 0.2 & 0.2 & Transition width $\alpha_p$ in the Heaviside
 in the density Eq.~\eqref{eq:density_profile} (with $rho_{p,b}=0$ and source profiles Eq.~\eqref{eq:electron_source} (should be
-small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes)
+small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes, must not be zero)
 \\
 damping & dict & & & magnetic and density damping region \\
 \qquad rate & float & 0    & 0   & Friction coefficient $\omega_d$ in density and velocity damping Eq.~\eqref{eq:velocity_source} \\
 \qquad boundary & float & 0.2  & 1.2 & Modification region boundary $\rho_{p,b}$: yields $\psi_0 = (1-\rho_{p,b}^2)\psi_{p,\min}$ in Eq.~\eqref{eq:modified_psip}.
 \\
-\qquad alpha   & float & 0.05 & - & Transition width $\alpha_p$: yields
+\qquad alpha   & float & 0.25 & 0 & Transition width $\alpha_p$: yields
 $\alpha=-2\rho_{p,b}\alpha_p+\alpha_p^2)\psi_{p,\min}$ for the Heaviside in the modified
 $\psi_p$ function \eqref{eq:modified_psip}. If zero, then we do not modify the
 magnetic field and damping is ignored.\\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 228aa506d..ce36238c4 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -150,9 +150,10 @@ int main( int argc, char* argv[])
 
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     //Wall damping has to be constructed before modification (!)
-    HVec damping_profile = feltor::wall_damping( grid, p, gp, mag);
+    HVec damping_profile = dg::evaluate( dg::zero, grid);
     if( p.damping_alpha > 0.)
     {
+        damping_profile = feltor::wall_damping( grid, p, gp, mag);
         double RO=mag.R0(), ZO=0.;
         dg::geo::findOpoint( mag.get_psip(), RO, ZO);
         double psipO = mag.psip()( RO, ZO);
diff --git a/src/feltor/init.h b/src/feltor/init.h
index aff82d192..a4ab868c4 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -40,6 +40,8 @@ HVec circular_damping( const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
+    if( p.profile_alpha == 0)
+        throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
     HVec circular = dg::pullback( dg::compose(
                 dg::PolynomialHeaviside( gp.a, gp.a*p.profile_alpha/2., -1),
                 Radius( mag.R0(), 0.)), grid);
@@ -66,9 +68,12 @@ HVec profile_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
+    if( p.profile_alpha == 0)
+        throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
     HVec profile_damping = dg::pullback( dg::compose(dg::PolynomialHeaviside(
         1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)), grid);
-    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag), profile_damping, profile_damping);
+    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag),
+        profile_damping, profile_damping);
     return profile_damping;
 }
 HVec profile(const Geometry& grid,
@@ -87,6 +92,8 @@ HVec source_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
+    if( p.source_alpha == 0)
+        throw dg::Error(dg::Message()<< "Invalid parameter: source alpha must not be 0\n");
     HVec source_damping = dg::pullback(
         dg::compose(dg::PolynomialHeaviside(
             p.source_boundary-p.source_alpha/2.,
@@ -124,6 +131,8 @@ HVec wall_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
+    if( p.source_alpha == 0)
+        throw dg::Error(dg::Message()<< "Invalid parameter: damping alpha must not be 0\n");
     HVec wall_damping = dg::pullback(dg::compose( dg::PolynomialHeaviside(
         p.damping_boundary+p.damping_alpha/2., p.damping_alpha/2., +1),
                 dg::geo::RhoP(mag)), grid);
@@ -148,7 +157,11 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
+            if( p.sigma_z == 0)
+                throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in straight blob initial condition\n");
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
+            if( p.sigma == 0)
+                throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 in straight blob initial condition\n");
             dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
             if( p.symmetric)
                 ntilde = dg::pullback( init0, grid);
@@ -176,7 +189,11 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
+            if( p.sigma_z == 0)
+                throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in straight blob initial condition\n");
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
+            if( p.sigma == 0)
+                throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 in straight blob initial condition\n");
             dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
             if( p.symmetric)
                 ntilde = dg::pullback( init0, grid);
@@ -204,6 +221,8 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
+            if( p.sigma_z == 0)
+                throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in turbulence initial condition\n");
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
             dg::BathRZ init0(16,16,grid.x0(),grid.y0(), 30.,2.,p.amp);
             if( p.symmetric)
@@ -249,6 +268,8 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             const Geometry& grid, const feltor::Parameters& p,
             const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
+            if( p.sigma == 0)
+                throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 in turbulence on gaussian\n");
             dg::Gaussian prof( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
                 p.sigma, p.nprofamp);
             std::array<std::array<DVec,2>,2> y0;
@@ -256,6 +277,8 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 dg::pullback( prof, grid) );
 
             HVec ntilde = dg::evaluate(dg::zero,grid);
+            if( p.sigma_z == 0)
+                throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in turbulence on gaussian\n");
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
             dg::BathRZ init0(16,16,grid.x0(),grid.y0(), 30.,2.,p.amp);
             if( p.symmetric)
@@ -314,6 +337,8 @@ std::map<std::string, std::function< HVec(
         Geometry& grid, const feltor::Parameters& p,
         const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
+            if( p.sigma == 0)
+                throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 for torpex source profile\n");
             dg::Gaussian prof( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
                 p.sigma, p.nprofamp);
             ne_profile = dg::pullback( prof, grid);
@@ -331,6 +356,8 @@ std::map<std::string, std::function< HVec(
         const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = false;
+            if( p.sigma == 0)
+                throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 for gaussian source profile\n");
             dg::Gaussian prof( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
                 p.sigma, 1.);
             return dg::pullback( prof, grid);
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 19d28e0b0..07424c2fb 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -94,15 +94,15 @@ struct Parameters
         sigma_z     = js["sigma_z"].asDouble();
         k_psi       = js["k_psi"].asDouble();
 
-        nprofamp   = js["profile"]["amp"].asDouble();
-        profile_alpha = js["profile"]["alpha"].asDouble();
+        nprofamp   = js["profile"].get("amp", 0.).asDouble();
+        profile_alpha = js["profile"].get("alpha", 0.2).asDouble();
 
         source_rate     = js["source"].get("rate", 0.).asDouble();
         source_type     = js["source"].get("type", "profile").asString();
         source_boundary = js["source"].get("boundary", 0.2).asDouble();
         source_alpha    = js["source"].get("alpha", 0.2).asDouble();
         damping_rate = js["damping"].get("rate", 0.).asDouble();
-        damping_alpha= js["damping"].get("alpha", 0.05).asDouble();
+        damping_alpha= js["damping"].get("alpha", 0.).asDouble();
         damping_boundary = js["damping"].get("boundary", 1.2).asDouble();
 
         bcxN = dg::str2bc(js["bc"]["density"][0].asString());
-- 
GitLab


From 4d33275857f2874b50dca3047b612d41695325b3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 12 May 2020 22:48:41 +0200
Subject: [PATCH 220/540] Fix feltordiag bugs in source terms

- maybe think about parallel momentum source (can we observe intrinsic
rotation with this?)
---
 src/feltor/feltor.h     | 19 ++++++++++++++-----
 src/feltor/feltor.tex   | 11 ++++++-----
 src/feltor/feltordiag.h | 34 +++++++++++++++++++---------------
 3 files changed, 39 insertions(+), 25 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index f990409a4..284a69d0e 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -199,9 +199,15 @@ struct Explicit
     const Container& density(int i)const{
         return m_fields[0][i];
     }
+    const Container& density_source(int i)const{
+        return m_s[0][i];
+    }
     const Container& velocity(int i)const{
         return m_fields[1][i];
     }
+    const Container& velocity_source(int i)const{
+        return m_s[1][i];
+    }
     const Container& potential(int i) const {
         return m_phi[i];
     }
@@ -220,11 +226,6 @@ struct Explicit
     const std::array<Container, 3> & gradA () const {
         return m_dA;
     }
-    void compute_gradS( int i, std::array<Container,3>& gradS) const{
-        dg::blas2::symv( m_dx_N, m_s[0][i], gradS[0]);
-        dg::blas2::symv( m_dy_N, m_s[1][i], gradS[1]);
-        if(!m_p.symmetric)dg::blas2::symv( m_dz, m_s[2][i], gradS[2]);
-    }
     const Container & dsN (int i) const {
         return m_dsN[i];
     }
@@ -237,6 +238,14 @@ struct Explicit
     const Container & dssU(int i) {
         return m_dssU[i];
     }
+    void compute_gradSN( int i, std::array<Container,3>& gradS) const{
+        // MW: don't like this function, if we need more gradients we might
+        // want a more flexible solution
+        // grad S_ne and grad S_ni
+        dg::blas2::symv( m_dx_N, m_s[0][i], gradS[0]);
+        dg::blas2::symv( m_dy_N, m_s[0][i], gradS[1]);
+        if(!m_p.symmetric)dg::blas2::symv( m_dz, m_s[0][i], gradS[2]);
+    }
     const Container & compute_dppN(int i) { //2nd varphi derivative
         dg::blas2::symv( m_dz, m_fields[0][i], m_temp0);
         dg::blas2::symv( m_dz, m_temp0, m_temp1);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 6e28c842e..d3ad3e768 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -606,6 +606,7 @@ accordingly.
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
 \subsection{Conservative form}
+%MW: don't we have a momentum source in the form we currently give?
 We scale all spatial lengths by $\rho_s = \sqrt{T_e m_i}/(eB_0)$ and time by the ion gyro-frequency $\Omega_0 = eB_0/m_i$.
 The magnetic field is scaled with $B_0$, densities with $n_0$ and the parallel velocity is scaled with $c_s = \sqrt{T_e/m_i}$.
 The potential is scaled with $\hat \phi = e/T_e$ and the vector potential with
@@ -999,7 +1000,7 @@ The relevant terms in the output file are
     jsneE\_tt & $ n_e \vec u_E\cn\psi_p$ \\
     lneperp\_tt &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ &
     lneparallel\_tt &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
-    sne & $S_{n_e}$ & \\
+    sne\_tt & $S_{n_e}$ & \\
 \bottomrule
 \end{longtable}
 
@@ -1063,8 +1064,8 @@ The relevant terms in the output file are
     ue2   &$z_i\mu_i N_i u_E^2 /2$ \\
     neue2 &$ z_e\mu_e n_e u_e^2/2$ \\
     niui2 &$ z_i\mu_i N_i U_i^2/2$ \\
-    see & $z_e(\tau_e (1+\ln n_e) + \phi + \frac{1}{2}\mu_e u_e^2) S_{n_e} $ \\
-    sei & $z_i(\tau_i (1+\ln N_i) + \psi + \frac{1}{2}\mu_i U_i^2) S_{N_i} $ \\
+    see\_tt & $z_e(\tau_e (1+\ln n_e) + \phi + \frac{1}{2}\mu_e u_e^2) S_{n_e} $ \\
+    sei\_tt & $z_i(\tau_i (1+\ln N_i) + \psi + \frac{1}{2}\mu_i U_i^2) S_{N_i} $ \\
     resistivity\_tt &-$\eta_\parallel n_e^2 (U_i-u_e)^2$ \\
     jsee\_tt &$z_e(\tau_e \ln n_e + \mu_e u_e^2/2 + \phi)n_e(\vec u_E + \vec u_C + \vec u_K)\cn \psi_p
         + z_e \tau_e n_e u_e^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
@@ -1199,7 +1200,7 @@ where we neglected the term $\RA{n\vec u\cn \mathcal I_0}$ in the continuity equ
 \textbf{Name} &  \textbf{Equation}\\
 \midrule
     nei0 &$n_e \mathcal I_0$ &
-    snei0 & $S_{n_e } \mathcal I_0$ \\
+    snei0\_tt & $S_{n_e } \mathcal I_0$ \\
 \bottomrule
 \end{longtable}
 
@@ -1410,7 +1411,7 @@ Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}
 \qquad alpha  & float & 0.2 & 0.2 & Transition width $\alpha_p$ in the Heaviside
 at the separatrix (must not be zero - even if amp is zero - it is also used for the perturbation)
 \\
-source & dict & & & Density source, cf. the output \texttt{sne\_ifs} in \texttt{feltordiag} (or \texttt{SourceProfile\_ifs} in \texttt{geometry\_diag}) to see how much mass the source with the parameters below generates.  \\
+source & dict & & & Density source, cf. the output \texttt{sne\_tt\_ifs} in \texttt{feltordiag} (or \texttt{SourceProfile\_ifs} in \texttt{geometry\_diag}) to see how much mass the source with the parameters below generates and compare to \texttt{jsne\_tt\_fsa} to see how much mass is lost.  \\
 \qquad rate & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}.
 \\
 \qquad type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 18b267174..0882f5a70 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -173,6 +173,8 @@ struct Record_static{
 ///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 //Here is a list of static (time-independent) 3d variables that go into the output
+//Except xc, yc, and zc these are redundant since we have geometry_diag.cu
+//MW: maybe it's a test of sorts
 std::vector<Record_static> diagnostics3d_static_list = {
     { "BR", "R-component of magnetic field in cylindrical coordinates",
         []( HVec& result, Variables& v, Geometry& grid){
@@ -278,6 +280,8 @@ std::vector<Record> diagnostics3d_list = {
 };
 
 //Here is a list of static (time-independent) 2d variables that go into the output
+//MW: These are redundant since we have geometry_diag.cu -> remove ? if geometry_diag works as expected (I guess it can also be a test of sorts)
+//MW: if they stay they should be documented in feltor.tex
 //( we make 3d variables here but only the first 2d slice is output)
 std::vector<Record_static> diagnostics2d_static_list = {
     { "Psip2d", "Flux-function psi",
@@ -508,9 +512,9 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., result);
         }
     },
-    {"sne", "Source term for electron density", false,
+    {"sne_tt", "Source term for electron density", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::copy( v.f.sources()[0][0], result);
+            dg::blas1::copy( v.f.density_source(0), result);
         }
     },
     /// ------------------- Energy terms ------------------------//
@@ -572,21 +576,21 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( -v.p.eta, result, result, 0., result);
         }
     },
-    {"see", "Energy sink/source for electrons", false,
+    {"see_tt", "Energy sink/source for electrons", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
                 v.f.density(0), v.f.velocity(0), v.f.potential(0),
-                v.f.sources()[0][0]
+                v.f.density_source(0)
             );
         }
     },
-    {"sei", "Energy sink/source for ions", false,
+    {"sei_tt", "Energy sink/source for ions", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
                 v.f.density(1), v.f.velocity(1), v.f.potential(1),
-                v.f.sources()[0][1]
+                v.f.density_source(1)
             );
         }
     },
@@ -825,12 +829,12 @@ std::vector<Record> diagnostics2d_list = {
         []( DVec& result, Variables& v){
             routines::dot( v.f.gradP(0), v.gradPsip, result);
             dg::blas1::pointwiseDot( 1., result, v.f.binv(), v.f.binv(), 0., result);
-            dg::blas1::pointwiseDot( v.p.mu[1], result, v.f.sources()[0][0], 0., result);
+            dg::blas1::pointwiseDot( v.p.mu[1], result, v.f.density_source(0), 0., result);
         }
     },
     {"sospi_tt", "Diamagnetic vorticity source term with electron source", true,
         []( DVec& result, Variables& v){
-            v.f.compute_gradS( 0, v.tmp);
+            v.f.compute_gradSN( 0, v.tmp);
             routines::dot( v.tmp, v.gradPsip, result);
             dg::blas1::scal( result, v.p.mu[1]*v.p.tau[1]);
         }
@@ -905,24 +909,24 @@ std::vector<Record> diagnostics2d_list = {
     {"sparsni_tt", "Parallel momentum source by density source", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.p.mu[1],
-                v.f.sources()[0][1], v.f.velocity(1), 0., result);
+                v.f.density_source(1), v.f.velocity(1), 0., result);
         }
     },
     {"sparsnibphi_tt", "Parallel angular momentum source by density source", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.p.mu[1],
-                v.f.sources()[0][1], v.f.velocity(0), v.f.bphi(), 0., result);
+                v.f.density_source(1), v.f.velocity(1), v.f.bphi(), 0., result);
         }
     },
     /// --------------------- Mirror force term ---------------------------//
     {"sparmirrore_tt", "Mirror force term with electron density (Time average)", true,
         []( DVec& result, Variables& v){
-            dg::blas1::pointwiseDot( -v.p.mu[0], v.f.divb(), v.f.density(0), 0., result);
+            dg::blas1::pointwiseDot( -v.p.tau[0], v.f.divb(), v.f.density(0), 0., result);
         }
     },
     {"sparmirrori_tt", "Mirror force term with ion density (Time average)", true,
         []( DVec& result, Variables& v){
-            dg::blas1::pointwiseDot( v.p.mu[1], v.f.divb(), v.f.density(1), 0., result);
+            dg::blas1::pointwiseDot( v.p.tau[1], v.f.divb(), v.f.density(1), 0., result);
         }
     },
     /// --------------------- Lorentz force terms ---------------------------//
@@ -959,10 +963,10 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( v.f.density(0), result, result);
         }
     },
-    {"snei0", "inertial factor source", false,
+    {"snei0_tt", "inertial factor source", true,
         []( DVec& result, Variables& v ) {
             result = dg::pullback( dg::geo::Hoo( v.mag), v.f.grid());
-            dg::blas1::pointwiseDot( v.f.sources()[0][0], result, result);
+            dg::blas1::pointwiseDot( v.f.density_source(0), result, result);
         }
     },
 
@@ -1001,7 +1005,7 @@ std::vector<Record> restart3d_list = {
 };
 // These two lists signify the quantities involved in accuracy computation
 std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
-std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt", "see", "sei"};
+std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt", "see_tt", "sei_tt"};
 
 template<class Container>
 void slice_vector3d( const Container& transfer, Container& transfer2d, size_t local_size2d)
-- 
GitLab


From 6dbea4fb9080520992d68395be9a9bc8dbe125e4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 May 2020 22:14:34 +0200
Subject: [PATCH 221/540] Add Json utilities in file

- without using them yet
---
 inc/dg/file/file.h           |  3 ++
 inc/dg/file/json_utilities.h |  3 ++
 inc/file/Doxyfile            |  4 +-
 inc/file/json_utilities.h    | 76 ++++++++++++++++++++++++++++++++++++
 inc/file/nc_utilities.h      | 40 +++++++++++++++++++
 5 files changed, 124 insertions(+), 2 deletions(-)
 create mode 100644 inc/dg/file/file.h
 create mode 100644 inc/dg/file/json_utilities.h
 create mode 100644 inc/file/json_utilities.h

diff --git a/inc/dg/file/file.h b/inc/dg/file/file.h
new file mode 100644
index 000000000..d26c0caa2
--- /dev/null
+++ b/inc/dg/file/file.h
@@ -0,0 +1,3 @@
+#pragma once
+#include "nc_utilities.h"
+#include "json_utilities.h"
diff --git a/inc/dg/file/json_utilities.h b/inc/dg/file/json_utilities.h
new file mode 100644
index 000000000..549d01aaa
--- /dev/null
+++ b/inc/dg/file/json_utilities.h
@@ -0,0 +1,3 @@
+#pragma once
+#define _FILE_JSON_INCLUDED_BY_DG_
+#include "../../file/json_utilities.h"
diff --git a/inc/file/Doxyfile b/inc/file/Doxyfile
index 65bbf86dd..d7047966e 100644
--- a/inc/file/Doxyfile
+++ b/inc/file/Doxyfile
@@ -32,7 +32,7 @@ DOXYFILE_ENCODING      = UTF-8
 # title of most generated pages and in a few other places.
 # The default value is: My Project.
 
-PROJECT_NAME           = "Extension: Netcdf output utilities"
+PROJECT_NAME           = "Extension: Json and NetCDF utilities"
 
 # The PROJECT_NUMBER tag can be used to enter a project or revision number. This
 # could be handy for archiving the generated documentation or if some version
@@ -44,7 +44,7 @@ PROJECT_NUMBER         =
 # for a project that appears at the top of each page and should give viewer a
 # quick idea about the purpose of the project. Keep the description short.
 
-PROJECT_BRIEF          = "Facilitate the creation of coordinate and time variables partly following CF conventions in \"dg/file/nc_utilities.h\""
+PROJECT_BRIEF          = "Facilitate the handling of Json and NetCDF files in \"dg/file/file.h\""
 
 # With the PROJECT_LOGO tag one can specify a logo or an icon that is included
 # in the documentation. The maximum height of the logo should not exceed 55
diff --git a/inc/file/json_utilities.h b/inc/file/json_utilities.h
new file mode 100644
index 000000000..29e0f1dfb
--- /dev/null
+++ b/inc/file/json_utilities.h
@@ -0,0 +1,76 @@
+#pragma once
+
+#include <iostream>
+#include <fstream>
+#include <string>
+
+#include "json/json.h"
+/*!@file
+ *
+ * Json utility functions
+ */
+
+//Note that the json utilities are separate from netcdf utilities because
+//of the different dependencies that they incur
+namespace file
+{
+
+/**
+ * @brief Convenience wrapper to open a file and parse it into a Json::Value
+ *
+ * @attention This function will print an error message to \c std::cerr and brutally call \c exit(EXIT_FAILURE) on any error that occurs.
+ * @note included in \c json_utilities.h
+ * @param filename Name of the JSON file to parse
+ * @param js Contains all the found Json variables on output
+ * @param mode Either "default" in which case comments are allowed or "strict" in which case they are not
+ */
+static inline void file2Json( std::string filename, Json::Value& js, std::string mode = "default")
+{
+    Json::CharReaderBuilder parser;
+    if( "strict" == mode )
+        Json::CharReaderBuilder::strictMode( &parser.settings_);
+    else
+        Json::CharReaderBuilder::setDefaults( &parser.settings_);
+
+    std::ifstream isI( filename);
+    if( !isI.good())
+    {
+        std::cerr << "\nAn error occured while parsing "<<filename<<"\n";
+        std::cerr << "*** File does not exist! *** \n\n";
+        exit( EXIT_FAILURE);
+    }
+    std::string errs;
+    if( !parseFromStream( parser, isI, &js, &errs))
+    {
+        std::cerr << "An error occured while parsing "<<filename<<"\n"<<errs;
+        exit( EXIT_FAILURE);
+    }
+}
+/**
+ * @brief Convenience wrapper to parse a string into a Json::Value
+ *
+ * Parse a string into a Json Value
+ * @attention This function will print an error message to \c std::cerr and brutally call \c exit(EXIT_FAILURE) on any error that occurs.
+ * @note included in \c json_utilities.h
+ * @param filename (a string to print when an error occurs, has no further use)
+ * @param input The string to interpret as a Json string
+ * @param js Contains all the found Json variables on output
+ * @param mode Either "default" in which case comments are allowed or "strict" in which case they are not
+ */
+static inline void string2Json( std::string filename, std::string input, Json::Value& js, std::string mode = "default")
+{
+    Json::CharReaderBuilder parser;
+    if( "strict" == mode )
+        Json::CharReaderBuilder::strictMode( &parser.settings_);
+    else
+        Json::CharReaderBuilder::setDefaults( &parser.settings_);
+    std::string errs;
+    std::stringstream ss(input);
+    if( !parseFromStream( parser, ss, &js, &errs) )
+    {
+        std::cerr << "An error occured while parsing "<<filename<<"\n"<<errs;
+        exit( EXIT_FAILURE);
+    }
+}
+
+}//namespace file
diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index 44c0c67bc..baa8781d0 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -50,6 +50,7 @@ inline int put_var_T<double>( int ncid, int varID, double* data){
 }
 ///@endcond
 
+///@copydoc define_time
 template<class T>
 inline int define_real_time( int ncid, const char* name, int* dimID, int* tvarID)
 {
@@ -307,4 +308,43 @@ inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRe
 }
 #endif //MPI_VERSION
 
+/*! @brief Read a netcdf string attribute into a std::string
+ *
+ * Open file, look for attribute, read attribute in string and close the file at the end.
+ * @param filename Name of the netcdf file to parse
+ * @param att_name Name of the netcdf attribute in filename
+ * @param att Contains the content of \c att_name on output
+    */
+static inline void netcdf2string( std::string filename, std::string att_name, std::string& att)
+{
+    int ncid;
+    int status = nc_open( filename.data(), NC_NOWRITE, &ncid);
+    if( status != NC_NOERR)
+    {
+        std::cerr << "\nAn error occured opening file "<<filename<<"\n";
+        std::cerr << nc_strerror(status) <<std::endl;
+        exit( EXIT_FAILURE);
+    }
+    size_t length;
+    status = nc_inq_attlen( ncid, NC_GLOBAL, att_name.data(), &length);
+    if( status != NC_NOERR)
+    {
+        std::cerr << "\nAn error occured parsing file *"<<filename<<"* for attribute *"<<att_name<<"*\n";
+        std::cerr << nc_strerror(status)<<std::endl;
+        nc_close(ncid);
+        exit( EXIT_FAILURE);
+    }
+    att.resize( length, 'x');
+    status = nc_get_att_text( ncid, NC_GLOBAL, att_name.data(), &att[0]);
+    if( status != NC_NOERR)
+    {
+        std::cerr << "\nAn error occured parsing file *"<<filename<<"* for attribute *"<<att_name<<"*\n";
+        std::cerr << nc_strerror(status)<<std::endl;
+        nc_close(ncid);
+        exit( EXIT_FAILURE);
+    }
+    nc_close(ncid);
+}
+
+
 } //namespace file
-- 
GitLab


From a5c36268b08650be05c8633552c7c4b6234f90e7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 May 2020 22:16:43 +0200
Subject: [PATCH 222/540] Make geometry_diag use new Json utilities

- makes it more resilient to file errors
---
 inc/geometries/geometry_diag.cu | 57 +++++++++++----------------------
 1 file changed, 19 insertions(+), 38 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 6e4071719..6f14d8e0b 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -7,10 +7,8 @@
 #include <ctime>
 #include <cmath>
 
-#include "json/json.h"
-
 #include "dg/algorithm.h"
-#include "dg/file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "solovev.h"
 //#include "taylor.h"
@@ -80,53 +78,36 @@ struct Parameters
     }
 };
 
+
 int main( int argc, char* argv[])
 {
-    if( !(argc == 4 || argc == 3))
-    {
-        std::cerr << "ERROR: Wrong number of arguments!\n";
-        std::cerr << " Usage: "<< argv[0]<<" [input.json] [geom.json] [output.nc]\n";
-        std::cerr << " ( Minimum input json file is { \"n\" : 3, \"Nx\": 100, \"Ny\":100 })\n";
-        std::cerr << "Or \n Usage: "<< argv[0]<<" [file.nc] [output.nc]\n";
-        std::cerr << " ( Program searches for string variables 'inputfile' and 'geomfile' in file.nc and tries a json parser)\n";
-        return -1;
-    }
     std::string newfilename;
     Json::Value input_js, geom_js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc == 4)
     {
         newfilename = argv[3];
         std::cout << argv[0]<< " "<<argv[1]<<" & "<<argv[2]<<" -> " <<argv[3]<<std::endl;
-        std::ifstream isI( argv[1]);
-        std::ifstream isG( argv[2]);
-        parseFromStream( parser, isI, &input_js, &errs); //read input without comments
-        parseFromStream( parser, isG, &geom_js, &errs); //read input without comments
+        file::file2Json( argv[1], input_js, "strict");
+        file::file2Json( argv[2], geom_js, "strict");
     }
-    else
+    else if( argc == 3)
     {
         newfilename = argv[2];
         std::cout << argv[0]<< " "<<argv[1]<<" -> " <<argv[2]<<std::endl;
-        //////////////////////////open nc file//////////////////////////////////
-        file::NC_Error_Handle err;
-        int ncid;
-        err = nc_open( argv[1], NC_NOWRITE, &ncid);
-        ///////////////read in and show inputfile und geomfile//////////////////
-        std::string input, geom;
-        size_t length;
-        err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-        input.resize( length, 'x');
-        err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
-        err = nc_inq_attlen( ncid, NC_GLOBAL, "geomfile", &length);
-        geom.resize( length, 'x');
-        err = nc_get_att_text( ncid, NC_GLOBAL, "geomfile", &geom[0]);
-        nc_close( ncid);
-        std::stringstream ss( input);
-        parseFromStream( parser, ss, &input_js, &errs); //read input without comments
-        ss.str( geom);
-        parseFromStream( parser, ss, &geom_js, &errs); //read input without comments
+        std::string temp;
+        file::netcdf2string( argv[1], "inputfile", temp);
+        file::string2Json( argv[1], temp, input_js, "strict");
+        file::netcdf2string( argv[1], "geomfile", temp);
+        file::string2Json( argv[1], temp, geom_js, "strict");
+    }
+    else
+    {
+        std::cerr << "ERROR: Wrong number of arguments!\n";
+        std::cerr << " Usage: "<< argv[0]<<" [input.json] [geom.json] [output.nc]\n";
+        std::cerr << " ( Minimum input json file is { \"n\" : 3, \"Nx\": 100, \"Ny\":100 })\n";
+        std::cerr << "Or \n Usage: "<< argv[0]<<" [file.nc] [output.nc]\n";
+        std::cerr << " ( Program searches for string variables 'inputfile' and 'geomfile' in file.nc and tries a json parser)\n";
+        return -1;
     }
     const Parameters p(input_js);
     const dg::geo::solovev::Parameters gp(geom_js);
-- 
GitLab


From e6a9d43637a2e47fd356a6113e17b841e17edd7b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 May 2020 23:31:25 +0200
Subject: [PATCH 223/540] Great modernizing of Json handling

- delete feltor::steadystate
---
 src/ep/toeflR.cu                              |  21 +-
 src/ep/toefl_mpi.cu                           |  14 +-
 .../{window_params.js => window_params.json}  |   0
 src/feltor/Makefile                           |   5 +-
 src/feltor/feltor.cu                          |  19 +-
 src/feltor/feltor.h                           |   5 +-
 src/feltor/feltor_hpc.cu                      |  11 +-
 src/feltor/feltordiag.cu                      |  34 +--
 src/feltor/init_from_file.h                   |  27 +--
 src/feltor/interpolate_in_3d.cu               |  44 ++--
 src/feltor/manufactured.cu                    |  19 +-
 src/feltor/steadystate.cu                     | 195 ------------------
 .../{window_params.js => window_params.json}  |   0
 src/feltorSH/feltor.cu                        |  21 +-
 src/feltorSH/feltor_hpc.cu                    |  18 +-
 src/feltorSH/feltor_mpi.cu                    |  14 +-
 .../input/{default.in => default.json}        |   0
 .../{verification.in => verification.json}    |   0
 .../{window_params.js => window_params.json}  |   0
 src/feltorSHp/feltor.cu                       |  21 +-
 src/feltorSHp/feltor_hpc.cu                   |  17 +-
 src/feltorSHp/feltor_mpi.cu                   |  14 +-
 .../input/{default.in => default.json}        |   0
 .../{verification.in => verification.json}    |   0
 .../{window_params.js => window_params.json}  |   0
 src/feltorSesol/feltor.cu                     |  23 +--
 src/feltorSesol/feltor_hpc.cu                 |  37 +---
 src/feltorSesol/feltor_mpi.cu                 |  31 +--
 .../input/{default.in => default.json}        |   0
 .../input/{default_fd.in => default_fd.json}  |   0
 src/feltorSesol/input/input.txt               |  53 -----
 .../{window_params.js => window_params.json}  |   0
 src/feltorShw/feltor.cu                       |  22 +-
 src/feltorShw/feltor_hpc.cu                   |  38 +---
 src/feltorShw/feltor_mpi.cu                   |  33 +--
 .../input/{default.in => default.json}        |   0
 .../{window_params.js => window_params.json}  |   0
 src/hasegawa/hw.cu                            |  20 +-
 src/hasegawa/mima.cu                          |  21 +-
 .../{window_params.js => window_params.json}  |   0
 ...eometry_params.js => geometry_params.json} |   0
 src/heat/heat.cu                              |  17 +-
 src/heat/heat_hpc.cu                          |  33 ++-
 .../{window_params.js => window_params.json}  |   0
 src/impurities/README                         |   4 -
 src/impurities/toeflI.cu                      |  21 +-
 src/impurities/toefl_hpc.cu                   |  13 +-
 src/impurities/toefl_mpi.cu                   |  16 +-
 .../{window_params.js => window_params.json}  |   0
 src/lamb_dipole/shu_b.cu                      |  11 +-
 src/lamb_dipole/shu_hpc.cu                    |  12 +-
 src/polar/polar.cu                            |  14 +-
 src/polar/polar_mpi.cu                        |  14 +-
 src/reco2D/reconnection.cu                    |  20 +-
 src/reco2D/reconnection_hpc.cu                |  16 +-
 src/reco2D/reconnection_mpi.cu                |  16 +-
 .../{window_params.js => window_params.json}  |   0
 src/toefl/toeflR.cu                           |  17 +-
 src/toefl/toefl_hpc.cu                        |  10 +-
 .../{window_params.js => window_params.json}  |   0
 60 files changed, 194 insertions(+), 817 deletions(-)
 rename src/ep/{window_params.js => window_params.json} (100%)
 delete mode 100644 src/feltor/steadystate.cu
 rename src/feltor/{window_params.js => window_params.json} (100%)
 rename src/feltorSH/input/{default.in => default.json} (100%)
 rename src/feltorSH/input/{verification.in => verification.json} (100%)
 rename src/feltorSH/{window_params.js => window_params.json} (100%)
 rename src/feltorSHp/input/{default.in => default.json} (100%)
 rename src/feltorSHp/input/{verification.in => verification.json} (100%)
 rename src/feltorSHp/{window_params.js => window_params.json} (100%)
 rename src/feltorSesol/input/{default.in => default.json} (100%)
 rename src/feltorSesol/input/{default_fd.in => default_fd.json} (100%)
 delete mode 100644 src/feltorSesol/input/input.txt
 rename src/feltorSesol/{window_params.js => window_params.json} (100%)
 rename src/feltorShw/input/{default.in => default.json} (100%)
 rename src/feltorShw/{window_params.js => window_params.json} (100%)
 rename src/hasegawa/{window_params.js => window_params.json} (100%)
 rename src/heat/geometry/{geometry_params.js => geometry_params.json} (100%)
 rename src/heat/{window_params.js => window_params.json} (100%)
 rename src/impurities/{window_params.js => window_params.json} (100%)
 rename src/reco2D/{window_params.js => window_params.json} (100%)
 rename src/toefl/{window_params.js => window_params.json} (100%)

diff --git a/src/ep/toeflR.cu b/src/ep/toeflR.cu
index 73e9172c3..a0f1fce0d 100644
--- a/src/ep/toeflR.cu
+++ b/src/ep/toeflR.cu
@@ -8,14 +8,9 @@
 
 #include "toeflR.cuh"
 #include "dg/algorithm.h"
+#include "dg/file/json_utilities.h"
 #include "parameters.h"
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - directly visualizes results on the screen using parameters in window_params.txt
-*/
-
 
 int main( int argc, char* argv[])
 {
@@ -23,15 +18,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json("input.json", js, "default");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json(argv[1], js, "default");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -40,9 +29,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json("window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/ep/toefl_mpi.cu b/src/ep/toefl_mpi.cu
index 0150e7231..5c6527a6e 100644
--- a/src/ep/toefl_mpi.cu
+++ b/src/ep/toefl_mpi.cu
@@ -5,10 +5,10 @@
 #include <mpi.h> //activate mpi
 
 #include "netcdf_par.h"
-#include "file/nc_utilities.h"
 
-#include "toeflR.cuh"
 #include "dg/algorithm.h"
+#include "dg/file/file.h"
+#include "toeflR.cuh"
 #include "parameters.h"
 
 
@@ -52,19 +52,13 @@ int main( int argc, char* argv[])
     MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3)
     {
         if(rank==0)std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
         return -1;
     }
-    else 
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+    else
+        file::file2Json( argv[1], js, "strict");
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const Parameters p( js);
     if(rank==0)p.display( std::cout);
diff --git a/src/ep/window_params.js b/src/ep/window_params.json
similarity index 100%
rename from src/ep/window_params.js
rename to src/ep/window_params.json
diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index 1d6856200..0a93b91da 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -8,7 +8,7 @@ include ../../config/devices/devices.mk
 INCLUDE+= -I../         # other src libraries
 INCLUDE+= -I../../inc   # other project libraries
 
-all: feltor_hpc manufactured
+all: feltor_hpc feltor feltor_mpi manufactured feltordiag
 
 manufactured: manufactured.cu manufactured.h feltor.h implicit.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
@@ -21,9 +21,6 @@ interpolate_in_3d: interpolate_in_3d.cu feltordiag.h
 feltor: feltor.cu feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
-steadystate: steadystate.cu feltor.h implicit.h init.h parameters.h init_from_file.h
-	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
-
 feltor_hpc: feltor_hpc.cu feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
 
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 9c9d2eb9d..ae5809250 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -5,6 +5,7 @@
 #include <cmath>
 
 #include "draw/host_window.h"
+#include "dg/file/json_utilities.h"
 
 #include "feltor.h"
 #include "implicit.h"
@@ -26,21 +27,17 @@ int main( int argc, char* argv[])
     Json::Value js, gs;
     if( argc == 1)
     {
-        std::ifstream is("input.json");
-        std::ifstream ks("geometry_params.json");
-        is >> js;
-        ks >> gs;
+        file::file2Json( "input.json", js, "strict");
+        file::file2Json( "geometry_params.json", gs, "strict");
     }
     else if( argc == 3)
     {
-        std::ifstream is(argv[1]);
-        std::ifstream ks(argv[2]);
-        is >> js;
-        ks >> gs;
+        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[2], gs, "strict");
     }
     else
     {
-        std::cerr << "ERROR: Too many arguments!\nUsage: "
+        std::cerr << "ERROR: Wrong number of arguments!\nUsage: "
                   << argv[0]<<" [inputfile] [geomfile] \n";
         return -1;
     }
@@ -157,9 +154,7 @@ int main( int argc, char* argv[])
     /////////glfw initialisation ////////////////////////////////////////////
     //
     std::stringstream title;
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json( "window_params.json", js, "default");
     unsigned red = js.get("reduction", 1).asUInt();
     double rows = js["rows"].asDouble(), cols = p.Nz/red+1,
            width = js["width"].asDouble(), height = js["height"].asDouble();
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 284a69d0e..4703747f2 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -170,7 +170,7 @@ struct Explicit
     using vector = std::array<std::array<Container,2>,2>;
     using container = Container;
     Explicit( const Geometry& g, feltor::Parameters p,
-        dg::geo::TokamakMagneticField mag, bool full_system );
+        dg::geo::TokamakMagneticField mag, bool full_system ); //full system means explicit AND implicit
 
     //Given N_i-1 initialize n_e-1 such that phi=0
     void initializene( const Container& ni, Container& ne);
@@ -184,6 +184,9 @@ struct Explicit
     const std::array<std::array<Container,2>,2>& fields() const{
         return m_fields;
     }
+    const std::array<Container,2>& potentials() const{
+        return m_phi;
+    }
     const std::array<std::array<Container,2>,2>& sources() const{
         return m_s;
     }
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index ce36238c4..c6f421297 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -10,7 +10,7 @@
 #include <mpi.h>
 #endif //FELTOR_MPI
 
-#include "dg/file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltor.h"
 #include "implicit.h"
 
@@ -101,9 +101,6 @@ int main( int argc, char* argv[])
 #endif //FELTOR_MPI
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js, gs;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 4 && argc != 5)
     {
         MPI_OUT std::cerr << "ERROR: Wrong number of arguments!\nUsage: "
@@ -113,10 +110,8 @@ int main( int argc, char* argv[])
     }
     else
     {
-        std::ifstream is(argv[1]);
-        std::ifstream ks(argv[2]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-        parseFromStream( parser, ks, &gs, &errs); //read input without comments
+        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[2], gs, "strict");
     }
     const feltor::Parameters p( js);
     const dg::geo::solovev::Parameters gp(gs);
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 07c4ff14b..a539edc4c 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -8,7 +8,7 @@
 
 #include "dg/algorithm.h"
 #include "dg/geometries/geometries.h"
-#include "dg/file/nc_utilities.h"
+#include "dg/file/file.h"
 using HVec = dg::HVec;
 using DVec = dg::DVec;
 using DMatrix = dg::DMatrix;
@@ -30,28 +30,12 @@ int main( int argc, char* argv[])
     std::cout << " -> "<<argv[argc-1]<<std::endl;
 
     //------------------------open input nc file--------------------------------//
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "geomfile", &length);
-    std::string geom( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "geomfile", &geom[0]);
-    err = nc_close(ncid);
-
-    //std::cout << "input "<<input<<std::endl;
-    //std::cout << "geome "<<geom <<std::endl;
     Json::Value js,gs;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss( input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
-    ss.str( geom);
-    parseFromStream( parser, ss, &gs, &errs); //read input without comments
+    std::string input, geom;
+    file::netcdf2string( argv[1], "inputfile", input);
+    file::string2Json( argv[1], input, js, "strict");
+    file::netcdf2string( argv[1], "geomfile", geom);
+    file::string2Json( argv[1], geom, gs, "strict");
     const feltor::Parameters p(js);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
@@ -62,6 +46,7 @@ int main( int argc, char* argv[])
 
     //-----------------Create Netcdf output file with attributes----------//
     int ncid_out;
+    file::NC_Error_Handle err;
     err = nc_create(argv[argc-1],NC_NETCDF4|NC_CLOBBER, &ncid_out);
 
     /// Set global attributes
@@ -225,7 +210,7 @@ int main( int argc, char* argv[])
     long_name="Cartesian y-coordinate";
     err = nc_put_att_text( ncid_out, yccID, "long_name",
         long_name.size(), long_name.data());
-    err = nc_enddef( ncid);
+    err = nc_enddef( ncid_out);
     err = nc_put_var_double( ncid_out, xccID, gridX2d.map()[0].data());
     err = nc_put_var_double( ncid_out, yccID, gridX2d.map()[1].data());
     err = nc_redef(ncid_out);
@@ -253,7 +238,7 @@ int main( int argc, char* argv[])
             &dim_ids1d[1], &vid);
         err = nc_put_att_text( ncid_out, vid, "long_name",
             std::get<2>(tp).size(), std::get<2>(tp).data());
-        err = nc_enddef( ncid);
+        err = nc_enddef( ncid_out);
         err = nc_put_var_double( ncid_out, vid, std::get<1>(tp).data());
         err = nc_redef(ncid_out);
     }
@@ -313,6 +298,7 @@ int main( int argc, char* argv[])
     }
     /////////////////////////////////////////////////////////////////////////
     size_t counter = 0;
+    int ncid;
     for( int j=1; j<argc-1; j++)
     {
         int timeID;
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index 58b66d676..097ec408b 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -15,27 +15,19 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
 #endif
     std::array<std::array<DVec,2>,2> y0;
     ///////////////////read in and show inputfile
-    file::NC_Error_Handle errIN;
-    int ncidIN;
-    errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
-    size_t lengthIN;
-    errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
-    std::string inputIN( lengthIN, 'x');
-    errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
 
     Json::Value jsIN;
-    std::stringstream is(inputIN);
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    parseFromStream( parser, is, &jsIN, &errs); //read input without comments
-    unsigned  pINn       = jsIN["n"].asUInt();
-    unsigned  pINNx      = jsIN["Nx"].asUInt();
-    unsigned  pINNy      = jsIN["Ny"].asUInt();
-    unsigned  pINNz      = jsIN["Nz"].asUInt();
+    std::string temp;
+    file::netcdf2string( file_name, "inputfile", temp);
+    file::string2Json(file_name, temp, jsIN, "strict");
+    unsigned  pINn  = jsIN["n"].asUInt();
+    unsigned  pINNx = jsIN["Nx"].asUInt();
+    unsigned  pINNy = jsIN["Ny"].asUInt();
+    unsigned  pINNz = jsIN["Nz"].asUInt();
     bool      pINsymmetric   = jsIN.get( "symmetric", false).asBool();
     MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
     MPI_OUT std::cout << " file parameters:" << std::endl;
+    MPI_OUT std::cout << pINn<<" x "<<pINNx<<" x "<<pINNy<<" x "<<pINNz<<" : symmetric "<<std::boolalpha<<pINsymmetric<<std::endl;
 
     // Now read in last timestep
     Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
@@ -69,6 +61,9 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     int timeIDIN;
     size_t size_time, count_time = 1;
     /////////////////////Get time length and initial data///////////////////////////
+    file::NC_Error_Handle errIN;
+    int ncidIN;
+    errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
     errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
     errIN = nc_inq_dimlen(ncidIN, timeIDIN, &size_time);
     errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index 3313b8a62..c35fcc62a 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -8,7 +8,7 @@
 
 #include "dg/algorithm.h"
 #include "dg/geometries/geometries.h"
-#include "dg/file/nc_utilities.h"
+#include "dg/file/file.h"
 using HVec = dg::HVec;
 using DVec = dg::DVec;
 using DMatrix = dg::DMatrix;
@@ -42,28 +42,12 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     //------------------------open input nc file--------------------------------//
-    file::NC_Error_Handle err;
-    int ncid_in;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid_in);
-    size_t length=0;
-    err = nc_inq_attlen( ncid_in, NC_GLOBAL, "inputfile", &length);
-    std::string inputfile( length, 'x');
-    err = nc_get_att_text( ncid_in, NC_GLOBAL, "inputfile", &inputfile[0]);
-    err = nc_inq_attlen( ncid_in, NC_GLOBAL, "geomfile", &length);
-    std::string geom( length, 'x');
-    err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geom[0]);
-    err = nc_close(ncid_in);
-
-    //std::cout << "inputfile "<<input<<std::endl;
-    //std::cout << "geome "<<geom <<std::endl;
     Json::Value js,gs;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss( inputfile);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
-    ss.str( geom);
-    parseFromStream( parser, ss, &gs, &errs); //read input without comments
+    std::string inputfile, geomfile;
+    file::netcdf2string( argv[1], "inputfile", inputfile);
+    file::string2Json(argv[1], inputfile, js, "strict");
+    file::netcdf2string( argv[1], "geomfile", geomfile);
+    file::string2Json(argv[1], geomfile, gs, "strict");
     const feltor::Parameters p(js);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
@@ -71,6 +55,7 @@ int main( int argc, char* argv[])
 
     //-----------------Create Netcdf output file with attributes----------//
     int ncid_out;
+    file::NC_Error_Handle err;
     err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
 
     /// Set global attributes
@@ -89,7 +74,7 @@ int main( int argc, char* argv[])
     att["source"] = "FELTOR";
     att["references"] = "https://github.com/feltor-dev/feltor";
     att["inputfile"] = inputfile;
-    att["geomfile"] = geom;
+    att["geomfile"] = geomfile;
     for( auto pair : att)
         err = nc_put_att_text( ncid_out, NC_GLOBAL,
             pair.first.data(), pair.second.size(), pair.second.data());
@@ -126,12 +111,21 @@ int main( int argc, char* argv[])
 
     /////////////////////////////////////////////////////////////////////////
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    if( p.alpha_mag > 0.)
-        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
+    if( p.damping_alpha > 0.)
+    {
+        double RO=mag.R0(), ZO=0.;
+        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+        double psipO = mag.psip()( RO, ZO);
+        double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
+        double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
+        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2.,
+                fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
+    }
     auto bhat = dg::geo::createBHat( mag);
     dg::geo::Fieldaligned<Geometry, IHMatrix, HVec> fieldaligned(
         bhat, g3d_out, dg::NEU, dg::NEU, dg::geo::NoLimiter(), //let's take NEU bc because N is not homogeneous
         p.rk4eps, 5, 5);
+    int ncid_in;
     err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
     dg::IHMatrix interpolate_in_2d = dg::create::interpolation( g3d_out_equidistant, g3d_out);
 
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 1e261855e..7f17d55c3 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -4,6 +4,7 @@
 
 #include "dg/algorithm.h"
 #include "dg/geometries/geometries.h"
+#include "dg/file/json_utilities.h"
 
 #include "parameters.h"
 #define DG_MANUFACTURED
@@ -19,15 +20,9 @@ int main( int argc, char* argv[])
 {
     Json::Value js, gs;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "strict");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "strict");
     else
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile]\n";
@@ -45,7 +40,7 @@ int main( int argc, char* argv[])
     //create RHS
     std::cout << "Initialize explicit" << std::endl;
     dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
-    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, mag);
+    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, mag, false);
     std::cout << "Initialize implicit" << std::endl;
     feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
 
@@ -123,7 +118,7 @@ int main( int argc, char* argv[])
     dg::blas1::plus(sol[0][0],-1); //ne-1
     dg::blas1::plus(sol[0][1],-1); //Ni-1
     const std::array<std::array<dg::DVec,2>,2>& num = feltor.fields();
-    const std::array<dg::DVec,2>& num_phi = feltor.potential();
+    const std::array<dg::DVec,2>& num_phi = feltor.potentials();
     const dg::DVec& num_apar = feltor.induction();
     double normne = sqrt(dg::blas2::dot( w3d, sol[0][0]));
     double normni = sqrt(dg::blas2::dot( w3d, sol[0][1]));
@@ -149,8 +144,8 @@ int main( int argc, char* argv[])
               <<"    phie: "<<sqrt(dg::blas2::dot( w3d,sol_phi[0]))/normphie<<"\t"<<normphie<<"\n"
               <<"    phii: "<<sqrt(dg::blas2::dot( w3d,sol_phi[1]))/normphii<<"\t"<<normphii<<"\n"
               <<"    apar: "<<sqrt(dg::blas2::dot( w3d,sol_apar))/normapar<<"\t"<<normapar<<"\n";
-    feltor.update_quantities();
-    feltor.quantities().display();
+    //feltor.update_quantities();
+    //feltor.quantities().display();
 
 
     return 0;
diff --git a/src/feltor/steadystate.cu b/src/feltor/steadystate.cu
deleted file mode 100644
index 12ac61e3a..000000000
--- a/src/feltor/steadystate.cu
+++ /dev/null
@@ -1,195 +0,0 @@
-#include <iostream>
-#include <iomanip>
-#include <vector>
-#include <sstream>
-#include <cmath>
-
-#include "draw/host_window.h"
-#include "feltor.h"
-#include "implicit.h"
-using HVec = dg::HVec;
-using DVec = dg::DVec;
-using DMatrix = dg::DMatrix;
-using IDMatrix = dg::IDMatrix;
-using IHMatrix = dg::IHMatrix;
-using Geometry = dg::CylindricalGrid3d;
-#define MPI_OUT
-
-#include "init.h" //for the source profiles
-
-namespace detail{
-template<class Explicit, class Implicit, class Container >
-struct FullSystem
-{
-    FullSystem() = default;
-    FullSystem( Explicit exp, Implicit imp, Container temp):
-            m_exp( exp), m_imp( imp), m_temp(temp){}
-
-    template<class Container2>
-    void operator()( const Container2& y, Container2& yp)
-    {
-        m_exp( 0, y, m_temp);
-        m_imp( 0, y, yp);
-        dg::blas1::axpby( 1., m_temp, 1., yp);
-    }
-    private:
-    Explicit m_exp;
-    Implicit m_imp;
-    Container m_temp;
-};
-}//namespace detail
-
-template<class Explicit, class Implicit, class Container, class ContainerType2>
-void solve_steady_state( Explicit& ex, Implicit& im, Container& x, const Container& b, const ContainerType2& weights, double damping)
-{
-    Container tmp(x);
-    detail::FullSystem<Explicit&, Implicit&, Container&> full(ex,im,tmp);
-    // allocate memory
-    unsigned mMax =10, restart = mMax;
-    dg::AndersonAcceleration<Container> acc( x, mMax);
-    // Evaluate right hand side and solution on the grid
-    const double eps = 1e-3;
-    unsigned max_iter =1000;
-    std::cout << "Type maximum iteration number\n";
-    std::cin >> max_iter;
-    std::cout << "Number of iterations "<< acc.solve( full, x, b, weights, eps, eps, max_iter, damping, restart, true)<<std::endl;
-
-}
-
-int main( int argc, char* argv[])
-{
-    ////Parameter initialisation ////////////////////////////////////////////
-    Json::Value js, gs;
-    if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        std::ifstream ks("geometry_params.json");
-        is >> js;
-        ks >> gs;
-    }
-    else if( argc == 3)
-    {
-        std::ifstream is(argv[1]);
-        std::ifstream ks(argv[2]);
-        is >> js;
-        ks >> gs;
-    }
-    else
-    {
-        std::cerr << "ERROR: Too many arguments!\nUsage: "
-                  << argv[0]<<" [inputfile] [geomfile] \n";
-        return -1;
-    }
-    js["symmetric"] =  true; //overwrite symmetric parameter
-    const feltor::Parameters p( js);
-    const dg::geo::solovev::Parameters gp(gs);
-    p.display( std::cout);
-    gp.display( std::cout);
-    /////////////////////////////////////////////////////////////////////////
-    double Rmin=gp.R_0-p.boxscaleRm*gp.a;
-    double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
-    double Rmax=gp.R_0+p.boxscaleRp*gp.a;
-    double Zmax=p.boxscaleZp*gp.a*gp.elongation;
-    //Make grid
-    dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
-        p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    if( p.alpha_mag > 0.)
-        mag = dg::geo::createModifiedSolovevField(gp, (1.-p.rho_damping)*mag.psip()(mag.R0(),0.), p.alpha_mag);
-
-    //create RHS
-    std::cout << "Constructing Explicit...\n";
-    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
-    std::cout << "Constructing Implicit...\n";
-    feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
-    std::cout << "Done!\n";
-
-    bool fixed_profile;
-    HVec profile = dg::evaluate( dg::zero, grid);
-    HVec source_profile;
-    try{
-        source_profile = feltor::source_profiles.at(p.source_type)(
-        fixed_profile, profile, grid, p, gp, mag);
-    }catch ( std::out_of_range& error){
-        std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
-        return -1;
-    }
-
-    feltor.set_source( fixed_profile, dg::construct<DVec>(profile), p.omega_source, dg::construct<DVec>(source_profile));
-
-    DVec result = dg::evaluate( dg::zero, grid);
-    std::array<std::array<DVec,2>,2> y0;
-    y0[0][0] = y0[0][1] =y0[1][0] =y0[1][1] = result;
-    std::array<std::array<DVec,2>,2> b(y0);
-    DVec weights = dg::create::weights( grid);
-
-
-    /////////////////////////////////////////////////
-    while(true)
-    {
-
-    /////////////////////////set up transfer for glfw
-    dg::DVec dvisual( grid.size(), 0.);
-    dg::HVec hvisual( grid.size(), 0.), visual(hvisual), avisual(hvisual);
-    dg::IHMatrix equi = dg::create::backscatter( grid);
-    draw::ColorMapRedBlueExtMinMax colors(-1.0, 1.0);
-    std::map<std::string, const dg::DVec* > v4d;
-    v4d["ne-1 / "] = &y0[0][0],               v4d["ni-1 / "] = &y0[0][1];
-    v4d["Ue / "]   = &feltor.fields()[1][0],  v4d["Ui / "]   = &feltor.fields()[1][1];
-    v4d["Ome / "] = &feltor.potential(0); v4d["Apar / "] = &feltor.induction();
-    /////////glfw initialisation ////////////////////////////////////////////
-    //
-    std::stringstream title;
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
-    unsigned red = js.get("reduction", 1).asUInt();
-    double rows = js["rows"].asDouble(), cols = p.Nz/red,
-           width = js["width"].asDouble(), height = js["height"].asDouble();
-    if ( p.symmetric ) cols = rows, rows = 1;
-    GLFWwindow* w = draw::glfwInitAndCreateWindow( cols*width, rows*height, "");
-    draw::RenderHostData render(rows, cols);
-
-    std::cout << "Begin computation \n";
-    std::cout << std::scientific << std::setprecision( 2);
-    title << std::setprecision(2) << std::scientific;
-    while ( !glfwWindowShouldClose( w ))
-    {
-        title << std::scientific;
-        solve_steady_state( feltor, im, y0, b, weights, p.dt);
-        for( auto pair : v4d)
-        {
-            if(pair.first == "Ome / ")
-            {
-                dg::assign( feltor.lapMperpP(0), hvisual);
-                dg::assign( *pair.second, hvisual);
-            }
-            else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
-            {
-                dg::assign( *pair.second, hvisual);
-                dg::blas1::axpby( 1., hvisual, -1., profile, hvisual);
-            }
-            else
-                dg::assign( *pair.second, hvisual);
-            dg::blas2::gemv( equi, hvisual, visual);
-            colors.scalemax() = (double)thrust::reduce(
-                visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
-            colors.scalemin() = -colors.scalemax();
-            title <<pair.first << colors.scalemax()<<"   ";
-            render.renderQuad( hvisual, grid.n()*grid.Nx(),
-                                        grid.n()*grid.Ny(), colors);
-        }
-        glfwSetWindowTitle(w,title.str().c_str());
-        title.str("");
-        glfwPollEvents();
-        glfwSwapBuffers( w);
-
-    }
-    }
-    glfwTerminate();
-    ////////////////////////////////////////////////////////////////////
-
-    ////////////////////////////////////////////////////////////////////
-    return 0;
-
-}
diff --git a/src/feltor/window_params.js b/src/feltor/window_params.json
similarity index 100%
rename from src/feltor/window_params.js
rename to src/feltor/window_params.json
diff --git a/src/feltorSH/feltor.cu b/src/feltorSH/feltor.cu
index f6fe3c81a..e7342f5fe 100644
--- a/src/feltorSH/feltor.cu
+++ b/src/feltorSH/feltor.cu
@@ -6,31 +6,20 @@
 // #define DG_DEBUG
 
 #include "draw/host_window.h"
+#include "dg/file/json_utilities.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 
-/*
-   - reads parameters from input.json or any other given file,
-   - integrates the Explicit - functor and
-   - directly visualizes results on the screen using parameters in window_params.js
-*/
-
 
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "strict");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "strict");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -39,10 +28,8 @@ int main( int argc, char* argv[])
     const eule::Parameters p(  js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
+    file::file2Json( "window_params.json", js, "default");
     std::stringstream title;
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorSH/feltor_hpc.cu b/src/feltorSH/feltor_hpc.cu
index 0a8a0481e..25f1e859f 100644
--- a/src/feltorSH/feltor_hpc.cu
+++ b/src/feltorSH/feltor_hpc.cu
@@ -4,36 +4,24 @@
 #include <sstream>
 #include <cmath>
 // #define DG_DEBUG
-#include "file/nc_utilities.h"
+#include "dg/algorithm.h"
+#include "dg/file/file.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3)
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     p.display( std::cout);
diff --git a/src/feltorSH/feltor_mpi.cu b/src/feltorSH/feltor_mpi.cu
index 16933d126..a3fd4795e 100644
--- a/src/feltorSH/feltor_mpi.cu
+++ b/src/feltorSH/feltor_mpi.cu
@@ -10,19 +10,12 @@
 #include "netcdf_par.h"
 
 #include "dg/algorithm.h"
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 int main( int argc, char* argv[])
 {
      ////////////////////////////////setup MPI///////////////////////////////
@@ -47,10 +40,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     if(rank==0)p.display( std::cout);
diff --git a/src/feltorSH/input/default.in b/src/feltorSH/input/default.json
similarity index 100%
rename from src/feltorSH/input/default.in
rename to src/feltorSH/input/default.json
diff --git a/src/feltorSH/input/verification.in b/src/feltorSH/input/verification.json
similarity index 100%
rename from src/feltorSH/input/verification.in
rename to src/feltorSH/input/verification.json
diff --git a/src/feltorSH/window_params.js b/src/feltorSH/window_params.json
similarity index 100%
rename from src/feltorSH/window_params.js
rename to src/feltorSH/window_params.json
diff --git a/src/feltorSHp/feltor.cu b/src/feltorSHp/feltor.cu
index 34cd3e059..ef6cfbb3c 100644
--- a/src/feltorSHp/feltor.cu
+++ b/src/feltorSHp/feltor.cu
@@ -6,31 +6,20 @@
 // #define DG_DEBUG
 
 #include "draw/host_window.h"
+#include "dg/file/json_utilities.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the Explicit - functor and 
-   - directly visualizes results on the screen using parameters in window_params.txt
-*/
-
 
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "strict");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "strict");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -40,9 +29,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json( "window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorSHp/feltor_hpc.cu b/src/feltorSHp/feltor_hpc.cu
index db3f64953..18fe39c8a 100644
--- a/src/feltorSHp/feltor_hpc.cu
+++ b/src/feltorSHp/feltor_hpc.cu
@@ -5,36 +5,23 @@
 #include <cmath>
 // #define DG_DEBUG
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3)
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     p.display( std::cout);
diff --git a/src/feltorSHp/feltor_mpi.cu b/src/feltorSHp/feltor_mpi.cu
index b3f47732f..6ea2e317d 100644
--- a/src/feltorSHp/feltor_mpi.cu
+++ b/src/feltorSHp/feltor_mpi.cu
@@ -10,19 +10,12 @@
 #include "netcdf_par.h"
 
 #include "dg/algorithm.h"
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 int main( int argc, char* argv[])
 {
      ////////////////////////////////setup MPI///////////////////////////////
@@ -47,10 +40,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     if(rank==0)p.display( std::cout);
diff --git a/src/feltorSHp/input/default.in b/src/feltorSHp/input/default.json
similarity index 100%
rename from src/feltorSHp/input/default.in
rename to src/feltorSHp/input/default.json
diff --git a/src/feltorSHp/input/verification.in b/src/feltorSHp/input/verification.json
similarity index 100%
rename from src/feltorSHp/input/verification.in
rename to src/feltorSHp/input/verification.json
diff --git a/src/feltorSHp/window_params.js b/src/feltorSHp/window_params.json
similarity index 100%
rename from src/feltorSHp/window_params.js
rename to src/feltorSHp/window_params.json
diff --git a/src/feltorSesol/feltor.cu b/src/feltorSesol/feltor.cu
index e8eff0a93..015b8b980 100644
--- a/src/feltorSesol/feltor.cu
+++ b/src/feltorSesol/feltor.cu
@@ -6,34 +6,21 @@
 // #define DG_DEBUG
 
 #include "draw/host_window.h"
+#include "dg/file/json_utilities.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 #include "../diag/probes.h"
 
 
-
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the Explicit - functor and 
-   - directly visualizes results on the screen using parameters in window_params.txt
-*/
-
-
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json("input.json", js, "default");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json(argv[1], js, "default");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -43,9 +30,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json( "window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorSesol/feltor_hpc.cu b/src/feltorSesol/feltor_hpc.cu
index 29b36ae3c..3261bba29 100644
--- a/src/feltorSesol/feltor_hpc.cu
+++ b/src/feltorSesol/feltor_hpc.cu
@@ -4,26 +4,16 @@
 #include <sstream>
 #include <cmath>
 // #define DG_DEBUG
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3 && argc != 4)
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n"; 
@@ -31,10 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     p.display( std::cout);
@@ -87,19 +74,12 @@ int main( int argc, char* argv[])
       std::cout << "Done!\n";
     }
     if (argc==4) {
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
-        size_t lengthIN;
-        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
-        std::string inputIN( lengthIN, 'x');
-        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);    
-
+        std::string inputIN;
+        file::netcdf2string( argv[3], "inputfile", inputIN);
         Json::Value jsIN;
-        std::stringstream is(inputIN);
-        parseFromStream( parser, is, &jsIN, &errs); //read input without comments
-        const eule::Parameters pIN(  jsIN);    
+        file::string2Json( argv[3], inputIN, jsIN, "strict");
+        const eule::Parameters pIN(  jsIN);
         std::cout << "[input.nc] file parameters" << std::endl;
         pIN.display( std::cout);    
 
@@ -115,6 +95,9 @@ int main( int argc, char* argv[])
         size_t stepsIN;
         /////////////////////The initial field///////////////////////////////////////////
         /////////////////////Get time length and initial data///////////////////////////
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         errIN = nc_inq_varid(ncidIN, namesIN[0].data(), &dataIDsIN[0]);
         errIN = nc_inq_dimlen(ncidIN, dataIDsIN[0], &stepsIN);
         stepsIN-=1;
@@ -183,7 +166,7 @@ int main( int argc, char* argv[])
     int dim_ids_probe[2];
     dim_ids_probe[0] = EtimeID;
     //dim_ids_probe[1] = 
-    file :: define_dimension(ncid, "X_probe", &dim_ids_probe[1], dg::evaluate(dg::LinearX(1.0, 0), grid_probe).data(), 8);
+    file :: define_dimension(ncid, &dim_ids_probe[1],  grid_probe, "X_probe" );
     for(unsigned i = 0; i < varname_probes.size(); i++)
     {
         err = nc_def_var(ncid, varname_probes[i].data(), NC_DOUBLE, 2, dim_ids_probe, &ID_probes[i]);
diff --git a/src/feltorSesol/feltor_mpi.cu b/src/feltorSesol/feltor_mpi.cu
index b2ea09771..2eb74ce04 100644
--- a/src/feltorSesol/feltor_mpi.cu
+++ b/src/feltorSesol/feltor_mpi.cu
@@ -16,13 +16,6 @@
 #include "parameters.h"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 int main( int argc, char* argv[])
 {
      ////////////////////////////////setup MPI///////////////////////////////
@@ -38,9 +31,6 @@ int main( int argc, char* argv[])
     MPI_Comm_size( MPI_COMM_WORLD, &size);
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3 && argc != 4)
     {
         if(rank==0)std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n"; 
@@ -48,10 +38,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     if(rank==0)p.display( std::cout);
@@ -116,18 +103,11 @@ int main( int argc, char* argv[])
         if(rank==0) std::cout << "Done!\n";
     }
     if (argc==4) {
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
-        size_t lengthIN;
-        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
-        std::string inputIN( lengthIN, 'x');
-        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);    
-
+        std::string inputIN;
+        file::netcdf2string( argv[3], "inputfile", inputIN);
         Json::Value jsIN;
-        std::stringstream is(inputIN);
-        parseFromStream( parser, is, &jsIN, &errs); //read input without comments
+        file::string2Json( argv[3], inputIN, jsIN, "strict");
         const eule::Parameters pIN(  jsIN);    
         std::cout << "[input.nc] file parameters" << std::endl;
         pIN.display( std::cout);   
@@ -147,6 +127,9 @@ int main( int argc, char* argv[])
         size_t stepsIN;
         /////////////////////The initial field///////////////////////////////////////////
         /////////////////////Get time length and initial data///////////////////////////
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         errIN = nc_inq_varid(ncidIN, namesIN[0].data(), &dataIDsIN[0]);
         errIN = nc_inq_dimlen(ncidIN, dataIDsIN[0], &stepsIN);
         stepsIN-=1;
diff --git a/src/feltorSesol/input/default.in b/src/feltorSesol/input/default.json
similarity index 100%
rename from src/feltorSesol/input/default.in
rename to src/feltorSesol/input/default.json
diff --git a/src/feltorSesol/input/default_fd.in b/src/feltorSesol/input/default_fd.json
similarity index 100%
rename from src/feltorSesol/input/default_fd.in
rename to src/feltorSesol/input/default_fd.json
diff --git a/src/feltorSesol/input/input.txt b/src/feltorSesol/input/input.txt
deleted file mode 100644
index 914ee0d8c..000000000
--- a/src/feltorSesol/input/input.txt
+++ /dev/null
@@ -1,53 +0,0 @@
-  GNU nano 2.4.2                                                                           Datei: inputAUGt1_f.txt                                                                                                                                                             
-
-                * Input-File for "FELTOR" *
-                ---------------------------
-
-
-@-----------------------------Space and Time discretization------------
-1)  n  (# of x,y-polynomials)            =  1 (3)
-2)  nx (grid points in x)                =  160(192)
-3)  ny (grid points in y)                =  160
-4)  dt (time step in units c_s/rho_s)    =  1.1(0.01)
-----------------------------------Output parameters--------------------
-5)  n_out (# of x-y polynomials in output)  =  1
-6)  nx_out (# grid points in output field)  =  160
-7)  ny_out (# grid points in output field)  =  160
-8)  itstp  (steps between outputs)          =  10
-9) total # of outputs (excluding first)     =  1000
--------------------------Algorithmic parameters------------------------
-10)  eps_pol (stop for polarisation)        =   1e-5 (1e-6)
-11)  eps_gamma (stop for Gamma CG)          =   1e-5 (1e-8)
-12)  eps_time ( stop for time inversion )   =   1e-12
--------------------------Physical parameters----------------------------
-13) mu_e (-m_e/m_i)                         = -0.000272121 (-0.000544617,-0.000272121,-0.000181372 )
-14) tau (Ti/Te)                             =  2.0   (0.0)
-15) mcv (curvature)                         =  0.00015   (0.0003)
-16) nu_perp                                 =  5e-3
-17) D  (coupling ~ mcv^2)                   =  0.005e-8
-18) para resistivity (C ~ D lx/d)           =  2.5e-6
-19) parallel Length (~ 2 pi q / mcv)        =  1000000   (2000000)
-------------------------Initial perturbation parameters---------------------
-20) amp (blob amplitude)                    =  0.025    (1.0)
-21) sigma (blob variance in units of rho_s) =  2.5     (10)
-22) x-position ( in units of lx)            =  0.333333   (0.4)
-22) y-position ( in units of ly)            =  0.5   (0.5)
-24) Profile amplitude                       =  4.0  (peak amplitude)
-25) Background Prof amplitude               =  1.0   (density on the boundary)
----------------------------------box setup----------------
-26) lx (in rho_s)                           =  150
-27) ly (in rho_s)                           =  150
-28) bc_x (0 periodic, 1 Dirichlet, 2DN )    =   2      (1.0)
-29) bc_y (0 periodic, 1 Dirichlet, 2DN )    =   0      (1.0)
----------------------------------Zonal flow dynamics and gradient length----------------
-30) ordinary/modified HW (0/1)              = 1
-31) ln (in rho_s)                           = 100
----------------------------------EDGE/SOL Dynamics-------------------------------------------
-32) SOL boundary in units of lx (>1 no SOL)    = 0.3333333
-33) damping width                              = 0.5
-34) profile source rate in units c_s/rho_s     = 0.1
-35) source dampingb in u of lx (<1 no Source)  = 0.1666666
-36) source damping width                       = 0.5
-@ ------------------------------------------------------------
-
-
diff --git a/src/feltorSesol/window_params.js b/src/feltorSesol/window_params.json
similarity index 100%
rename from src/feltorSesol/window_params.js
rename to src/feltorSesol/window_params.json
diff --git a/src/feltorShw/feltor.cu b/src/feltorShw/feltor.cu
index a82061bea..bcad81c3b 100644
--- a/src/feltorShw/feltor.cu
+++ b/src/feltorShw/feltor.cu
@@ -6,6 +6,7 @@
 // #define DG_DEBUG
 
 #include "draw/host_window.h"
+#include "dg/file/json_utilities.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
@@ -13,27 +14,14 @@
 
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the Explicit - functor and 
-   - directly visualizes results on the screen using parameters in window_params.txt
-*/
-
-
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "strict");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "strict");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -43,9 +31,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json( "window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorShw/feltor_hpc.cu b/src/feltorShw/feltor_hpc.cu
index 3fad5a358..86a43acd8 100644
--- a/src/feltorShw/feltor_hpc.cu
+++ b/src/feltorShw/feltor_hpc.cu
@@ -4,26 +4,16 @@
 #include <sstream>
 #include <cmath>
 // #define DG_DEBUG
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3 && argc != 4)
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n"; 
@@ -31,10 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     p.display( std::cout);
@@ -111,18 +98,12 @@ int main( int argc, char* argv[])
       }
     }
     if (argc==4) {
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
-        size_t lengthIN;
-        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
-        std::string inputIN( lengthIN, 'x');
-        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);    
-        
         Json::Value jsIN;
-        std::stringstream is(inputIN);
-        parseFromStream( parser, is, &jsIN, &errs); //read input without comments
+        std::string inputIN;
+        file::netcdf2string( argv[3], "inputfile", inputIN);
+        file::string2Json(argv[3], inputIN, jsIN, "strict");
+
         const eule::Parameters pIN(  jsIN);    
         std::cout << "[input.nc] file parameters" << std::endl;
         pIN.display( std::cout);       
@@ -139,6 +120,9 @@ int main( int argc, char* argv[])
         size_t stepsIN;
         /////////////////////The initial field///////////////////////////////////////////
         /////////////////////Get time length and initial data///////////////////////////
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         errIN = nc_inq_varid(ncidIN, namesIN[0].data(), &dataIDsIN[0]);
         errIN = nc_inq_dimlen(ncidIN, dataIDsIN[0], &stepsIN);
         stepsIN-=1;
@@ -152,7 +136,7 @@ int main( int argc, char* argv[])
         dg::IHMatrix interpolateIN = dg::create::interpolation( grid,grid_IN); 
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[0], start2dIN, count2dIN, transferINH.data());
         dg::blas2::gemv( interpolateIN, transferINH,temp);
-        dg::blas1::transfer(temp,y0[0]);
+        dg::blas1::transfer(temp, y0[0]);
         errIN = nc_inq_varid(ncidIN, namesIN[1].data(), &dataIDsIN[1]);
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[1], start2dIN, count2dIN, transferINH.data());
         dg::blas2::gemv( interpolateIN, transferINH,temp);
@@ -207,7 +191,7 @@ int main( int argc, char* argv[])
     int dim_ids_probe[2];
     dim_ids_probe[0] = EtimeID;
     //dim_ids_probe[1] = 
-    file :: define_dimension(ncid, "X_probe", &dim_ids_probe[1], dg::evaluate(dg::LinearX(1.0, 0), grid_probe).data(), 8);
+    file :: define_dimension(ncid,  &dim_ids_probe[1], grid_probe, "X_probe");
     for(unsigned i = 0; i < varname_probes.size(); i++)
     {
         err = nc_def_var(ncid, varname_probes[i].data(), NC_DOUBLE, 2, dim_ids_probe, &ID_probes[i]);
diff --git a/src/feltorShw/feltor_mpi.cu b/src/feltorShw/feltor_mpi.cu
index 2975cd46d..03c5363d8 100644
--- a/src/feltorShw/feltor_mpi.cu
+++ b/src/feltorShw/feltor_mpi.cu
@@ -11,19 +11,12 @@
 #include "netcdf_par.h"
 
 #include "dg/algorithm.h"
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "feltor.cuh"
 #include "parameters.h"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - writes outputs to a given outputfile using hdf5. 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-*/
-
 
 namespace ns_ncid
 {
@@ -61,9 +54,6 @@ int main( int argc, char* argv[])
     MPI_Comm_size( MPI_COMM_WORLD, &size);
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3 && argc != 4)
     {
         if(rank==0)std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n"; 
@@ -71,10 +61,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     if(rank==0) p.display( std::cout);
@@ -164,18 +151,11 @@ int main( int argc, char* argv[])
       }
     }
     if (argc==4) {
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
-        size_t lengthIN;
-        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &lengthIN);
-        std::string inputIN( lengthIN, 'x');
-        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);    
-
         Json::Value jsIN;
-        std::stringstream is(inputIN);
-        parseFromStream( parser, is, &jsIN, &errs); //read input without comments
+        std::string temp;
+        file::netcdf2string( file_name, "inputfile", temp);
+        file::string2Json(file_name, temp, jsIN, "strict");
         const eule::Parameters pIN(  jsIN);    
         if(rank==0) std::cout << "[input.nc] file parameters" << std::endl;
         if(rank==0) pIN.display( std::cout);   
@@ -195,6 +175,9 @@ int main( int argc, char* argv[])
         size_t stepsIN;
         /////////////////////The initial field///////////////////////////////////////////
         /////////////////////Get time length and initial data///////////////////////////
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         errIN = nc_inq_varid(ncidIN, namesIN[0].data(), &dataIDsIN[0]);
         errIN = nc_inq_dimlen(ncidIN, dataIDsIN[0], &stepsIN);
         stepsIN-=1;
diff --git a/src/feltorShw/input/default.in b/src/feltorShw/input/default.json
similarity index 100%
rename from src/feltorShw/input/default.in
rename to src/feltorShw/input/default.json
diff --git a/src/feltorShw/window_params.js b/src/feltorShw/window_params.json
similarity index 100%
rename from src/feltorShw/window_params.js
rename to src/feltorShw/window_params.json
diff --git a/src/hasegawa/hw.cu b/src/hasegawa/hw.cu
index a78391a3f..0c5452fcb 100644
--- a/src/hasegawa/hw.cu
+++ b/src/hasegawa/hw.cu
@@ -8,12 +8,8 @@
 
 #include "hw.cuh"
 #include "../toefl/parameters.h"
+#include "dg/file/json_utilities.h"
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - directly visualizes results on the screen using parameters in window_params.txt
-*/
 
 int main( int argc, char* argv[])
 {
@@ -21,15 +17,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "default");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "default");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -38,9 +28,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json( "window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/hasegawa/mima.cu b/src/hasegawa/mima.cu
index ec7e5dbce..ef3ede527 100644
--- a/src/hasegawa/mima.cu
+++ b/src/hasegawa/mima.cu
@@ -8,12 +8,7 @@
 
 #include "mima.cuh"
 #include "../toefl/parameters.h"
-
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - directly visualizes results on the screen using parameters in window_params.txt
-*/
+#include "dg/file/json_utilities.h"
 
 int main( int argc, char* argv[])
 {
@@ -21,15 +16,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "default");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "default");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -38,9 +27,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json( "window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/hasegawa/window_params.js b/src/hasegawa/window_params.json
similarity index 100%
rename from src/hasegawa/window_params.js
rename to src/hasegawa/window_params.json
diff --git a/src/heat/geometry/geometry_params.js b/src/heat/geometry/geometry_params.json
similarity index 100%
rename from src/heat/geometry/geometry_params.js
rename to src/heat/geometry/geometry_params.json
diff --git a/src/heat/heat.cu b/src/heat/heat.cu
index c55b3976a..4a81c691e 100644
--- a/src/heat/heat.cu
+++ b/src/heat/heat.cu
@@ -9,6 +9,7 @@
 #include "draw/host_window.h"
 //#include "draw/device_window.cuh"
 #include "dg/algorithm.h"
+#include "dg/file/json_utilities.h"
 #include "dg/geometries/geometries.h"
 
 #include "parameters.h"
@@ -21,17 +22,13 @@ int main( int argc, char* argv[])
     Json::Value js, gs;
     if( argc == 1)
     {
-        std::ifstream is("input.json");
-        std::ifstream ks("geometry_params.js");
-        is >> js;
-        ks >> gs;
+        file::file2Json("input/default.json", js, "default");
+        file::file2Json("geometry/geometry_params.json", gs, "default");
     }
     else if( argc == 3)
     {
-        std::ifstream is(argv[1]);
-        std::ifstream ks(argv[2]);
-        is >> js;
-        ks >> gs;
+        file::file2Json(argv[1], js, "strict");
+        file::file2Json(argv[2], gs, "strict");
     }
     else
     {
@@ -56,9 +53,7 @@ int main( int argc, char* argv[])
     std::cout << "Initialize implicit" << std::endl;
     heat::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
     /////////glfw initialisation ////////////////////////////////////////
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json("window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     //////////////////////////////////////////////////////////////////////
diff --git a/src/heat/heat_hpc.cu b/src/heat/heat_hpc.cu
index 8e1741588..358e6a2bd 100644
--- a/src/heat/heat_hpc.cu
+++ b/src/heat/heat_hpc.cu
@@ -9,8 +9,8 @@
 #include <cusp/print.h>
 #include "json/json.h"
 
-#include "dg/file/nc_utilities.h"
 #include "dg/algorithm.h"
+#include "dg/file/file.h"
 #include "dg/geometries/geometries.h"
 
 #include "parameters.h"
@@ -32,10 +32,8 @@ int main( int argc, char* argv[])
     }
     else
     {
-        std::ifstream is(argv[1]);
-        std::ifstream ks(argv[2]);
-        parseFromStream( parser, is, &js, &errs);
-        parseFromStream( parser, ks, &gs, &errs);
+        file::file2Json(argv[1], js, "strict");
+        file::file2Json(argv[2], gs, "strict");
     }
     const heat::Parameters p( js); p.display( std::cout);
     const dg::geo::solovev::Parameters gp(gs); gp.display( std::cout);
@@ -60,25 +58,14 @@ int main( int argc, char* argv[])
     //////////////////////////////open nc file//////////////////////////////////
     if (argc == 5)
     {
-        file::NC_Error_Handle errin;
-        int ncidin;
-        errin = nc_open( argv[4], NC_NOWRITE, &ncidin);
         //////////////read in and show inputfile und geomfile////////////
-        size_t length;
-        errin = nc_inq_attlen( ncidin, NC_GLOBAL, "inputfile", &length);
-        std::string inputin( length, 'x');
-        errin = nc_get_att_text( ncidin, NC_GLOBAL, "inputfile", &inputin[0]);
-        errin = nc_inq_attlen( ncidin, NC_GLOBAL, "geomfile", &length);
-        std::string geomin( length, 'x');
-        errin = nc_get_att_text( ncidin, NC_GLOBAL, "geomfile", &geomin[0]);
+        std::string inputin, geomin;
+        file::netcdf2string( argv[4], "inputfile", inputin);
+        file::string2Json( argv[4],  inputin, js, "strict");
+        file::netcdf2string( argv[4], "geomfile", geomin);
+        file::string2Json( argv[4], geomin, gs, "strict");
         std::cout << "input in"<<inputin<<std::endl;
         std::cout << "geome in"<<geomin <<std::endl;
-        //Now read Tend and interpolate from input grid to our grid
-        std::stringstream is;
-        is.str( inputin);
-        parseFromStream( parser, is, &js, &errs);
-        is.str( geomin);
-        parseFromStream( parser, is, &gs, &errs);
         const heat::Parameters pin(js);
         const dg::geo::solovev::Parameters gpin(gs);
         size_t start3din[4]  = {pin.maxout, 0, 0, 0};
@@ -87,6 +74,10 @@ int main( int argc, char* argv[])
             pin.n_out, pin.Nx_out, pin.Ny_out, pin.Nz_out, p.bcx, p.bcy, dg::PER);
         dg::IHMatrix interpolatef2c = dg::create::interpolation( grid, grid_in);//f2c
         dg::HVec TendIN = dg::evaluate( dg::zero, grid_in);
+        //Now read Tend and interpolate from input grid to our grid
+        file::NC_Error_Handle errin;
+        int ncidin;
+        errin = nc_open( argv[4], NC_NOWRITE, &ncidin);
         int dataIDin;
         errin = nc_inq_varid(ncidin, "T", &dataIDin);
         errin = nc_get_vara_double( ncidin, dataIDin, start3din, count3din,
diff --git a/src/heat/window_params.js b/src/heat/window_params.json
similarity index 100%
rename from src/heat/window_params.js
rename to src/heat/window_params.json
diff --git a/src/impurities/README b/src/impurities/README
index 6e55dd666..a98c0c4c1 100644
--- a/src/impurities/README
+++ b/src/impurities/README
@@ -1,5 +1 @@
 Project description: 2d blobs with impurities
-
-Problems: The file output is still the
-old hdf5 file.h format and the inversion of the matrices should be 
-changed to the invert class. 
diff --git a/src/impurities/toeflI.cu b/src/impurities/toeflI.cu
index 0bb358382..e3a851caf 100644
--- a/src/impurities/toeflI.cu
+++ b/src/impurities/toeflI.cu
@@ -6,31 +6,20 @@
 #include "draw/host_window.h"
 //#include "draw/device_window.cuh"
 
+#include "dg/file/json_utilities.h"
 
 #include "toeflI.cuh"
 #include "parameters.h"
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflI - functor and 
-   - directly visualizes results on the screen using parameters in window_params.txt
-*/
-
 int main( int argc, char* argv[])
 {
     //Parameter initialisation
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "default");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "default");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -39,9 +28,7 @@ int main( int argc, char* argv[])
     const imp::Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json( "window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/impurities/toefl_hpc.cu b/src/impurities/toefl_hpc.cu
index bde8a5492..ee9d8d74f 100644
--- a/src/impurities/toefl_hpc.cu
+++ b/src/impurities/toefl_hpc.cu
@@ -5,13 +5,7 @@
 #include "toeflI.cuh"
 #include "parameters.h"
 
-#include "file/nc_utilities.h"
-
-/*
-   - reads parameters from input.txt or any other given file,
-   - integrates the ToeflR - functor and
-   - writes outputs to a given outputfile using netcdf4.
-*/
+#include "dg/file/file.h"
 
 int main( int argc, char* argv[])
 {
@@ -26,10 +20,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     std::cout << js<<std::endl;
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const imp::Parameters p( js);
diff --git a/src/impurities/toefl_mpi.cu b/src/impurities/toefl_mpi.cu
index f98950614..0d7d4ccc0 100644
--- a/src/impurities/toefl_mpi.cu
+++ b/src/impurities/toefl_mpi.cu
@@ -8,13 +8,7 @@
 #include "toeflI.cuh"
 #include "parameters.h"
 
-#include "file/nc_utilities.h"
-
-/*
-   - reads parameters from input.txt or any other given file,
-   - integrates the ToeflR - functor and
-   - writes outputs to a given outputfile using netcdf4.
-*/
+#include "dg/file/file.h"
 
 int main( int argc, char* argv[])
 {   ////////////////////////////////setup MPI///////////////////////////////
@@ -49,18 +43,12 @@ int main( int argc, char* argv[])
     MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3)
     {   if(rank==0)std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
         return -1;
     }
     else
-    {   
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json(argv[1], js, "strict");
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const imp::Parameters p( js);
     if(rank==0)p.display( std::cout);
diff --git a/src/impurities/window_params.js b/src/impurities/window_params.json
similarity index 100%
rename from src/impurities/window_params.js
rename to src/impurities/window_params.json
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index da0834e53..3dd6948ee 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -6,6 +6,7 @@
 #include <thrust/host_vector.h>
 
 #include "dg/algorithm.h"
+#include "dg/file/json_utilities.h"
 
 #include "draw/host_window.h"
 
@@ -28,15 +29,9 @@ int main( int argc, char* argv[])
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input/default.json");
-        is >> js;
-    }
+        file::file2Json( "input/default.json", js, "default");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
diff --git a/src/lamb_dipole/shu_hpc.cu b/src/lamb_dipole/shu_hpc.cu
index 8f09a0ad0..dd8106378 100644
--- a/src/lamb_dipole/shu_hpc.cu
+++ b/src/lamb_dipole/shu_hpc.cu
@@ -1,7 +1,7 @@
 #include <iostream>
 #include <iomanip>
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "shu.cuh"
 #include "parameters.h"
 
@@ -18,19 +18,13 @@ int main( int argc, char * argv[])
     dg::Timer t;
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3)
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
         return -1;
     }
     else
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs);
-    }
+        file::file2Json( argv[1], js);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const Parameters p( js);
     p.display( std::cout);
@@ -39,7 +33,7 @@ int main( int argc, char * argv[])
     dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
     dg::DVec w2d( dg::create::weights(grid));
     dg::Lamb lamb( p.posX*p.lx, p.posY*p.ly, p.R, p.U);
-    dg::HVec omega; 
+    dg::HVec omega;
     if( p.initial == "lamb")
         omega = dg::evaluate ( lamb, grid);
     else if ( p.initial == "shear")
diff --git a/src/polar/polar.cu b/src/polar/polar.cu
index 7dee63f37..b6531d6f9 100644
--- a/src/polar/polar.cu
+++ b/src/polar/polar.cu
@@ -1,13 +1,13 @@
 #include <iostream>
 #include <iomanip>
 #include <sstream>
-#include <limits.h>  // UINT_MAX is needed in cusp (v0.5.1) but limits.h is not included
 #include <thrust/remove.h>
 #include <thrust/host_vector.h>
 
 
 #include "dg/algorithm.h"
-#include "geometries/geometries.h"
+#include "dg/geometries/geometries.h"
+#include "dg/file/json_utilities.h"
 
 #ifdef OPENGL_WINDOW
 #include "draw/host_window.h"
@@ -98,15 +98,9 @@ int main(int argc, char* argv[])
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "default");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "default");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
diff --git a/src/polar/polar_mpi.cu b/src/polar/polar_mpi.cu
index d41daf48b..1f250da99 100644
--- a/src/polar/polar_mpi.cu
+++ b/src/polar/polar_mpi.cu
@@ -1,13 +1,13 @@
 #include <iostream>
 #include <iomanip>
 #include <sstream>
-#include <limits.h>  // UINT_MAX is needed in cusp (v0.5.1) but limits.h is not included
 #include <mpi.h>
 #include <thrust/remove.h>
 #include <thrust/host_vector.h>
 
 #include "dg/algorithm.h"
-#include "geometries/geometries.h"
+#include "dg/geometries/geometries.h"
+#include "dg/file/json_utilities.h"
 
 #include "ns.h"
 #include "parameters.h"
@@ -38,15 +38,9 @@ int main(int argc, char* argv[])
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "default");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "default");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
diff --git a/src/reco2D/reconnection.cu b/src/reco2D/reconnection.cu
index 4650df80c..69743fe01 100644
--- a/src/reco2D/reconnection.cu
+++ b/src/reco2D/reconnection.cu
@@ -9,13 +9,9 @@
 //#include "draw/device_window.cuh"
 
 #include "reconnection.cuh"
+#include "dg/file/json_utilities.h"
 
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - integrates the ToeflR - functor and 
-   - directly visualizes results on the screen using parameters in window_params.txt
-*/
 
 int main( int argc, char* argv[])
 {
@@ -23,15 +19,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "default");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "default");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -40,9 +30,7 @@ int main( int argc, char* argv[])
     const asela::Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json( "window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/reco2D/reconnection_hpc.cu b/src/reco2D/reconnection_hpc.cu
index 2e4276302..66e71ec7d 100644
--- a/src/reco2D/reconnection_hpc.cu
+++ b/src/reco2D/reconnection_hpc.cu
@@ -5,22 +5,13 @@
 #include <cmath>
 
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "reconnection.cuh"
 
-/*
-   - reads parameters from input.txt or any other given file, 
-   - Initializes and integrates Asela and 
-   - writes outputs to a given outputfile using netcdf 
-        density fields are the real densities in XSPACE ( not logarithmic values)
-
-*/
-
 int main( int argc, char* argv[])
 {
     ////////////////////////Parameter initialisation//////////////////////////
-    Json::Reader reader;
     Json::Value js;
     if( argc != 3)
     {
@@ -28,10 +19,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        reader.parse(is,js,false);
-    }
+        file::file2Json( argv[1], js, "strict");
     const asela::Parameters p( js);
     p.display( std::cout);
     std::string input = js.toStyledString();
diff --git a/src/reco2D/reconnection_mpi.cu b/src/reco2D/reconnection_mpi.cu
index 9b4097a1b..3562f3062 100644
--- a/src/reco2D/reconnection_mpi.cu
+++ b/src/reco2D/reconnection_mpi.cu
@@ -9,18 +9,10 @@
 #include "dg/algorithm.h"
 
 #include "netcdf_par.h" //exclude if par netcdf=OFF
-#include "dg/file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "reconnection.cuh"
 
-/*
-    - the only difference to the asela_hpc.cu file is that this program 
-        uses the MPI backend and
-        the parallel netcdf output 
-    - pay attention that both the grid dimensions as well as the 
-        output dimensions must be divisible by the mpi process numbers
-*/
-
 int main( int argc, char* argv[])
 {
     ////////////////////////////////setup MPI///////////////////////////////
@@ -53,7 +45,6 @@ int main( int argc, char* argv[])
     MPI_Comm comm;
     MPI_Cart_create( MPI_COMM_WORLD, 2, np, periods, true, &comm);
     ////////////////////////Parameter initialisation//////////////////////////
-    Json::Reader reader;
     Json::Value js, gs;
     if( argc != 4)
     {
@@ -61,10 +52,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-    {
-        std::ifstream is(argv[1]);
-        reader.parse(is,js,false);
-    }
+        file::file2Json( argv[1], js, "strict");
     const asela::Parameters p( js);
     if(rank==0)p.display( std::cout);
     std::string input = js.toStyledString();
diff --git a/src/reco2D/window_params.js b/src/reco2D/window_params.json
similarity index 100%
rename from src/reco2D/window_params.js
rename to src/reco2D/window_params.json
diff --git a/src/toefl/toeflR.cu b/src/toefl/toeflR.cu
index 3ee2346dd..2ec250ee7 100644
--- a/src/toefl/toeflR.cu
+++ b/src/toefl/toeflR.cu
@@ -8,12 +8,13 @@
 
 #include "toeflR.cuh"
 #include "dg/algorithm.h"
+#include "dg/file/json_utilities.h"
 #include "parameters.h"
 
 /*
    - reads parameters from input.json or any other given file,
    - integrates the ToeflR - functor and
-   - directly visualizes results on the screen using parameters in window_params.txt
+   - directly visualizes results on the screen using parameters in window_params.json
 */
 
 
@@ -23,15 +24,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-    {
-        std::ifstream is("input.json");
-        is >> js;
-    }
+        file::file2Json( "input.json", js, "strict");
     else if( argc == 2)
-    {
-        std::ifstream is(argv[1]);
-        is >> js;
-    }
+        file::file2Json( argv[1], js, "strict");
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -40,9 +35,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    std::ifstream is( "window_params.js");
-    is >> js;
-    is.close();
+    file::file2Json( "window_params.json", js, "default");
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/toefl/toefl_hpc.cu b/src/toefl/toefl_hpc.cu
index a27541c9a..7362dfaae 100644
--- a/src/toefl/toefl_hpc.cu
+++ b/src/toefl/toefl_hpc.cu
@@ -7,7 +7,7 @@
 #endif //FELTOR_MPI
 
 #include "dg/algorithm.h"
-#include "dg/file/nc_utilities.h"
+#include "dg/file/file.h"
 
 #include "toeflR.cuh"
 #include "parameters.h"
@@ -67,19 +67,13 @@ int main( int argc, char* argv[])
 #endif//TOEFL_MPI
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
     if( argc != 3)
     {
         MPI_OUT std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
         return -1;
     }
     else
-    {
-        std::ifstream is(argv[1]);
-        parseFromStream( parser, is, &js, &errs); //read input without comments
-    }
+        file::file2Json( argv[1], js, "strict");
     MPI_OUT std::cout << js<<std::endl;
     const Parameters p( js);
     MPI_OUT p.display( std::cout);
diff --git a/src/toefl/window_params.js b/src/toefl/window_params.json
similarity index 100%
rename from src/toefl/window_params.js
rename to src/toefl/window_params.json
-- 
GitLab


From b9d6a71e7eb44d8fcbfbf7a3f7dfc8131399ab84 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 14 May 2020 21:59:48 +0200
Subject: [PATCH 224/540] Bring diag codes up-to-date concerning Json

---
 diag/crosscoherencdiag.cpp | 26 ++++++++---------------
 diag/feltorSHdiag.cpp      | 28 +++++++++----------------
 diag/feltorSHdiag.cu       | 31 ++++++++++-----------------
 diag/feltorSHvmaxdiag.cu   |  8 ++-----
 diag/feltorSesoldiag.cpp   | 25 ++++++++--------------
 diag/feltorShwdiag.cpp     | 28 +++++++++----------------
 diag/feltorShwmerger.cpp   | 11 ++++------
 diag/feltorShwradstat.cpp  | 10 +++------
 diag/feltorShwstat.cpp     | 10 +++------
 diag/fftwdiag.cpp          | 22 +++++++------------
 diag/growthrate.cpp        | 26 +++++++----------------
 diag/histdiag.cpp          |  7 ++++---
 diag/impRdiag.cu           | 43 +++++++++++++++++++-------------------
 diag/normdiag.cu           |  5 ++---
 diag/probes.h              |  2 +-
 diag/reco2Ddiag.cu         | 25 +++++++++-------------
 diag/toeflEPdiag.cu        | 27 ++++++++----------------
 diag/toeflRdiag.cu         | 32 ++++++++--------------------
 diag/vmaxnc.cu             |  3 +--
 19 files changed, 132 insertions(+), 237 deletions(-)

diff --git a/diag/crosscoherencdiag.cpp b/diag/crosscoherencdiag.cpp
index b292d5436..5022c05f1 100644
--- a/diag/crosscoherencdiag.cpp
+++ b/diag/crosscoherencdiag.cpp
@@ -6,7 +6,7 @@
 #include <algorithm> 
 
 #include "dg/algorithm.h"
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorShw/parameters.h"
 
 /**
@@ -42,21 +42,12 @@ int main( int argc, char* argv[])
         return -1;
     }
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;   
-     file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     ///////////////////read in and show inputfile//////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]); 
-    std::cout << "input "<<input<<std::endl;    
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
+    std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    file::string2Json( argv[1], input, js, "strict");
     const eule::Parameters p(js);
     p.display(std::cout);
 
@@ -73,12 +64,13 @@ int main( int argc, char* argv[])
     std::cout << "Gathering time data " << std::endl;
     size_t Estart[] = {0};
     size_t Ecount[] = {1};
-    err = nc_close(ncid);
     double time=0.;
     int NepID,phipID;
     double Nep,phip;
     unsigned step=0;
     //read in values into vectors
+    file::NC_Error_Handle err;
+    int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
 
     unsigned imin,imax;
@@ -136,7 +128,7 @@ int main( int argc, char* argv[])
     errout = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncidout); 
     //plot 1
     std::cout << "1d plot of Ne"<<std::endl;
-    errout = file::define_dimension( ncidout,"Ne_", &dim_ids1[0],  g1d1);
+    errout = file::define_dimension( ncidout, &dim_ids1[0],  g1d1, "Ne_");
     errout = nc_def_var( ncidout, "P(Ne)",   NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[0]);
     errout = nc_def_var( ncidout, "Ne",    NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[1]);
     errout = nc_enddef( ncidout);
@@ -145,7 +137,7 @@ int main( int argc, char* argv[])
     //plot 2
     std::cout << "1d plot of Phi"<<std::endl;
     errout = nc_redef(ncidout);
-    errout = file::define_dimension( ncidout,"Phi_", &dim_ids2[0],  g1d2);
+    errout = file::define_dimension( ncidout, &dim_ids2[0],  g1d2,"Phi_");
     errout = nc_def_var( ncidout, "P(Phi)",   NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[0]);
     errout = nc_def_var( ncidout, "Phi",    NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[1]);
     errout = nc_enddef( ncidout);
diff --git a/diag/feltorSHdiag.cpp b/diag/feltorSHdiag.cpp
index 0900166d0..88e71c708 100644
--- a/diag/feltorSHdiag.cpp
+++ b/diag/feltorSHdiag.cpp
@@ -7,7 +7,7 @@
 
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorSH/parameters.h"
 // #include "probes.h"
 
@@ -21,25 +21,14 @@ int main( int argc, char* argv[])
 //     std::ofstream os( argv[2]);
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
-    //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    ///////////////////read in and show inputfile und geomfile//////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
+    ///////////////////read in and show inputfile//////////////////
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
-    
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    file::string2Json( argv[1], input, js, "strict");
     const eule::Parameters p(js);
-    p.display();
+    p.display(std::cout);
     
     ///////////////////////////////////////////////////////////////////////////
     //Grids
@@ -93,7 +82,10 @@ int main( int argc, char* argv[])
 
     
     
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);   
+    //////////////////////////////open nc file//////////////////////////////////
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err1d = nc_open( argv[2], NC_WRITE, &ncid1d);
 
     for( unsigned i=0; i<p.maxout; i++)//timestepping
diff --git a/diag/feltorSHdiag.cu b/diag/feltorSHdiag.cu
index f10982373..8077a434c 100644
--- a/diag/feltorSHdiag.cu
+++ b/diag/feltorSHdiag.cu
@@ -6,10 +6,10 @@
 #include <sstream>
 
 #include "dg/algorithm.h"
-#include "geometries/geometries.h"
+#include "dg/geometries/geometries.h"
 
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorSH/parameters.h"
 // #include "probes.h"
 
@@ -44,25 +44,14 @@ int main( int argc, char* argv[])
 //     std::ofstream os( argv[2]);
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
-    //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    ///////////////////read in and show inputfile und geomfile//////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
+    ///////////////////read in and show inputfile//////////////////
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
-    
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    file::string2Json( argv[1], input, js, "strict");
     const eule::Parameters p(js);
-    p.display();
+    p.display(std::cout);
     
     ///////////////////////////////////////////////////////////////////////////
     //Grids
@@ -158,8 +147,10 @@ int main( int argc, char* argv[])
     Heaviside2d heavi(2.0* p.sigma);
     double normalize = 1.;
     dg::DVec heavy;
-    //open netcdf files
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);   
+    //////////////////////////////open nc file//////////////////////////////////
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
     unsigned position = 0;
     double posX_max = 0.0,posY_max = 0.0,posX_max_old = 0.0,posY_max_old = 0.0,velX_max=0.0, velY_max=0.0,posX_max_hs=0.0,posY_max_hs=0.0,velCOM=0.0;
diff --git a/diag/feltorSHvmaxdiag.cu b/diag/feltorSHvmaxdiag.cu
index e652e2fc1..30f95286f 100644
--- a/diag/feltorSHvmaxdiag.cu
+++ b/diag/feltorSHvmaxdiag.cu
@@ -8,7 +8,7 @@
 
 #include "dg/algorithm.h"
 #include "feltorSH/parameters.h"
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 
 //scan all imputfiles for maximum radial velocity and write to std::out
 int main( int argc, char* argv[])
@@ -34,11 +34,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
 //         std::cout << "input "<<input<<std::endl;
         Json::Value js;
-        Json::CharReaderBuilder parser;
-        parser["collectComments"] = false;
-        std::string errs;
-        std::stringstream ss(input);
-        parseFromStream( parser, ss, &js, &errs); //read input without comments
+        file::string2Json( argv[i], input, js, "strict");
         const eule::Parameters p(js);
         err = nc_inq_dimid( ncid, "time", &timeID);
         err = nc_inq_dimlen( ncid, timeID, &numOut);
diff --git a/diag/feltorSesoldiag.cpp b/diag/feltorSesoldiag.cpp
index c60956386..a6c441f41 100644
--- a/diag/feltorSesoldiag.cpp
+++ b/diag/feltorSesoldiag.cpp
@@ -7,7 +7,7 @@
 
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorSesol/parameters.h"
 // #include "probes.h"
 
@@ -21,24 +21,14 @@ int main( int argc, char* argv[])
 //     std::ofstream os( argv[2]);
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
-    //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    ///////////////////read in and show inputfile und geomfile//////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
-
+    ///////////////////read in and show inputfile//////////////////
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    file::string2Json( argv[1], input, js, "strict");
     const eule::Parameters p(js);
+    p.display(std::cout);
 
     ///////////////////////////////////////////////////////////////////////////
 
@@ -104,6 +94,8 @@ int main( int argc, char* argv[])
 //     std::cout << "enter new imin(>0) and imax(<maxout):" << std::endl;
 //     std::cin >> imin >> imax;
     time = imin*p.itstp;
+    file::NC_Error_Handle err;
+    int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
 
@@ -152,6 +144,7 @@ int main( int argc, char* argv[])
     err = nc_inq_dimlen(ncid, dataIDs[0], &steps);
     steps-=1;
     imax = steps/p.itstp;
+    //////////////////////////////open nc file//////////////////////////////////
     for( unsigned i=imin; i<imax; i++)//timestepping
     {
             start2d[0] = i;
diff --git a/diag/feltorShwdiag.cpp b/diag/feltorShwdiag.cpp
index cce55b419..7215bea2c 100644
--- a/diag/feltorShwdiag.cpp
+++ b/diag/feltorShwdiag.cpp
@@ -7,7 +7,7 @@
 
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorShw/parameters.h"
 // #include "probes.h"
 
@@ -21,25 +21,14 @@ int main( int argc, char* argv[])
 //     std::ofstream os( argv[2]);
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
-    //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    ///////////////////read in and show inputfile und geomfile//////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
-    
+    ///////////////////read in and show inputfile//////////////////
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
-    
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
-    const eule::Parameters p(js);   
+    file::string2Json( argv[1], input, js, "strict");
+    const eule::Parameters p(js);
+    p.display(std::cout);
     ///////////////////////////////////////////////////////////////////////////
     
     //Grids
@@ -117,6 +106,9 @@ int main( int argc, char* argv[])
     unsigned imin,imax;
     imin= 0;
     time = imin*p.itstp;
+    //////////////////////////////open nc file//////////////////////////////////
+    file::NC_Error_Handle err;
+    int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
 
diff --git a/diag/feltorShwmerger.cpp b/diag/feltorShwmerger.cpp
index 34231860d..624360dbb 100644
--- a/diag/feltorShwmerger.cpp
+++ b/diag/feltorShwmerger.cpp
@@ -7,9 +7,10 @@
 
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorShw/parameters.h"
 
+//MW: the command line argument ncrcat should do the same doesn't it?
 //merge inputfiles together to a new output file
 //be aware of grids!!! Should be equal for all input files
 int main( int argc, char* argv[])
@@ -43,12 +44,8 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        Json::CharReaderBuilder parser;
-        parser["collectComments"] = false;
-        std::string errs;
-        std::stringstream ss(input);
-        parseFromStream( parser, ss, &js, &errs); //read input without comments
-        const eule::Parameters p(js);   
+        file::string2Json( argv[i], input, js, "strict");
+        const eule::Parameters p(js);
         
         dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
         size_t count2d[3]  = {1, g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
diff --git a/diag/feltorShwradstat.cpp b/diag/feltorShwradstat.cpp
index c2a6d74d9..a69a36d94 100644
--- a/diag/feltorShwradstat.cpp
+++ b/diag/feltorShwradstat.cpp
@@ -8,7 +8,7 @@
 #include <iterator>
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorShw/parameters.h"
 
 int main( int argc, char* argv[])
@@ -35,12 +35,8 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        Json::CharReaderBuilder parser;
-        parser["collectComments"] = false;
-        std::string errs;
-        std::stringstream ss(input);
-        parseFromStream( parser, ss, &js, &errs); //read input without comments
-        const eule::Parameters p(js);   
+        file::string2Json( argv[i], input, js, "strict");
+        const eule::Parameters p(js);
         
         dg::Grid1d g1d( 0., p.lx,p.n_out, p.Nx_out, p.bc_x);
 	size_t count1d[2]  = {1, g1d.n()*g1d.N()};
diff --git a/diag/feltorShwstat.cpp b/diag/feltorShwstat.cpp
index e99df7f18..d3ef27ab9 100644
--- a/diag/feltorShwstat.cpp
+++ b/diag/feltorShwstat.cpp
@@ -8,7 +8,7 @@
 #include <iterator>
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorShw/parameters.h"
 
 double Mean(std::vector<double> v, unsigned imin, unsigned imax)
@@ -49,12 +49,8 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        Json::CharReaderBuilder parser;
-        parser["collectComments"] = false;
-        std::string errs;
-        std::stringstream ss(input);
-        parseFromStream( parser, ss, &js, &errs); //read input without comments
-        const eule::Parameters p(js);   
+        file::string2Json( argv[i], input, js, "strict");
+        const eule::Parameters p(js);
         
 	size_t start0d  = 0;    
         //get maxtime of input file
diff --git a/diag/fftwdiag.cpp b/diag/fftwdiag.cpp
index 7fd5a88dc..e1bacf0ee 100644
--- a/diag/fftwdiag.cpp
+++ b/diag/fftwdiag.cpp
@@ -10,7 +10,7 @@
 
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorShw/parameters.h"
 int main( int argc, char* argv[])
 {
@@ -26,22 +26,14 @@ int main( int argc, char* argv[])
     file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    ///////////////////read in and show inputfile //////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);    
-    err = nc_close(ncid); 
-
-//     std::cout << "input "<<input<<std::endl;    
+    ///////////////////read in and show inputfile//////////////////
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
+    std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    file::string2Json( argv[1], input, js, "strict");
     const eule::Parameters p(js);
-//     p.display(std::cout);
+    p.display(std::cout);
     
     //////////////////////////////Grids//////////////////////////////////////
     //input grid
diff --git a/diag/growthrate.cpp b/diag/growthrate.cpp
index 13be0c149..a467724f7 100644
--- a/diag/growthrate.cpp
+++ b/diag/growthrate.cpp
@@ -10,7 +10,7 @@
 
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorShw/parameters.h"
 int main( int argc, char* argv[])
 {
@@ -20,26 +20,14 @@ int main( int argc, char* argv[])
         return -1;
     }
 
-    //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    ///////////////////read in and show inputfile //////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);    
-    err = nc_close(ncid); 
-
-//     std::cout << "input "<<input<<std::endl;    
+    ///////////////////read in and show inputfile//////////////////
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
+    std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    file::string2Json( argv[1], input, js, "strict");
     const eule::Parameters p(js);
-//     p.display(std::cout);
+    p.display(std::cout);
     
     //////////////////////////////Grids//////////////////////////////////////
     //input grid
diff --git a/diag/histdiag.cpp b/diag/histdiag.cpp
index ad56f6ad8..784a5293d 100644
--- a/diag/histdiag.cpp
+++ b/diag/histdiag.cpp
@@ -4,9 +4,10 @@
 #include <vector>
 #include <string>
 #include <algorithm> 
+#include <thrust/random.h>
 
 #include "dg/algorithm.h"
-#include "file/nc_utilities.h"
+#include "dg/file/nc_utilities.h"
 
 /**
  * @brief normalizes input vector 
@@ -97,7 +98,7 @@ int main( int argc, char* argv[])
     file::NC_Error_Handle err; 
     err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid); 
     //plot 1
-    err = file::define_dimension( ncid,"A1_", &dim_ids1[0],  g1d1);
+    err = file::define_dimension( ncid, &dim_ids1[0],  g1d1,"A1_");
     err = nc_def_var( ncid, "P(A1)",   NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[0]);
     err = nc_def_var( ncid, "A1",    NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[1]);
     err = nc_enddef( ncid);
@@ -105,7 +106,7 @@ int main( int argc, char* argv[])
     err = nc_put_var_double( ncid, dataIDs1[1], A1.data() );
     err = nc_redef(ncid);
     //plot 2
-    err = file::define_dimension( ncid,"A2_", &dim_ids2[0],  g1d2);
+    err = file::define_dimension( ncid, &dim_ids2[0],  g1d2,"A2_");
     err = nc_def_var( ncid, "P(A2)",   NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[0]);
     err = nc_def_var( ncid, "A2",    NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[1]);
     err = nc_enddef( ncid);
diff --git a/diag/impRdiag.cu b/diag/impRdiag.cu
index e6eb2b0e0..018c6fd6d 100644
--- a/diag/impRdiag.cu
+++ b/diag/impRdiag.cu
@@ -6,7 +6,7 @@
 #include <sstream>
 
 #include "dg/algorithm.h"
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "impurities/parameters.h"
 
 struct Heaviside2d
@@ -26,28 +26,27 @@ private:
 };
 
 int main( int argc, char* argv[])
-{ if( argc != 3)   // lazy check: command line parameters
-  { std::cerr << "Usage: "<< argv[0] <<" [input.nc] [output.nc]\n";
-    return -1;
-  }
-  std::cout << argv[1] << " -> " << argv[2]<<std::endl;
-  ////////process parameter from .nc datafile////////
-  file::NC_Error_Handle err_in;
-  int ncid_in;
-  err_in = nc_open(argv[1], NC_NOWRITE, &ncid_in);
-  //read & print parameter string
-  size_t length;
-  err_in = nc_inq_attlen(ncid_in, NC_GLOBAL, "inputfile", &length);
-  std::string input(length, 'x');
-  err_in = nc_get_att_text(ncid_in, NC_GLOBAL, "inputfile", &input[0]);
-  std::cout << "input "<< input << std::endl;
-  //parse: parameter string--json-->p.xxx
+{
+    if( argc != 3)   // lazy check: command line parameters
+    {
+        std::cerr << "Usage: "<< argv[0] <<" [input.nc] [output.nc]\n";
+        return -1;
+    }
+    std::cout << argv[1] << " -> " << argv[2]<<std::endl;
+    ////////process parameter from .nc datafile////////
+    file::NC_Error_Handle err_in;
+    int ncid_in;
+    err_in = nc_open(argv[1], NC_NOWRITE, &ncid_in);
+    //read & print parameter string
+    size_t length;
+    err_in = nc_inq_attlen(ncid_in, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err_in = nc_get_att_text(ncid_in, NC_GLOBAL, "inputfile", &input[0]);
+    std::cout << "input "<< input << std::endl;
+    //parse: parameter string--json-->p.xxx
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    file::string2Json( argv[1], input, js, "strict");
+
   const imp::Parameters p(js);
   p.display(std::cout);
   // dimensions
diff --git a/diag/normdiag.cu b/diag/normdiag.cu
index c018c3690..65baaaf5d 100644
--- a/diag/normdiag.cu
+++ b/diag/normdiag.cu
@@ -11,7 +11,7 @@
 
 #include "dg/functors.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "feltorShw/parameters.h"
 int main( int argc, char* argv[])
 {
@@ -32,9 +32,8 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);    
     err = nc_close(ncid); 
 
-    Json::Reader reader;
     Json::Value js;
-    reader.parse( input, js, false);
+    file::string2Json( argv[1], input, js, "strict");
     const eule::Parameters p(js);
     
     //////////////////////////////Grids//////////////////////////////////////
diff --git a/diag/probes.h b/diag/probes.h
index d48da888e..eb77dda2c 100644
--- a/diag/probes.h
+++ b/diag/probes.h
@@ -8,12 +8,12 @@
 #ifndef PROBES_H
 #define PROBES_H
 
-using namespace std;
 
 #include "dg/algorithm.h"
 #include <fstream>
 #include <sstream>
 
+//using namespace std;
 /* 
  * Class that takes care of probe output
  * 
diff --git a/diag/reco2Ddiag.cu b/diag/reco2Ddiag.cu
index 9c872d05f..c6e25d8c1 100644
--- a/diag/reco2Ddiag.cu
+++ b/diag/reco2Ddiag.cu
@@ -11,7 +11,7 @@
 
 #include "dg/functors.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "reco2D/parameters.h"
 int main( int argc, char* argv[])
 {
@@ -21,22 +21,14 @@ int main( int argc, char* argv[])
         return -1;
     }
 
-    //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    ///////////////////read in and show inputfile //////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);    
-    err = nc_close(ncid); 
-
-    Json::Reader reader;
+    ///////////////////read in and show inputfile//////////////////
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
+    std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    reader.parse( input, js, false);
+    file::string2Json( argv[1], input, js, "strict");
     const asela::Parameters p(js);
-    
+
     //////////////////////////////Grids//////////////////////////////////////
     //input grid
     dg::Grid2d g2d( -p.lxhalf, p.lxhalf, -p.lyhalf, p.lyhalf , p.n, p.Nx, p.Ny, dg::DIR, dg::PER);
@@ -56,6 +48,9 @@ int main( int argc, char* argv[])
     dg::DVec apareq(dg::evaluate( init0, g2d));
 
     //open netcdf files
+    //////////////////////////////open nc file//////////////////////////////////
+    file::NC_Error_Handle err;
+    int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     //set min and max timesteps
     double time = 0.;
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index f90739c01..137925e47 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -7,7 +7,7 @@
 
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "ep/parameters.h"
 // #include "probes.h"
 
@@ -62,26 +62,14 @@ int main( int argc, char* argv[])
 //     std::ofstream os( argv[2]);
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
-    //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    ///////////////////read in and show inputfile und geomfile//////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
+    ///////////////////read in and show inputfile//////////////////
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
-    
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    file::string2Json( argv[1], input, js, "strict");
     const Parameters p(js);
     p.display(std::cout);
-    err = nc_close( ncid);
     
     ///////////////////////////////////////////////////////////////////////////
     //Grids
@@ -169,7 +157,10 @@ int main( int argc, char* argv[])
     double posX_max = 0.0,posY_max = 0.0,posX_max_old = 0.0,posY_max_old = 0.0,velX_max=0.0, velY_max=0.0,posX_max_hs=0.0,posY_max_hs=0.0,velCOM=0.0;
     double compactness_ne=0.0;
     //-----------------Start timestepping
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);   
+    //////////////////////////////open nc file//////////////////////////////////
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
     for( unsigned i=0; i<=p.maxout; i++)
     {
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index 0da35c7f8..992121a36 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -7,17 +7,12 @@
 
 #include "dg/algorithm.h"
 
-#include "file/nc_utilities.h"
+#include "dg/file/file.h"
 #include "toefl/parameters.h"
-// #include "probes.h"
-
-double X( double x, double y) {return x;}
-double Y( double x, double y) {return y;}
 
 struct Heaviside2d
 {
     Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){}
-//     Heaviside2d( double sigma):sigma2_(sigma*sigma), x_(0), y_(0){}
     void set_origin( double x0, double y0){ x_=x0, y_=y0;}
     double operator()(double x, double y)const
     {
@@ -62,26 +57,14 @@ int main( int argc, char* argv[])
 //     std::ofstream os( argv[2]);
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
-    //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
-    ///////////////////read in and show inputfile und geomfile//////////////////
-    size_t length;
-    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
-    std::string input( length, 'x');
-    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
+    ///////////////////read in and show inputfile//////////////////
+    std::string input;
+    file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
-    
     Json::Value js;
-    Json::CharReaderBuilder parser;
-    parser["collectComments"] = false;
-    std::string errs;
-    std::stringstream ss(input);
-    parseFromStream( parser, ss, &js, &errs); //read input without comments
+    file::string2Json( argv[1], input, js, "strict");
     const Parameters p(js);
     p.display(std::cout);
-    err = nc_close( ncid);
     
     ///////////////////////////////////////////////////////////////////////////
     //Grids
@@ -169,7 +152,10 @@ int main( int argc, char* argv[])
     double posX_max = 0.0,posY_max = 0.0,posX_max_old = 0.0,posY_max_old = 0.0,velX_max=0.0, velY_max=0.0,posX_max_hs=0.0,posY_max_hs=0.0,velCOM=0.0;
     double compactness_ne=0.0;
     //-----------------Start timestepping
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);   
+    //////////////////////////////open nc file//////////////////////////////////
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
     for( unsigned i=0; i<=p.maxout; i++)
     {
diff --git a/diag/vmaxnc.cu b/diag/vmaxnc.cu
index 9b65f0ae1..3895d27f1 100644
--- a/diag/vmaxnc.cu
+++ b/diag/vmaxnc.cu
@@ -5,7 +5,7 @@
 #include <string>
 #include <algorithm>
 
-#include "file/nc_utilities.h"
+#include "dg/file/nc_utilities.h"
 
 //scan all imputfiles for maximum radial velocity and write to std::out
 int main( int argc, char* argv[])
@@ -32,7 +32,6 @@ int main( int argc, char* argv[])
 
     }
 
-    
     return 0;
 }
 
-- 
GitLab


From 42d62c3b4869866558512ca68df35919308bec3b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 15 May 2020 23:16:39 +0200
Subject: [PATCH 225/540] Create Distance functor

---
 inc/dg/functors.h | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index ae03412c0..7ebbb190f 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -805,6 +805,21 @@ struct Heaviside
     int m_s;
 };
 
+
+/**
+ * @brief \f[ \sqrt{ (x-x_0)^2 + (y-y_0)^2} \f]
+ */
+struct Distance
+{
+    Distance( double x0, double y0): m_x0(x0), m_y0(y0){}
+    DG_DEVICE
+    double operator()(double x, double y){
+        return sqrt( (x-m_x0)*(x-m_x0) + (y-m_y0)*(y-m_y0));
+    }
+    private:
+    double m_x0, m_y0;
+};
+
 /**
  * @brief One up to \c psimax, then a Gaussian down to zero
      \f[ \begin{cases}
-- 
GitLab


From ad7768d3cf2520fb21dd15b84e7c8f36b26a9d6c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 15 May 2020 23:28:06 +0200
Subject: [PATCH 226/540] Make ep/toefl_mpi use serial netcdf

- start an attempt to unify the outline of our src projects
---
 inc/file/easy_output.h    |  2 +-
 src/ep/input/default.json | 52 +++++++++++------------
 src/ep/toefl_mpi.cu       | 89 ++++++++++++++++++---------------------
 3 files changed, 67 insertions(+), 76 deletions(-)

diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
index 6b7413561..212c109a4 100644
--- a/inc/file/easy_output.h
+++ b/inc/file/easy_output.h
@@ -87,7 +87,7 @@ struct NC_Error_Handle
 * or each process funnels its data through the master rank (\c false),
 * which involves communication but may be faster than the former method.
 * @attention In the MPI version, if \c parallel==true a **parallel netcdf** must be
-* linked while if \c parallel==false we need **serial netcdf**.
+* linked, the file opened with the \c NC_MPIIO flag from the \c netcdf_par.h header and the variable be marked with \c NC_COLLECTIVE access while if \c parallel==false we need **serial netcdf** and only the master thread needs to open and access the file.
 * Note that serious performance penalties have been observed on some platforms for parallel netcdf.
 */
 template<class host_vector>
diff --git a/src/ep/input/default.json b/src/ep/input/default.json
index beeca52e0..07c7468db 100644
--- a/src/ep/input/default.json
+++ b/src/ep/input/default.json
@@ -1,31 +1,29 @@
-/* This is a comment: Remember that comments are not conformal 
-with the JSON standard but are allowed only with some specifig JSON parsers. So remove them when you automate the process */
 {
-    "n" :  3,       //# x-y-polynomials
-    "Nx" : 100,     //# grid points in x
-    "Ny" : 100,     //# grid points in y
-    "dt" : 3,     //time step in units of c_s/rho_s
-    "n_out"  : 3,   //# x-y polynomials in output
-    "Nx_out" : 100, //# grid points in x in output fields
-    "Ny_out" : 100, //# grid points in y in output fields
-    "itstp"  : 2,   //steps between outputs
-    "maxout" : 100, //# outputs excluding first
-    "eps_pol"   : 1e-6,   //accuracy of polarisation solver
-    "eps_gamma" : 1e-7,   //accuracy of Gamma 
-    "eps_time"  : 1e-10,  //accuracy of implicit time-stepper
-    "curvature"  : 0.0001, //magnetic curvature
-    "tau"  : [-1.0, 1.0], //T_s/(Z_s Te)
-    "mu"   : [-1.0, 1.0], // m_s/(Z_s m_e)
+    "n" :  3,
+    "Nx" : 100,
+    "Ny" : 100,
+    "dt" : 3,
+    "n_out"  : 3,
+    "Nx_out" : 100,
+    "Ny_out" : 100,
+    "itstp"  : 2,
+    "maxout" : 100,
+    "eps_pol"   : 1e-6,
+    "eps_gamma" : 1e-7,
+    "eps_time"  : 1e-10,
+    "curvature"  : 0.0001,
+    "tau"  : [-1.0, 1.0],
+    "mu"   : [-1.0, 1.0],
     "z"    : [-1, 1],
-    "nu_perp"  : 5e-3, //pependicular viscosity
-    "amplitude"  : 0.5,//amplitude of the blob
-    "sigma"  : 10,     //blob variance in units of rho_s
-    "posX"  : 0.3,     //blob x-position in units of lx
-    "posY"  : 0.5,     //blob y-position in units of ly
-    "lx"  : 200,       //lx in units of rho_s
-    "ly"  : 200,       //ly in units of rho_s
-    "bc_x"  : "DIR",   //boundary condition in x
-    "bc_y"  : "PER",   //boundary condition in y
-    "debye":  0.0   //debye length parameter
+    "nu_perp"  : 5e-3,
+    "amplitude"  : 0.5,
+    "sigma"  : 10,
+    "posX"  : 0.3,
+    "posY"  : 0.5,
+    "lx"  : 200,
+    "ly"  : 200,
+    "bc_x"  : "DIR",
+    "bc_y"  : "PER",
+    "debye":  0.0
 
 }
diff --git a/src/ep/toefl_mpi.cu b/src/ep/toefl_mpi.cu
index 5c6527a6e..08f3234fb 100644
--- a/src/ep/toefl_mpi.cu
+++ b/src/ep/toefl_mpi.cu
@@ -4,8 +4,6 @@
 
 #include <mpi.h> //activate mpi
 
-#include "netcdf_par.h"
-
 #include "dg/algorithm.h"
 #include "dg/file/file.h"
 #include "toeflR.cuh"
@@ -90,34 +88,25 @@ int main( int argc, char* argv[])
     /////////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
     int ncid;
-    MPI_Info info = MPI_INFO_NULL;
-    err = nc_create_par( argv[2],NC_NETCDF4|NC_MPIIO|NC_CLOBBER,comm,info, &ncid);
-    err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
+    if(rank==0)err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
+    if(rank==0)err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
+    if(rank==0)err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
     //field IDs
     std::string names[4] = {"electrons", "positrons", "potential", "vorticity"}; 
     int dataIDs[4]; 
     for( unsigned i=0; i<4; i++){
-        err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);}
+        if(rank==0)err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);}
 
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    if(rank==0)err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, dissID, dEdtID;
-    err = nc_def_var( ncid, "energy",      NC_DOUBLE, 1, &EtimeID, &energyID);
-    err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
-    err = nc_def_var( ncid, "dissipation", NC_DOUBLE, 1, &EtimeID, &dissID);
-    err = nc_def_var( ncid, "dEdt",        NC_DOUBLE, 1, &EtimeID, &dEdtID);
-    for(unsigned i=0; i<4; i++)
-        err = nc_var_par_access( ncid, dataIDs[i], NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, tvarID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, EtimevarID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, energyID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, massID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, dissID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, dEdtID, NC_COLLECTIVE);
-    err = nc_enddef(ncid);
+    if(rank==0)err = nc_def_var( ncid, "energy",      NC_DOUBLE, 1, &EtimeID, &energyID);
+    if(rank==0)err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
+    if(rank==0)err = nc_def_var( ncid, "dissipation", NC_DOUBLE, 1, &EtimeID, &dissID);
+    if(rank==0)err = nc_def_var( ncid, "dEdt",        NC_DOUBLE, 1, &EtimeID, &dEdtID);
+    if(rank==0)err = nc_enddef(ncid);
 
     ///////////////////////////////////first output/////////////////////////
     int dims[2],  coords[2];
@@ -127,28 +116,28 @@ int main( int argc, char* argv[])
     size_t Estart[] = {0};
     size_t Ecount[] = {1};
     dg::MDVec transfer( dg::evaluate(dg::zero, grid));
-    dg::DVec transferD( dg::evaluate(dg::zero, grid_out.local()));
-    dg::HVec transferH( dg::evaluate(dg::zero, grid_out.local()));
-    dg::IDMatrix interpolate = dg::create::interpolation( grid_out.local(), grid.local()); //create local interpolation matrix
+    dg::MDVec transferD( dg::evaluate(dg::zero, grid_out));
+    dg::MHVec transferH( dg::evaluate(dg::zero, grid_out));
+    dg::MIDMatrix interpolate = dg::create::interpolation( grid_out, grid);
     for( unsigned i=0; i<2; i++)
     {
-        dg::blas2::gemv( interpolate, y0[i].data(), transferD);
+        dg::blas2::gemv( interpolate, y0[i], transferD);
         dg::blas1::transfer( transferD, transferH);
-        err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
+        file::put_vara_double( ncid, dataIDs[i], 0, grid_out, transferH);
     }
     //pot
     transfer = test.potential();
-    dg::blas2::gemv( interpolate, transfer.data(), transferD);
+    dg::blas2::gemv( interpolate, transfer, transferD);
     dg::blas1::transfer( transferD, transferH);
-    err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
+    file::put_vara_double( ncid, dataIDs[2], 0, grid_out, transferH );
     //Vor
     transfer = test.potential();
-    dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);        
-    dg::blas2::gemv( interpolate,y1[1].data(), transferD);
+    dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);
+    dg::blas2::gemv( interpolate,y1[1], transferD);
     dg::blas1::transfer( transferD, transferH);
-    err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
-    err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-    //err = nc_close(ncid);
+    file::put_vara_double( ncid, dataIDs[3], 0, grid_out, transferH );
+    if(rank==0)err = nc_put_vara_double( ncid, tvarID, start, count, &time);
+    if(rank==0)err = nc_close(ncid);
     ///////////////////////////////////////Timeloop/////////////////////////////////
     const double mass0 = test.mass(), mass_blob0 = mass0 - grid.lx()*grid.ly();
     double E0 = test.energy(), energy0 = E0, E1 = 0, diff = 0;
@@ -183,34 +172,39 @@ int main( int argc, char* argv[])
             time+=p.dt;
             Estart[0] += 1;
             {
-                //err = nc_open(argv[2], NC_WRITE, &ncid);
                 double ener=test.energy(), mass=test.mass(), diff=test.mass_diffusion(), dEdt=test.energy_diffusion();
-                err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
-                err = nc_put_vara_double( ncid, energyID,   Estart, Ecount, &ener);
-                err = nc_put_vara_double( ncid, massID,     Estart, Ecount, &mass);
-                err = nc_put_vara_double( ncid, dissID,     Estart, Ecount, &diff);
-                err = nc_put_vara_double( ncid, dEdtID,     Estart, Ecount, &dEdt);
-                //err = nc_close(ncid);
+                if(rank==0)
+                {
+                    err = nc_open(argv[2], NC_WRITE, &ncid);
+                    err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
+                    err = nc_put_vara_double( ncid, energyID,   Estart, Ecount, &ener);
+                    err = nc_put_vara_double( ncid, massID,     Estart, Ecount, &mass);
+                    err = nc_put_vara_double( ncid, dissID,     Estart, Ecount, &diff);
+                    err = nc_put_vara_double( ncid, dEdtID,     Estart, Ecount, &dEdt);
+                    err = nc_close(ncid);
+                }
             }
         }
         //////////////////////////write fields////////////////////////
+        if(rank==0)err = nc_open(argv[2], NC_WRITE, &ncid);
         start[0] = i;
         for( unsigned j=0; j<2; j++)
         {
-            dg::blas2::gemv( interpolate, y0[j].data(), transferD);
+            dg::blas2::gemv( interpolate, y0[j], transferD);
             dg::blas1::transfer( transferD, transferH);
-            err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
+            file::put_vara_double( ncid, dataIDs[j], i, grid, transferH);
         }
         transfer = test.potential();
-        dg::blas2::gemv( interpolate, transfer.data(), transferD);
+        dg::blas2::gemv( interpolate, transfer, transferD);
         dg::blas1::transfer( transferD, transferH);
-        err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
+        file::put_vara_double( ncid, dataIDs[2], i, grid, transferH );
         transfer = test.potential();
         dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);        //correct?    
-        dg::blas2::gemv( interpolate,y1[1].data(), transferD);
+        dg::blas2::gemv( interpolate,y1[1], transferD);
         dg::blas1::transfer( transferD, transferH);
-        err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
-        err = nc_put_vara_double( ncid, tvarID, start, count, &time);
+        file::put_vara_double( ncid, dataIDs[3], i, grid, transferH );
+        if(rank==0)err = nc_put_vara_double( ncid, tvarID, start, count, &time);
+        if(rank==0)err = nc_close(ncid);
 
 #ifdef DG_BENCHMARK
         ti.toc();
@@ -231,7 +225,6 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << std::fixed << std::setprecision(2) <<std::setfill('0');
     if(rank==0)std::cout <<"Computation Time \t"<<hour<<":"<<std::setw(2)<<minute<<":"<<second<<"\n";
     if(rank==0)std::cout <<"which is         \t"<<t.diff()/p.itstp/p.maxout<<"s/step\n";
-    nc_close(ncid);
     MPI_Finalize();
 
     return 0;
-- 
GitLab


From 9d4265e82589ed8145f568d6a548d18ef68d80a3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 16 May 2020 12:57:02 +0200
Subject: [PATCH 227/540] Fix excessive creation of new communicators

---
 inc/dg/backend/exblas/mpi_accumulate.h | 49 +++++++++++++++++---------
 1 file changed, 32 insertions(+), 17 deletions(-)

diff --git a/inc/dg/backend/exblas/mpi_accumulate.h b/inc/dg/backend/exblas/mpi_accumulate.h
index c450bb9bf..cc94f5de7 100644
--- a/inc/dg/backend/exblas/mpi_accumulate.h
+++ b/inc/dg/backend/exblas/mpi_accumulate.h
@@ -12,6 +12,10 @@
 
 namespace exblas {
 
+namespace detail{
+//we keep track of communicators that were created in the past
+static std::map<MPI_Comm, std::array<MPI_Comm, 2>> comm_mods;
+}
 /**
  * @brief This function can be used to partition communicators for the \c exblas::reduce_mpi_cpu function
  *
@@ -19,26 +23,37 @@ namespace exblas {
  * @param comm the input communicator (unmodified, may not be \c MPI_COMM_NULL)
  * @param comm_mod a subgroup of comm (comm is split)
  * @param comm_mod_reduce a subgroup of comm, consists of all rank 0 processes in comm_mod
- * @note the creation of new communicators involves communication between all participation processes (comm in this case)
+ * @note the creation of new communicators involves communication between all participation processes (comm in this case).
+ * @attention In order to avoid excessive creation of new MPI communicators (there is a limit to how many a program can create), the function keeps record of which communicators it has been called with. If you repeatedly call this function with the same \c comm only the first call will actually create new communicators.
  */
 static void mpi_reduce_communicator(MPI_Comm comm, MPI_Comm* comm_mod, MPI_Comm* comm_mod_reduce){
     assert( comm != MPI_COMM_NULL);
-    int mod = 128;
-    int rank, size;
-    MPI_Comm_rank( comm, &rank);
-    MPI_Comm_size( comm, &size);
-    MPI_Comm_split( comm, rank/mod, rank%mod, comm_mod); //collective call
-    MPI_Group group, reduce_group;
-    MPI_Comm_group( comm, &group); //local call
-    int reduce_size=(int)ceil((double)size/(double)mod);
-    std::vector<int> reduce_ranks(reduce_size);
-    for( int i=0; i<reduce_size; i++)
-        reduce_ranks[i] = i*mod;
-    MPI_Group_incl( group, reduce_size, reduce_ranks.data(), &reduce_group); //local
-    MPI_Comm_create( comm, reduce_group, comm_mod_reduce); //collective
-    MPI_Group_free( &group);
-    MPI_Group_free( &reduce_group);
-    //returns MPI_COMM_NULL to processes that are not in the group
+    if( detail::comm_mods.count(comm) == 1 )
+    {
+        *comm_mod = detail::comm_mods[comm][0];
+        *comm_mod_reduce = detail::comm_mods[comm][1];
+        return;
+    }
+    else
+    {
+        int mod = 128;
+        int rank, size;
+        MPI_Comm_rank( comm, &rank);
+        MPI_Comm_size( comm, &size);
+        MPI_Comm_split( comm, rank/mod, rank%mod, comm_mod); //collective call
+        MPI_Group group, reduce_group;
+        MPI_Comm_group( comm, &group); //local call
+        int reduce_size=(int)ceil((double)size/(double)mod);
+        std::vector<int> reduce_ranks(reduce_size);
+        for( int i=0; i<reduce_size; i++)
+            reduce_ranks[i] = i*mod;
+        MPI_Group_incl( group, reduce_size, reduce_ranks.data(), &reduce_group); //local
+        MPI_Comm_create( comm, reduce_group, comm_mod_reduce); //collective
+        MPI_Group_free( &group);
+        MPI_Group_free( &reduce_group);
+        detail::comm_mods[comm] = {*comm_mod, *comm_mod_reduce};
+        //returns MPI_COMM_NULL to processes that are not in the group
+    }
 }
 
 /*! @brief reduce a number of superaccumulators distributed among mpi processes
-- 
GitLab


From fbf35349839cba77a9e7536033e5038a3e0431f2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 16 May 2020 13:36:58 +0200
Subject: [PATCH 228/540] Make json and nc utilities throw instead of EXIT

- also change parameters of netcdf2string
---
 diag/crosscoherencdiag.cpp      |  2 +-
 diag/feltorSHdiag.cpp           |  2 +-
 diag/feltorSHdiag.cu            |  2 +-
 diag/feltorSHvmaxdiag.cu        |  2 +-
 diag/feltorSesoldiag.cpp        |  2 +-
 diag/feltorShwdiag.cpp          |  2 +-
 diag/feltorShwmerger.cpp        |  2 +-
 diag/feltorShwradstat.cpp       |  2 +-
 diag/feltorShwstat.cpp          |  2 +-
 diag/fftwdiag.cpp               |  2 +-
 diag/growthrate.cpp             |  2 +-
 diag/impRdiag.cu                |  2 +-
 diag/normdiag.cu                |  2 +-
 diag/reco2Ddiag.cu              |  2 +-
 diag/toeflEPdiag.cu             |  2 +-
 diag/toeflRdiag.cu              |  2 +-
 inc/file/easy_output.h          | 27 +++++++++++++++++++++++++--
 inc/file/json_utilities.h       | 23 ++++++++++-------------
 inc/file/nc_utilities.h         | 10 +++++-----
 src/feltor/feltordiag.cu        |  4 ++--
 src/feltor/init_from_file.h     |  2 +-
 src/feltor/interpolate_in_3d.cu |  4 ++--
 src/feltorSesol/feltor_hpc.cu   |  2 +-
 src/feltorSesol/feltor_mpi.cu   |  2 +-
 src/feltorShw/feltor_hpc.cu     |  2 +-
 src/feltorShw/feltor_mpi.cu     |  2 +-
 src/heat/heat_hpc.cu            |  4 ++--
 27 files changed, 67 insertions(+), 47 deletions(-)

diff --git a/diag/crosscoherencdiag.cpp b/diag/crosscoherencdiag.cpp
index 5022c05f1..e826c722f 100644
--- a/diag/crosscoherencdiag.cpp
+++ b/diag/crosscoherencdiag.cpp
@@ -47,7 +47,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const eule::Parameters p(js);
     p.display(std::cout);
 
diff --git a/diag/feltorSHdiag.cpp b/diag/feltorSHdiag.cpp
index 88e71c708..dfb1fd483 100644
--- a/diag/feltorSHdiag.cpp
+++ b/diag/feltorSHdiag.cpp
@@ -26,7 +26,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const eule::Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/feltorSHdiag.cu b/diag/feltorSHdiag.cu
index 8077a434c..a5af65475 100644
--- a/diag/feltorSHdiag.cu
+++ b/diag/feltorSHdiag.cu
@@ -49,7 +49,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const eule::Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/feltorSHvmaxdiag.cu b/diag/feltorSHvmaxdiag.cu
index 30f95286f..674002331 100644
--- a/diag/feltorSHvmaxdiag.cu
+++ b/diag/feltorSHvmaxdiag.cu
@@ -34,7 +34,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
 //         std::cout << "input "<<input<<std::endl;
         Json::Value js;
-        file::string2Json( argv[i], input, js, "strict");
+        file::string2Json( input, js, "strict");
         const eule::Parameters p(js);
         err = nc_inq_dimid( ncid, "time", &timeID);
         err = nc_inq_dimlen( ncid, timeID, &numOut);
diff --git a/diag/feltorSesoldiag.cpp b/diag/feltorSesoldiag.cpp
index a6c441f41..423f5da6d 100644
--- a/diag/feltorSesoldiag.cpp
+++ b/diag/feltorSesoldiag.cpp
@@ -26,7 +26,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const eule::Parameters p(js);
     p.display(std::cout);
 
diff --git a/diag/feltorShwdiag.cpp b/diag/feltorShwdiag.cpp
index 7215bea2c..cfb6ea9b4 100644
--- a/diag/feltorShwdiag.cpp
+++ b/diag/feltorShwdiag.cpp
@@ -26,7 +26,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const eule::Parameters p(js);
     p.display(std::cout);
     ///////////////////////////////////////////////////////////////////////////
diff --git a/diag/feltorShwmerger.cpp b/diag/feltorShwmerger.cpp
index 624360dbb..443b17f94 100644
--- a/diag/feltorShwmerger.cpp
+++ b/diag/feltorShwmerger.cpp
@@ -44,7 +44,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        file::string2Json( argv[i], input, js, "strict");
+        file::string2Json( input, js, "strict");
         const eule::Parameters p(js);
         
         dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
diff --git a/diag/feltorShwradstat.cpp b/diag/feltorShwradstat.cpp
index a69a36d94..e524ee24e 100644
--- a/diag/feltorShwradstat.cpp
+++ b/diag/feltorShwradstat.cpp
@@ -35,7 +35,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        file::string2Json( argv[i], input, js, "strict");
+        file::string2Json( input, js, "strict");
         const eule::Parameters p(js);
         
         dg::Grid1d g1d( 0., p.lx,p.n_out, p.Nx_out, p.bc_x);
diff --git a/diag/feltorShwstat.cpp b/diag/feltorShwstat.cpp
index d3ef27ab9..d382670a9 100644
--- a/diag/feltorShwstat.cpp
+++ b/diag/feltorShwstat.cpp
@@ -49,7 +49,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        file::string2Json( argv[i], input, js, "strict");
+        file::string2Json( input, js, "strict");
         const eule::Parameters p(js);
         
 	size_t start0d  = 0;    
diff --git a/diag/fftwdiag.cpp b/diag/fftwdiag.cpp
index e1bacf0ee..083d29f0b 100644
--- a/diag/fftwdiag.cpp
+++ b/diag/fftwdiag.cpp
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const eule::Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/growthrate.cpp b/diag/growthrate.cpp
index a467724f7..8af6e1cd1 100644
--- a/diag/growthrate.cpp
+++ b/diag/growthrate.cpp
@@ -25,7 +25,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const eule::Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/impRdiag.cu b/diag/impRdiag.cu
index 018c6fd6d..ce814c5ba 100644
--- a/diag/impRdiag.cu
+++ b/diag/impRdiag.cu
@@ -45,7 +45,7 @@ int main( int argc, char* argv[])
     std::cout << "input "<< input << std::endl;
     //parse: parameter string--json-->p.xxx
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
 
   const imp::Parameters p(js);
   p.display(std::cout);
diff --git a/diag/normdiag.cu b/diag/normdiag.cu
index 65baaaf5d..53dcd792f 100644
--- a/diag/normdiag.cu
+++ b/diag/normdiag.cu
@@ -33,7 +33,7 @@ int main( int argc, char* argv[])
     err = nc_close(ncid); 
 
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const eule::Parameters p(js);
     
     //////////////////////////////Grids//////////////////////////////////////
diff --git a/diag/reco2Ddiag.cu b/diag/reco2Ddiag.cu
index c6e25d8c1..5f324bf35 100644
--- a/diag/reco2Ddiag.cu
+++ b/diag/reco2Ddiag.cu
@@ -26,7 +26,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const asela::Parameters p(js);
 
     //////////////////////////////Grids//////////////////////////////////////
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index 137925e47..ad9ac9082 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -67,7 +67,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index 992121a36..856f595ac 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -62,7 +62,7 @@ int main( int argc, char* argv[])
     file::netcdf2string( argv[1], "inputfile", input);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     const Parameters p(js);
     p.display(std::cout);
     
diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
index 212c109a4..483d5de88 100644
--- a/inc/file/easy_output.h
+++ b/inc/file/easy_output.h
@@ -8,10 +8,16 @@
 #include "dg/topology/mpi_grid.h"
 #endif //MPI_VERSION
 
+/*!@file
+ *
+ * Contains Error handling class
+ */
+
 namespace file
 {
+
 /**
- * @brief Class thrown by the NC_ErrorClonePtr
+ * @brief Class thrown by the NC_Error_Handle
  */
 struct NC_Error : public std::exception
 {
@@ -35,7 +41,24 @@ struct NC_Error : public std::exception
 
 /**
  * @brief Empty utitlity class that handles return values of netcdf
- * functions and throws NC_Error if something goes wrong
+ * functions and throws NC_Error(status) if( status != NC_NOERR)
+ *
+ * For example
+ * @code
+ * file::NC_Error_Handle err;
+ * int ncid = -1;
+ * try{
+ *      err = nc_open( "file.nc", NC_WRITE, &ncid);
+ * //throws if for example "file.nc" does not exist
+ * } catch ( std::exception& e)
+ * {
+ *      //log the error and exit
+ *      std::cerr << "An error occured opening file.nc !\n"<<e.what()<<std::endl;
+ *      exit( EXIT_FAILURE);
+ * }
+ * @endcode
+ *
+ * This allows for a C++ style error handling of netcdf errors in that the program either terminates if the NC_Error is not caught or does something graceful in a try catch statement.
  */
 struct NC_Error_Handle
 {
diff --git a/inc/file/json_utilities.h b/inc/file/json_utilities.h
index 29e0f1dfb..316d5e91b 100644
--- a/inc/file/json_utilities.h
+++ b/inc/file/json_utilities.h
@@ -3,6 +3,7 @@
 #include <iostream>
 #include <fstream>
 #include <string>
+#include <stdexcept> //std::runtime_error
 
 #include "json/json.h"
 /*!@file
@@ -18,7 +19,7 @@ namespace file
 /**
  * @brief Convenience wrapper to open a file and parse it into a Json::Value
  *
- * @attention This function will print an error message to \c std::cerr and brutally call \c exit(EXIT_FAILURE) on any error that occurs.
+ * @attention This function will throw a \c std::runtime_error containing an error message on any error that occurs on parsing.
  * @note included in \c json_utilities.h
  * @param filename Name of the JSON file to parse
  * @param js Contains all the found Json variables on output
@@ -35,29 +36,28 @@ static inline void file2Json( std::string filename, Json::Value& js, std::string
     std::ifstream isI( filename);
     if( !isI.good())
     {
-        std::cerr << "\nAn error occured while parsing "<<filename<<"\n";
-        std::cerr << "*** File does not exist! *** \n\n";
-        exit( EXIT_FAILURE);
+        std::string message = "\nAn error occured while parsing "+filename+"\n";
+        message +=  "*** File does not exist! *** \n\n";
+        throw std::runtime_error( message);
     }
     std::string errs;
     if( !parseFromStream( parser, isI, &js, &errs))
     {
-        std::cerr << "An error occured while parsing "<<filename<<"\n"<<errs;
-        exit( EXIT_FAILURE);
+        std::string message = "An error occured while parsing "+filename+"\n"+errs;
+        throw std::runtime_error( message);
     }
 }
 /**
  * @brief Convenience wrapper to parse a string into a Json::Value
  *
  * Parse a string into a Json Value
- * @attention This function will print an error message to \c std::cerr and brutally call \c exit(EXIT_FAILURE) on any error that occurs.
+ * @attention This function will throw a \c std::runtime_error with the Json error string on any error that occurs on parsing.
  * @note included in \c json_utilities.h
- * @param filename (a string to print when an error occurs, has no further use)
  * @param input The string to interpret as a Json string
  * @param js Contains all the found Json variables on output
  * @param mode Either "default" in which case comments are allowed or "strict" in which case they are not
  */
-static inline void string2Json( std::string filename, std::string input, Json::Value& js, std::string mode = "default")
+static inline void string2Json( std::string input, Json::Value& js, std::string mode = "default")
 {
     Json::CharReaderBuilder parser;
     if( "strict" == mode )
@@ -67,10 +67,7 @@ static inline void string2Json( std::string filename, std::string input, Json::V
     std::string errs;
     std::stringstream ss(input);
     if( !parseFromStream( parser, ss, &js, &errs) )
-    {
-        std::cerr << "An error occured while parsing "<<filename<<"\n"<<errs;
-        exit( EXIT_FAILURE);
-    }
+        throw std::runtime_error( errs);
 }
 
 }//namespace file
diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index baa8781d0..d167ae5ed 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -16,7 +16,7 @@
 
 /*!@file
  *
- * Contains Error handling class and the define_dimensions functions
+ * The define_dimensions functions
  */
 
 
@@ -314,7 +314,7 @@ inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRe
  * @param filename Name of the netcdf file to parse
  * @param att_name Name of the netcdf attribute in filename
  * @param att Contains the content of \c att_name on output
-    */
+*/
 static inline void netcdf2string( std::string filename, std::string att_name, std::string& att)
 {
     int ncid;
@@ -323,7 +323,7 @@ static inline void netcdf2string( std::string filename, std::string att_name, st
     {
         std::cerr << "\nAn error occured opening file "<<filename<<"\n";
         std::cerr << nc_strerror(status) <<std::endl;
-        exit( EXIT_FAILURE);
+        throw NC_Error( status);
     }
     size_t length;
     status = nc_inq_attlen( ncid, NC_GLOBAL, att_name.data(), &length);
@@ -332,7 +332,7 @@ static inline void netcdf2string( std::string filename, std::string att_name, st
         std::cerr << "\nAn error occured parsing file *"<<filename<<"* for attribute *"<<att_name<<"*\n";
         std::cerr << nc_strerror(status)<<std::endl;
         nc_close(ncid);
-        exit( EXIT_FAILURE);
+        throw NC_Error( status);
     }
     att.resize( length, 'x');
     status = nc_get_att_text( ncid, NC_GLOBAL, att_name.data(), &att[0]);
@@ -341,7 +341,7 @@ static inline void netcdf2string( std::string filename, std::string att_name, st
         std::cerr << "\nAn error occured parsing file *"<<filename<<"* for attribute *"<<att_name<<"*\n";
         std::cerr << nc_strerror(status)<<std::endl;
         nc_close(ncid);
-        exit( EXIT_FAILURE);
+        throw NC_Error( status);
     }
     nc_close(ncid);
 }
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index a539edc4c..ba1f561c8 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -33,9 +33,9 @@ int main( int argc, char* argv[])
     Json::Value js,gs;
     std::string input, geom;
     file::netcdf2string( argv[1], "inputfile", input);
-    file::string2Json( argv[1], input, js, "strict");
+    file::string2Json( input, js, "strict");
     file::netcdf2string( argv[1], "geomfile", geom);
-    file::string2Json( argv[1], geom, gs, "strict");
+    file::string2Json( geom, gs, "strict");
     const feltor::Parameters p(js);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index 097ec408b..a9e630ed3 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -19,7 +19,7 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     Json::Value jsIN;
     std::string temp;
     file::netcdf2string( file_name, "inputfile", temp);
-    file::string2Json(file_name, temp, jsIN, "strict");
+    file::string2Json(temp, jsIN, "strict");
     unsigned  pINn  = jsIN["n"].asUInt();
     unsigned  pINNx = jsIN["Nx"].asUInt();
     unsigned  pINNy = jsIN["Ny"].asUInt();
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index c35fcc62a..2ed18fe0b 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -45,9 +45,9 @@ int main( int argc, char* argv[])
     Json::Value js,gs;
     std::string inputfile, geomfile;
     file::netcdf2string( argv[1], "inputfile", inputfile);
-    file::string2Json(argv[1], inputfile, js, "strict");
+    file::string2Json(inputfile, js, "strict");
     file::netcdf2string( argv[1], "geomfile", geomfile);
-    file::string2Json(argv[1], geomfile, gs, "strict");
+    file::string2Json(geomfile, gs, "strict");
     const feltor::Parameters p(js);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
diff --git a/src/feltorSesol/feltor_hpc.cu b/src/feltorSesol/feltor_hpc.cu
index 3261bba29..ffe1e03f5 100644
--- a/src/feltorSesol/feltor_hpc.cu
+++ b/src/feltorSesol/feltor_hpc.cu
@@ -78,7 +78,7 @@ int main( int argc, char* argv[])
         std::string inputIN;
         file::netcdf2string( argv[3], "inputfile", inputIN);
         Json::Value jsIN;
-        file::string2Json( argv[3], inputIN, jsIN, "strict");
+        file::string2Json( inputIN, jsIN, "strict");
         const eule::Parameters pIN(  jsIN);
         std::cout << "[input.nc] file parameters" << std::endl;
         pIN.display( std::cout);    
diff --git a/src/feltorSesol/feltor_mpi.cu b/src/feltorSesol/feltor_mpi.cu
index 2eb74ce04..0b2d2ab73 100644
--- a/src/feltorSesol/feltor_mpi.cu
+++ b/src/feltorSesol/feltor_mpi.cu
@@ -107,7 +107,7 @@ int main( int argc, char* argv[])
         std::string inputIN;
         file::netcdf2string( argv[3], "inputfile", inputIN);
         Json::Value jsIN;
-        file::string2Json( argv[3], inputIN, jsIN, "strict");
+        file::string2Json( inputIN, jsIN, "strict");
         const eule::Parameters pIN(  jsIN);    
         std::cout << "[input.nc] file parameters" << std::endl;
         pIN.display( std::cout);   
diff --git a/src/feltorShw/feltor_hpc.cu b/src/feltorShw/feltor_hpc.cu
index 86a43acd8..18dbedcde 100644
--- a/src/feltorShw/feltor_hpc.cu
+++ b/src/feltorShw/feltor_hpc.cu
@@ -102,7 +102,7 @@ int main( int argc, char* argv[])
         Json::Value jsIN;
         std::string inputIN;
         file::netcdf2string( argv[3], "inputfile", inputIN);
-        file::string2Json(argv[3], inputIN, jsIN, "strict");
+        file::string2Json(inputIN, jsIN, "strict");
 
         const eule::Parameters pIN(  jsIN);    
         std::cout << "[input.nc] file parameters" << std::endl;
diff --git a/src/feltorShw/feltor_mpi.cu b/src/feltorShw/feltor_mpi.cu
index 03c5363d8..adf11d202 100644
--- a/src/feltorShw/feltor_mpi.cu
+++ b/src/feltorShw/feltor_mpi.cu
@@ -155,7 +155,7 @@ int main( int argc, char* argv[])
         Json::Value jsIN;
         std::string temp;
         file::netcdf2string( file_name, "inputfile", temp);
-        file::string2Json(file_name, temp, jsIN, "strict");
+        file::string2Json(temp, jsIN, "strict");
         const eule::Parameters pIN(  jsIN);    
         if(rank==0) std::cout << "[input.nc] file parameters" << std::endl;
         if(rank==0) pIN.display( std::cout);   
diff --git a/src/heat/heat_hpc.cu b/src/heat/heat_hpc.cu
index 358e6a2bd..0752d9601 100644
--- a/src/heat/heat_hpc.cu
+++ b/src/heat/heat_hpc.cu
@@ -61,9 +61,9 @@ int main( int argc, char* argv[])
         //////////////read in and show inputfile und geomfile////////////
         std::string inputin, geomin;
         file::netcdf2string( argv[4], "inputfile", inputin);
-        file::string2Json( argv[4],  inputin, js, "strict");
+        file::string2Json( inputin, js, "strict");
         file::netcdf2string( argv[4], "geomfile", geomin);
-        file::string2Json( argv[4], geomin, gs, "strict");
+        file::string2Json( geomin, gs, "strict");
         std::cout << "input in"<<inputin<<std::endl;
         std::cout << "geome in"<<geomin <<std::endl;
         const heat::Parameters pin(js);
-- 
GitLab


From 63c1fef9be65e686f29e1859ce466a6432be3448 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 16 May 2020 22:43:29 +0200
Subject: [PATCH 229/540] Delete file::netcdf2string function

- 2 code lines less per call is not worth the issue of error handling
---
 diag/crosscoherencdiag.cpp      | 12 +++++-----
 diag/feltorSHdiag.cpp           | 12 +++++-----
 diag/feltorSHdiag.cu            | 12 +++++-----
 diag/feltorSesoldiag.cpp        | 12 +++++-----
 diag/feltorShwdiag.cpp          | 12 +++++-----
 diag/fftwdiag.cpp               |  7 +++---
 diag/growthrate.cpp             | 11 ++++++----
 diag/reco2Ddiag.cu              | 12 +++++-----
 diag/toeflEPdiag.cu             | 12 +++++-----
 diag/toeflRdiag.cu              | 12 +++++-----
 inc/file/nc_utilities.h         | 39 ---------------------------------
 inc/geometries/geometry_diag.cu | 19 +++++++++++-----
 src/feltor/feltor_hpc.cu        | 10 ++++++++-
 src/feltor/feltordiag.cu        | 23 ++++++++++++-------
 src/feltor/init_from_file.h     | 14 +++++++-----
 src/feltor/interpolate_in_3d.cu | 16 +++++++++-----
 src/feltorSesol/feltor_hpc.cu   | 15 ++++++++-----
 src/feltorSesol/feltor_mpi.cu   | 14 +++++++-----
 src/feltorShw/feltor_hpc.cu     | 12 +++++-----
 src/feltorShw/feltor_mpi.cu     | 14 +++++++-----
 src/heat/heat_hpc.cu            | 21 +++++++++++-------
 21 files changed, 168 insertions(+), 143 deletions(-)

diff --git a/diag/crosscoherencdiag.cpp b/diag/crosscoherencdiag.cpp
index e826c722f..297ed39b6 100644
--- a/diag/crosscoherencdiag.cpp
+++ b/diag/crosscoherencdiag.cpp
@@ -43,8 +43,13 @@ int main( int argc, char* argv[])
     }
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;   
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -69,9 +74,6 @@ int main( int argc, char* argv[])
     double Nep,phip;
     unsigned step=0;
     //read in values into vectors
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
 
     unsigned imin,imax;
     imin=1;
diff --git a/diag/feltorSHdiag.cpp b/diag/feltorSHdiag.cpp
index dfb1fd483..23d7a11c5 100644
--- a/diag/feltorSHdiag.cpp
+++ b/diag/feltorSHdiag.cpp
@@ -22,8 +22,13 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -83,9 +88,6 @@ int main( int argc, char* argv[])
     
     
     //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err1d = nc_open( argv[2], NC_WRITE, &ncid1d);
 
     for( unsigned i=0; i<p.maxout; i++)//timestepping
diff --git a/diag/feltorSHdiag.cu b/diag/feltorSHdiag.cu
index a5af65475..f0641aca4 100644
--- a/diag/feltorSHdiag.cu
+++ b/diag/feltorSHdiag.cu
@@ -45,8 +45,13 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -148,9 +153,6 @@ int main( int argc, char* argv[])
     double normalize = 1.;
     dg::DVec heavy;
     //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
     unsigned position = 0;
     double posX_max = 0.0,posY_max = 0.0,posX_max_old = 0.0,posY_max_old = 0.0,velX_max=0.0, velY_max=0.0,posX_max_hs=0.0,posY_max_hs=0.0,velCOM=0.0;
diff --git a/diag/feltorSesoldiag.cpp b/diag/feltorSesoldiag.cpp
index 423f5da6d..e063bd412 100644
--- a/diag/feltorSesoldiag.cpp
+++ b/diag/feltorSesoldiag.cpp
@@ -22,8 +22,13 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -94,9 +99,6 @@ int main( int argc, char* argv[])
 //     std::cout << "enter new imin(>0) and imax(<maxout):" << std::endl;
 //     std::cin >> imin >> imax;
     time = imin*p.itstp;
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
 
     unsigned num_probes = 5;
diff --git a/diag/feltorShwdiag.cpp b/diag/feltorShwdiag.cpp
index cfb6ea9b4..7b74dd263 100644
--- a/diag/feltorShwdiag.cpp
+++ b/diag/feltorShwdiag.cpp
@@ -22,8 +22,13 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -107,9 +112,6 @@ int main( int argc, char* argv[])
     imin= 0;
     time = imin*p.itstp;
     //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
 
     unsigned num_probes = 5;
diff --git a/diag/fftwdiag.cpp b/diag/fftwdiag.cpp
index 083d29f0b..75310399c 100644
--- a/diag/fftwdiag.cpp
+++ b/diag/fftwdiag.cpp
@@ -27,8 +27,10 @@ int main( int argc, char* argv[])
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -127,7 +129,6 @@ int main( int argc, char* argv[])
     spectral::DRT_DFT trafo( Ny, Nx, kind);
     
     //open netcdf files
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err2d_f = nc_open( argv[2], NC_WRITE, &ncid2d_f);
     err1d_f = nc_open( argv[3], NC_WRITE, &ncid1d_f);
     //set min and max timesteps
diff --git a/diag/growthrate.cpp b/diag/growthrate.cpp
index 8af6e1cd1..a9d922291 100644
--- a/diag/growthrate.cpp
+++ b/diag/growthrate.cpp
@@ -21,8 +21,13 @@ int main( int argc, char* argv[])
     }
 
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -41,8 +46,6 @@ int main( int argc, char* argv[])
     
     //dg stuff
     dg::HVec phi(dg::evaluate(dg::zero,g2d));
-    //open netcdf files
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     //set min and max timesteps
     double time = 0.;
     unsigned imin,imax;    
diff --git a/diag/reco2Ddiag.cu b/diag/reco2Ddiag.cu
index 5f324bf35..5c9b0c6e4 100644
--- a/diag/reco2Ddiag.cu
+++ b/diag/reco2Ddiag.cu
@@ -22,8 +22,13 @@ int main( int argc, char* argv[])
     }
 
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -49,9 +54,6 @@ int main( int argc, char* argv[])
 
     //open netcdf files
     //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     //set min and max timesteps
     double time = 0.;
     unsigned imin,imax;    
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index ad9ac9082..c624bd87d 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -63,8 +63,13 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -158,9 +163,6 @@ int main( int argc, char* argv[])
     double compactness_ne=0.0;
     //-----------------Start timestepping
     //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
     for( unsigned i=0; i<=p.maxout; i++)
     {
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index 856f595ac..d39fed8a4 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -58,8 +58,13 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    std::string input;
-    file::netcdf2string( argv[1], "inputfile", input);
+    file::NC_Error_Handle err;
+    int ncid;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid);
+    size_t length;
+    err = nc_inq_attlen( ncid, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
     file::string2Json( input, js, "strict");
@@ -153,9 +158,6 @@ int main( int argc, char* argv[])
     double compactness_ne=0.0;
     //-----------------Start timestepping
     //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid);
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
     for( unsigned i=0; i<=p.maxout; i++)
     {
diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index d167ae5ed..1879c2ed5 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -308,43 +308,4 @@ inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRe
 }
 #endif //MPI_VERSION
 
-/*! @brief Read a netcdf string attribute into a std::string
- *
- * Open file, look for attribute, read attribute in string and close the file at the end.
- * @param filename Name of the netcdf file to parse
- * @param att_name Name of the netcdf attribute in filename
- * @param att Contains the content of \c att_name on output
-*/
-static inline void netcdf2string( std::string filename, std::string att_name, std::string& att)
-{
-    int ncid;
-    int status = nc_open( filename.data(), NC_NOWRITE, &ncid);
-    if( status != NC_NOERR)
-    {
-        std::cerr << "\nAn error occured opening file "<<filename<<"\n";
-        std::cerr << nc_strerror(status) <<std::endl;
-        throw NC_Error( status);
-    }
-    size_t length;
-    status = nc_inq_attlen( ncid, NC_GLOBAL, att_name.data(), &length);
-    if( status != NC_NOERR)
-    {
-        std::cerr << "\nAn error occured parsing file *"<<filename<<"* for attribute *"<<att_name<<"*\n";
-        std::cerr << nc_strerror(status)<<std::endl;
-        nc_close(ncid);
-        throw NC_Error( status);
-    }
-    att.resize( length, 'x');
-    status = nc_get_att_text( ncid, NC_GLOBAL, att_name.data(), &att[0]);
-    if( status != NC_NOERR)
-    {
-        std::cerr << "\nAn error occured parsing file *"<<filename<<"* for attribute *"<<att_name<<"*\n";
-        std::cerr << nc_strerror(status)<<std::endl;
-        nc_close(ncid);
-        throw NC_Error( status);
-    }
-    nc_close(ncid);
-}
-
-
 } //namespace file
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 6f14d8e0b..f3934b4a9 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -94,11 +94,20 @@ int main( int argc, char* argv[])
     {
         newfilename = argv[2];
         std::cout << argv[0]<< " "<<argv[1]<<" -> " <<argv[2]<<std::endl;
-        std::string temp;
-        file::netcdf2string( argv[1], "inputfile", temp);
-        file::string2Json( argv[1], temp, input_js, "strict");
-        file::netcdf2string( argv[1], "geomfile", temp);
-        file::string2Json( argv[1], temp, geom_js, "strict");
+        file::NC_Error_Handle err;
+        int ncid_in;
+        err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
+        size_t length;
+        err = nc_inq_attlen( ncid_in, NC_GLOBAL, "inputfile", &length);
+        std::string inputfile(length, 'x');
+        err = nc_get_att_text( ncid_in, NC_GLOBAL, "inputfile", &inputfile[0]);
+        err = nc_inq_attlen( ncid_in, NC_GLOBAL, "geomfile", &length);
+        std::string geomfile(length, 'x');
+        err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geomfile[0]);
+        err = nc_close( ncid_in);
+        Json::Value js,gs;
+        file::string2Json(inputfile, input_js, "strict");
+        file::string2Json(geomfile, geom_js, "strict");
     }
     else
     {
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index c6f421297..abe465745 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -207,7 +207,15 @@ int main( int argc, char* argv[])
         }
     }
     if( argc == 5)
-        y0 = feltor::init_from_file(argv[4], grid, p,time);
+    {
+        try{
+            y0 = feltor::init_from_file(argv[4], grid, p,time);
+        }catch (std::exception& e){
+            MPI_OUT std::cerr << "An error occured initializing from file "<<argv[4]<<std::endl;
+            MPI_OUT std::cerr << e.what();
+            return -1;
+        }
+    }
 
     bool fixed_profile;
     {
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index ba1f561c8..daa1c41b0 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -30,12 +30,20 @@ int main( int argc, char* argv[])
     std::cout << " -> "<<argv[argc-1]<<std::endl;
 
     //------------------------open input nc file--------------------------------//
+    file::NC_Error_Handle err;
+    int ncid_in;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
+    size_t length;
+    err = nc_inq_attlen( ncid_in, NC_GLOBAL, "inputfile", &length);
+    std::string inputfile(length, 'x');
+    err = nc_get_att_text( ncid_in, NC_GLOBAL, "inputfile", &inputfile[0]);
+    err = nc_inq_attlen( ncid_in, NC_GLOBAL, "geomfile", &length);
+    std::string geomfile(length, 'x');
+    err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geomfile[0]);
+    err = nc_close( ncid_in);
     Json::Value js,gs;
-    std::string input, geom;
-    file::netcdf2string( argv[1], "inputfile", input);
-    file::string2Json( input, js, "strict");
-    file::netcdf2string( argv[1], "geomfile", geom);
-    file::string2Json( geom, gs, "strict");
+    file::string2Json(inputfile, js, "strict");
+    file::string2Json(geomfile, gs, "strict");
     const feltor::Parameters p(js);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
@@ -46,7 +54,6 @@ int main( int argc, char* argv[])
 
     //-----------------Create Netcdf output file with attributes----------//
     int ncid_out;
-    file::NC_Error_Handle err;
     err = nc_create(argv[argc-1],NC_NETCDF4|NC_CLOBBER, &ncid_out);
 
     /// Set global attributes
@@ -64,8 +71,8 @@ int main( int argc, char* argv[])
     att["comment"] = "Find more info in feltor/src/feltor.tex";
     att["source"] = "FELTOR";
     att["references"] = "https://github.com/feltor-dev/feltor";
-    att["inputfile"] = input;
-    att["geomfile"] = geom;
+    att["inputfile"] = inputfile;
+    att["geomfile"] = geomfile;
     for( auto pair : att)
         err = nc_put_att_text( ncid_out, NC_GLOBAL,
             pair.first.data(), pair.second.size(), pair.second.data());
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index a9e630ed3..789a9b723 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -16,10 +16,15 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     std::array<std::array<DVec,2>,2> y0;
     ///////////////////read in and show inputfile
 
+    file::NC_Error_Handle errIN;
+    int ncidIN;
+    errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
     Json::Value jsIN;
-    std::string temp;
-    file::netcdf2string( file_name, "inputfile", temp);
-    file::string2Json(temp, jsIN, "strict");
+    size_t length;
+    errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &length);
+    std::string input(length, 'x');
+    errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &input[0]);
+    file::string2Json( input, jsIN, "strict");
     unsigned  pINn  = jsIN["n"].asUInt();
     unsigned  pINNx = jsIN["Nx"].asUInt();
     unsigned  pINNy = jsIN["Ny"].asUInt();
@@ -61,9 +66,6 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     int timeIDIN;
     size_t size_time, count_time = 1;
     /////////////////////Get time length and initial data///////////////////////////
-    file::NC_Error_Handle errIN;
-    int ncidIN;
-    errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
     errIN = nc_inq_dimid( ncidIN, "time", &timeIDIN);
     errIN = nc_inq_dimlen(ncidIN, timeIDIN, &size_time);
     errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index 2ed18fe0b..ca7b41b76 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -42,11 +42,18 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     //------------------------open input nc file--------------------------------//
+    file::NC_Error_Handle err;
+    int ncid_in;
+    err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
+    size_t length;
+    err = nc_inq_attlen( ncid_in, NC_GLOBAL, "inputfile", &length);
+    std::string inputfile(length, 'x');
+    err = nc_get_att_text( ncid_in, NC_GLOBAL, "inputfile", &inputfile[0]);
+    err = nc_inq_attlen( ncid_in, NC_GLOBAL, "geomfile", &length);
+    std::string geomfile(length, 'x');
+    err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geomfile[0]);
     Json::Value js,gs;
-    std::string inputfile, geomfile;
-    file::netcdf2string( argv[1], "inputfile", inputfile);
     file::string2Json(inputfile, js, "strict");
-    file::netcdf2string( argv[1], "geomfile", geomfile);
     file::string2Json(geomfile, gs, "strict");
     const feltor::Parameters p(js);
     const dg::geo::solovev::Parameters gp(gs);
@@ -55,7 +62,6 @@ int main( int argc, char* argv[])
 
     //-----------------Create Netcdf output file with attributes----------//
     int ncid_out;
-    file::NC_Error_Handle err;
     err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
 
     /// Set global attributes
@@ -125,8 +131,6 @@ int main( int argc, char* argv[])
     dg::geo::Fieldaligned<Geometry, IHMatrix, HVec> fieldaligned(
         bhat, g3d_out, dg::NEU, dg::NEU, dg::geo::NoLimiter(), //let's take NEU bc because N is not homogeneous
         p.rk4eps, 5, 5);
-    int ncid_in;
-    err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
     dg::IHMatrix interpolate_in_2d = dg::create::interpolation( g3d_out_equidistant, g3d_out);
 
 
diff --git a/src/feltorSesol/feltor_hpc.cu b/src/feltorSesol/feltor_hpc.cu
index ffe1e03f5..d4f11e172 100644
--- a/src/feltorSesol/feltor_hpc.cu
+++ b/src/feltorSesol/feltor_hpc.cu
@@ -74,11 +74,17 @@ int main( int argc, char* argv[])
       std::cout << "Done!\n";
     }
     if (argc==4) {
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
-        std::string inputIN;
-        file::netcdf2string( argv[3], "inputfile", inputIN);
+        size_t length;
+        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &length);
+        std::string inputIN(length, 'x');
+        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json( inputIN, jsIN, "strict");
+        file::string2Json(inputIN, jsIN, "strict");
+
         const eule::Parameters pIN(  jsIN);
         std::cout << "[input.nc] file parameters" << std::endl;
         pIN.display( std::cout);    
@@ -95,9 +101,6 @@ int main( int argc, char* argv[])
         size_t stepsIN;
         /////////////////////The initial field///////////////////////////////////////////
         /////////////////////Get time length and initial data///////////////////////////
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         errIN = nc_inq_varid(ncidIN, namesIN[0].data(), &dataIDsIN[0]);
         errIN = nc_inq_dimlen(ncidIN, dataIDsIN[0], &stepsIN);
         stepsIN-=1;
diff --git a/src/feltorSesol/feltor_mpi.cu b/src/feltorSesol/feltor_mpi.cu
index 0b2d2ab73..a57736f16 100644
--- a/src/feltorSesol/feltor_mpi.cu
+++ b/src/feltorSesol/feltor_mpi.cu
@@ -103,11 +103,16 @@ int main( int argc, char* argv[])
         if(rank==0) std::cout << "Done!\n";
     }
     if (argc==4) {
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
-        std::string inputIN;
-        file::netcdf2string( argv[3], "inputfile", inputIN);
+        size_t length;
+        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &length);
+        std::string inputIN(length, 'x');
+        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json( inputIN, jsIN, "strict");
+        file::string2Json(inputIN, jsIN, "strict");
         const eule::Parameters pIN(  jsIN);    
         std::cout << "[input.nc] file parameters" << std::endl;
         pIN.display( std::cout);   
@@ -127,9 +132,6 @@ int main( int argc, char* argv[])
         size_t stepsIN;
         /////////////////////The initial field///////////////////////////////////////////
         /////////////////////Get time length and initial data///////////////////////////
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         errIN = nc_inq_varid(ncidIN, namesIN[0].data(), &dataIDsIN[0]);
         errIN = nc_inq_dimlen(ncidIN, dataIDsIN[0], &stepsIN);
         stepsIN-=1;
diff --git a/src/feltorShw/feltor_hpc.cu b/src/feltorShw/feltor_hpc.cu
index 18dbedcde..0c7bdb027 100644
--- a/src/feltorShw/feltor_hpc.cu
+++ b/src/feltorShw/feltor_hpc.cu
@@ -98,10 +98,15 @@ int main( int argc, char* argv[])
       }
     }
     if (argc==4) {
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
+        size_t length;
+        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &length);
+        std::string inputIN(length, 'x');
+        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        std::string inputIN;
-        file::netcdf2string( argv[3], "inputfile", inputIN);
         file::string2Json(inputIN, jsIN, "strict");
 
         const eule::Parameters pIN(  jsIN);    
@@ -120,9 +125,6 @@ int main( int argc, char* argv[])
         size_t stepsIN;
         /////////////////////The initial field///////////////////////////////////////////
         /////////////////////Get time length and initial data///////////////////////////
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         errIN = nc_inq_varid(ncidIN, namesIN[0].data(), &dataIDsIN[0]);
         errIN = nc_inq_dimlen(ncidIN, dataIDsIN[0], &stepsIN);
         stepsIN-=1;
diff --git a/src/feltorShw/feltor_mpi.cu b/src/feltorShw/feltor_mpi.cu
index adf11d202..2d7b2c0cf 100644
--- a/src/feltorShw/feltor_mpi.cu
+++ b/src/feltorShw/feltor_mpi.cu
@@ -151,11 +151,16 @@ int main( int argc, char* argv[])
       }
     }
     if (argc==4) {
+        file::NC_Error_Handle errIN;
+        int ncidIN;
+        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
+        size_t length;
+        errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &length);
+        std::string inputIN(length, 'x');
+        errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        std::string temp;
-        file::netcdf2string( file_name, "inputfile", temp);
-        file::string2Json(temp, jsIN, "strict");
+        file::string2Json(inputIN, jsIN, "strict");
         const eule::Parameters pIN(  jsIN);    
         if(rank==0) std::cout << "[input.nc] file parameters" << std::endl;
         if(rank==0) pIN.display( std::cout);   
@@ -175,9 +180,6 @@ int main( int argc, char* argv[])
         size_t stepsIN;
         /////////////////////The initial field///////////////////////////////////////////
         /////////////////////Get time length and initial data///////////////////////////
-        file::NC_Error_Handle errIN;
-        int ncidIN;
-        errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         errIN = nc_inq_varid(ncidIN, namesIN[0].data(), &dataIDsIN[0]);
         errIN = nc_inq_dimlen(ncidIN, dataIDsIN[0], &stepsIN);
         stepsIN-=1;
diff --git a/src/heat/heat_hpc.cu b/src/heat/heat_hpc.cu
index 0752d9601..01e1c88d8 100644
--- a/src/heat/heat_hpc.cu
+++ b/src/heat/heat_hpc.cu
@@ -58,12 +58,20 @@ int main( int argc, char* argv[])
     //////////////////////////////open nc file//////////////////////////////////
     if (argc == 5)
     {
+        file::NC_Error_Handle errin;
+        int ncidin;
+        errin = nc_open( argv[4], NC_NOWRITE, &ncidin);
         //////////////read in and show inputfile und geomfile////////////
-        std::string inputin, geomin;
-        file::netcdf2string( argv[4], "inputfile", inputin);
-        file::string2Json( inputin, js, "strict");
-        file::netcdf2string( argv[4], "geomfile", geomin);
-        file::string2Json( geomin, gs, "strict");
+        size_t length;
+        errin = nc_inq_attlen( ncidin, NC_GLOBAL, "inputfile", &length);
+        std::string inputin(length, 'x');
+        errin = nc_get_att_text( ncidin, NC_GLOBAL, "inputfile", &inputin[0]);
+        errin = nc_inq_attlen( ncidin, NC_GLOBAL, "geomfile", &length);
+        std::string geomin(length, 'x');
+        errin = nc_get_att_text( ncidin, NC_GLOBAL, "geomfile", &geomin[0]);
+        Json::Value js,gs;
+        file::string2Json(inputin, js, "strict");
+        file::string2Json(geomin, gs, "strict");
         std::cout << "input in"<<inputin<<std::endl;
         std::cout << "geome in"<<geomin <<std::endl;
         const heat::Parameters pin(js);
@@ -75,9 +83,6 @@ int main( int argc, char* argv[])
         dg::IHMatrix interpolatef2c = dg::create::interpolation( grid, grid_in);//f2c
         dg::HVec TendIN = dg::evaluate( dg::zero, grid_in);
         //Now read Tend and interpolate from input grid to our grid
-        file::NC_Error_Handle errin;
-        int ncidin;
-        errin = nc_open( argv[4], NC_NOWRITE, &ncidin);
         int dataIDin;
         errin = nc_inq_varid(ncidin, "T", &dataIDin);
         errin = nc_get_vara_double( ncidin, dataIDin, start3din, count3din,
-- 
GitLab


From 3342b15fecde3d93b9495a3cdf317102899d63f6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 17 May 2020 00:52:08 +0200
Subject: [PATCH 230/540] Error handling in feltor_hpc

---
 src/feltor/feltor.tex    | 85 +++++++++++++++++++++++++++++++++++-----
 src/feltor/feltor_hpc.cu | 82 ++++++++++++++++++++++++++++----------
 2 files changed, 136 insertions(+), 31 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index d3ad3e768..b49b6ef83 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -39,7 +39,6 @@ we can define various differential operations.
 %vector potential $A_\parallel$ via
 %${ \vec b }_\perp := ({\vn \times A_\parallel \bhat)}/{B}$
 \rowcolors{2}{gray!25}{white}
-%\begin{longtable}{>{\RaggedRight}p{7cm}>{\RaggedRight}p{7cm}}
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
 %\toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Symbol} & \textbf{Definition} \\
@@ -1232,7 +1231,7 @@ Unfortunately, we were unable to find a closed solution for the energy integrals
 \section{Numerical methods}
 discontinuous Galerkin on structured grid
 \rowcolors{2}{gray!25}{white} %%% Use this line in front of longtable
-\begin{longtable}{p{3cm}l>{\RaggedRight}p{8cm}}
+\begin{longtable}{p{3cm}p{3cm}p{8cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Term} &  \textbf{Method} & \textbf{Description}  \\ \midrule
     coordinate system & Cylindrical & equidistant discretization of $[R_{\min},R_{\max}] \times [Z_{\min},Z_{\max}] \times [0,2\pi]$ (Eq.~\eqref{eq:box}, equal number of Gaussian nodes in $R$ and $Z$, equidistant planes in $\varphi$ with one Gaussian node \\
@@ -1276,7 +1275,7 @@ the output file \texttt{output.nc}.
 Input file format: json
 
 %%This is a booktabs table
-\begin{longtable}{llll>{\RaggedRight}p{6cm}}
+\begin{longtable}{llllp{6cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
 n      & integer & 3 & - &Number of Gaussian nodes in R and Z (we practically always take 3)
@@ -1435,9 +1434,6 @@ $\psi_p$ function \eqref{eq:modified_psip}. If zero, then we do not modify the
 magnetic field and damping is ignored.\\
 \bottomrule
 \end{longtable}
-The default value is taken if the value name is not found in the input file. If there is no default and
-the value is not found,
-the program exits with an error message.
 \subsection{Geometry file structure} \label{sec:geometry_file}
 File format: json
 
@@ -1455,9 +1451,6 @@ File format: json
     inverseaspectratio & float & 0.16667 & - & minor to major radius $a/R_0$ (used to compute $a$ from $R_0$) \\
 \bottomrule
 \end{longtable}
-The default value is taken if the value name is not found in the input file. If there is no default and
-the value is not found,
-the program exits with an error message.
 
 \subsection{Output} \label{sec:output_file}
 Output file format: netcdf-4/hdf5; A \textit{coordinate variable (Coord. Var.)} is a Dataset with the same name as a dimension.
@@ -1547,7 +1540,7 @@ the whole simulation is lost. It is safer to just merge files afterwards with fo
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Diagnostics}\label{sec:diagnostics}
 \texttt{feltor/diag/feltordiag.cu}
- reads one or more previously generated simulation file(s) \texttt{input0.nc ... inputN.nc} described in Section~\ref{sec:output_file} and writes into a second output file \texttt{output.nc} described as follows. \\
+ reads one or more previously generated simulation file(s) \texttt{input0.nc ... inputN.nc} described in Section~\ref{sec:output_file} and writes into a single second output file \texttt{output.nc} described as follows. \\
 Compilation\\
 \texttt{make feltordiag} \\
 Usage \\
@@ -1600,6 +1593,78 @@ Usage \\
 \texttt{./geometry\_diag input.json geometry.json diag\_geometry.nc} \\
 The program outputs a host of static 1d, 2d and 3d geometric quantities.
 The output file is for example useful in connection with the ``Group Datasets'' filter in paraview, which merges Datasets from different files into one using shallow copy only.
+\section{Error conditions}
+All previously mentioned codes can crash for various reasons. Here,
+we list and describe situations, which generally may lead to program
+termination
+\begin{longtable}{p{6cm}p{8cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Error condition} &  \textbf{Handling} \\ \midrule
+An input file does not exist or is otherwise invalid
+&
+Program terminates with an error message to \texttt{std::cerr}
+    \\
+An input netcdf file misses a required field
+&
+Program terminates with a NetCDF error message to \texttt{std::cerr}
+    \\
+No write permission for the output file location
+&
+Program terminates with an error message to \texttt{std::cerr}
+    \\
+An input Json file misses a key or contains a typo in a key
+&
+If a key is not found the parser uses a default value,
+which is $0$ if not otherwise specified
+and the program continues execution.
+This is because the program cannot distinguish between intentional
+omission of a parameter and unintentional
+(a user can intentionally choose to use the default value
+or an older input file is used which does not contain parameters introduced afterward but have a reasonable default)
+Depending on which parameter is wrongly set this may either lead to early termination or actually lead to a long simulation. Usually the program prints the parameters
+it read from the input file into \texttt{std::cout} and into the output file
+for inspection so double check.
+    \\
+    An input Json file has an invalid value, e.g. a  typo in a string value
+&
+Invalid values lead to termination with an error message to \texttt{std::cerr}, once and if program tries to use the value
+    \\
+Number of processes in $x$, $y$ and $z$ direction does not match total number of Processes
+&
+Program terminates with an error message to \texttt{std::cerr}
+    \\
+An MPI error occurs
+&
+Program crashes horribly printing cryptic error messages (stack trace) to \texttt{std::cerr}
+    \\
+A numerical instability occurs
+&
+The program terminates usually caused by a NaN exception raised. However,
+the cause for the instability has to be determined inspecting the
+last output in the output file.
+    \\
+\qquad large fieldaligned oscillations in $u_e$ paired with instability in the edge of the box
+&
+Apply damping region
+    \\
+\qquad Perpendicular grid oscillations in $u_e$ and $\Delta_\perp \phi$ in the damping region, symmetric in $\varphi$
+&
+Increase damping $alpha$, increase damping boundary, make the box larger/smaller
+    \\
+\qquad Spike in $u_e$ shortly after simulation start
+&
+Increase $\nu_\perp$, increase $N_x$, $N_y$, decrease perturbation amplitude
+    \\
+\qquad Grid oscillations far away from the edge
+&
+Probably caused by the perpendicular transport that goes unstable. Increase $\nu_\perp$ and/or $N_x$, $N_y$
+\\
+\qquad Oscillations where fieldlines intersect the wall
+&
+Caused by boundary conditions in FCI method and necessarily underresolved toroidal direction. Increase $N_z$, decrease $N_x$, $N_y$ or better decrease $q$ value by decreasing $\mathcal P_\psi$ in geometry input file
+\\
+\bottomrule
+\end{longtable}
 
 %..................................................................
 \bibliography{../../doc/related_pages/references}
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index abe465745..2adbbbc90 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -89,8 +89,12 @@ int main( int argc, char* argv[])
                   << num_threads<<" threads = "
                   <<size*num_threads<<" total"<<std::endl;
 ;
-        assert( size == np[0]*np[1]*np[2] &&
-        "Partition needs to match total number of processes!");
+        if( size != np[0]*np[1]*np[2])
+        {
+            std::cerr << "ERROR: Process partition needs to match total number of processes!"<<std::endl;
+            MPI_Abort(MPI_COMM_WORLD, -1);
+            return -1;
+        }
     }
     MPI_Bcast( np, 3, MPI_INT, 0, MPI_COMM_WORLD);
     MPI_Comm comm;
@@ -110,8 +114,17 @@ int main( int argc, char* argv[])
     }
     else
     {
-        file::file2Json( argv[1], js, "strict");
-        file::file2Json( argv[2], gs, "strict");
+        try{
+            file::file2Json( argv[1], js, "strict");
+            file::file2Json( argv[2], gs, "strict");
+        } catch( std::exception& e) {
+            MPI_OUT std::cerr << e.what();
+#ifdef FELTOR_MPI
+            MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
+            return -1;
+        }
+
     }
     const feltor::Parameters p( js);
     const dg::geo::solovev::Parameters gp(gs);
@@ -203,7 +216,10 @@ int main( int argc, char* argv[])
             y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
         }catch ( std::out_of_range& error){
             MPI_OUT std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
-        return -1;
+#ifdef FELTOR_MPI
+            MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
+            return -1;
         }
     }
     if( argc == 5)
@@ -211,34 +227,47 @@ int main( int argc, char* argv[])
         try{
             y0 = feltor::init_from_file(argv[4], grid, p,time);
         }catch (std::exception& e){
-            MPI_OUT std::cerr << "An error occured initializing from file "<<argv[4]<<std::endl;
-            MPI_OUT std::cerr << e.what();
+            MPI_OUT std::cerr << "ERROR occured initializing from file "<<argv[4]<<std::endl;
+            MPI_OUT std::cerr << e.what()<<std::endl;
+#ifdef FELTOR_MPI
+            MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
             return -1;
         }
     }
 
-    bool fixed_profile;
-    {
-    HVec profile, source_profile;
     try{
+        bool fixed_profile;
+        HVec profile, source_profile;
         source_profile = feltor::source_profiles.at(p.source_type)(
-        fixed_profile, profile, grid, p, gp, mag);
+            fixed_profile, profile, grid, p, gp, mag);
+        feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
+            p.source_rate, dg::construct<DVec>(source_profile),
+            p.damping_rate, dg::construct<DVec>(damping_profile)
+        );
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
+#ifdef FELTOR_MPI
+        MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
         return -1;
     }
 
-    feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
-        p.source_rate, dg::construct<DVec>(source_profile),
-        p.damping_rate, dg::construct<DVec>(damping_profile)
-    );
-    }
-
     /// //////////////////////////set up netcdf/////////////////////////////////////
     file::NC_Error_Handle err;
     std::string file_name = argv[3];
     int ncid=-1;
-    MPI_OUT err = nc_create( file_name.data(), NC_NETCDF4|NC_CLOBBER, &ncid);
+    try{
+        MPI_OUT err = nc_create( file_name.data(), NC_NETCDF4|NC_CLOBBER, &ncid);
+    }catch( std::exception& e)
+    {
+        std::cerr << "ERROR creating file "<<file_name<<std::endl;
+        std::cerr << e.what()<<std::endl;
+#ifdef FELTOR_MPI
+        MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
+       return -1;
+    }
     /// Set global attributes
     std::map<std::string, std::string> att;
     att["title"] = "Output file of feltor/src/feltor_hpc.cu";
@@ -434,9 +463,20 @@ int main( int argc, char* argv[])
                     karniadakis.step( feltor, im, time, y0);
                     //bdf.step( feltor, time, y0);
                 }
-                catch( dg::Fail& fail) {
-                    MPI_OUT std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-                    MPI_OUT std::cerr << "Does Simulation respect CFL condition?\n";
+                catch( dg::Fail& fail){
+                    MPI_OUT std::cerr << "ERROR failed to converge to "<<fail.epsilon()<<"\n";
+                    MPI_OUT std::cerr << "Does simulation respect CFL condition?"<<std::endl;
+#ifdef FELTOR_MPI
+                    MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
+                    return -1;
+                }
+                catch( std::exception& fail) {
+                    MPI_OUT std::cerr << "ERROR in timestepper\n";
+                    MPI_OUT std::cerr << fail.what()<<std::endl;
+#ifdef FELTOR_MPI
+                    MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
                     return -1;
                 }
                 step++;
-- 
GitLab


From 6442b68cacc45f86deca7d4ad2458a902f417837 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 17 May 2020 00:59:35 +0200
Subject: [PATCH 231/540] FIX bug of missing headers in mpi_accumulate.h

---
 inc/dg/backend/exblas/mpi_accumulate.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/inc/dg/backend/exblas/mpi_accumulate.h b/inc/dg/backend/exblas/mpi_accumulate.h
index cc94f5de7..03eac6593 100644
--- a/inc/dg/backend/exblas/mpi_accumulate.h
+++ b/inc/dg/backend/exblas/mpi_accumulate.h
@@ -7,7 +7,9 @@
  */
 #pragma once
 #include <mpi.h>
+#include <array>
 #include <vector>
+#include <map>
 #include "accumulate.h"
 
 namespace exblas {
-- 
GitLab


From 1050754fb2f075063eaf53d6510630ef94f26a15 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 May 2020 12:04:00 +0200
Subject: [PATCH 232/540] Fix construct and assign implementation for vector

- it should be possible to assign vectors to vectors
---
 inc/dg/backend/blas1_dispatch_vector.h | 74 ++++++++++++++++++++++++--
 1 file changed, 70 insertions(+), 4 deletions(-)

diff --git a/inc/dg/backend/blas1_dispatch_vector.h b/inc/dg/backend/blas1_dispatch_vector.h
index 2d4eb96bc..31b58f928 100644
--- a/inc/dg/backend/blas1_dispatch_vector.h
+++ b/inc/dg/backend/blas1_dispatch_vector.h
@@ -23,8 +23,9 @@ template<class from_ContainerType, class to_ContainerType, class ...Params>
 inline void assign( const from_ContainerType&, to_ContainerType&, Params&& ...ps);
 
 namespace detail{
+
 template<class To, class From, class ...Params>
-To doConstruct( const From& src, ArrayVectorTag, AnyVectorTag, Params&&...ps )
+To doConstruct( const From& src, ArrayVectorTag, SharedVectorTag, Params&&...ps )
 {
     To t;
     using inner_vector = typename To::value_type;
@@ -32,9 +33,36 @@ To doConstruct( const From& src, ArrayVectorTag, AnyVectorTag, Params&&...ps )
         t[i] = dg::construct<inner_vector>(src, std::forward<Params>(ps)...);
     return t;
 }
+template<class To, class From, class ...Params>
+To doConstruct( const From& src, ArrayVectorTag, MPIVectorTag, Params&&...ps )
+{
+    To t;
+    using inner_vector = typename To::value_type;
+    for (unsigned i=0; i<t.size(); i++)
+        t[i] = dg::construct<inner_vector>(src, std::forward<Params>(ps)...);
+    return t;
+}
+template<class To, class From, class ...Params>
+To doConstruct( const From& src, ArrayVectorTag, RecursiveVectorTag, Params&&...ps )
+{
+    To t;
+    using inner_vector = typename To::value_type;
+    for (unsigned i=0; i<t.size(); i++)
+        t[i] = dg::construct<inner_vector>(src[i], std::forward<Params>(ps)...);
+    return t;
+}
 
 template<class To, class From, class Size, class ...Params>
-To doConstruct( const From& src, RecursiveVectorTag, AnyVectorTag, Size size, Params&&... ps )
+To doConstruct( const From& src, RecursiveVectorTag, SharedVectorTag, Size size, Params&&... ps )
+{
+    To t(size);
+    using inner_vector = typename To::value_type;
+    for (int i=0; i<(int)size; i++)
+        t[i] = dg::construct<inner_vector>(src, std::forward<Params>(ps)...);
+    return t;
+}
+template<class To, class From, class Size, class ...Params>
+To doConstruct( const From& src, RecursiveVectorTag, MPIVectorTag, Size size, Params&&... ps )
 {
     To t(size);
     using inner_vector = typename To::value_type;
@@ -42,20 +70,58 @@ To doConstruct( const From& src, RecursiveVectorTag, AnyVectorTag, Size size, Pa
         t[i] = dg::construct<inner_vector>(src, std::forward<Params>(ps)...);
     return t;
 }
+template<class To, class From, class ...Params>
+To doConstruct( const From& src, RecursiveVectorTag, RecursiveVectorTag, Params&&...ps )
+{
+    unsigned size = src.size();
+    To t(size);
+    using inner_vector = typename To::value_type;
+    for (unsigned i=0; i<size; i++)
+        t[i] = dg::construct<inner_vector>(src[i], std::forward<Params>(ps)...);
+    return t;
+}
+
 template<class From, class To, class ...Params>
-void doAssign( const From& src, To& to, AnyVectorTag, ArrayVectorTag, Params&&...ps )
+void doAssign( const From& src, To& to, SharedVectorTag, ArrayVectorTag, Params&&...ps )
 {
     for (unsigned i=0; i<to.size(); i++)
         dg::assign(src, to[i], std::forward<Params>(ps)...);
 }
+template<class From, class To, class ...Params>
+void doAssign( const From& src, To& to, MPIVectorTag, ArrayVectorTag, Params&&...ps )
+{
+    for (unsigned i=0; i<to.size(); i++)
+        dg::assign(src, to[i], std::forward<Params>(ps)...);
+}
+template<class From, class To, class ...Params>
+void doAssign( const From& src, To& to, RecursiveVectorTag, ArrayVectorTag, Params&&...ps )
+{
+    for (unsigned i=0; i<to.size(); i++)
+        dg::assign(src[i], to[i], std::forward<Params>(ps)...);
+}
 
 template<class From, class To, class Size, class ...Params>
-void doAssign( const From& src, To& to, AnyVectorTag, RecursiveVectorTag, Size size, Params&&... ps )
+void doAssign( const From& src, To& to, SharedVectorTag, RecursiveVectorTag, Size size, Params&&... ps )
+{
+    to.resize(size);
+    for (int i=0; i<(int)size; i++)
+        dg::assign(src, to[i], std::forward<Params>(ps)...);
+}
+template<class From, class To, class Size, class ...Params>
+void doAssign( const From& src, To& to, MPIVectorTag, RecursiveVectorTag, Size size, Params&&... ps )
 {
     to.resize(size);
     for (int i=0; i<(int)size; i++)
         dg::assign(src, to[i], std::forward<Params>(ps)...);
 }
+template<class From, class To, class ...Params>
+void doAssign( const From& src, To& to, RecursiveVectorTag, RecursiveVectorTag, Params&&...ps )
+{
+    unsigned size = src.size();
+    to.resize(size);
+    for (unsigned i=0; i<size; i++)
+        dg::assign(src[i], to[i], std::forward<Params>(ps)...);
+}
 
 } //namespace detail
 
-- 
GitLab


From f44a944e10cb08a56f7b929072c05c58d26f7f80 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 May 2020 12:49:24 +0200
Subject: [PATCH 233/540] Clean implementation of host_vector in grids

There is no need to use a Traits system here, it's not like users start
writing their own topology classes, we just need the typedef to work
---
 inc/dg/topology/grid.h           | 51 ++++++++++++++------------------
 inc/dg/topology/gridX.h          | 28 +++++++++++-------
 inc/dg/topology/mpi_base.h       | 49 ++++++++++++++----------------
 inc/dg/topology/mpi_grid.h       | 33 +++++++++------------
 inc/geometries/curvilinear.h     | 10 +++----
 inc/geometries/curvilinearX.h    | 20 ++++++-------
 inc/geometries/mpi_curvilinear.h | 22 +++++++-------
 7 files changed, 102 insertions(+), 111 deletions(-)

diff --git a/inc/dg/topology/grid.h b/inc/dg/topology/grid.h
index 4f45a4298..38a269bd2 100644
--- a/inc/dg/topology/grid.h
+++ b/inc/dg/topology/grid.h
@@ -65,6 +65,13 @@
 
 namespace dg{
 
+///@cond
+template<class real_type>
+struct RealGrid2d;
+template<class real_type>
+struct RealGrid3d;
+///@endcond
+
 /**
 * @brief 1D grid
 * @ingroup grid
@@ -73,9 +80,10 @@ namespace dg{
 template<class real_type>
 struct RealGrid1d
 {
-    typedef SharedTag memory_category;
-    typedef OneDimensionalTag dimensionality;
-    typedef real_type value_type;
+    using value_type = real_type;
+    /// The host vector type used by host functions like evaluate
+    using host_vector = thrust::host_vector<real_type>;
+    using host_grid = RealGrid1d<real_type>;
     /**
      * @brief construct an empty grid
      * this leaves the access functions undefined
@@ -270,9 +278,10 @@ struct RealGrid1d
 template<class real_type>
 struct aRealTopology2d
 {
-    typedef SharedTag memory_category; //!< tag for choosing default host vector type
-    typedef TwoDimensionalTag dimensionality;
-    typedef real_type value_type;
+    using value_type = real_type;
+    /// The host vector type used by host functions like evaluate
+    using host_vector = thrust::host_vector<real_type>;
+    using host_grid = RealGrid2d<real_type>;
 
     /**
      * @brief Left boundary in x
@@ -488,9 +497,10 @@ struct aRealTopology2d
 template<class real_type>
 struct aRealTopology3d
 {
-    typedef SharedTag memory_category;
-    typedef ThreeDimensionalTag dimensionality;
-    typedef real_type value_type;
+    using value_type = real_type;
+    /// The host vector type used by host functions like evaluate
+    using host_vector = thrust::host_vector<real_type>;
+    using host_grid = RealGrid3d<real_type>;
 
     /**
      * @brief left boundary in x
@@ -807,29 +817,12 @@ void aRealTopology3d<real_type>::do_set(unsigned new_n, unsigned new_Nx,unsigned
     gy_.set(new_n, new_Ny);
     gz_.set(1,new_Nz);
 }
-template<class MemoryTag, class DimensionalityTag, class real_type>
-struct MemoryTraits { };
-
-template<class real_type>
-struct MemoryTraits< SharedTag, OneDimensionalTag, real_type> {
-    using host_vector = thrust::host_vector<real_type>;
-    using host_grid   = RealGrid1d<real_type>;
-};
-template<class real_type>
-struct MemoryTraits< SharedTag, TwoDimensionalTag, real_type> {
-    using host_vector = thrust::host_vector<real_type>;
-    using host_grid   = RealGrid2d<real_type>;
-};
-template<class real_type>
-struct MemoryTraits< SharedTag, ThreeDimensionalTag,real_type> {
-    using host_vector = thrust::host_vector<real_type>;
-    using host_grid   = RealGrid3d<real_type>;
-};
 
 template<class Topology>
-using get_host_vector = typename MemoryTraits< typename TopologyTraits<Topology>::memory_category, typename TopologyTraits<Topology>::dimensionality, typename TopologyTraits<Topology>::value_type>::host_vector;
+using get_host_vector = typename Topology::host_vector;
+
 template<class Topology>
-using get_host_grid = typename MemoryTraits< typename TopologyTraits<Topology>::memory_category, typename TopologyTraits<Topology>::dimensionality, typename TopologyTraits<Topology>::value_type>::host_grid;
+using get_host_grid = typename Topology::host_grid;
 
 ///@endcond
 
diff --git a/inc/dg/topology/gridX.h b/inc/dg/topology/gridX.h
index f82ab9581..5fa1a6443 100644
--- a/inc/dg/topology/gridX.h
+++ b/inc/dg/topology/gridX.h
@@ -42,8 +42,13 @@
  * @param Nz # of points in z
  */
 
-
 namespace dg{
+///@cond
+template<class real_type>
+struct RealGridX2d;
+template<class real_type>
+struct RealGridX3d;
+///@endcond
 
 /**
 * @brief 1D grid for X-point topology
@@ -61,9 +66,10 @@ namespace dg{
 template<class real_type>
 struct RealGridX1d
 {
-    typedef SharedTag memory_category;
-    typedef OneDimensionalTag dimensionality;
-    typedef real_type value_type;
+    using value_type = real_type;
+    /// The host vector type used by host functions like evaluate
+    using host_vector = thrust::host_vector<real_type>;
+    using host_grid = RealGridX1d<real_type>;
     /**
      * @brief 1D X-point grid
      *
@@ -249,9 +255,10 @@ struct RealGridX1d
 template<class real_type>
 struct aRealTopologyX2d
 {
-    typedef SharedTag memory_category;
-    typedef TwoDimensionalTag dimensionality;
-    typedef real_type value_type;
+    using value_type = real_type;
+    /// The host vector type used by host functions like evaluate
+    using host_vector = thrust::host_vector<real_type>;
+    using host_grid = RealGridX2d<real_type>;
 
     /**
      * @brief Left boundary in x
@@ -532,9 +539,10 @@ struct RealGridX2d : public aRealTopologyX2d<real_type>
 template<class real_type>
 struct aRealTopologyX3d
 {
-    typedef SharedTag memory_category;
-    typedef ThreeDimensionalTag dimensionality;
-    typedef real_type value_type;
+    using value_type = real_type;
+    /// The host vector type used by host functions like evaluate
+    using host_vector = thrust::host_vector<real_type>;
+    using host_grid = RealGridX3d<real_type>;
     /**
      * @brief left boundary in x
      *
diff --git a/inc/dg/topology/mpi_base.h b/inc/dg/topology/mpi_base.h
index a085bd961..f0eff86d4 100644
--- a/inc/dg/topology/mpi_base.h
+++ b/inc/dg/topology/mpi_base.h
@@ -16,17 +16,16 @@ namespace dg
 template<class real_type>
 struct aRealMPIGeometry2d : public aRealMPITopology2d<real_type>
 {
-    typedef MPI_Vector<thrust::host_vector<real_type> > host_vector;
     ///@copydoc aRealGeometry2d::jacobian()
-    SparseTensor<host_vector > jacobian()const {
+    SparseTensor<MPI_Vector<thrust::host_vector<real_type>> > jacobian()const {
         return do_compute_jacobian();
     }
     ///@copydoc aRealGeometry2d::metric()
-    SparseTensor<host_vector > metric()const {
+    SparseTensor<MPI_Vector<thrust::host_vector<real_type>> > metric()const {
         return do_compute_metric();
     }
     ///@copydoc aRealGeometry2d::map()
-    std::vector<host_vector > map()const{
+    std::vector<MPI_Vector<thrust::host_vector<real_type>> > map()const{
         return do_compute_map();
     }
     ///Geometries are cloneable
@@ -42,14 +41,14 @@ struct aRealMPIGeometry2d : public aRealMPITopology2d<real_type>
     ///@copydoc aRealMPITopology2d::operator=(const aRealMPITopology2d&)
     aRealMPIGeometry2d& operator=( const aRealMPIGeometry2d& src) = default;
     private:
-    virtual SparseTensor<host_vector > do_compute_metric()const {
-        return SparseTensor<host_vector >(*this);
+    virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>> > do_compute_metric()const {
+        return SparseTensor<MPI_Vector<thrust::host_vector<real_type>> >(*this);
     }
-    virtual SparseTensor<host_vector > do_compute_jacobian()const {
-        return SparseTensor<host_vector >(*this);
+    virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>> > do_compute_jacobian()const {
+        return SparseTensor<MPI_Vector<thrust::host_vector<real_type>> >(*this);
     }
-    virtual std::vector<host_vector > do_compute_map()const{
-        std::vector<host_vector> map(2);
+    virtual std::vector<MPI_Vector<thrust::host_vector<real_type>> > do_compute_map()const{
+        std::vector<MPI_Vector<thrust::host_vector<real_type>>> map(2);
         map[0] = dg::evaluate(dg::cooX2d, *this);
         map[1] = dg::evaluate(dg::cooY2d, *this);
         return map;
@@ -62,17 +61,16 @@ struct aRealMPIGeometry2d : public aRealMPITopology2d<real_type>
 template<class real_type>
 struct aRealMPIGeometry3d : public aRealMPITopology3d<real_type>
 {
-    typedef MPI_Vector<thrust::host_vector<real_type> > host_vector;
     ///@copydoc aRealGeometry3d::jacobian()
-    SparseTensor<host_vector > jacobian()const{
+    SparseTensor<MPI_Vector<thrust::host_vector<real_type>> > jacobian()const{
         return do_compute_jacobian();
     }
     ///@copydoc aRealGeometry3d::metric()
-    SparseTensor<host_vector > metric()const {
+    SparseTensor<MPI_Vector<thrust::host_vector<real_type>> > metric()const {
         return do_compute_metric();
     }
     ///@copydoc aRealGeometry3d::map()
-    std::vector<host_vector > map()const{
+    std::vector<MPI_Vector<thrust::host_vector<real_type>> > map()const{
         return do_compute_map();
     }
     ///Geometries are cloneable
@@ -88,14 +86,14 @@ struct aRealMPIGeometry3d : public aRealMPITopology3d<real_type>
     ///@copydoc aRealMPITopology3d::operator=(const aRealMPITopology3d&)
     aRealMPIGeometry3d& operator=( const aRealMPIGeometry3d& src) = default;
     private:
-    virtual SparseTensor<host_vector > do_compute_metric()const {
-        return SparseTensor<host_vector >(*this);
+    virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>> > do_compute_metric()const {
+        return SparseTensor<MPI_Vector<thrust::host_vector<real_type>> >(*this);
     }
-    virtual SparseTensor<host_vector > do_compute_jacobian()const {
-        return SparseTensor<host_vector >(*this);
+    virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>> > do_compute_jacobian()const {
+        return SparseTensor<MPI_Vector<thrust::host_vector<real_type>> >(*this);
     }
-    virtual std::vector<host_vector > do_compute_map()const{
-        std::vector<host_vector> map(3);
+    virtual std::vector<MPI_Vector<thrust::host_vector<real_type>> > do_compute_map()const{
+        std::vector<MPI_Vector<thrust::host_vector<real_type>>> map(3);
         map[0] = dg::evaluate(dg::cooX3d, *this);
         map[1] = dg::evaluate(dg::cooY3d, *this);
         map[2] = dg::evaluate(dg::cooZ3d, *this);
@@ -173,7 +171,7 @@ struct RealCartesianMPIGrid2d : public aRealMPIGeometry2d<real_type>
 template<class real_type>
 struct RealCartesianMPIGrid3d : public aRealProductMPIGeometry3d<real_type>
 {
-    typedef RealCartesianMPIGrid2d<real_type> perpendicular_grid;
+    using perpendicular_grid = RealCartesianMPIGrid2d<real_type>;
     ///@copydoc hide_grid_parameters3d
     ///@copydoc hide_comm_parameters3d
     RealCartesianMPIGrid3d( real_type x0, real_type x1, real_type y0, real_type y1, real_type z0, real_type z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, MPI_Comm comm): aRealProductMPIGeometry3d<real_type>( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, dg::PER,dg::PER,dg::PER, comm){}
@@ -213,7 +211,7 @@ struct RealCartesianMPIGrid3d : public aRealProductMPIGeometry3d<real_type>
 template<class real_type>
 struct RealCylindricalMPIGrid3d: public aRealProductMPIGeometry3d<real_type>
 {
-    typedef RealCartesianMPIGrid2d<real_type> perpendicular_grid;
+    using perpendicular_grid = RealCartesianMPIGrid2d<real_type>;
     ///@copydoc hide_grid_parameters3d
     ///@copydoc hide_bc_parameters3d
     ///@copydoc hide_comm_parameters3d
@@ -238,13 +236,12 @@ struct RealCylindricalMPIGrid3d: public aRealProductMPIGeometry3d<real_type>
                 this->global().bcx(), this->global().bcy(), this->global().bcz());
     }
     private:
-    using host_vector = MPI_Vector<thrust::host_vector<real_type>>;
     virtual RealCartesianMPIGrid2d<real_type>* do_perp_grid()const override final{
         return new RealCartesianMPIGrid2d<real_type>( this->global().x0(), this->global().x1(), this->global().y0(), this->global().y1(), this->global().n(), this->global().Nx(), this->global().Ny(), this->global().bcx(), this->global().bcy(), this->get_perp_comm( ));
     }
-    virtual SparseTensor<host_vector > do_compute_metric()const override final{
-        SparseTensor<host_vector> metric(*this);
-        host_vector R = dg::evaluate(dg::cooX3d, *this);
+    virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>> > do_compute_metric()const override final{
+        SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> metric(*this);
+        MPI_Vector<thrust::host_vector<real_type>> R = dg::evaluate(dg::cooX3d, *this);
         for( unsigned i = 0; i<this->local().size(); i++)
             R.data()[i] = 1./R.data()[i]/R.data()[i];
         metric.idx(2,2)=2;
diff --git a/inc/dg/topology/mpi_grid.h b/inc/dg/topology/mpi_grid.h
index 26056f0fa..7f8ea29d6 100644
--- a/inc/dg/topology/mpi_grid.h
+++ b/inc/dg/topology/mpi_grid.h
@@ -21,7 +21,12 @@ namespace dg
  * @note the paramateres given in the constructor are global parameters
  */
 
-
+///@cond
+template<class real_type>
+struct RealMPIGrid2d;
+template<class real_type>
+struct RealMPIGrid3d;
+///@endcond
 
 
 /**
@@ -38,9 +43,10 @@ namespace dg
 template<class real_type>
 struct aRealMPITopology2d
 {
-    typedef MPITag memory_category;
-    typedef TwoDimensionalTag dimensionality;
-    typedef real_type value_type;
+    using value_type = real_type;
+    /// The host vector type used by host functions like evaluate
+    using host_vector = MPI_Vector<thrust::host_vector<real_type>>;
+    using host_grid = RealMPIGrid2d<real_type>;
 
     /**
      * @brief Return global x0
@@ -325,9 +331,10 @@ struct aRealMPITopology2d
 template<class real_type>
 struct aRealMPITopology3d
 {
-    typedef MPITag memory_category;
-    typedef ThreeDimensionalTag dimensionality;
-    typedef real_type value_type;
+    using value_type = real_type;
+    /// The host vector type used by host functions like evaluate
+    using host_vector = MPI_Vector<thrust::host_vector<real_type>>;
+    using host_grid = RealMPIGrid3d<real_type>;
 
     /**
      * @brief Return global x0
@@ -728,18 +735,6 @@ struct RealMPIGrid3d : public aRealMPITopology3d<real_type>
     }
 };
 
-///@cond
-template<class real_type>
-struct MemoryTraits< MPITag, TwoDimensionalTag, real_type> {
-    using host_vector = MPI_Vector<thrust::host_vector<real_type>>;
-    using host_grid   = RealMPIGrid2d<real_type>;
-};
-template<class real_type>
-struct MemoryTraits< MPITag, ThreeDimensionalTag, real_type> {
-    using host_vector = MPI_Vector<thrust::host_vector<real_type>>;
-    using host_grid   = RealMPIGrid3d<real_type>;
-};
-///@endcond
 ///@addtogroup gridtypes
 ///@{
 using MPIGrid2d         = dg::RealMPIGrid2d<double>;
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 64fcda05a..3cd5c0bf0 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -113,12 +113,12 @@ struct RealCurvilinearGrid2d : public dg::aRealGeometry2d<real_type>
     virtual SparseTensor<thrust::host_vector<real_type> > do_compute_jacobian( ) const override final{
         return jac_;
     }
-    virtual SparseTensor<thrust::host_vector<real_type> > do_compute_metric( ) const override final{
+    virtual SparseTensor<thrust::host_vector<real_type>> do_compute_metric( ) const override final{
         return metric_;
     }
-    virtual std::vector<thrust::host_vector<real_type> > do_compute_map()const override final{return map_;}
-    dg::SparseTensor<thrust::host_vector<real_type> > jac_, metric_;
-    std::vector<thrust::host_vector<real_type> > map_;
+    virtual std::vector<thrust::host_vector<real_type>> do_compute_map()const override final{return map_;}
+    dg::SparseTensor<thrust::host_vector<real_type>> jac_, metric_;
+    std::vector<thrust::host_vector<real_type>> map_;
     dg::ClonePtr<aRealGenerator2d<real_type>> handle_;
 };
 
@@ -132,7 +132,7 @@ struct RealCurvilinearGrid2d : public dg::aRealGeometry2d<real_type>
 template<class real_type>
 struct RealCurvilinearProductGrid3d : public dg::aRealProductGeometry3d<real_type>
 {
-    typedef RealCurvilinearGrid2d<real_type> perpendicular_grid;
+    using perpendicular_grid = RealCurvilinearGrid2d<real_type>;
 
     ///@copydoc hide_grid_parameters3d
     RealCurvilinearProductGrid3d( const aRealGenerator2d<real_type>& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
diff --git a/inc/geometries/curvilinearX.h b/inc/geometries/curvilinearX.h
index ff8a5c652..dae331ca4 100644
--- a/inc/geometries/curvilinearX.h
+++ b/inc/geometries/curvilinearX.h
@@ -83,16 +83,16 @@ struct RealCurvilinearProductGridX3d : public dg::aRealGeometryX3d<real_type>
         handle_->generate( x_vec, y_vec, gY1d.n()*gY1d.outer_N(), gY1d.n()*(gY1d.inner_N()+gY1d.outer_N()), map_[0], map_[1], jac_.values()[2], jac_.values()[3], jac_.values()[4], jac_.values()[5]);
         jac_.idx(0,0) = 2, jac_.idx(0,1) = 3, jac_.idx(1,0)=4, jac_.idx(1,1) = 5;
     }
-    virtual SparseTensor<thrust::host_vector<real_type> > do_compute_jacobian( ) const override final{
+    virtual SparseTensor<thrust::host_vector<real_type>> do_compute_jacobian( ) const override final{
         return jac_;
     }
-    virtual SparseTensor<thrust::host_vector<real_type> > do_compute_metric( ) const override final
+    virtual SparseTensor<thrust::host_vector<real_type>> do_compute_metric( ) const override final
     {
         return detail::square( jac_, map_[0], handle_->isOrthogonal());
     }
-    virtual std::vector<thrust::host_vector<real_type> > do_compute_map()const override final{return map_;}
-    std::vector<thrust::host_vector<real_type> > map_;
-    SparseTensor<thrust::host_vector<real_type> > jac_;
+    virtual std::vector<thrust::host_vector<real_type>> do_compute_map()const override final{return map_;}
+    std::vector<thrust::host_vector<real_type>> map_;
+    SparseTensor<thrust::host_vector<real_type>> jac_;
     dg::ClonePtr<aRealGeneratorX2d<real_type>> handle_;
 };
 
@@ -131,15 +131,15 @@ struct RealCurvilinearGridX2d : public dg::aRealGeometryX2d<real_type>
         dg::blas1::copy( 1., metric_.values()[3]); //set pp to 1
         map_.pop_back();
     }
-    virtual SparseTensor<thrust::host_vector<real_type> > do_compute_jacobian( ) const override final{
+    virtual SparseTensor<thrust::host_vector<real_type>> do_compute_jacobian( ) const override final{
         return jac_;
     }
-    virtual SparseTensor<thrust::host_vector<real_type> > do_compute_metric( ) const override final{
+    virtual SparseTensor<thrust::host_vector<real_type>> do_compute_metric( ) const override final{
         return metric_;
     }
-    virtual std::vector<thrust::host_vector<real_type> > do_compute_map()const override final{return map_;}
-    dg::SparseTensor<thrust::host_vector<real_type> > jac_, metric_;
-    std::vector<thrust::host_vector<real_type> > map_;
+    virtual std::vector<thrust::host_vector<real_type>> do_compute_map()const override final{return map_;}
+    dg::SparseTensor<thrust::host_vector<real_type>> jac_, metric_;
+    std::vector<thrust::host_vector<real_type>> map_;
     dg::ClonePtr<aRealGeneratorX2d<real_type>> handle_;
 };
 
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index c0e6719af..cce08129f 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -49,7 +49,6 @@ struct RealCurvilinearMPIGrid2d : public dg::aRealMPIGeometry2d<real_type>
                 global().bcx(), global().bcy());
     }
     //These are necessary to help compiler find inherited names
-    using typename dg::aRealMPIGeometry2d<real_type>::host_vector;
     using dg::aRealMPIGeometry2d<real_type>::global;
     private:
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) override final
@@ -80,15 +79,15 @@ struct RealCurvilinearMPIGrid2d : public dg::aRealMPIGeometry2d<real_type>
             map_[i] = global2local( map[i], *this);
     }
 
-    virtual SparseTensor<host_vector> do_compute_jacobian( ) const override final{
+    virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> do_compute_jacobian( ) const override final{
         return jac_;
     }
-    virtual SparseTensor<host_vector> do_compute_metric( ) const override final{
+    virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> do_compute_metric( ) const override final{
         return metric_;
     }
-    virtual std::vector<host_vector > do_compute_map()const override final{return map_;}
-    dg::SparseTensor<host_vector > jac_, metric_;
-    std::vector<host_vector > map_;
+    virtual std::vector<MPI_Vector<thrust::host_vector<real_type>>> do_compute_map()const override final{return map_;}
+    dg::SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> jac_, metric_;
+    std::vector<MPI_Vector<thrust::host_vector<real_type>>> map_;
     dg::ClonePtr<aRealGenerator2d<real_type>> handle_;
 };
 
@@ -125,7 +124,6 @@ struct RealCurvilinearProductMPIGrid3d : public dg::aRealProductMPIGeometry3d<re
                 global().bcx(), global().bcy(), global().bcz());
     }
     //These are necessary to help compiler find inherited names
-    using typename dg::aRealMPIGeometry3d<real_type>::host_vector;
     using dg::aRealMPIGeometry3d<real_type>::global;
     private:
     virtual perpendicular_grid* do_perp_grid() const override final{ return new perpendicular_grid(*this);}
@@ -172,15 +170,15 @@ struct RealCurvilinearProductMPIGrid3d : public dg::aRealProductMPIGeometry3d<re
                 map_[1].data()[k*size2d+i] = map_[1].data()[(k-1)*size2d+i];
             }
     }
-    virtual SparseTensor<host_vector> do_compute_jacobian( ) const override final{
+    virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> do_compute_jacobian( ) const override final{
         return jac_;
     }
-    virtual SparseTensor<host_vector> do_compute_metric( ) const override final{
+    virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> do_compute_metric( ) const override final{
         return detail::square( jac_, map_[0], handle_->isOrthogonal());
     }
-    virtual std::vector<host_vector > do_compute_map()const override final{return map_;}
-    dg::SparseTensor<host_vector > jac_;
-    std::vector<host_vector > map_;
+    virtual std::vector<MPI_Vector<thrust::host_vector<real_type>>> do_compute_map()const override final{return map_;}
+    dg::SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> jac_;
+    std::vector<MPI_Vector<thrust::host_vector<real_type>>> map_;
     ClonePtr<dg::geo::aRealGenerator2d<real_type>> handle_;
 };
 ///@cond
-- 
GitLab


From 5e466cd0622100dfdf53e1ac51fcb8b1e35eecbe Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 May 2020 12:55:18 +0200
Subject: [PATCH 234/540] Document remarks on MPI calls for the file functions

---
 inc/file/easy_output.h  | 6 +++---
 inc/file/nc_utilities.h | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
index 483d5de88..3205bf8eb 100644
--- a/inc/file/easy_output.h
+++ b/inc/file/easy_output.h
@@ -109,7 +109,7 @@ struct NC_Error_Handle
 * writes to the file independently in parallel (\c true)
 * or each process funnels its data through the master rank (\c false),
 * which involves communication but may be faster than the former method.
-* @attention In the MPI version, if \c parallel==true a **parallel netcdf** must be
+* @attention In the MPI version (i) all processes must call this function and (ii) if \c parallel==true a **parallel netcdf** must be
 * linked, the file opened with the \c NC_MPIIO flag from the \c netcdf_par.h header and the variable be marked with \c NC_COLLECTIVE access while if \c parallel==false we need **serial netcdf** and only the master thread needs to open and access the file.
 * Note that serious performance penalties have been observed on some platforms for parallel netcdf.
 */
@@ -143,8 +143,8 @@ void put_var_double(int ncid, int varid, dg::aTopology2d& grid,
 * writes to the file independently in parallel (\c true)
 * or each process funnels its data through the master rank (\c false),
 * which involves communication but may be faster than the former method.
-* @attention In the MPI version, if \c parallel==true a **parallel netcdf** must be
-* linked while if \c parallel==false we need **serial netcdf**.
+* @attention In the MPI version (i) all processes must call this function and (ii) if \c parallel==true a **parallel netcdf** must be
+* linked, the file opened with the \c NC_MPIIO flag from the \c netcdf_par.h header and the variable be marked with \c NC_COLLECTIVE access while if \c parallel==false we need **serial netcdf** and only the master thread needs to open and access the file.
 * Note that serious performance penalties have been observed on some platforms for parallel netcdf.
 */
 template<class host_vector>
diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index 1879c2ed5..d9ae4d094 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -282,25 +282,25 @@ inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRe
 
 #ifdef MPI_VERSION
 
-/// Convenience function that just calls the corresponding serial version with the global grid
+/// Only master process should call this!! Convenience function that just calls the corresponding serial version with the global grid.
 template<class T>
 inline int define_dimensions( int ncid, int* dimsIDs, const dg::aRealMPITopology2d<T>& g, std::array<std::string,2> name_dims = {"y", "x"})
 {
     return define_dimensions( ncid, dimsIDs, g.global(), name_dims);
 }
-/// Convenience function that just calls the corresponding serial version with the global grid
+///Only master process should call this!! Convenience function that just calls the corresponding serial version with the global grid
 template<class T>
 inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRealMPITopology2d<T>& g, std::array<std::string,3> name_dims = {"time", "y", "x"})
 {
     return define_dimensions( ncid, dimsIDs, tvarID, g.global(), name_dims);
 }
-/// Convenience function that just calls the corresponding serial version with the global grid
+///Only master process should call this!! Convenience function that just calls the corresponding serial version with the global grid
 template<class T>
 inline int define_dimensions( int ncid, int* dimsIDs, const dg::aRealMPITopology3d<T>& g, std::array<std::string, 3> name_dims = {"z", "y", "x"})
 {
     return define_dimensions( ncid, dimsIDs, g.global(), name_dims);
 }
-/// Convenience function that just calls the corresponding serial version with the global grid
+///Only master process should call this!! Convenience function that just calls the corresponding serial version with the global grid
 template<class T>
 inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRealMPITopology3d<T>& g, std::array<std::string, 4> name_dims = {"time", "z", "y", "x"})
 {
-- 
GitLab


From 91813a235c7be492fd31103ab1168ecccd8d177a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 May 2020 15:41:42 +0200
Subject: [PATCH 235/540] Get rid of netcdf_par in geometries codes

---
 inc/geometries/flux_t.cu                  |  2 +-
 inc/geometries/geometry_elliptic_mpib.cu  | 53 ++++++++--------------
 inc/geometries/ribeiroX_t.cu              |  2 +-
 inc/geometries/ribeiro_mpit.cu            | 55 +++++++----------------
 inc/geometries/ribeiro_t.cu               | 12 ++---
 inc/geometries/separatrix_orthogonal_t.cu |  7 +--
 6 files changed, 40 insertions(+), 91 deletions(-)

diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 06e70d001..62c7abff6 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -97,7 +97,7 @@ int main( int argc, char* argv[])
     int coordsID[2];
     err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
-    dg::HVec X=dg::pullback(dg::cooX2d, g2d), Y=dg::pullback(dg::cooY2d, g2d); //P = dg::pullback( dg::coo3, g);
+    dg::HVec X=g2d.map()[0], Y=g2d.map()[1];
     err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
     err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
 
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 93ffc775f..1921d4c32 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -2,7 +2,6 @@
 #include <memory>
 #include <mpi.h>
 
-#include <netcdf_par.h>
 #include "json/json.h"
 
 #include "dg/file/nc_utilities.h"
@@ -56,36 +55,19 @@ int main(int argc, char**argv)
     ///////////////////////////////////////////////////////////////////////////
     int ncid;
     file::NC_Error_Handle ncerr;
-    MPI_Info info = MPI_INFO_NULL;
-    ncerr = nc_create_par( "testE_mpi.nc", NC_NETCDF4|NC_MPIIO|NC_CLOBBER, comm, info, &ncid); //MPI ON
+    if(rank==0)ncerr = nc_create( "testE_mpi.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
-    ncerr = file::define_dimensions(  ncid, dim2d, g2d->global());
+    if(rank==0)ncerr = file::define_dimensions(  ncid, dim2d, *g2d);
     int coordsID[2], psiID, functionID, function2ID;
-    ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
-    ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
-    ncerr = nc_def_var( ncid, "error", NC_DOUBLE, 2, dim2d, &psiID);
-    ncerr = nc_def_var( ncid, "num_solution", NC_DOUBLE, 2, dim2d, &functionID);
-    ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
+    if(rank==0)ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
+    if(rank==0)ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
+    if(rank==0)ncerr = nc_def_var( ncid, "error", NC_DOUBLE, 2, dim2d, &psiID);
+    if(rank==0)ncerr = nc_def_var( ncid, "num_solution", NC_DOUBLE, 2, dim2d, &functionID);
+    if(rank==0)ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
 
-    int dims[2], periods[2],  coords[2];
-    MPI_Cart_get( g2d->communicator(), 2, dims, periods, coords);
-    size_t count[2] = {g2d->local().n()*g2d->local().Ny(), g2d->local().n()*g2d->local().Nx()};
-    size_t start[2] = {coords[1]*count[0], coords[0]*count[1]};
-
-    ncerr = nc_var_par_access( ncid, coordsID[0], NC_COLLECTIVE);
-    ncerr = nc_var_par_access( ncid, coordsID[1], NC_COLLECTIVE);
-    ncerr = nc_var_par_access( ncid, psiID, NC_COLLECTIVE);
-    ncerr = nc_var_par_access( ncid, functionID, NC_COLLECTIVE);
-    ncerr = nc_var_par_access( ncid, function2ID, NC_COLLECTIVE);
-
-    dg::HVec X( g2d->local().size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d->local().size(); i++)
-    {
-        X[i] = g2d->map()[0].data()[i];
-        Y[i] = g2d->map()[1].data()[i];
-    }
-    ncerr = nc_put_vara_double( ncid, coordsID[0], start, count, X.data());
-    ncerr = nc_put_vara_double( ncid, coordsID[1], start, count, Y.data());
+    dg::MHVec X( g2d->map()[0]), Y( g2d->map()[1]);
+    file::put_var_double( ncid, coordsID[0], *g2d, X);
+    file::put_var_double( ncid, coordsID[1], *g2d, Y);
     ///////////////////////////////////////////////////////////////////////////
     dg::MDVec x =    dg::evaluate( dg::zero, *g2d);
     const dg::MDVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), *g2d);
@@ -121,13 +103,14 @@ int main(int argc, char**argv)
     if(rank==0)std::cout << *thrust::max_element( gyy.data().begin(), gyy.data().end()) << "\t";
     if(rank==0)std::cout<<t.diff()/(double)number<<"s"<<std::endl;
 
-    dg::blas1::transfer( error.data(), X );
-    ncerr = nc_put_vara_double( ncid, psiID, start, count, X.data());
-    dg::blas1::transfer( x.data(), X );
-    ncerr = nc_put_vara_double( ncid, functionID, start, count, X.data());
-    dg::blas1::transfer( solution.data(), X );
-    ncerr = nc_put_vara_double( ncid, function2ID, start, count, X.data());
-    ncerr = nc_close( ncid);
+    dg::MHVec transfer;
+    dg::assign( error, transfer);
+    file::put_var_double( ncid, psiID, *g2d, transfer);
+    dg::assign( x, transfer);
+    file::put_var_double( ncid, functionID, *g2d, transfer);
+    dg::assign( solution, transfer);
+    file::put_var_double( ncid, function2ID, *g2d, transfer);
+    if(rank==0)ncerr = nc_close( ncid);
     MPI_Finalize();
 
 
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index 1b6408760..0d19dfb72 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -198,7 +198,7 @@ int main( int argc, char* argv[])
     std::cout << "TEST VOLUME IS:\n";
     dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, -2.0*gp.a*gp.elongation, 1.2*gp.a*gp.elongation, 1, 5e3, 1e4, dg::PER, dg::PER);
     gp.psipmax = 0., gp.psipmin = psi_0;
-    dg::geo::Iris iris( psip.f(), gp.psipmin, gp.psipmax);
+    auto iris = dg::compose( dg::Iris(  gp.psipmin, gp.psipmax), psip.f());
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::DVec cutter = dg::pullback( iris, g2d), cut_vol( cutter);
     dg::blas1::pointwiseDot(cutter, w2d, cut_vol);
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 7753f470f..f1c3dfb20 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -16,7 +16,6 @@
 #include "simple_orthogonal.h"
 //#include "ds.h"
 
-#include <netcdf_par.h>
 #include "dg/file/nc_utilities.h"
 
 double sineX( double x, double y) {return sin(x)*sin(y);}
@@ -64,56 +63,35 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     int ncid;
     file::NC_Error_Handle err;
-    MPI_Info info = MPI_INFO_NULL;
-    err = nc_create_par( "test_mpi.nc", NC_NETCDF4|NC_MPIIO|NC_CLOBBER, g2d->communicator(), info, &ncid); //MPI ON
+    if(rank==0)err = nc_create( "test_mpi.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim3d[2];
-    err = file::define_dimensions(  ncid, dim3d, g2d->global());
+    if(rank==0)err = file::define_dimensions(  ncid, dim3d, *g2d);
     int coordsID[2], onesID, defID,confID, volID, divBID;
-    err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
-    err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
-    //err = nc_def_var( ncid, "z_XYP", NC_DOUBLE, 3, dim3d, &coordsID[2]);
-    err = nc_def_var( ncid, "psi", NC_DOUBLE, 2, dim3d, &onesID);
-    err = nc_def_var( ncid, "deformation", NC_DOUBLE, 2, dim3d, &defID);
-    err = nc_def_var( ncid, "conformal", NC_DOUBLE, 2, dim3d, &confID);
-    err = nc_def_var( ncid, "volume", NC_DOUBLE, 2, dim3d, &volID);
-    err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim3d, &divBID);
-
-    int dims[2], periods[2],  coords[2];
-    MPI_Cart_get( g2d->communicator(), 2, dims, periods, coords);
-    size_t count[2] = {g2d->local().n()*g2d->local().Ny(), g2d->local().n()*g2d->local().Nx()};
-    size_t start[2] = {coords[1]*count[0], coords[0]*count[1]};
-    err = nc_var_par_access( ncid, coordsID[0], NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, coordsID[1], NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, onesID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, defID, NC_COLLECTIVE);
-    err = nc_var_par_access( ncid, divBID, NC_COLLECTIVE);
+    if(rank==0)err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
+    if(rank==0)err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
+    if(rank==0)err = nc_def_var( ncid, "psi", NC_DOUBLE, 2, dim3d, &onesID);
+    if(rank==0)err = nc_def_var( ncid, "deformation", NC_DOUBLE, 2, dim3d, &defID);
+    if(rank==0)err = nc_def_var( ncid, "conformal", NC_DOUBLE, 2, dim3d, &confID);
+    if(rank==0)err = nc_def_var( ncid, "volume", NC_DOUBLE, 2, dim3d, &volID);
+    if(rank==0)err = nc_def_var( ncid, "divB", NC_DOUBLE, 2, dim3d, &divBID);
 
     dg::MHVec psi_p = dg::pullback( psip.f(), *g2d);
     //g.display();
-    err = nc_put_vara_double( ncid, onesID, start, count, psi_p.data().data());
-    dg::HVec X( g2d->local().size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d->local().size(); i++)
-    {
-        X[i] = g2d->map()[0].data()[i];
-        Y[i] = g2d->map()[0].data()[i];
-    }
+    file::put_var_double( ncid, onesID, *g2d, psi_p);
+    dg::MHVec X = g2d->map()[0], Y = g2d->map()[1];
+    file::put_var_double( ncid, coordsID[0], *g2d, X);
+    file::put_var_double( ncid, coordsID[1], *g2d, Y);
 
     dg::MHVec temp0( dg::evaluate(dg::zero, *g2d)), temp1(temp0);
     dg::MHVec w2d = dg::create::weights( *g2d);
 
-    err = nc_put_vara_double( ncid, coordsID[0], start,count, X.data());
-    err = nc_put_vara_double( ncid, coordsID[1], start,count, Y.data());
-
     dg::SparseTensor<dg::MHVec> metric = g2d->metric();
     dg::MHVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
     dg::MHVec vol = dg::tensor::volume(metric);
-    //err = nc_put_vara_double( ncid, coordsID[2], g.z().data());
-    //dg::blas1::pointwiseDivide( g2d->g_xy(), g2d->g_xx(), temp0);
     dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     const dg::MHVec ones = dg::evaluate( dg::one, *g2d);
     dg::blas1::axpby( 1., ones, -1., temp0, temp0);
-    dg::blas1::transfer( temp0.data(), X);
-    err = nc_put_vara_double( ncid, defID, start,count, X.data());
+    file::put_var_double( ncid, defID, *g2d, temp0);
 
     if(rank==0)std::cout << "Construction successful!\n";
 
@@ -133,8 +111,7 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
-    dg::blas1::transfer( temp0.data(), X);
-    err = nc_put_var_double( ncid, volID, X.data());
+    file::put_var_double( ncid, volID, *g2d, temp0);
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     if(rank==0)std::cout << "Rel Consistency  of volume is "<<error<<"\n";
@@ -165,7 +142,7 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "relative difference in volume is "<<fabs(volumeRZP - volume)/volume<<std::endl;
     if(rank==0)std::cout << "Note that the error might also come from the volume in RZP!\n"; //since integration of jacobian is fairly good probably
 
-    err = nc_close( ncid);
+    if(rank==0)err = nc_close( ncid);
     MPI_Finalize();
 
 
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index a138125b0..549f4e3fe 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -91,19 +91,13 @@ int main( int argc, char* argv[])
     thrust::host_vector<double> psi_p = dg::pullback( psip.f(), *g2d);
     //g.display();
     err = nc_put_var_double( ncid, onesID, periodify(psi_p, g2d_periodic).data());
-    dg::HVec X( g2d->size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d->size(); i++)
-    {
-        X[i] = g2d->map()[0][i];
-        Y[i] = g2d->map()[1][i];
-    }
+    dg::HVec X( g2d->map()[0]), Y(g2d->map()[1]);
+    err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
+    err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
 
     dg::HVec temp0( g2d->size()), temp1(temp0);
     dg::HVec w2d = dg::create::weights( *g2d);
 
-    err = nc_put_var_double( ncid, coordsID[0], periodify(X, g2d_periodic).data());
-    err = nc_put_var_double( ncid, coordsID[1], periodify(Y, g2d_periodic).data());
-
     dg::SparseTensor<dg::HVec> metric = g2d->metric();
     dg::HVec g_xx = metric.value(0,0), g_xy = metric.value(0,1), g_yy=metric.value(1,1);
     dg::HVec vol = dg::tensor::volume(metric);
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index a7e344b74..24c542dd0 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -154,12 +154,7 @@ int main( int argc, char* argv[])
     }
     map2d.emplace_back( "Psi_p_interpolated", psip_X, "Poloidal flux function");
     g2d.display();
-    dg::HVec X( g2d.size()), Y(X), P = dg::evaluate( dg::zero, g2d);
-    for( unsigned i=0; i<g2d.size(); i++)
-    {
-        X[i] = g2d.map()[0][i];
-        Y[i] = g2d.map()[1][i];
-    }
+    dg::HVec X( g2d.map()[0]), Y(g2d.map()[1]), P = dg::evaluate( dg::zero, g2d);
     map2d.emplace_back( "xc", X, "X-coordinate Cartesian");
     map2d.emplace_back( "yc", Y, "Y-coordinate Cartesian");
     map2d.emplace_back( "zc", P, "Z-coordinate Cartesian");
-- 
GitLab


From 8abff53aab4c7389951e9c1d3e193ac2d2a42d69 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 May 2020 15:57:53 +0200
Subject: [PATCH 236/540] Clear separation of feltor diagnostics and init

- it is better to have independent files not depending on some
undocumented typedefs
---
 src/feltor/feltor.cu            |  15 +++--
 src/feltor/feltor_hpc.cu        |  51 ++++++++--------
 src/feltor/feltordiag.cu        |   9 ++-
 src/feltor/feltordiag.h         | 104 +++++++++++++++++---------------
 src/feltor/init.h               |  69 ++++++++++++++-------
 src/feltor/init_from_file.h     |  38 +++++++-----
 src/feltor/interpolate_in_3d.cu |   9 +--
 7 files changed, 174 insertions(+), 121 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index ae5809250..0cfc71efb 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -9,6 +9,8 @@
 
 #include "feltor.h"
 #include "implicit.h"
+#include "init.h"
+#include "feltordiag.h"
 
 using HVec = dg::HVec;
 using DVec = dg::DVec;
@@ -18,8 +20,9 @@ using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 
-#include "init.h"
-#include "feltordiag.h"
+using Initialize = feltor::Initialize<Geometry, IHMatrix, IDMatrix, DMatrix, DVec>;
+using Sources = feltor::Sources<Geometry, IDMatrix, DMatrix, DVec>;
+using Diagnostics = feltor::Diagnostics<Geometry, IDMatrix, DMatrix, DVec>;
 
 int main( int argc, char* argv[])
 {
@@ -85,7 +88,7 @@ int main( int argc, char* argv[])
     gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
     gradPsip[2] =  result; //zero
-    feltor::Variables var = {
+    Diagnostics::Variables var = {
         feltor, p,gp,mag, gradPsip, gradPsip
     };
 
@@ -95,7 +98,7 @@ int main( int argc, char* argv[])
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
     try{
-        y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+        y0 = Initialize::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
         return -1;
@@ -105,7 +108,7 @@ int main( int argc, char* argv[])
     HVec profile = dg::evaluate( dg::zero, grid);
     HVec source_profile;
     try{
-        source_profile = feltor::source_profiles.at(p.source_type)(
+        source_profile = Sources::source_profiles.at(p.source_type)(
             fixed_profile, profile, grid, p, gp, mag);
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
@@ -234,7 +237,7 @@ int main( int argc, char* argv[])
             }
             double deltat = time - previous_time;
             double energy = 0, ediff = 0.;
-            for( auto& record : feltor::diagnostics2d_list)
+            for( auto& record : Diagnostics::list_2d)
             {
                 if( std::find( feltor::energies.begin(), feltor::energies.end(), record.name) != feltor::energies.end())
                 {
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 2adbbbc90..08a1837e1 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -14,6 +14,10 @@
 #include "feltor.h"
 #include "implicit.h"
 
+#include "init.h"
+#include "init_from_file.h"
+#include "feltordiag.h"
+
 #ifdef FELTOR_MPI
 using HVec = dg::MHVec;
 using DVec = dg::MDVec;
@@ -34,9 +38,9 @@ using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #endif //FELTOR_MPI
 
-#include "init.h"
-#include "init_from_file.h"
-#include "feltordiag.h"
+using Initialize = feltor::Initialize<Geometry, IHMatrix, IDMatrix, DMatrix, DVec>;
+using Sources = feltor::Sources<Geometry, IDMatrix, DMatrix, DVec>;
+using Diagnostics = feltor::Diagnostics<Geometry, IDMatrix, DMatrix, DVec>;
 
 #ifdef FELTOR_MPI
 //ATTENTION: in slurm should be used with --signal=SIGINT@30 (<signal>@<time in seconds>)
@@ -198,7 +202,7 @@ int main( int argc, char* argv[])
     gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
     gradPsip[2] =  resultD; //zero
-    feltor::Variables var = {
+    Diagnostics::Variables var = {
         feltor, p, gp, mag, gradPsip, gradPsip
     };
     // the vector ids
@@ -213,7 +217,7 @@ int main( int argc, char* argv[])
     if( argc == 4)
     {
         try{
-            y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+            y0 = Initialize::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
         }catch ( std::out_of_range& error){
             MPI_OUT std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
 #ifdef FELTOR_MPI
@@ -225,7 +229,7 @@ int main( int argc, char* argv[])
     if( argc == 5)
     {
         try{
-            y0 = feltor::init_from_file(argv[4], grid, p,time);
+            dg::assign( feltor::init_from_file<Geometry, IHMatrix>(argv[4], grid, p,time), y0);
         }catch (std::exception& e){
             MPI_OUT std::cerr << "ERROR occured initializing from file "<<argv[4]<<std::endl;
             MPI_OUT std::cerr << e.what()<<std::endl;
@@ -239,7 +243,7 @@ int main( int argc, char* argv[])
     try{
         bool fixed_profile;
         HVec profile, source_profile;
-        source_profile = feltor::source_profiles.at(p.source_type)(
+        source_profile = Sources::source_profiles.at(p.source_type)(
             fixed_profile, profile, grid, p, gp, mag);
         feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
             p.source_rate, dg::construct<DVec>(source_profile),
@@ -302,7 +306,7 @@ int main( int argc, char* argv[])
 #endif //FELTOR_MPI
 
     //create & output static 3d variables into file
-    for ( auto& record : feltor::diagnostics3d_static_list)
+    for ( auto& record : Diagnostics::static_list_3d)
     {
         int vecID;
         MPI_OUT err = nc_def_var( ncid, record.name.data(), NC_DOUBLE, 3,
@@ -317,7 +321,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_redef(ncid);
     }
     //create & output static 2d variables into file
-    for ( auto& record : feltor::diagnostics2d_static_list)
+    for ( auto& record : Diagnostics::static_list_2d)
     {
         int vecID;
         MPI_OUT err = nc_def_var( ncid, record.name.data(), NC_DOUBLE, 2,
@@ -333,7 +337,7 @@ int main( int argc, char* argv[])
     }
 
     //Create field IDs
-    for( auto& record : feltor::diagnostics3d_list)
+    for( auto& record : Diagnostics::list_3d)
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
@@ -343,7 +347,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_put_att_text( ncid, id4d.at(name), "long_name", long_name.size(),
             long_name.data());
     }
-    for( auto& record : feltor::restart3d_list)
+    for( auto& record : Diagnostics::restart3d_list)
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
@@ -353,7 +357,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_put_att_text( ncid, restart_ids.at(name), "long_name", long_name.size(),
             long_name.data());
     }
-    for( auto& record : feltor::diagnostics2d_list)
+    for( auto& record : Diagnostics::list_2d)
     {
         std::string name = record.name + "_ta2d";
         std::string long_name = record.long_name + " (Toroidal average)";
@@ -389,20 +393,20 @@ int main( int argc, char* argv[])
 
     size_t start = 0, count = 1;
     MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
-    for( auto& record : feltor::diagnostics3d_list)
+    for( auto& record : Diagnostics::list_3d)
     {
         record.function( resultD, var);
         dg::blas2::symv( projectD, resultD, transferD);
         dg::assign( transferD, transferH);
         file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
     }
-    for( auto& record : feltor::restart3d_list)
+    for( auto& record : Diagnostics::restart3d_list)
     {
         record.function( resultD, var);
         dg::assign( resultD, resultH);
         file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
     }
-    for( auto& record : feltor::diagnostics2d_list)
+    for( auto& record : Diagnostics::list_2d)
     {
         dg::Timer tti;
         tti.tic();
@@ -485,18 +489,13 @@ int main( int argc, char* argv[])
             tti.tic();
             double deltat = time - previous_time;
             double energy = 0, ediff = 0.;
-            for( auto& record : feltor::diagnostics2d_list)
+            for( auto& record : Diagnostics::list_2d)
             {
                 if( std::find( feltor::energies.begin(), feltor::energies.end(), record.name) != feltor::energies.end())
                 {
                     record.function( resultD, var);
                     energy += dg::blas1::dot( resultD, feltor.vol3d());
                 }
-                if( std::find( feltor::energy_diff.begin(), feltor::energy_diff.end(), record.name) != feltor::energy_diff.end())
-                {
-                    record.function( resultD, var);
-                    ediff += dg::blas1::dot( resultD, feltor.vol3d());
-                }
                 if( record.integral)
                 {
                     record.function( resultD, var);
@@ -510,6 +509,10 @@ int main( int argc, char* argv[])
                     feltor::slice_vector3d( transferD, transferD2d, local_size2d);
                     dg::assign( transferD2d, transferH2d);
                     time_integrals.at(record.name+"_2d").add( time, transferH2d);
+                    if( std::find( feltor::energy_diff.begin(), feltor::energy_diff.end(), record.name) != feltor::energy_diff.end())
+                    {
+                        ediff += dg::blas1::dot( resultD, feltor.vol3d());
+                    }
                 }
 
             }
@@ -550,20 +553,20 @@ int main( int argc, char* argv[])
         start = i;
         MPI_OUT err = nc_open(file_name.data(), NC_WRITE, &ncid);
         MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
-        for( auto& record : feltor::diagnostics3d_list)
+        for( auto& record : Diagnostics::list_3d)
         {
             record.function( resultD, var);
             dg::blas2::symv( projectD, resultD, transferD);
             dg::assign( transferD, transferH);
             file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
         }
-        for( auto& record : feltor::restart3d_list)
+        for( auto& record : Diagnostics::restart3d_list)
         {
             record.function( resultD, var);
             dg::assign( resultD, resultH);
             file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
         }
-        for( auto& record : feltor::diagnostics2d_list)
+        for( auto& record : Diagnostics::list_2d)
         {
             if(record.integral) // we already computed the output...
             {
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index daa1c41b0..cdb82ac06 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -9,6 +9,8 @@
 #include "dg/algorithm.h"
 #include "dg/geometries/geometries.h"
 #include "dg/file/file.h"
+#include "feltordiag.h"
+
 using HVec = dg::HVec;
 using DVec = dg::DVec;
 using DMatrix = dg::DMatrix;
@@ -16,7 +18,8 @@ using IDMatrix = dg::IDMatrix;
 using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
-#include "feltordiag.h"
+
+using Diagnostics = feltor::Diagnostics<Geometry, IDMatrix, DMatrix, DVec>;
 
 int main( int argc, char* argv[])
 {
@@ -250,7 +253,7 @@ int main( int argc, char* argv[])
         err = nc_redef(ncid_out);
     }
 
-    for( auto& record : feltor::diagnostics2d_list)
+    for( auto& record : Diagnostics::list_2d)
     {
         std::string record_name = record.name;
         if( record_name[0] == 'j')
@@ -329,7 +332,7 @@ int main( int argc, char* argv[])
             std::cout << counter << " Timestep = " << i <<"/"<<steps-1 << "  time = " << time << std::endl;
             counter++;
             err = nc_put_vara_double( ncid_out, tvarID, start2d_out, count2d, &time);
-            for( auto& record : feltor::diagnostics2d_list)
+            for( auto& record : Diagnostics::list_2d)
             {
                 std::string record_name = record.name;
                 if( record_name[0] == 'j')
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 0882f5a70..c5dfbbc7d 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -13,9 +13,8 @@
 namespace feltor{
 
 // This file constitutes the diagnostics module for feltor
-// The way it works is that it allocates global lists of Records that describe what goes into the file
-// You can register you own diagnostics in one of three diagnostics lists (static 3d, dynamic 3d and
-// dynamic 2d) further down
+// The way it works is that it defines a Dignostics class that holds lists of Records that describe what goes into the file
+// You can register you own diagnostics in one of the diagnostics lists
 // which will then be applied during a simulation
 
 namespace routines{
@@ -145,29 +144,48 @@ void jacobian(
 }
 }//namespace routines
 
-//From here on, we use the typedefs to ease the notation
-
-struct Variables{
-    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
-    feltor::Parameters p;
-    dg::geo::solovev::Parameters gp;
-    dg::geo::TokamakMagneticField mag;
-    std::array<DVec, 3> gradPsip;
-    std::array<DVec, 3> tmp;
-};
-
-struct Record{
-    std::string name;
-    std::string long_name;
-    bool integral; //indicates whether the function should be time-integrated
-    std::function<void( DVec&, Variables&)> function;
-};
+template< class Geometry, class IDMatrix, class DMatrix, class DVec >
+struct Diagnostics
+{
+    using HVec = typename Geometry::host_vector;
 
-struct Record_static{
-    std::string name;
-    std::string long_name;
-    std::function<void( HVec&, Variables&, Geometry& grid)> function;
-};
+    struct Variables{
+        feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+        feltor::Parameters p;
+        dg::geo::solovev::Parameters gp;
+        dg::geo::TokamakMagneticField mag;
+        std::array<DVec, 3> gradPsip;
+        std::array<DVec, 3> tmp;
+    };
+    struct Record{
+        std::string name;
+        std::string long_name;
+        bool integral; //indicates whether the function should be time-integrated
+        std::function<void( DVec&, Variables&)> function;
+    };
+    struct Record_static{
+        std::string name;
+        std::string long_name;
+        std::function<void( HVec&, Variables&, Geometry& grid)> function;
+    };
+    static std::array<std::tuple<std::string, std::string, HVec>, 3> generate_cyl2cart( Geometry& grid)
+    {
+        HVec xc = dg::evaluate( dg::cooRZP2X, grid);
+        HVec yc = dg::evaluate( dg::cooRZP2Y, grid);
+        HVec zc = dg::evaluate( dg::cooRZP2Z, grid);
+        std::array<std::tuple<std::string, std::string, HVec>, 3> list = {{
+            { "xc", "x-coordinate in Cartesian coordinate system", xc },
+            { "yc", "y-coordinate in Cartesian coordinate system", yc },
+            { "zc", "z-coordinate in Cartesian coordinate system", zc }
+        }};
+        return list;
+    }
+    static std::vector<Record_static> static_list_3d;
+    static std::vector<Record_static> static_list_2d;
+    static std::vector<Record> list_3d;
+    static std::vector<Record> list_2d;
+    static std::vector<Record> restart3d_list;
+};//Diagnostics
 
 ///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -175,7 +193,8 @@ struct Record_static{
 //Here is a list of static (time-independent) 3d variables that go into the output
 //Except xc, yc, and zc these are redundant since we have geometry_diag.cu
 //MW: maybe it's a test of sorts
-std::vector<Record_static> diagnostics3d_static_list = {
+template< class Geometry, class IDMatrix, class DMatrix, class DVec >
+std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record_static> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::static_list_3d = {
     { "BR", "R-component of magnetic field in cylindrical coordinates",
         []( HVec& result, Variables& v, Geometry& grid){
             dg::geo::BFieldR fieldR(v.mag);
@@ -203,7 +222,7 @@ std::vector<Record_static> diagnostics3d_static_list = {
         []( HVec& result, Variables& v, Geometry& grid ){
             result = dg::evaluate( dg::zero, grid);
             bool fixed_profile;
-            HVec source = feltor::source_profiles.at(v.p.source_type)(
+            HVec source = feltor::Sources<Geometry, IDMatrix, DMatrix, DVec>::source_profiles.at(v.p.source_type)(
                 fixed_profile, result, grid, v.p, v.gp, v.mag);
         }
     },
@@ -211,7 +230,7 @@ std::vector<Record_static> diagnostics3d_static_list = {
         []( HVec& result, Variables& v, Geometry& grid ){
             bool fixed_profile;
             HVec profile;
-            result = feltor::source_profiles.at(v.p.source_type)(
+            result = feltor::Sources<Geometry, IDMatrix, DMatrix, DVec>::source_profiles.at(v.p.source_type)(
                 fixed_profile, profile, grid, v.p, v.gp, v.mag);
         }
     },
@@ -232,21 +251,9 @@ std::vector<Record_static> diagnostics3d_static_list = {
     },
 };
 
-std::array<std::tuple<std::string, std::string, HVec>, 3> generate_cyl2cart( Geometry& grid)
-{
-    HVec xc = dg::evaluate( dg::cooRZP2X, grid);
-    HVec yc = dg::evaluate( dg::cooRZP2Y, grid);
-    HVec zc = dg::evaluate( dg::cooRZP2Z, grid);
-    std::array<std::tuple<std::string, std::string, HVec>, 3> list = {{
-        { "xc", "x-coordinate in Cartesian coordinate system", xc },
-        { "yc", "y-coordinate in Cartesian coordinate system", yc },
-        { "zc", "z-coordinate in Cartesian coordinate system", zc }
-    }};
-    return list;
-}
-
 // Here are all 3d outputs we want to have
-std::vector<Record> diagnostics3d_list = {
+template< class Geometry, class IDMatrix, class DMatrix, class DVec >
+std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::list_3d = {
     {"electrons", "electron density", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(0), result);
@@ -283,7 +290,8 @@ std::vector<Record> diagnostics3d_list = {
 //MW: These are redundant since we have geometry_diag.cu -> remove ? if geometry_diag works as expected (I guess it can also be a test of sorts)
 //MW: if they stay they should be documented in feltor.tex
 //( we make 3d variables here but only the first 2d slice is output)
-std::vector<Record_static> diagnostics2d_static_list = {
+template< class Geometry, class IDMatrix, class DMatrix, class DVec >
+std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record_static> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::static_list_2d = {
     { "Psip2d", "Flux-function psi",
         []( HVec& result, Variables& v, Geometry& grid ){
             result = dg::pullback( v.mag.psip(), grid);
@@ -351,7 +359,8 @@ std::vector<Record_static> diagnostics2d_static_list = {
     }
 };
 // and here are all the 2d outputs we want to produce
-std::vector<Record> diagnostics2d_list = {
+template< class Geometry, class IDMatrix, class DMatrix, class DVec >
+std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::list_2d = {
     {"electrons", "Electron density", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(0), result);
@@ -976,7 +985,8 @@ std::vector<Record> diagnostics2d_list = {
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-std::vector<Record> restart3d_list = {
+template< class Geometry, class IDMatrix, class DMatrix, class DVec >
+std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::restart3d_list = {
     {"restart_electrons", "electron density", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(0), result);
@@ -1004,8 +1014,8 @@ std::vector<Record> restart3d_list = {
     }
 };
 // These two lists signify the quantities involved in accuracy computation
-std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
-std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt", "see_tt", "sei_tt"};
+static std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
+static std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt", "see_tt", "sei_tt"};
 
 template<class Container>
 void slice_vector3d( const Container& transfer, Container& transfer2d, size_t local_size2d)
diff --git a/src/feltor/init.h b/src/feltor/init.h
index a4ab868c4..914e7043d 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -36,24 +36,26 @@ struct Radius : public dg::geo::aCylindricalFunctor<Radius>
     private:
     double m_R0, m_Z0;
 };
-HVec circular_damping( const Geometry& grid,
+template<class Geometry>
+dg::get_host_vector<Geometry> circular_damping( const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     if( p.profile_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
-    HVec circular = dg::pullback( dg::compose(
+    dg::get_host_vector<Geometry> circular = dg::pullback( dg::compose(
                 dg::PolynomialHeaviside( gp.a, gp.a*p.profile_alpha/2., -1),
                 Radius( mag.R0(), 0.)), grid);
     return circular;
 }
 
 
-HVec xpoint_damping(const Geometry& grid,
+template<class Geometry>
+dg::get_host_vector<Geometry> xpoint_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
-    HVec xpoint_damping = dg::evaluate( dg::one, grid);
+    dg::get_host_vector<Geometry> xpoint_damping = dg::evaluate( dg::one, grid);
     if( gp.hasXpoint() )
     {
         double RX = gp.R_0 - 1.1*gp.triangularity*gp.a;
@@ -64,37 +66,40 @@ HVec xpoint_damping(const Geometry& grid,
     }
     return xpoint_damping;
 }
-HVec profile_damping(const Geometry& grid,
+template<class Geometry>
+dg::get_host_vector<Geometry> profile_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     if( p.profile_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
-    HVec profile_damping = dg::pullback( dg::compose(dg::PolynomialHeaviside(
+    dg::get_host_vector<Geometry> profile_damping = dg::pullback( dg::compose(dg::PolynomialHeaviside(
         1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)), grid);
     dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag),
         profile_damping, profile_damping);
     return profile_damping;
 }
-HVec profile(const Geometry& grid,
+template<class Geometry>
+dg::get_host_vector<Geometry> profile(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag ){
     double RO=mag.R0(), ZO=0.;
     dg::geo::findOpoint( mag.get_psip(), RO, ZO);
     double psipO = mag.psip()( RO, ZO);
     //First the profile and the source (on the host since we want to output those)
-    HVec profile = dg::pullback( dg::compose(dg::LinearX(
+    dg::get_host_vector<Geometry> profile = dg::pullback( dg::compose(dg::LinearX(
         p.nprofamp/psipO, 0.), mag.psip()), grid);
     dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), profile, profile);
     return profile;
 }
-HVec source_damping(const Geometry& grid,
+template<class Geometry>
+dg::get_host_vector<Geometry> source_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     if( p.source_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: source alpha must not be 0\n");
-    HVec source_damping = dg::pullback(
+    dg::get_host_vector<Geometry> source_damping = dg::pullback(
         dg::compose(dg::PolynomialHeaviside(
             p.source_boundary-p.source_alpha/2.,
         p.source_alpha/2., -1 ), dg::geo::RhoP(mag)), grid);
@@ -104,20 +109,15 @@ HVec source_damping(const Geometry& grid,
 }
 
 
+template<class Geometry, class IDMatrix, class DMatrix, class DVec>
 void init_ni(
     std::array<std::array<DVec,2>,2>& y0,
     Explicit<Geometry, IDMatrix, DMatrix, DVec>& feltor,
     const Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
 {
-#ifdef FELTOR_MPI
-    int rank;
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-#endif
-    MPI_OUT std::cout << "initialize ni with "<<p.initphi << std::endl;
     feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
     double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
-    MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
     if( minimalni <= -1)
     {
         throw dg::Error(dg::Message()<< "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n"
@@ -127,13 +127,14 @@ void init_ni(
 }//namespace detail
 
 //for wall shadow
-HVec wall_damping(const Geometry& grid,
+template<class Geometry>
+dg::get_host_vector<Geometry> wall_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     if( p.source_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: damping alpha must not be 0\n");
-    HVec wall_damping = dg::pullback(dg::compose( dg::PolynomialHeaviside(
+    dg::get_host_vector<Geometry> wall_damping = dg::pullback(dg::compose( dg::PolynomialHeaviside(
         p.damping_boundary+p.damping_alpha/2., p.damping_alpha/2., +1),
                 dg::geo::RhoP(mag)), grid);
     return wall_damping;
@@ -143,11 +144,34 @@ HVec wall_damping(const Geometry& grid,
  * source profiles.  Just add your own to the relevant map below.
  */
 
+template< class Geometry, class IHMatrix, class IDMatrix, class DMatrix, class DVec >
+struct Initialize
+{
+    using HVec = typename Geometry::host_vector;
+    static std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
+        Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
+        const Geometry& grid, const feltor::Parameters& p,
+        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+    > > initial_conditions;
+};
+template< class Geometry, class IDMatrix, class DMatrix, class DVec >
+struct Sources
+{
+    using HVec = typename Geometry::host_vector;
+    static std::map<std::string, std::function< HVec(
+        bool& fixed_profile,
+        HVec& ne_profile,
+        Geometry& grid, const feltor::Parameters& p,
+        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+    > > source_profiles;
+};
+
+template< class Geometry, class IHMatrix, class IDMatrix, class DMatrix, class DVec >
 std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
     const Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
-> > initial_conditions =
+> > Initialize<Geometry, IHMatrix, IDMatrix, DMatrix, DVec>::initial_conditions =
 {
     { "blob",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
@@ -303,12 +327,13 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     }
 };
 
-std::map<std::string, std::function< HVec(
+template< class Geometry, class IDMatrix, class DMatrix, class DVec >
+std::map<std::string, std::function< typename Geometry::host_vector(
     bool& fixed_profile, //indicate whether a profile should be forced (yes or no)
-    HVec& ne_profile,    // if fixed_profile is yes you need to construct something here, if no then you can ignore the parameter; if you construct something it will show in the output file in any case
+    typename Geometry::host_vector& ne_profile,    // if fixed_profile is yes you need to construct something here, if no then you can ignore the parameter; if you construct something it will show in the output file in any case
     Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
-> > source_profiles =
+> > Sources<Geometry, IDMatrix, DMatrix, DVec>::source_profiles =
 {
     {"profile",
         []( bool& fixed_profile, HVec& ne_profile,
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index 789a9b723..779860664 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -4,16 +4,12 @@
 #include "dg/file/nc_utilities.h"
 
 namespace feltor
-{//We use the typedefs and MPI_OUT
-//
+{
+    //
 //everyone reads their portion of the input data
-//don't forget to also read source profiles
-std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Geometry& grid, const Parameters& p, double& time){
-#ifdef FELTOR_MPI
-    int rank;
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-#endif
-    std::array<std::array<DVec,2>,2> y0;
+template<class Geometry, class IHMatrix>
+std::array<std::array<typename Geometry::host_vector,2>,2> init_from_file( std::string file_name, const Geometry& grid, const Parameters& p, double& time){
+
     ///////////////////read in and show inputfile
 
     file::NC_Error_Handle errIN;
@@ -30,9 +26,17 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     unsigned  pINNy = jsIN["Ny"].asUInt();
     unsigned  pINNz = jsIN["Nz"].asUInt();
     bool      pINsymmetric   = jsIN.get( "symmetric", false).asBool();
-    MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
-    MPI_OUT std::cout << " file parameters:" << std::endl;
-    MPI_OUT std::cout << pINn<<" x "<<pINNx<<" x "<<pINNy<<" x "<<pINNz<<" : symmetric "<<std::boolalpha<<pINsymmetric<<std::endl;
+#ifdef FELTOR_MPI
+    int rank;
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    if(rank==0){
+#endif
+    std::cout << "RESTART from file "<<file_name<< std::endl;
+    std::cout << " file parameters:" << std::endl;
+    std::cout << pINn<<" x "<<pINNx<<" x "<<pINNy<<" x "<<pINNz<<" : symmetric "<<std::boolalpha<<pINsymmetric<<std::endl;
+#ifdef FELTOR_MPI
+    }
+#endif
 
     // Now read in last timestep
     Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
@@ -58,8 +62,8 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     size_t countIN[3] = {grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
         grid_IN.n()*grid_IN.Nx()};
     #endif //FELTOR_MPI
-    std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
-    HVec transferINH( dg::evaluate(dg::zero, grid_IN));
+    std::vector<typename Geometry::host_vector> transferINHvec( 5, dg::evaluate( dg::zero, grid));
+    typename Geometry::host_vector transferINH( dg::evaluate(dg::zero, grid_IN));
 
     std::string namesIN[5] = {"restart_electrons", "restart_ions", "restart_Ue", "restart_Ui", "restart_induction"};
 
@@ -71,7 +75,10 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
     size_time -= 1;
     errIN = nc_get_vara_double( ncidIN, timeIDIN, &size_time, &count_time, &time);
-    MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
+#ifdef FELTOR_MPI
+    if(rank==0)
+#endif
+    std::cout << " Current time = "<< time <<  std::endl;
     for( unsigned i=0; i<5; i++)
     {
         int dataID;
@@ -94,6 +101,7 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
     dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
 
+    std::array<std::array<typename Geometry::host_vector,2>,2> y0;
     dg::assign( transferINHvec[0], y0[0][0]); //ne-1
     dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
     dg::assign( transferINHvec[2], y0[1][0]); //We
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index ca7b41b76..cc2df9977 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -18,6 +18,7 @@ using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #include "feltordiag.h"
+using Diagnostics = feltor::Diagnostics<Geometry, IDMatrix, DMatrix, DVec>;
 
 thrust::host_vector<float> append( const thrust::host_vector<float>& in, const dg::aRealTopology3d<double>& g)
 {
@@ -134,7 +135,7 @@ int main( int argc, char* argv[])
     dg::IHMatrix interpolate_in_2d = dg::create::interpolation( g3d_out_equidistant, g3d_out);
 
 
-    for( auto& record : feltor::diagnostics3d_static_list)
+    for( auto& record : Diagnostics::static_list_3d)
     {
         if( record.name != "xc" && record.name != "yc" && record.name != "zc" )
         {
@@ -157,7 +158,7 @@ int main( int argc, char* argv[])
             err = nc_redef(ncid_out);
         }
     }
-    for( auto record : feltor::generate_cyl2cart( g3d_out_equidistant) )
+    for( auto record : Diagnostics::generate_cyl2cart( g3d_out_equidistant) )
     {
         int vID;
         err = nc_def_var( ncid_out, std::get<0>(record).data(), NC_FLOAT, 3, &dim_ids[1],
@@ -170,7 +171,7 @@ int main( int argc, char* argv[])
         err = nc_redef(ncid_out);
 
     }
-    for( auto& record : feltor::diagnostics3d_list)
+    for( auto& record : Diagnostics::list_3d)
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
@@ -202,7 +203,7 @@ int main( int argc, char* argv[])
         std::cout << "  time = " << time << std::endl;
         start3d[0] = i/10;
         err = nc_put_vara_double( ncid_out, tvarID, start3d, count3d_out, &time);
-        for( auto& record : feltor::diagnostics3d_list)
+        for( auto& record : Diagnostics::list_3d)
         {
             std::string record_name = record.name;
             int dataID =0;
-- 
GitLab


From 6f05c98afca65fa03e3a707c7515ccfefefb7a8e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 May 2020 16:20:58 +0200
Subject: [PATCH 237/540] Speed-up Hoo pullback in feltordiag

---
 src/feltor/feltor.cu     | 3 ++-
 src/feltor/feltor_hpc.cu | 3 ++-
 src/feltor/feltordiag.h  | 7 +++----
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 0cfc71efb..a0669f8f9 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -88,8 +88,9 @@ int main( int argc, char* argv[])
     gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
     gradPsip[2] =  result; //zero
+    DVec hoo = dg::pullback( dg::geo::Hoo( mag), grid);
     Diagnostics::Variables var = {
-        feltor, p,gp,mag, gradPsip, gradPsip
+        feltor, p,gp,mag, gradPsip, gradPsip, hoo
     };
 
 
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 08a1837e1..3ec0b5398 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -202,8 +202,9 @@ int main( int argc, char* argv[])
     gradPsip[0] =  dg::evaluate( mag.psipR(), grid);
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
     gradPsip[2] =  resultD; //zero
+    DVec hoo = dg::pullback( dg::geo::Hoo( mag), grid);
     Diagnostics::Variables var = {
-        feltor, p, gp, mag, gradPsip, gradPsip
+        feltor, p, gp, mag, gradPsip, gradPsip, hoo
     };
     // the vector ids
     std::map<std::string, int> id3d, id4d, restart_ids;
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index c5dfbbc7d..1e2c692d6 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -156,6 +156,7 @@ struct Diagnostics
         dg::geo::TokamakMagneticField mag;
         std::array<DVec, 3> gradPsip;
         std::array<DVec, 3> tmp;
+        DVec hoo; //keep hoo there to avoid pullback
     };
     struct Record{
         std::string name;
@@ -968,14 +969,12 @@ std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> D
     /// --------------------- Zonal flow energy terms------------------------//
     {"nei0", "inertial factor", false,
         []( DVec& result, Variables& v ) {
-            result = dg::pullback( dg::geo::Hoo( v.mag), v.f.grid());
-            dg::blas1::pointwiseDot( v.f.density(0), result, result);
+            dg::blas1::pointwiseDot( v.f.density(0), v.hoo, result);
         }
     },
     {"snei0_tt", "inertial factor source", true,
         []( DVec& result, Variables& v ) {
-            result = dg::pullback( dg::geo::Hoo( v.mag), v.f.grid());
-            dg::blas1::pointwiseDot( v.f.density_source(0), result, result);
+            dg::blas1::pointwiseDot( v.f.density_source(0), v.hoo, result);
         }
     },
 
-- 
GitLab


From cde2e5139c91fb9bc71610574e7324ff8e5154f7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 May 2020 21:55:07 +0200
Subject: [PATCH 238/540] Revert "Clear separation of feltor diagnostics and
 init"

This reverts commit 8abff53aab4c7389951e9c1d3e193ac2d2a42d69.
Because on Marconi100 it triggered a bug when compiling
---
 src/feltor/feltor.cu            |  15 ++---
 src/feltor/feltor_hpc.cu        |  44 ++++++-------
 src/feltor/feltordiag.cu        |   9 +--
 src/feltor/feltordiag.h         | 106 +++++++++++++++-----------------
 src/feltor/init.h               |  69 +++++++--------------
 src/feltor/init_from_file.h     |  38 +++++-------
 src/feltor/interpolate_in_3d.cu |   9 ++-
 7 files changed, 117 insertions(+), 173 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index a0669f8f9..df7810b12 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -9,8 +9,6 @@
 
 #include "feltor.h"
 #include "implicit.h"
-#include "init.h"
-#include "feltordiag.h"
 
 using HVec = dg::HVec;
 using DVec = dg::DVec;
@@ -20,9 +18,8 @@ using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 
-using Initialize = feltor::Initialize<Geometry, IHMatrix, IDMatrix, DMatrix, DVec>;
-using Sources = feltor::Sources<Geometry, IDMatrix, DMatrix, DVec>;
-using Diagnostics = feltor::Diagnostics<Geometry, IDMatrix, DMatrix, DVec>;
+#include "init.h"
+#include "feltordiag.h"
 
 int main( int argc, char* argv[])
 {
@@ -89,7 +86,7 @@ int main( int argc, char* argv[])
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
     gradPsip[2] =  result; //zero
     DVec hoo = dg::pullback( dg::geo::Hoo( mag), grid);
-    Diagnostics::Variables var = {
+    feltor::Variables var = {
         feltor, p,gp,mag, gradPsip, gradPsip, hoo
     };
 
@@ -99,7 +96,7 @@ int main( int argc, char* argv[])
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
     try{
-        y0 = Initialize::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+        y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
         return -1;
@@ -109,7 +106,7 @@ int main( int argc, char* argv[])
     HVec profile = dg::evaluate( dg::zero, grid);
     HVec source_profile;
     try{
-        source_profile = Sources::source_profiles.at(p.source_type)(
+        source_profile = feltor::source_profiles.at(p.source_type)(
             fixed_profile, profile, grid, p, gp, mag);
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
@@ -238,7 +235,7 @@ int main( int argc, char* argv[])
             }
             double deltat = time - previous_time;
             double energy = 0, ediff = 0.;
-            for( auto& record : Diagnostics::list_2d)
+            for( auto& record : feltor::diagnostics2d_list)
             {
                 if( std::find( feltor::energies.begin(), feltor::energies.end(), record.name) != feltor::energies.end())
                 {
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 3ec0b5398..7a01776be 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -14,10 +14,6 @@
 #include "feltor.h"
 #include "implicit.h"
 
-#include "init.h"
-#include "init_from_file.h"
-#include "feltordiag.h"
-
 #ifdef FELTOR_MPI
 using HVec = dg::MHVec;
 using DVec = dg::MDVec;
@@ -38,9 +34,9 @@ using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #endif //FELTOR_MPI
 
-using Initialize = feltor::Initialize<Geometry, IHMatrix, IDMatrix, DMatrix, DVec>;
-using Sources = feltor::Sources<Geometry, IDMatrix, DMatrix, DVec>;
-using Diagnostics = feltor::Diagnostics<Geometry, IDMatrix, DMatrix, DVec>;
+#include "init.h"
+#include "init_from_file.h"
+#include "feltordiag.h"
 
 #ifdef FELTOR_MPI
 //ATTENTION: in slurm should be used with --signal=SIGINT@30 (<signal>@<time in seconds>)
@@ -203,7 +199,7 @@ int main( int argc, char* argv[])
     gradPsip[1] =  dg::evaluate( mag.psipZ(), grid);
     gradPsip[2] =  resultD; //zero
     DVec hoo = dg::pullback( dg::geo::Hoo( mag), grid);
-    Diagnostics::Variables var = {
+    feltor::Variables var = {
         feltor, p, gp, mag, gradPsip, gradPsip, hoo
     };
     // the vector ids
@@ -218,7 +214,7 @@ int main( int argc, char* argv[])
     if( argc == 4)
     {
         try{
-            y0 = Initialize::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+            y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
         }catch ( std::out_of_range& error){
             MPI_OUT std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
 #ifdef FELTOR_MPI
@@ -230,7 +226,7 @@ int main( int argc, char* argv[])
     if( argc == 5)
     {
         try{
-            dg::assign( feltor::init_from_file<Geometry, IHMatrix>(argv[4], grid, p,time), y0);
+            y0 = feltor::init_from_file(argv[4], grid, p,time);
         }catch (std::exception& e){
             MPI_OUT std::cerr << "ERROR occured initializing from file "<<argv[4]<<std::endl;
             MPI_OUT std::cerr << e.what()<<std::endl;
@@ -244,7 +240,7 @@ int main( int argc, char* argv[])
     try{
         bool fixed_profile;
         HVec profile, source_profile;
-        source_profile = Sources::source_profiles.at(p.source_type)(
+        source_profile = feltor::source_profiles.at(p.source_type)(
             fixed_profile, profile, grid, p, gp, mag);
         feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
             p.source_rate, dg::construct<DVec>(source_profile),
@@ -307,7 +303,7 @@ int main( int argc, char* argv[])
 #endif //FELTOR_MPI
 
     //create & output static 3d variables into file
-    for ( auto& record : Diagnostics::static_list_3d)
+    for ( auto& record : feltor::diagnostics3d_static_list)
     {
         int vecID;
         MPI_OUT err = nc_def_var( ncid, record.name.data(), NC_DOUBLE, 3,
@@ -322,7 +318,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_redef(ncid);
     }
     //create & output static 2d variables into file
-    for ( auto& record : Diagnostics::static_list_2d)
+    for ( auto& record : feltor::diagnostics2d_static_list)
     {
         int vecID;
         MPI_OUT err = nc_def_var( ncid, record.name.data(), NC_DOUBLE, 2,
@@ -338,7 +334,7 @@ int main( int argc, char* argv[])
     }
 
     //Create field IDs
-    for( auto& record : Diagnostics::list_3d)
+    for( auto& record : feltor::diagnostics3d_list)
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
@@ -348,7 +344,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_put_att_text( ncid, id4d.at(name), "long_name", long_name.size(),
             long_name.data());
     }
-    for( auto& record : Diagnostics::restart3d_list)
+    for( auto& record : feltor::restart3d_list)
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
@@ -358,7 +354,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_put_att_text( ncid, restart_ids.at(name), "long_name", long_name.size(),
             long_name.data());
     }
-    for( auto& record : Diagnostics::list_2d)
+    for( auto& record : feltor::diagnostics2d_list)
     {
         std::string name = record.name + "_ta2d";
         std::string long_name = record.long_name + " (Toroidal average)";
@@ -394,20 +390,20 @@ int main( int argc, char* argv[])
 
     size_t start = 0, count = 1;
     MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
-    for( auto& record : Diagnostics::list_3d)
+    for( auto& record : feltor::diagnostics3d_list)
     {
         record.function( resultD, var);
         dg::blas2::symv( projectD, resultD, transferD);
         dg::assign( transferD, transferH);
         file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
     }
-    for( auto& record : Diagnostics::restart3d_list)
+    for( auto& record : feltor::restart3d_list)
     {
         record.function( resultD, var);
         dg::assign( resultD, resultH);
         file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
     }
-    for( auto& record : Diagnostics::list_2d)
+    for( auto& record : feltor::diagnostics2d_list)
     {
         dg::Timer tti;
         tti.tic();
@@ -490,7 +486,7 @@ int main( int argc, char* argv[])
             tti.tic();
             double deltat = time - previous_time;
             double energy = 0, ediff = 0.;
-            for( auto& record : Diagnostics::list_2d)
+            for( auto& record : feltor::diagnostics2d_list)
             {
                 if( std::find( feltor::energies.begin(), feltor::energies.end(), record.name) != feltor::energies.end())
                 {
@@ -511,9 +507,7 @@ int main( int argc, char* argv[])
                     dg::assign( transferD2d, transferH2d);
                     time_integrals.at(record.name+"_2d").add( time, transferH2d);
                     if( std::find( feltor::energy_diff.begin(), feltor::energy_diff.end(), record.name) != feltor::energy_diff.end())
-                    {
                         ediff += dg::blas1::dot( resultD, feltor.vol3d());
-                    }
                 }
 
             }
@@ -554,20 +548,20 @@ int main( int argc, char* argv[])
         start = i;
         MPI_OUT err = nc_open(file_name.data(), NC_WRITE, &ncid);
         MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
-        for( auto& record : Diagnostics::list_3d)
+        for( auto& record : feltor::diagnostics3d_list)
         {
             record.function( resultD, var);
             dg::blas2::symv( projectD, resultD, transferD);
             dg::assign( transferD, transferH);
             file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
         }
-        for( auto& record : Diagnostics::restart3d_list)
+        for( auto& record : feltor::restart3d_list)
         {
             record.function( resultD, var);
             dg::assign( resultD, resultH);
             file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
         }
-        for( auto& record : Diagnostics::list_2d)
+        for( auto& record : feltor::diagnostics2d_list)
         {
             if(record.integral) // we already computed the output...
             {
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index cdb82ac06..daa1c41b0 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -9,8 +9,6 @@
 #include "dg/algorithm.h"
 #include "dg/geometries/geometries.h"
 #include "dg/file/file.h"
-#include "feltordiag.h"
-
 using HVec = dg::HVec;
 using DVec = dg::DVec;
 using DMatrix = dg::DMatrix;
@@ -18,8 +16,7 @@ using IDMatrix = dg::IDMatrix;
 using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
-
-using Diagnostics = feltor::Diagnostics<Geometry, IDMatrix, DMatrix, DVec>;
+#include "feltordiag.h"
 
 int main( int argc, char* argv[])
 {
@@ -253,7 +250,7 @@ int main( int argc, char* argv[])
         err = nc_redef(ncid_out);
     }
 
-    for( auto& record : Diagnostics::list_2d)
+    for( auto& record : feltor::diagnostics2d_list)
     {
         std::string record_name = record.name;
         if( record_name[0] == 'j')
@@ -332,7 +329,7 @@ int main( int argc, char* argv[])
             std::cout << counter << " Timestep = " << i <<"/"<<steps-1 << "  time = " << time << std::endl;
             counter++;
             err = nc_put_vara_double( ncid_out, tvarID, start2d_out, count2d, &time);
-            for( auto& record : Diagnostics::list_2d)
+            for( auto& record : feltor::diagnostics2d_list)
             {
                 std::string record_name = record.name;
                 if( record_name[0] == 'j')
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 1e2c692d6..3d289ce23 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -13,8 +13,9 @@
 namespace feltor{
 
 // This file constitutes the diagnostics module for feltor
-// The way it works is that it defines a Dignostics class that holds lists of Records that describe what goes into the file
-// You can register you own diagnostics in one of the diagnostics lists
+// The way it works is that it allocates global lists of Records that describe what goes into the file
+// You can register you own diagnostics in one of three diagnostics lists (static 3d, dynamic 3d and
+// dynamic 2d) further down
 // which will then be applied during a simulation
 
 namespace routines{
@@ -144,49 +145,30 @@ void jacobian(
 }
 }//namespace routines
 
-template< class Geometry, class IDMatrix, class DMatrix, class DVec >
-struct Diagnostics
-{
-    using HVec = typename Geometry::host_vector;
+//From here on, we use the typedefs to ease the notation
 
-    struct Variables{
-        feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
-        feltor::Parameters p;
-        dg::geo::solovev::Parameters gp;
-        dg::geo::TokamakMagneticField mag;
-        std::array<DVec, 3> gradPsip;
-        std::array<DVec, 3> tmp;
-        DVec hoo; //keep hoo there to avoid pullback
-    };
-    struct Record{
-        std::string name;
-        std::string long_name;
-        bool integral; //indicates whether the function should be time-integrated
-        std::function<void( DVec&, Variables&)> function;
-    };
-    struct Record_static{
-        std::string name;
-        std::string long_name;
-        std::function<void( HVec&, Variables&, Geometry& grid)> function;
-    };
-    static std::array<std::tuple<std::string, std::string, HVec>, 3> generate_cyl2cart( Geometry& grid)
-    {
-        HVec xc = dg::evaluate( dg::cooRZP2X, grid);
-        HVec yc = dg::evaluate( dg::cooRZP2Y, grid);
-        HVec zc = dg::evaluate( dg::cooRZP2Z, grid);
-        std::array<std::tuple<std::string, std::string, HVec>, 3> list = {{
-            { "xc", "x-coordinate in Cartesian coordinate system", xc },
-            { "yc", "y-coordinate in Cartesian coordinate system", yc },
-            { "zc", "z-coordinate in Cartesian coordinate system", zc }
-        }};
-        return list;
-    }
-    static std::vector<Record_static> static_list_3d;
-    static std::vector<Record_static> static_list_2d;
-    static std::vector<Record> list_3d;
-    static std::vector<Record> list_2d;
-    static std::vector<Record> restart3d_list;
-};//Diagnostics
+struct Variables{
+    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
+    feltor::Parameters p;
+    dg::geo::solovev::Parameters gp;
+    dg::geo::TokamakMagneticField mag;
+    std::array<DVec, 3> gradPsip;
+    std::array<DVec, 3> tmp;
+    DVec hoo; //keep hoo there to avoid pullback
+};
+
+struct Record{
+    std::string name;
+    std::string long_name;
+    bool integral; //indicates whether the function should be time-integrated
+    std::function<void( DVec&, Variables&)> function;
+};
+
+struct Record_static{
+    std::string name;
+    std::string long_name;
+    std::function<void( HVec&, Variables&, Geometry& grid)> function;
+};
 
 ///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ///%%%%%%%%%%%%%%%%%%%%%%%EXTEND LISTS WITH YOUR DIAGNOSTICS HERE%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -194,8 +176,7 @@ struct Diagnostics
 //Here is a list of static (time-independent) 3d variables that go into the output
 //Except xc, yc, and zc these are redundant since we have geometry_diag.cu
 //MW: maybe it's a test of sorts
-template< class Geometry, class IDMatrix, class DMatrix, class DVec >
-std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record_static> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::static_list_3d = {
+std::vector<Record_static> diagnostics3d_static_list = {
     { "BR", "R-component of magnetic field in cylindrical coordinates",
         []( HVec& result, Variables& v, Geometry& grid){
             dg::geo::BFieldR fieldR(v.mag);
@@ -223,7 +204,7 @@ std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record_st
         []( HVec& result, Variables& v, Geometry& grid ){
             result = dg::evaluate( dg::zero, grid);
             bool fixed_profile;
-            HVec source = feltor::Sources<Geometry, IDMatrix, DMatrix, DVec>::source_profiles.at(v.p.source_type)(
+            HVec source = feltor::source_profiles.at(v.p.source_type)(
                 fixed_profile, result, grid, v.p, v.gp, v.mag);
         }
     },
@@ -231,7 +212,7 @@ std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record_st
         []( HVec& result, Variables& v, Geometry& grid ){
             bool fixed_profile;
             HVec profile;
-            result = feltor::Sources<Geometry, IDMatrix, DMatrix, DVec>::source_profiles.at(v.p.source_type)(
+            result = feltor::source_profiles.at(v.p.source_type)(
                 fixed_profile, profile, grid, v.p, v.gp, v.mag);
         }
     },
@@ -252,9 +233,21 @@ std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record_st
     },
 };
 
+std::array<std::tuple<std::string, std::string, HVec>, 3> generate_cyl2cart( Geometry& grid)
+{
+    HVec xc = dg::evaluate( dg::cooRZP2X, grid);
+    HVec yc = dg::evaluate( dg::cooRZP2Y, grid);
+    HVec zc = dg::evaluate( dg::cooRZP2Z, grid);
+    std::array<std::tuple<std::string, std::string, HVec>, 3> list = {{
+        { "xc", "x-coordinate in Cartesian coordinate system", xc },
+        { "yc", "y-coordinate in Cartesian coordinate system", yc },
+        { "zc", "z-coordinate in Cartesian coordinate system", zc }
+    }};
+    return list;
+}
+
 // Here are all 3d outputs we want to have
-template< class Geometry, class IDMatrix, class DMatrix, class DVec >
-std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::list_3d = {
+std::vector<Record> diagnostics3d_list = {
     {"electrons", "electron density", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(0), result);
@@ -291,8 +284,7 @@ std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> D
 //MW: These are redundant since we have geometry_diag.cu -> remove ? if geometry_diag works as expected (I guess it can also be a test of sorts)
 //MW: if they stay they should be documented in feltor.tex
 //( we make 3d variables here but only the first 2d slice is output)
-template< class Geometry, class IDMatrix, class DMatrix, class DVec >
-std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record_static> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::static_list_2d = {
+std::vector<Record_static> diagnostics2d_static_list = {
     { "Psip2d", "Flux-function psi",
         []( HVec& result, Variables& v, Geometry& grid ){
             result = dg::pullback( v.mag.psip(), grid);
@@ -360,8 +352,7 @@ std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record_st
     }
 };
 // and here are all the 2d outputs we want to produce
-template< class Geometry, class IDMatrix, class DMatrix, class DVec >
-std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::list_2d = {
+std::vector<Record> diagnostics2d_list = {
     {"electrons", "Electron density", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(0), result);
@@ -984,8 +975,7 @@ std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> D
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-template< class Geometry, class IDMatrix, class DMatrix, class DVec >
-std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::restart3d_list = {
+std::vector<Record> restart3d_list = {
     {"restart_electrons", "electron density", false,
         []( DVec& result, Variables& v ) {
              dg::blas1::copy(v.f.density(0), result);
@@ -1013,8 +1003,8 @@ std::vector<typename Diagnostics< Geometry, IDMatrix, DMatrix, DVec >::Record> D
     }
 };
 // These two lists signify the quantities involved in accuracy computation
-static std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
-static std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt", "see_tt", "sei_tt"};
+std::vector<std::string> energies = { "nelnne", "nilnni", "aperp2", "ue2","neue2","niui2"};
+std::vector<std::string> energy_diff = { "resistivity_tt", "leeperp_tt", "leiperp_tt", "leeparallel_tt", "leiparallel_tt", "see_tt", "sei_tt"};
 
 template<class Container>
 void slice_vector3d( const Container& transfer, Container& transfer2d, size_t local_size2d)
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 914e7043d..a4ab868c4 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -36,26 +36,24 @@ struct Radius : public dg::geo::aCylindricalFunctor<Radius>
     private:
     double m_R0, m_Z0;
 };
-template<class Geometry>
-dg::get_host_vector<Geometry> circular_damping( const Geometry& grid,
+HVec circular_damping( const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     if( p.profile_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
-    dg::get_host_vector<Geometry> circular = dg::pullback( dg::compose(
+    HVec circular = dg::pullback( dg::compose(
                 dg::PolynomialHeaviside( gp.a, gp.a*p.profile_alpha/2., -1),
                 Radius( mag.R0(), 0.)), grid);
     return circular;
 }
 
 
-template<class Geometry>
-dg::get_host_vector<Geometry> xpoint_damping(const Geometry& grid,
+HVec xpoint_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
-    dg::get_host_vector<Geometry> xpoint_damping = dg::evaluate( dg::one, grid);
+    HVec xpoint_damping = dg::evaluate( dg::one, grid);
     if( gp.hasXpoint() )
     {
         double RX = gp.R_0 - 1.1*gp.triangularity*gp.a;
@@ -66,40 +64,37 @@ dg::get_host_vector<Geometry> xpoint_damping(const Geometry& grid,
     }
     return xpoint_damping;
 }
-template<class Geometry>
-dg::get_host_vector<Geometry> profile_damping(const Geometry& grid,
+HVec profile_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     if( p.profile_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
-    dg::get_host_vector<Geometry> profile_damping = dg::pullback( dg::compose(dg::PolynomialHeaviside(
+    HVec profile_damping = dg::pullback( dg::compose(dg::PolynomialHeaviside(
         1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)), grid);
     dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag),
         profile_damping, profile_damping);
     return profile_damping;
 }
-template<class Geometry>
-dg::get_host_vector<Geometry> profile(const Geometry& grid,
+HVec profile(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag ){
     double RO=mag.R0(), ZO=0.;
     dg::geo::findOpoint( mag.get_psip(), RO, ZO);
     double psipO = mag.psip()( RO, ZO);
     //First the profile and the source (on the host since we want to output those)
-    dg::get_host_vector<Geometry> profile = dg::pullback( dg::compose(dg::LinearX(
+    HVec profile = dg::pullback( dg::compose(dg::LinearX(
         p.nprofamp/psipO, 0.), mag.psip()), grid);
     dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), profile, profile);
     return profile;
 }
-template<class Geometry>
-dg::get_host_vector<Geometry> source_damping(const Geometry& grid,
+HVec source_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     if( p.source_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: source alpha must not be 0\n");
-    dg::get_host_vector<Geometry> source_damping = dg::pullback(
+    HVec source_damping = dg::pullback(
         dg::compose(dg::PolynomialHeaviside(
             p.source_boundary-p.source_alpha/2.,
         p.source_alpha/2., -1 ), dg::geo::RhoP(mag)), grid);
@@ -109,15 +104,20 @@ dg::get_host_vector<Geometry> source_damping(const Geometry& grid,
 }
 
 
-template<class Geometry, class IDMatrix, class DMatrix, class DVec>
 void init_ni(
     std::array<std::array<DVec,2>,2>& y0,
     Explicit<Geometry, IDMatrix, DMatrix, DVec>& feltor,
     const Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
 {
+#ifdef FELTOR_MPI
+    int rank;
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+#endif
+    MPI_OUT std::cout << "initialize ni with "<<p.initphi << std::endl;
     feltor.initializeni( y0[0][0], y0[0][1], p.initphi);
     double minimalni = dg::blas1::reduce( y0[0][1], 1, thrust::minimum<double>());
+    MPI_OUT std::cerr << "Minimum Ni value "<<minimalni+1<<std::endl;
     if( minimalni <= -1)
     {
         throw dg::Error(dg::Message()<< "ERROR: invalid initial condition. Increase value for alpha since now the ion gyrocentre density is negative!\n"
@@ -127,14 +127,13 @@ void init_ni(
 }//namespace detail
 
 //for wall shadow
-template<class Geometry>
-dg::get_host_vector<Geometry> wall_damping(const Geometry& grid,
+HVec wall_damping(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     if( p.source_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: damping alpha must not be 0\n");
-    dg::get_host_vector<Geometry> wall_damping = dg::pullback(dg::compose( dg::PolynomialHeaviside(
+    HVec wall_damping = dg::pullback(dg::compose( dg::PolynomialHeaviside(
         p.damping_boundary+p.damping_alpha/2., p.damping_alpha/2., +1),
                 dg::geo::RhoP(mag)), grid);
     return wall_damping;
@@ -144,34 +143,11 @@ dg::get_host_vector<Geometry> wall_damping(const Geometry& grid,
  * source profiles.  Just add your own to the relevant map below.
  */
 
-template< class Geometry, class IHMatrix, class IDMatrix, class DMatrix, class DVec >
-struct Initialize
-{
-    using HVec = typename Geometry::host_vector;
-    static std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
-        Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
-        const Geometry& grid, const feltor::Parameters& p,
-        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
-    > > initial_conditions;
-};
-template< class Geometry, class IDMatrix, class DMatrix, class DVec >
-struct Sources
-{
-    using HVec = typename Geometry::host_vector;
-    static std::map<std::string, std::function< HVec(
-        bool& fixed_profile,
-        HVec& ne_profile,
-        Geometry& grid, const feltor::Parameters& p,
-        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
-    > > source_profiles;
-};
-
-template< class Geometry, class IHMatrix, class IDMatrix, class DMatrix, class DVec >
 std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
     const Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
-> > Initialize<Geometry, IHMatrix, IDMatrix, DMatrix, DVec>::initial_conditions =
+> > initial_conditions =
 {
     { "blob",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
@@ -327,13 +303,12 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     }
 };
 
-template< class Geometry, class IDMatrix, class DMatrix, class DVec >
-std::map<std::string, std::function< typename Geometry::host_vector(
+std::map<std::string, std::function< HVec(
     bool& fixed_profile, //indicate whether a profile should be forced (yes or no)
-    typename Geometry::host_vector& ne_profile,    // if fixed_profile is yes you need to construct something here, if no then you can ignore the parameter; if you construct something it will show in the output file in any case
+    HVec& ne_profile,    // if fixed_profile is yes you need to construct something here, if no then you can ignore the parameter; if you construct something it will show in the output file in any case
     Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
-> > Sources<Geometry, IDMatrix, DMatrix, DVec>::source_profiles =
+> > source_profiles =
 {
     {"profile",
         []( bool& fixed_profile, HVec& ne_profile,
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index 779860664..789a9b723 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -4,12 +4,16 @@
 #include "dg/file/nc_utilities.h"
 
 namespace feltor
-{
-    //
+{//We use the typedefs and MPI_OUT
+//
 //everyone reads their portion of the input data
-template<class Geometry, class IHMatrix>
-std::array<std::array<typename Geometry::host_vector,2>,2> init_from_file( std::string file_name, const Geometry& grid, const Parameters& p, double& time){
-
+//don't forget to also read source profiles
+std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Geometry& grid, const Parameters& p, double& time){
+#ifdef FELTOR_MPI
+    int rank;
+    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+#endif
+    std::array<std::array<DVec,2>,2> y0;
     ///////////////////read in and show inputfile
 
     file::NC_Error_Handle errIN;
@@ -26,17 +30,9 @@ std::array<std::array<typename Geometry::host_vector,2>,2> init_from_file( std::
     unsigned  pINNy = jsIN["Ny"].asUInt();
     unsigned  pINNz = jsIN["Nz"].asUInt();
     bool      pINsymmetric   = jsIN.get( "symmetric", false).asBool();
-#ifdef FELTOR_MPI
-    int rank;
-    MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-    if(rank==0){
-#endif
-    std::cout << "RESTART from file "<<file_name<< std::endl;
-    std::cout << " file parameters:" << std::endl;
-    std::cout << pINn<<" x "<<pINNx<<" x "<<pINNy<<" x "<<pINNz<<" : symmetric "<<std::boolalpha<<pINsymmetric<<std::endl;
-#ifdef FELTOR_MPI
-    }
-#endif
+    MPI_OUT std::cout << "RESTART from file "<<file_name<< std::endl;
+    MPI_OUT std::cout << " file parameters:" << std::endl;
+    MPI_OUT std::cout << pINn<<" x "<<pINNx<<" x "<<pINNy<<" x "<<pINNz<<" : symmetric "<<std::boolalpha<<pINsymmetric<<std::endl;
 
     // Now read in last timestep
     Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
@@ -62,8 +58,8 @@ std::array<std::array<typename Geometry::host_vector,2>,2> init_from_file( std::
     size_t countIN[3] = {grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
         grid_IN.n()*grid_IN.Nx()};
     #endif //FELTOR_MPI
-    std::vector<typename Geometry::host_vector> transferINHvec( 5, dg::evaluate( dg::zero, grid));
-    typename Geometry::host_vector transferINH( dg::evaluate(dg::zero, grid_IN));
+    std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
+    HVec transferINH( dg::evaluate(dg::zero, grid_IN));
 
     std::string namesIN[5] = {"restart_electrons", "restart_ions", "restart_Ue", "restart_Ui", "restart_induction"};
 
@@ -75,10 +71,7 @@ std::array<std::array<typename Geometry::host_vector,2>,2> init_from_file( std::
     errIN = nc_inq_varid( ncidIN, "time", &timeIDIN);
     size_time -= 1;
     errIN = nc_get_vara_double( ncidIN, timeIDIN, &size_time, &count_time, &time);
-#ifdef FELTOR_MPI
-    if(rank==0)
-#endif
-    std::cout << " Current time = "<< time <<  std::endl;
+    MPI_OUT std::cout << " Current time = "<< time <<  std::endl;
     for( unsigned i=0; i<5; i++)
     {
         int dataID;
@@ -101,7 +94,6 @@ std::array<std::array<typename Geometry::host_vector,2>,2> init_from_file( std::
     dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
     dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
 
-    std::array<std::array<typename Geometry::host_vector,2>,2> y0;
     dg::assign( transferINHvec[0], y0[0][0]); //ne-1
     dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
     dg::assign( transferINHvec[2], y0[1][0]); //We
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index cc2df9977..ca7b41b76 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -18,7 +18,6 @@ using IHMatrix = dg::IHMatrix;
 using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #include "feltordiag.h"
-using Diagnostics = feltor::Diagnostics<Geometry, IDMatrix, DMatrix, DVec>;
 
 thrust::host_vector<float> append( const thrust::host_vector<float>& in, const dg::aRealTopology3d<double>& g)
 {
@@ -135,7 +134,7 @@ int main( int argc, char* argv[])
     dg::IHMatrix interpolate_in_2d = dg::create::interpolation( g3d_out_equidistant, g3d_out);
 
 
-    for( auto& record : Diagnostics::static_list_3d)
+    for( auto& record : feltor::diagnostics3d_static_list)
     {
         if( record.name != "xc" && record.name != "yc" && record.name != "zc" )
         {
@@ -158,7 +157,7 @@ int main( int argc, char* argv[])
             err = nc_redef(ncid_out);
         }
     }
-    for( auto record : Diagnostics::generate_cyl2cart( g3d_out_equidistant) )
+    for( auto record : feltor::generate_cyl2cart( g3d_out_equidistant) )
     {
         int vID;
         err = nc_def_var( ncid_out, std::get<0>(record).data(), NC_FLOAT, 3, &dim_ids[1],
@@ -171,7 +170,7 @@ int main( int argc, char* argv[])
         err = nc_redef(ncid_out);
 
     }
-    for( auto& record : Diagnostics::list_3d)
+    for( auto& record : feltor::diagnostics3d_list)
     {
         std::string name = record.name;
         std::string long_name = record.long_name;
@@ -203,7 +202,7 @@ int main( int argc, char* argv[])
         std::cout << "  time = " << time << std::endl;
         start3d[0] = i/10;
         err = nc_put_vara_double( ncid_out, tvarID, start3d, count3d_out, &time);
-        for( auto& record : Diagnostics::list_3d)
+        for( auto& record : feltor::diagnostics3d_list)
         {
             std::string record_name = record.name;
             int dataID =0;
-- 
GitLab


From 5e6e3b0c066c9fba1d1a3ac4b5e4d83ed3739408 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 20 May 2020 15:13:46 +0200
Subject: [PATCH 239/540] Move eta,psi from feltordiag into geometry_diag

---
 inc/geometries/geometry_diag.cu | 50 ++++++++++++++++++++++++++-------
 src/feltor/feltordiag.cu        | 29 ++-----------------
 2 files changed, 43 insertions(+), 36 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index f3934b4a9..35def55d3 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -118,6 +118,7 @@ int main( int argc, char* argv[])
         std::cerr << " ( Program searches for string variables 'inputfile' and 'geomfile' in file.nc and tries a json parser)\n";
         return -1;
     }
+    std::cout << input_js<<std::endl;
     const Parameters p(input_js);
     const dg::geo::solovev::Parameters gp(geom_js);
     p.display( std::cout);
@@ -231,6 +232,7 @@ int main( int argc, char* argv[])
 
     /// -------  Elements for fsa on X-point grid ----------------
     double psipmax = dg::blas1::reduce( psipog2d, 0., thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
+    std::unique_ptr<dg::geo::CurvilinearGridX2d> gX2d;
     if( gp.hasXpoint())
     {
         std::cout << "Generate X-point flux-aligned grid ... \n";
@@ -242,14 +244,14 @@ int main( int argc, char* argv[])
         double fx_0 = 1./8.;
         psipmax = -fx_0/(1.-fx_0)*psipmin;
         //std::cout << "psi 1 is          "<<psipmax<<"\n";
-        dg::geo::CurvilinearGridX2d gX2d( generator, fx_0, 0., npsi, Npsi, 640, dg::DIR, dg::NEU);
+        gX2d = std::make_unique<dg::geo::CurvilinearGridX2d>(generator, fx_0, 0., npsi, Npsi, 640, dg::DIR, dg::NEU);
         std::cout << "DONE! \n";
-        dg::Average<dg::HVec > avg_eta( gX2d.grid(), dg::coo2d::y);
-        std::vector<dg::HVec> coordsX = gX2d.map();
-        dg::SparseTensor<dg::HVec> metricX = gX2d.metric();
+        dg::Average<dg::HVec > avg_eta( gX2d->grid(), dg::coo2d::y);
+        std::vector<dg::HVec> coordsX = gX2d->map();
+        dg::SparseTensor<dg::HVec> metricX = gX2d->metric();
         dg::HVec volX2d = dg::tensor::volume2d( metricX);
         dg::blas1::pointwiseDot( coordsX[0], volX2d, volX2d); //R\sqrt{g}
-        const double f0 = (gX2d.x1()-gX2d.x0())/ ( psipmax - psipO);
+        const double f0 = (gX2d->x1()-gX2d->x0())/ ( psipmax - psipO);
         dg::HVec dvdpsip;
         avg_eta( volX2d, dvdpsip, false);
         dg::blas1::scal( dvdpsip, 4.*M_PI*M_PI*f0);
@@ -275,7 +277,7 @@ int main( int argc, char* argv[])
         dg::HVec transferH, transferH1d;
         for( auto tp : map)
         {
-            transferH = dg::pullback( std::get<2>(tp), gX2d);
+            transferH = dg::pullback( std::get<2>(tp), *gX2d);
             dg::blas1::pointwiseDot( volX2d, transferH, transferH);
             avg_eta( transferH, transferH1d, false);
             dg::blas1::scal( transferH1d, 4*M_PI*M_PI*f0); //
@@ -352,10 +354,38 @@ int main( int argc, char* argv[])
             pair.first.data(), pair.second.size(), pair.second.data());
 
     int dim1d_ids[1], dim2d_ids[2], dim3d_ids[3] ;
-    err = file::define_dimension( ncid, &dim1d_ids[0], grid1d, "psi");
-    std::string psi_long_name = "Flux surface label";
-    err = nc_put_att_text( ncid, dim1d_ids[0], "long_name",
-        psi_long_name.size(), psi_long_name.data());
+    if( gp.hasXpoint())
+    {
+        int dim_idsX[2] = {0,0};
+        err = file::define_dimensions( ncid, dim_idsX, gX2d->grid(), {"eta", "psi"} );
+        std::string long_name = "Flux surface label";
+        err = nc_put_att_text( ncid, dim_idsX[0], "long_name",
+            long_name.size(), long_name.data());
+        long_name = "Flux angle";
+        err = nc_put_att_text( ncid, dim_idsX[1], "long_name",
+            long_name.size(), long_name.data());
+        int xccID, yccID;
+        err = nc_def_var( ncid, "xcc", NC_DOUBLE, 2, dim_idsX, &xccID);
+        err = nc_def_var( ncid, "ycc", NC_DOUBLE, 2, dim_idsX, &yccID);
+        long_name="Cartesian x-coordinate";
+        err = nc_put_att_text( ncid, xccID, "long_name",
+            long_name.size(), long_name.data());
+        long_name="Cartesian y-coordinate";
+        err = nc_put_att_text( ncid, yccID, "long_name",
+            long_name.size(), long_name.data());
+        err = nc_enddef( ncid);
+        err = nc_put_var_double( ncid, xccID, gX2d->map()[0].data());
+        err = nc_put_var_double( ncid, yccID, gX2d->map()[1].data());
+        err = nc_redef(ncid);
+        dim1d_ids[0] = dim_idsX[1];
+    }
+    else
+    {
+        err = file::define_dimension( ncid, &dim1d_ids[0], grid1d, "psi");
+        std::string psi_long_name = "Flux surface label";
+        err = nc_put_att_text( ncid, dim1d_ids[0], "long_name",
+            psi_long_name.size(), psi_long_name.data());
+    }
     dg::CylindricalGrid3d grid3d(Rmin,Rmax,Zmin,Zmax, 0, 2.*M_PI, n,Nx,Ny,Nz);
     dg::RealCylindricalGrid3d<float> fgrid3d(Rmin,Rmax,Zmin,Zmax, 0, 2.*M_PI, n,Nx,Ny,Nz);
 
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index daa1c41b0..c66001aa6 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -199,38 +199,15 @@ int main( int argc, char* argv[])
     dg::blas2::symv( fsa2rzmatrix, dvdpsip, dvdpsip2d);
     dg::HMatrix dpsi = dg::create::dx( g1d_out, dg::DIR_NEU);
 
-    //define eta, psi
-    int dim_idsX[2] = {0,0};
-    err = file::define_dimensions( ncid_out, dim_idsX, gridX2d.grid(), {"eta", "psi"} );
-    std::string long_name = "Flux surface label";
-    err = nc_put_att_text( ncid_out, dim_idsX[0], "long_name",
-        long_name.size(), long_name.data());
-    long_name = "Flux angle";
-    err = nc_put_att_text( ncid_out, dim_idsX[1], "long_name",
-        long_name.size(), long_name.data());
-    int xccID, yccID;
-    err = nc_def_var( ncid_out, "xcc", NC_DOUBLE, 2, dim_idsX, &xccID);
-    err = nc_def_var( ncid_out, "ycc", NC_DOUBLE, 2, dim_idsX, &yccID);
-    long_name="Cartesian x-coordinate";
-    err = nc_put_att_text( ncid_out, xccID, "long_name",
-        long_name.size(), long_name.data());
-    long_name="Cartesian y-coordinate";
-    err = nc_put_att_text( ncid_out, yccID, "long_name",
-        long_name.size(), long_name.data());
-    err = nc_enddef( ncid_out);
-    err = nc_put_var_double( ncid_out, xccID, gridX2d.map()[0].data());
-    err = nc_put_var_double( ncid_out, yccID, gridX2d.map()[1].data());
-    err = nc_redef(ncid_out);
-
     // define 2d and 1d and 0d dimensions and variables
     int dim_ids[3], tvarID;
     err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
     //Write long description
-    long_name = "Time at which 2d fields are written";
+    std::string long_name = "Time at which 2d fields are written";
     err = nc_put_att_text( ncid_out, tvarID, "long_name", long_name.size(),
             long_name.data());
-    int dim_ids1d[2] = {dim_ids[0], dim_idsX[1]}; //time,  psi
-
+    int dim_ids1d[2] = {dim_ids[0], 0}; //time,  psi
+    err = file::define_dimension( ncid_out, &dim_ids1d[1], g1d_out, {"psi"} );
     std::map<std::string, int> id0d, id1d, id2d;
 
     size_t count1d[2] = {1, g1d_out.n()*g1d_out.N()};
-- 
GitLab


From 8f084f7e56957ceac9119e6feb84d0f986afa129 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 20 May 2020 17:40:41 +0200
Subject: [PATCH 240/540] Make parameter init throw in feltor

- the danger of wasting CPU time due to a type is just too large
---
 inc/file/json_utilities.h       | 192 ++++++++++++++++++++++++++++++++
 src/feltor/feltor.cu            |  25 +++--
 src/feltor/feltor.tex           |  32 +++---
 src/feltor/feltor_hpc.cu        |  12 +-
 src/feltor/feltordiag.cu        |   2 +-
 src/feltor/interpolate_in_3d.cu |   2 +-
 src/feltor/manufactured.cu      |   2 +-
 src/feltor/parameters.h         | 134 +++++++++++-----------
 8 files changed, 304 insertions(+), 97 deletions(-)

diff --git a/inc/file/json_utilities.h b/inc/file/json_utilities.h
index 316d5e91b..f897dc519 100644
--- a/inc/file/json_utilities.h
+++ b/inc/file/json_utilities.h
@@ -16,6 +16,198 @@
 namespace file
 {
 
+/**
+ * @brief Switch between modes how to handle missing keys in a Json file
+ */
+enum ErrorMode{
+    throwOnError, //!< If a key is missing throw
+    warning, //!< If a key is missing, write a warning to std::cerr and continue
+    silent //!< If a key is missing, continue
+};
+
+
+/**
+ * @brief Wrapper around Json::Value::get function that handles missing keys
+ *
+ * @tparam T value type
+ * @param mode determines what to do when a key is missing
+ * @param js the input Json value
+ * @param key the key to look for in js
+ * @param value the value to take if key is missing
+ *
+ * @return js[key] if key is present, else value
+ */
+template<class T>
+Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, T value)
+{
+    if( js.isMember(key))
+        return js[key];
+    else
+    {
+        std::stringstream message;
+        message <<"*** "<<key<<" not found.";
+        if( throwOnError == mode)
+            throw std::runtime_error( message.str());
+        else if (warning == mode)
+            std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
+        else
+            ;
+        return value;
+    }
+}
+
+/**
+ * @brief Wrapper around Json::Value::get function that handles missing keys
+ *
+ * @tparam T value type
+ * @param mode determines what to do when a key or index is missing
+ * @param js the input Json value
+ * @param key the key to look for in js
+ * @param idx the idx within key to look for in js
+ * @param value the value to take if key is missing
+ *
+ * @return js[key][idx] if key is present, else value
+ */
+template<class T>
+Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key, unsigned idx, T value)
+{
+    if( js.isMember(key))
+    {
+        if( js[key].isValidIndex(idx))
+            return js[key][idx];
+        else
+        {
+            std::stringstream message;
+            message << "*** Index "<<idx<<" not present in "<<key;
+            if( throwOnError == mode)
+                throw std::runtime_error( message.str());
+            else if (warning == mode)
+                std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
+            else
+                ;
+            return value;
+        }
+    }
+    else
+    {
+        std::stringstream message;
+        message << "*** "<<key<<"["<<idx<<"] not found.";
+        if( throwOnError == mode)
+            throw std::runtime_error( message.str());
+        else if (warning == mode)
+            std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
+        else
+            ;
+        return value;
+    }
+}
+/**
+ * @brief Wrapper around Json::Value::get function that handles missing keys
+ *
+ * @tparam T value type
+ * @param mode determines what to do when a key is missing
+ * @param js the input Json value
+ * @param key the key to look for in js
+ * @param key2 the key to look for in \c key
+ * @param value the value to take if key is missing
+ *
+ * @return js[key][key2] if key is present, else value
+ */
+template<class T>
+Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, std::string key2, T value)
+{
+    if( js.isMember(key))
+    {
+        if( js[key].isMember(key2))
+            return js[key][key2];
+        else
+        {
+            std::stringstream message;
+            message << "*** "<<key2<<" not found in "<<key;
+            if( throwOnError == mode)
+                throw std::runtime_error( message.str());
+            else if (warning == mode)
+                std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
+            else
+                ;
+            return value;
+        }
+    }
+    else
+    {
+        std::stringstream message;
+        message << "*** "<<key<<" : "<<key2<<" not found.";
+        if( throwOnError == mode)
+            throw std::runtime_error( message.str());
+        else if (warning == mode)
+            std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
+        else
+            ;
+        return value;
+    }
+}
+/**
+ * @brief Wrapper around Json::Value::get function that handles missing keys
+ *
+ * @tparam T value type
+ * @param mode determines what to do when a key or index is missing
+ * @param js the input Json value
+ * @param key the key to look for in js
+ * @param key2 the key to look for in \c key
+ * @param idx the index to look for in \c key2
+ * @param value the value to take if key is missing
+ *
+ * @return js[key][key2][idx] if key is present, else value
+ */
+template<class T>
+Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key, std::string key2, unsigned idx, T value)
+{
+    if( js.isMember(key))
+    {
+        if( js[key].isMember(key2))
+        {
+            if( js[key][key2].isValidIndex(idx))
+                return js[key][key2][idx];
+            else
+            {
+                std::stringstream message;
+                message << "*** Index "<<idx<<" not present in "<<key<<" : "<<key2;
+                if( throwOnError == mode)
+                    throw std::runtime_error( message.str());
+                else if (warning == mode)
+                    std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
+                else
+                    ;
+                return value;
+            }
+        }
+        else
+        {
+            std::stringstream message;
+            message << "*** "<<key2<<"["<<idx<<"] not found in "<<key;
+            if( throwOnError == mode)
+                throw std::runtime_error( message.str());
+            else if (warning == mode)
+                std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
+            else
+                ;
+            return value;
+        }
+    }
+    else
+    {
+        std::stringstream message;
+        message << "*** "<<key<<" : "<<key2<<"["<<idx<<"] not found.";
+        if( throwOnError == mode)
+            throw std::runtime_error( message.str());
+        else if (warning == mode)
+            std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
+        else
+            ;
+        return value;
+    }
+}
+
 /**
  * @brief Convenience wrapper to open a file and parse it into a Json::Value
  *
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index df7810b12..14c039a3c 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -25,23 +25,30 @@ int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js, gs;
+    std::string inputfile, geomfile;
     if( argc == 1)
-    {
-        file::file2Json( "input.json", js, "strict");
-        file::file2Json( "geometry_params.json", gs, "strict");
-    }
+        inputfile = "input.json", geomfile= "geometry_params.json";
     else if( argc == 3)
-    {
-        file::file2Json( argv[1], js, "strict");
-        file::file2Json( argv[2], gs, "strict");
-    }
+        inputfile = argv[1], geomfile= argv[2];
     else
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "
                   << argv[0]<<" [inputfile] [geomfile] \n";
         return -1;
     }
-    const feltor::Parameters p( js);
+    try{
+        file::file2Json( inputfile, js, "strict");
+        feltor::Parameters(js, file::throwOnError);
+    }catch(std::runtime_error& e)
+    {
+
+        std::cerr << "ERROR in input file "<<inputfile<<std::endl;
+        std::cerr <<e.what()<<std::endl;
+        return -1;
+    }
+    file::file2Json( geomfile, gs, "strict");
+
+    const feltor::Parameters p(js);
     const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
     gp.display( std::cout);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index b49b6ef83..68eb740c9 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1419,7 +1419,7 @@ source & dict & & & Density source, cf. the output \texttt{sne\_tt\_ifs} in \tex
 "gaussian": Gaussian shaped source profile - uses \texttt{posX}, \texttt{posY} and \texttt{sigma},
     See the file {\tt init.h} to add your own custom source.
 \\
-\qquad boundary & float & 0.2  & 0.2 & Source region boundary $\rho_{p,b}$: yields in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
+\qquad boundary & float & 0.2  & 0.5 & Source region boundary $\rho_{p,b}$: yields in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
 \qquad alpha  & float & 0.2 & 0.2 & Transition width $\alpha_p$ in the Heaviside
 in the density Eq.~\eqref{eq:density_profile} (with $rho_{p,b}=0$ and source profiles Eq.~\eqref{eq:electron_source} (should be
 small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes, must not be zero)
@@ -1539,10 +1539,10 @@ the whole simulation is lost. It is safer to just merge files afterwards with fo
 \texttt{ncrcat output1.nc output2.nc output.nc}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Diagnostics}\label{sec:diagnostics}
-\texttt{feltor/diag/feltordiag.cu}
+\texttt{feltor/src/feltor/feltordiag.cu}
  reads one or more previously generated simulation file(s) \texttt{input0.nc ... inputN.nc} described in Section~\ref{sec:output_file} and writes into a single second output file \texttt{output.nc} described as follows. \\
 Compilation\\
-\texttt{make feltordiag} \\
+\texttt{make feltordiag device=\{gpu,omp\}} \\
 Usage \\
 \texttt{./feltordiag input0.nc ... inputN.nc output.nc} \\
 
@@ -1556,10 +1556,7 @@ geomfile   &     text attribute & - & verbose geometry input file as a string (v
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( default size: $3\cdot 64$) \\
-eta              & Coord. var. & 1 (eta) & $\eta$-coordinate of the X-point grid ( size: $3\cdot 640$) \\
 time             & Coord. Var. & 1 (time)& time at which fields are written (variable size: maxout$+1$, dimension size: unlimited) \\
-xcc              & Dataset & 2 (eta,psi) & Cartesian x-coordinate $x(\psi,\eta)$ \\
-ycc              & Dataset & 2 (eta,psi) & Cartesian y-coordinate $y(\psi,\eta)$ \\
 dvdpsip          & Dataset & 1 (psi) & $\d v/\d\psi_p$ \\
 psi\_vol         & Dataset & 1 (psi) & The volume enclosed by the flux surfaces $v(\psi_p) = \int_{\psi_p} \dV $ \\
 psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p) = 2\pi \int_\Omega |\vn\psi_p| \delta(\psi_p - \psi_{p0}) H(Z-Z_X) R\d R\d Z$ \\
@@ -1588,7 +1585,7 @@ We also have a useful geometry diagnostic program:
 generated simulation file \texttt{input.nc} or the input json files
 \texttt{input.json} and \texttt{geometry.json} and writes an output file \texttt{diag\_geometry.nc} as\\
 Compilation\\
-\texttt{make geometry\_diag device=omp} \\
+\texttt{make geometry\_diag device=\{gpu,omp\}} \\
 Usage \\
 \texttt{./geometry\_diag input.json geometry.json diag\_geometry.nc} \\
 The program outputs a host of static 1d, 2d and 3d geometric quantities.
@@ -1614,18 +1611,17 @@ Program terminates with an error message to \texttt{std::cerr}
     \\
 An input Json file misses a key or contains a typo in a key
 &
-If a key is not found the parser uses a default value,
-which is $0$ if not otherwise specified
-and the program continues execution.
-This is because the program cannot distinguish between intentional
-omission of a parameter and unintentional
-(a user can intentionally choose to use the default value
-or an older input file is used which does not contain parameters introduced afterward but have a reasonable default)
-Depending on which parameter is wrongly set this may either lead to early termination or actually lead to a long simulation. Usually the program prints the parameters
-it read from the input file into \texttt{std::cout} and into the output file
-for inspection so double check.
+The programs \texttt{feltor.cu} and \texttt{feltor\_hpc.cu}
+will exit with an error message. (The reason why we do not
+silently use the default value is that the danger of wasting
+valuable computing time on the cluster due to a typo is bigger than the
+added convenience. We want to be sure that the program
+does what the user wants).
+The other programs just issue warnings
+if a key is not found and just use a default value
+which is $0$ if not otherwise specified.
     \\
-    An input Json file has an invalid value, e.g. a  typo in a string value
+    An input Json file has an invalid value, e.g. a typo in a string value
 &
 Invalid values lead to termination with an error message to \texttt{std::cerr}, once and if program tries to use the value
     \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 7a01776be..f7674b514 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -116,15 +116,25 @@ int main( int argc, char* argv[])
     {
         try{
             file::file2Json( argv[1], js, "strict");
+            feltor::Parameters( js, file::throwOnError);
+        } catch( std::exception& e) {
+            MPI_OUT std::cerr << "ERROR in input parameter file "<<argv[1]<<std::endl;
+            MPI_OUT std::cerr << e.what();
+#ifdef FELTOR_MPI
+            MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
+            return -1;
+        }
+        try{
             file::file2Json( argv[2], gs, "strict");
         } catch( std::exception& e) {
+            MPI_OUT std::cerr << "ERROR in geometry file "<<argv[1]<<std::endl;
             MPI_OUT std::cerr << e.what();
 #ifdef FELTOR_MPI
             MPI_Abort(MPI_COMM_WORLD, -1);
 #endif //FELTOR_MPI
             return -1;
         }
-
     }
     const feltor::Parameters p( js);
     const dg::geo::solovev::Parameters gp(gs);
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index c66001aa6..a7413ebab 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -44,7 +44,7 @@ int main( int argc, char* argv[])
     Json::Value js,gs;
     file::string2Json(inputfile, js, "strict");
     file::string2Json(geomfile, gs, "strict");
-    const feltor::Parameters p(js);
+    const feltor::Parameters p(js, file::warning);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index ca7b41b76..9de52c5f1 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -55,7 +55,7 @@ int main( int argc, char* argv[])
     Json::Value js,gs;
     file::string2Json(inputfile, js, "strict");
     file::string2Json(geomfile, gs, "strict");
-    const feltor::Parameters p(js);
+    const feltor::Parameters p(js, file::warning);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 7f17d55c3..b59024e77 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -28,7 +28,7 @@ int main( int argc, char* argv[])
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile]\n";
         return -1;
     }
-    const feltor::Parameters p( js);// p.display( std::cout);
+    const feltor::Parameters p( js, file::throwOnError);// p.display( std::cout);
     std::cout << "# "<<p.n<<" x "<<p.Nx<<" x "<<p.Ny<<" x "<<p.Nz<<"\n";
     const double R_0 = 10;
     const double I_0 = 20; //q factor at r=1 is I_0/R_0
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 07424c2fb..aa63476fd 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -4,6 +4,7 @@
 #include <string>
 #include "dg/enums.h"
 #include "json/json.h"
+#include "dg/file/json_utilities.h"
 
 namespace feltor{
 /// If you need more parameters, just go ahead and extend the list
@@ -50,75 +51,76 @@ struct Parameters
     std::string source_type;
     bool symmetric, periodify;
     Parameters() = default;
-    Parameters( const Json::Value& js) {
-        n       = js["n"].asUInt();
-        Nx      = js["Nx"].asUInt();
-        Ny      = js["Ny"].asUInt();
-        Nz      = js["Nz"].asUInt();
-        dt      = js["dt"].asDouble();
-        cx      = js["compression"].get(0u,1).asUInt();
-        cy      = js["compression"].get(1u,1).asUInt();
+    Parameters( const Json::Value& js, enum file::ErrorMode mode = file::silent ) {
+        //We need to check if a member is present
+        n       = file::get(mode, js,"n", 3).asUInt();
+        Nx      = file::get(mode, js,"Nx", 0).asUInt();
+        Ny      = file::get(mode, js,"Ny", 0).asUInt();
+        Nz      = file::get(mode, js,"Nz", 0).asUInt();
+        dt      = file::get(mode, js,"dt", 0.).asDouble();
+        cx      = file::get_idx(mode, js,"compression",0u,1).asUInt();
+        cy      = file::get_idx(mode, js,"compression",1u,1).asUInt();
         n_out = n, Nx_out = Nx/cx, Ny_out = Ny/cy, Nz_out = Nz;
-        inner_loop = js.get("inner_loop",1).asUInt();
-        itstp   = js["itstp"].asUInt();
-        maxout  = js["maxout"].asUInt();
-        eps_time    = js["eps_time"].asDouble();
-        rtol        = js["rtol"].asDouble();
-
-        eps_pol     = js["eps_pol"].asDouble();
-        jfactor     = js.get("jumpfactor",1).asDouble();
-
-        eps_gamma   = js["eps_gamma"].asDouble();
-        stages      = js.get( "stages", 3).asUInt();
-        mx          = js["FCI"]["refine"].get( 0u, 1).asUInt();
-        my          = js["FCI"]["refine"].get( 1u, 1).asUInt();
-        rk4eps      = js["FCI"].get( "rk4eps", 1e-6).asDouble();
-        periodify   = js["FCI"].get( "periodify", true).asBool();
-
-        mu[0]       = js["mu"].asDouble();
+        inner_loop = file::get(mode, js, "inner_loop",1).asUInt();
+        itstp   = file::get( mode, js, "itstp", 0).asUInt();
+        maxout  = file::get( mode, js, "maxout", 0).asUInt();
+        eps_time    = file::get( mode, js, "eps_time", 1e-10).asDouble();
+        rtol        = file::get( mode, js, "rtol", 1e-5).asDouble();
+
+        eps_pol     = file::get( mode, js, "eps_pol", 1e-6).asDouble();
+        jfactor     = file::get( mode, js, "jumpfactor", 1).asDouble();
+
+        eps_gamma   = file::get( mode, js, "eps_gamma", 1e-6).asDouble();
+        stages      = file::get( mode, js, "stages", 3).asUInt();
+        mx          = file::get_idx( mode, js,"FCI","refine", 0u, 1).asUInt();
+        my          = file::get_idx( mode, js,"FCI","refine", 1u, 1).asUInt();
+        rk4eps      = file::get( mode, js,"FCI", "rk4eps", 1e-6).asDouble();
+        periodify   = file::get( mode, js,"FCI", "periodify", true).asBool();
+
+        mu[0]       = file::get( mode, js, "mu", -0.000272121).asDouble();
         mu[1]       = +1.;
         tau[0]      = -1.;
-        tau[1]      = js["tau"].asDouble();
-        beta        = js.get("beta",0.).asDouble();
-        nu_perp     = js["nu_perp"].asDouble();
-        perp_diff   = js.get("perp_diff", "viscous").asString();
-        nu_parallel = js["nu_parallel"].asDouble();
-        eta         = js["resistivity"].asDouble();
-
-        initne      = js.get( "initne", "blob").asString();
-        initphi     = js.get( "initphi", "zero").asString();
-        amp         = js["amp"].asDouble();
-        sigma       = js["sigma"].asDouble();
-        posX        = js["posX"].asDouble();
-        posY        = js["posY"].asDouble();
-        sigma_z     = js["sigma_z"].asDouble();
-        k_psi       = js["k_psi"].asDouble();
-
-        nprofamp   = js["profile"].get("amp", 0.).asDouble();
-        profile_alpha = js["profile"].get("alpha", 0.2).asDouble();
-
-        source_rate     = js["source"].get("rate", 0.).asDouble();
-        source_type     = js["source"].get("type", "profile").asString();
-        source_boundary = js["source"].get("boundary", 0.2).asDouble();
-        source_alpha    = js["source"].get("alpha", 0.2).asDouble();
-        damping_rate = js["damping"].get("rate", 0.).asDouble();
-        damping_alpha= js["damping"].get("alpha", 0.).asDouble();
-        damping_boundary = js["damping"].get("boundary", 1.2).asDouble();
-
-        bcxN = dg::str2bc(js["bc"]["density"][0].asString());
-        bcyN = dg::str2bc(js["bc"]["density"][1].asString());
-        bcxU = dg::str2bc(js["bc"]["velocity"][0].asString());
-        bcyU = dg::str2bc(js["bc"]["velocity"][1].asString());
-        bcxP = dg::str2bc(js["bc"]["potential"][0].asString());
-        bcyP = dg::str2bc(js["bc"]["potential"][1].asString());
-
-        boxscaleRm  = js["box"]["scaleR"].get(0u,1.05).asDouble();
-        boxscaleRp  = js["box"]["scaleR"].get(1u,1.05).asDouble();
-        boxscaleZm  = js["box"]["scaleZ"].get(0u,1.05).asDouble();
-        boxscaleZp  = js["box"]["scaleZ"].get(1u,1.05).asDouble();
-
-        curvmode    = js.get( "curvmode", "toroidal").asString();
-        symmetric   = js.get( "symmetric", false).asBool();
+        tau[1]      = file::get( mode, js, "tau", 0.).asDouble();
+        beta        = file::get( mode, js, "beta", 0.).asDouble();
+        nu_perp     = file::get( mode, js, "nu_perp", 0.).asDouble();
+        perp_diff   = file::get( mode, js, "perp_diff", "viscous").asString();
+        nu_parallel = file::get( mode, js, "nu_parallel", 0.).asDouble();
+        eta         = file::get( mode, js, "resistivity", 0.).asDouble();
+
+        initne      = file::get( mode, js, "initne", "blob").asString();
+        initphi     = file::get( mode, js, "initphi", "zero").asString();
+        amp         = file::get( mode, js, "amplitude", 0.).asDouble();
+        sigma       = file::get( mode, js, "sigma", 0.).asDouble();
+        posX        = file::get( mode, js, "posX", 0.).asDouble();
+        posY        = file::get( mode, js, "posY", 0.).asDouble();
+        sigma_z     = file::get( mode, js, "sigma_z", 0.).asDouble();
+        k_psi       = file::get( mode, js, "k_psi", 0.).asDouble();
+
+        nprofamp   = file::get( mode, js, "profile", "amp", 0.).asDouble();
+        profile_alpha = file::get( mode, js, "profile", "alpha", 0.2).asDouble();
+
+        source_rate     = file::get( mode, js, "source", "rate", 0.).asDouble();
+        source_type     = file::get( mode, js, "source", "type", "profile").asString();
+        source_boundary = file::get( mode, js, "source", "boundary", 0.5).asDouble();
+        source_alpha    = file::get( mode, js, "source", "alpha", 0.2).asDouble();
+        damping_rate = file::get( mode, js, "damping", "rate", 0.).asDouble();
+        damping_alpha= file::get( mode, js, "damping", "alpha", 0.).asDouble();
+        damping_boundary = file::get( mode, js, "damping", "boundary", 1.2).asDouble();
+
+        bcxN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 0, "").asString());
+        bcyN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 1, "").asString());
+        bcxU = dg::str2bc(file::get_idx( mode, js, "bc", "velocity", 0, "").asString());
+        bcyU = dg::str2bc(file::get_idx( mode, js, "bc", "velocity", 1, "").asString());
+        bcxP = dg::str2bc(file::get_idx( mode, js, "bc", "potential", 0, "").asString());
+        bcyP = dg::str2bc(file::get_idx( mode, js, "bc", "potential", 1, "").asString());
+
+        boxscaleRm  = file::get_idx( mode, js, "box", "scaleR", 0u, 1.05).asDouble();
+        boxscaleRp  = file::get_idx( mode, js, "box", "scaleR", 1u, 1.05).asDouble();
+        boxscaleZm  = file::get_idx( mode, js, "box", "scaleZ", 0u, 1.05).asDouble();
+        boxscaleZp  = file::get_idx( mode, js, "box", "scaleZ", 1u, 1.05).asDouble();
+
+        curvmode    = file::get( mode, js, "curvmode", "toroidal").asString();
+        symmetric   = file::get( mode, js, "symmetric", false).asBool();
     }
     void display( std::ostream& os = std::cout ) const
     {
-- 
GitLab


From 2e3fff717c82158f68c43a2efab0a6acca082ccc Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 20 May 2020 18:08:04 +0200
Subject: [PATCH 241/540] Remove parameters from solovev::Parameters

---
 inc/geometries/flux_t.cu                      |  7 +--
 inc/geometries/geometry_diag.cu               |  9 ++--
 inc/geometries/ribeiroX_t.cu                  |  4 +-
 inc/geometries/ribeiro_mpit.cu                |  7 +--
 inc/geometries/ribeiro_t.cu                   |  7 +--
 inc/geometries/separatrix_orthogonal_t.cu     |  4 +-
 inc/geometries/simple_orthogonal_t.cu         |  7 +--
 inc/geometries/solovev_parameters.h           | 54 ++++++-------------
 src/feltor/feltor.cu                          |  2 +-
 src/feltor/geometry/geometry_params.json      |  9 +---
 .../geometry/geometry_paramsXpoint.json       |  2 +
 11 files changed, 45 insertions(+), 67 deletions(-)

diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 62c7abff6..5b0e0eaef 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -188,9 +188,10 @@ int main( int argc, char* argv[])
     double volume2d = dg::blas1::dot( vol2d, ones2d);
 
     std::cout << "TEST VOLUME IS:\n";
-    if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
-    else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    auto iris = dg::compose( dg::Iris( gp.psipmin, gp.psipmax), c.psip());
+    double psipmin, psipmax;
+    if( psi_0 < psi_1) psipmax = psi_1, psipmin = psi_0;
+    else               psipmax = psi_0, psipmin = psi_1;
+    auto iris = dg::compose( dg::Iris(psipmin, psipmax), c.psip());
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,3, 2e2, 2e2, dg::PER, dg::PER);
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::HVec R  = dg::evaluate( dg::cooX2d, g2dC);
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 35def55d3..2b88fac8d 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -206,11 +206,11 @@ int main( int argc, char* argv[])
         {"TrueCurvatureNablaBGradPsip", "True Nabla B curvature dot the gradient of Psip", dg::geo::ScalarProduct( dg::geo::createTrueCurvatureNablaB(mag), dg::geo::createGradPsip(mag))},
         {"TrueCurvatureKappaGradPsip", "True Kappa curvature dot the gradient of Psip", dg::geo::ScalarProduct( dg::geo::createTrueCurvatureKappa(mag), dg::geo::createGradPsip(mag))},
         //////////////////////////////////
-        {"Iris", "A flux aligned Iris", dg::compose( dg::Iris( gp.psipmin, gp.psipmax), mag.psip())},
-        {"Pupil", "A flux aligned Pupil", dg::compose( dg::Pupil(gp.psipmaxcut), mag.psip()) },
-        {"GaussianDamping", "A flux aligned Heaviside with Gaussian damping", dg::compose( dg::GaussianDamping( gp.psipmaxcut, p.source_alpha), mag.psip()) },
+        {"Iris", "A flux aligned Iris", dg::compose( dg::Iris( 0.5, 0.7), dg::geo::RhoP(mag))},
+        {"Pupil", "A flux aligned Pupil", dg::compose( dg::Pupil(0.7), dg::geo::RhoP(mag)) },
+        {"GaussianDamping", "A flux aligned Heaviside with Gaussian damping", dg::compose( dg::GaussianDamping( 0.8, p.source_alpha), dg::geo::RhoP(mag)) },
         {"ZonalFlow",  "Flux aligned Sine function", dg::compose( dg::SinX ( p.amp, 0., 2.*M_PI*p.k_psi ), mag.psip())},
-        {"PsiLimiter", "A flux aligned Heaviside", dg::compose( dg::Heaviside( gp.psipmaxlim), mag.psip() )},
+        {"PsiLimiter", "A flux aligned Heaviside", dg::compose( dg::Heaviside( 1.03), dg::geo::RhoP(mag) )},
         {"SourceProfile", "A source profile", dg::compose( dg::PolynomialHeaviside(
                     p.source_boundary-p.source_alpha/2., p.source_alpha/2., -1 ),
                 dg::geo::RhoP(mag))},
@@ -227,7 +227,6 @@ int main( int argc, char* argv[])
         {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
             M_PI, p.sigma, p.sigma, p.sigma, p.amp)},
         { "Hoo", "The novel h02 factor", dg::geo::Hoo( mag) }
-
     };
 
     /// -------  Elements for fsa on X-point grid ----------------
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index 0d19dfb72..e1618ef13 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -197,8 +197,8 @@ int main( int argc, char* argv[])
 
     std::cout << "TEST VOLUME IS:\n";
     dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, -2.0*gp.a*gp.elongation, 1.2*gp.a*gp.elongation, 1, 5e3, 1e4, dg::PER, dg::PER);
-    gp.psipmax = 0., gp.psipmin = psi_0;
-    auto iris = dg::compose( dg::Iris(  gp.psipmin, gp.psipmax), psip.f());
+    double psipmax = 0., psipmin = psi_0;
+    auto iris = dg::compose( dg::Iris(  psipmin, psipmax), psip.f());
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::DVec cutter = dg::pullback( iris, g2d), cut_vol( cutter);
     dg::blas1::pointwiseDot(cutter, w2d, cut_vol);
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index f1c3dfb20..4d9352690 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -128,9 +128,10 @@ int main( int argc, char* argv[])
     double volume = dg::blas1::dot( vol, ones3d);
 
     if(rank==0)std::cout << "TEST VOLUME IS:\n";
-    if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
-    else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    auto iris = dg::compose( dg::Iris(gp.psipmin, gp.psipmax), psip.f());
+    double psipmin, psipmax;
+    if( psi_0 < psi_1) psipmax = psi_1, psipmin = psi_0;
+    else               psipmax = psi_0, psipmin = psi_1;
+    auto iris = dg::compose( dg::Iris(psipmin, psipmax), psip.f());
     //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
     dg::CartesianMPIGrid2d g2dC( gp.R_0 -2.*gp.a, gp.R_0 + 2.*gp.a, -2.*gp.a, 2.*gp.a, 1, 2e3, 2e3, dg::DIR, dg::PER, g2d->communicator());
     dg::MHVec vec  = dg::evaluate( iris, g2dC);
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index 549f4e3fe..d6ac73499 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -148,9 +148,10 @@ int main( int argc, char* argv[])
     double volume = dg::blas1::dot( vol, ones3d);
 
     std::cout << "TEST VOLUME IS:\n";
-    if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
-    else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    auto iris = dg::compose( dg::Iris(gp.psipmin, gp.psipmax), psip.f()) ;
+    double psipmin, psipmax;
+    if( psi_0 < psi_1) psipmax = psi_1, psipmin = psi_0;
+    else               psipmax = psi_0, psipmin = psi_1;
+    auto iris = dg::compose( dg::Iris(psipmin, psipmax), psip.f());
     //dg::CylindricalGrid3d<dg::HVec> g3d( gp.R_0 -2.*gp.a, gp.R_0 + 2*gp.a, -2*gp.a, 2*gp.a, 0, 2*M_PI, 3, 2200, 2200, 1, dg::PER, dg::PER, dg::PER);
 //     dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, -1.2*gp.a, 1.2*gp.a, 1, 1e3, 1e3, dg::PER, dg::PER);
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a, 2.0*gp.a, 1, 2e3, 2e3, dg::PER, dg::PER);
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 24c542dd0..c2bb15a72 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -196,8 +196,8 @@ int main( int argc, char* argv[])
 
     std::cout << "TEST VOLUME IS:\n";
     dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, Z_X, 1.2*gp.a*gp.elongation, 1, 5e2, 5e2, dg::PER, dg::PER);
-    gp.psipmax = 0., gp.psipmin = psi_0;
-    auto iris = dg::compose( dg::Iris( gp.psipmin, gp.psipmax), mag.psip());
+    double psipmax = 0., psipmin_iris = psi_0;
+    auto iris = dg::compose( dg::Iris( psipmin_iris, psipmax), mag.psip());
     dg::HVec vec  = dg::evaluate( iris, g2dC);
     dg::HVec g2d_weights = dg::create::volume( g2dC);
     double volumeRZP = dg::blas1::dot( vec, g2d_weights);
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 49ca08e5d..004109dba 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -162,9 +162,10 @@ int main( int argc, char* argv[])
     vol = dg::create::volume( g3d);
     dg::HVec ones3d = dg::evaluate( dg::one, g3d);
     double volume = dg::blas1::dot( vol, ones3d);
-    if( psi_0 < psi_1) gp.psipmax = psi_1, gp.psipmin = psi_0;
-    else               gp.psipmax = psi_0, gp.psipmin = psi_1;
-    auto iris = dg::compose( dg::Iris(gp.psipmin, gp.psipmax), psip.f());
+    double psipmax, psipmin;
+    if( psi_0 < psi_1) psipmax = psi_1, psipmin = psi_0;
+    else               psipmax = psi_0, psipmin = psi_1;
+    auto iris = dg::compose( dg::Iris(psipmin, psipmax), psip.f());
     dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a, 2.0*gp.a, 1, 2e3, 2e3, dg::PER, dg::PER);
 
     dg::HVec vec  = dg::evaluate( iris, g2dC);
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index c4ad22c9b..7f7d771ba 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -1,6 +1,9 @@
 #pragma once
 #include <string>
 #include <vector>
+#ifdef JSONCPP_VERSION_STRING
+#include <dg/file/json_utilities.h>
+#endif
 /*!@file
  *
  * Geometry parameters
@@ -24,13 +27,7 @@ struct Parameters
            pi, //!< prefactor for current I
            a,  //!<  little tokamak radius
            elongation, //!< elongation of the magnetic surfaces
-           triangularity, //!< triangularity of the magnetic surfaces
-           alpha, //!< damping width
-           rk4eps,  //!< accuracy for the field line integration
-           psipmin, //!< for source
-           psipmax, //!< for profile
-           psipmaxcut, //!< for cutting
-           psipmaxlim;  //!< for limiter
+           triangularity; //!< triangularity of the magnetic surfaces
     std::vector<double> c;  //!< 12 coefficients for the solovev equilibrium;
     std::string equilibrium;
 #ifdef JSONCPP_VERSION_STRING
@@ -38,30 +35,23 @@ struct Parameters
      * @brief Construct from Json dataset
      * @param js Can contain the variables "A" (0), "c" (0), "PP" (1.), "PI"
      * (1.), "R_0" , "inverseaspectratio" , "elongation" (1), "triangularity"
-     * (0), "alpha"  (0.), "rk4eps" (1e-5), "psip_min" (0), "psip_max" (0),
-     * "psip_max_cut" (0), "psip_max_lim" (1e10), "equilibrium" ("solovev")
+     * (0), "equilibrium" ("solovev")
      * @note the default values in brackets are taken if the variables are not found in the input file
      * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
      */
-    Parameters( const Json::Value& js) {
-        A  = js.get("A", 0).asDouble();
-        pp  = js.get("PP", 1).asDouble();
-        pi  = js.get("PI", 1).asDouble();
+    Parameters( const Json::Value& js, file::ErrorMode mode = file::silent) {
+        A  = file::get( mode, js, "A", 0).asDouble();
+        pp  = file::get( mode, js, "PP", 1).asDouble();
+        pi  = file::get( mode, js, "PI", 1).asDouble();
         c.resize(12);
         for (unsigned i=0;i<12;i++)
-            c[i] = js["c"].get(i,0).asDouble();
+            c[i] = file::get_idx( mode, js, "c", i, 0.).asDouble();
 
-        R_0  = js["R_0"].asDouble();
-        a  = R_0*js["inverseaspectratio"].asDouble();
-        elongation=js.get("elongation",1).asDouble();
-        triangularity=js.get("triangularity",0).asDouble();
-        alpha=js.get("alpha",0.).asDouble();
-        rk4eps=js.get("rk4eps",1e-5).asDouble();
-        psipmin= js.get("psip_min",0).asDouble();
-        psipmax= js.get("psip_max",0).asDouble();
-        psipmaxcut= js.get("psip_max_cut",0).asDouble();
-        psipmaxlim= js.get("psip_max_lim",1e10).asDouble();
-        equilibrium = js.get( "equilibrium", "solovev").asString();
+        R_0  = file::get( mode, js, "R_0", 0.).asDouble();
+        a  = R_0*file::get( mode, js, "inverseaspectratio", 0.).asDouble();
+        elongation=file::get( mode, js, "elongation", 1.).asDouble();
+        triangularity=file::get( mode, js, "triangularity", 0.).asDouble();
+        equilibrium = file::get( mode, js, "equilibrium", "solovev").asString();
     }
     /**
      * @brief Put values into a json string
@@ -80,12 +70,6 @@ struct Parameters
         js["inverseaspectratio"] = a/R_0;
         js["elongation"] = elongation;
         js["triangularity"] = triangularity;
-        js["alpha"] = alpha;
-        js["rk4eps"] = rk4eps;
-        js["psip_min"] = psipmin;
-        js["psip_max"] = psipmax;
-        js["psip_max_cut"] = psipmaxcut;
-        js["psip_max_lim"] = psipmaxlim;
         js[ "equilibrium"] = equilibrium;
         return js;
     }
@@ -130,13 +114,7 @@ struct Parameters
             <<" a             = "<<a<<"\n"
             <<" epsilon_a     = "<<a/R_0<<"\n"
             <<" elongation    = "<<elongation<<"\n"
-            <<" triangularity = "<<triangularity<<"\n"
-            <<" alpha         = "<<alpha<<"\n"
-            <<" rk4 epsilon   = "<<rk4eps<<"\n"
-            <<" psipmin       = "<<psipmin<<"\n"
-            <<" psipmax       = "<<psipmax<<"\n"
-            <<" psipmaxcut    = "<<psipmaxcut<<"\n"
-            <<" psipmaxlim    = "<<psipmaxlim<<"\n";
+            <<" triangularity = "<<triangularity<<"\n";
         os << std::flush;
 
     }
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 14c039a3c..fadd8d8d4 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -46,7 +46,7 @@ int main( int argc, char* argv[])
         std::cerr <<e.what()<<std::endl;
         return -1;
     }
-    file::file2Json( geomfile, gs, "strict");
+    file::file2Json( geomfile, gs, "default"); //comments allowed
 
     const feltor::Parameters p(js);
     const dg::geo::solovev::Parameters gp(gs);
diff --git a/src/feltor/geometry/geometry_params.json b/src/feltor/geometry/geometry_params.json
index 7d160c23f..23f05ee17 100644
--- a/src/feltor/geometry/geometry_params.json
+++ b/src/feltor/geometry/geometry_params.json
@@ -5,7 +5,8 @@
 {
 	"A" : 1,
 	"R_0" : 91.884200000000007,
-	"alpha" : 0.02,
+   	"PP": 1,
+   	"PI": 1,
 	"c" :
 	[
 		0.13104522088367745,
@@ -24,11 +25,5 @@
 	"elongation" : 1,
 	"equilibrium" : "solovev",
 	"inverseaspectratio" : 0.1666666666666667,
-	"psip_max" : 0,
-	"psip_max_cut" : 10000000000,
-	"psip_max_lim" : 10000000000,
-	"psip_min" : -6,
-	"qampl" : 1,
-	"rk4eps" : 0.01,
 	"triangularity" : 0
 }
diff --git a/src/feltor/geometry/geometry_paramsXpoint.json b/src/feltor/geometry/geometry_paramsXpoint.json
index 651059ae0..336ca0790 100644
--- a/src/feltor/geometry/geometry_paramsXpoint.json
+++ b/src/feltor/geometry/geometry_paramsXpoint.json
@@ -5,6 +5,8 @@
 {
 	"A" : 0,
 	"R_0" : 91.884233908188605,
+   	"PP": 1,
+   	"PI": 1,
 	"c" :
 	[
 		0.58136855188608583,
-- 
GitLab


From 5900cee8bfaa5fe7d5e5771f1562b5390822e8ad Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 20 May 2020 18:31:11 +0200
Subject: [PATCH 242/540] Make and use discardComments mode in json_utils

---
 inc/file/json_utilities.h        | 12 ++++++++++--
 src/feltor/feltor.cu             | 11 ++++++++++-
 src/feltor/feltor.tex            |  8 ++++----
 src/feltor/feltor_hpc.cu         | 11 ++++++-----
 src/feltor/geometry/compass.json |  1 +
 5 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/inc/file/json_utilities.h b/inc/file/json_utilities.h
index f897dc519..40989b0eb 100644
--- a/inc/file/json_utilities.h
+++ b/inc/file/json_utilities.h
@@ -215,13 +215,21 @@ Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key
  * @note included in \c json_utilities.h
  * @param filename Name of the JSON file to parse
  * @param js Contains all the found Json variables on output
- * @param mode Either "default" in which case comments are allowed or "strict" in which case they are not
+ * @param mode "default": comments are allowed and read;
+ * "discardComments": comments are allowed but ignored;
+ * "strict": comments are not allowed;
  */
-static inline void file2Json( std::string filename, Json::Value& js, std::string mode = "default")
+static inline void file2Json( std::string filename, Json::Value& js, std::string mode = "discardComments")
 {
     Json::CharReaderBuilder parser;
     if( "strict" == mode )
         Json::CharReaderBuilder::strictMode( &parser.settings_);
+    else if( "discardComments" == mode )
+    {
+        Json::CharReaderBuilder::strictMode( &parser.settings_);
+        parser.settings_["allowComments"] = true;
+        parser.settings_["collectComments"] = false;
+    }
     else
         Json::CharReaderBuilder::setDefaults( &parser.settings_);
 
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index fadd8d8d4..5428e824a 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -46,7 +46,16 @@ int main( int argc, char* argv[])
         std::cerr <<e.what()<<std::endl;
         return -1;
     }
-    file::file2Json( geomfile, gs, "default"); //comments allowed
+    try{
+        file::file2Json( geomfile, gs, "default");
+        dg::geo::solovev::Parameters(gs, file::throwOnError);
+    }catch(std::runtime_error& e)
+    {
+
+        std::cerr << "ERROR in geometry file "<<geomfile<<std::endl;
+        std::cerr <<e.what()<<std::endl;
+        return -1;
+    }
 
     const feltor::Parameters p(js);
     const dg::geo::solovev::Parameters gp(gs);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 68eb740c9..cd6b1137a 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1466,8 +1466,8 @@ since the internal netcdf information does not display equations.
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
-inputfile  &     text attribute & - & verbose input file as a string (valid JSON, no comments) \\
-geomfile   &     text attribute & - & verbose geometry input file as a string (valid JSON, no comments) \\
+inputfile  &     text attribute & - & verbose input file as a string (valid JSON, C-style comments are allowed but discarded) \\
+geomfile   &     text attribute & - & verbose geometry input file as a string (valid JSON, C-style comments are allowed but discarded) \\
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 z                & Coord. Var. & 1 (z) & $\varphi$-coordinate (computational space, size: $N_z$) \\
@@ -1551,8 +1551,8 @@ Output file format: netcdf-4/hdf5, Conventions: CF-1.7; A \textit{coordinate var
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
-inputfile  &     text attribute & - & verbose input file as a string (valid JSON, no comments) \\
-geomfile   &     text attribute & - & verbose geometry input file as a string (valid JSON, no comments) \\
+inputfile  &     text attribute & - & verbose input file as a string (valid JSON, C-style comments are allowed but discarded) \\
+geomfile   &     text attribute & - & verbose geometry input file as a string (valid JSON, C-style comments are allowed but discarded) \\
 x                & Coord. Var. & 1 (x) & $R$-coordinate (computational space, compressed size: $nN_x/c_x$)\\
 y                & Coord. Var. & 1 (y) & $Z$-coordinate (computational space, compressed size: $nN_y/c_y$)\\
 psi              & Coord. Var. & 1 (psi) & $\psi_p$-coordinate ( default size: $3\cdot 64$) \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index f7674b514..a3c2188f9 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -115,21 +115,22 @@ int main( int argc, char* argv[])
     else
     {
         try{
-            file::file2Json( argv[1], js, "strict");
+            file::file2Json( argv[1], js, "discardComments");
             feltor::Parameters( js, file::throwOnError);
         } catch( std::exception& e) {
             MPI_OUT std::cerr << "ERROR in input parameter file "<<argv[1]<<std::endl;
-            MPI_OUT std::cerr << e.what();
+            MPI_OUT std::cerr << e.what()<<std::endl;
 #ifdef FELTOR_MPI
             MPI_Abort(MPI_COMM_WORLD, -1);
 #endif //FELTOR_MPI
             return -1;
         }
         try{
-            file::file2Json( argv[2], gs, "strict");
+            file::file2Json( argv[2], gs, "discardComments");
+            dg::geo::solovev::Parameters( gs, file::throwOnError);
         } catch( std::exception& e) {
-            MPI_OUT std::cerr << "ERROR in geometry file "<<argv[1]<<std::endl;
-            MPI_OUT std::cerr << e.what();
+            MPI_OUT std::cerr << "ERROR in geometry file "<<argv[2]<<std::endl;
+            MPI_OUT std::cerr << e.what()<<std::endl;
 #ifdef FELTOR_MPI
             MPI_Abort(MPI_COMM_WORLD, -1);
 #endif //FELTOR_MPI
diff --git a/src/feltor/geometry/compass.json b/src/feltor/geometry/compass.json
index a487c4869..9a6328b9a 100644
--- a/src/feltor/geometry/compass.json
+++ b/src/feltor/geometry/compass.json
@@ -1,3 +1,4 @@
+//Compass geometry file T_e = 200eV, B = 0.8T, Deuterium plasma
 {
     "A": 0,
    	"R_0": 2.191566859511476e2,
-- 
GitLab


From 82098b9b2da79cb42fffc3a17f5375abe0f76629 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 26 May 2020 20:30:46 +0200
Subject: [PATCH 243/540] Add two more timer output lines in feltor.h

---
 src/feltor/feltor.h | 28 +++++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 4703747f2..03ac6a37f 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -858,12 +858,25 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
        y[1][1] := W_i
     */
 
+    dg::Timer timer;
+    double accu = 0.;//accumulated time
+    timer.tic();
     // set m_phi[0]
     compute_phi( t, y[0]);
 
     // set m_phi[1], m_dP[0], m_dP[1] and m_UE2 --- needs m_phi[0]
     compute_psi( t);
 
+    timer.toc();
+    accu += timer.diff();
+    #ifdef MPI_VERSION
+        int rank;
+        MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+        if(rank==0)
+    #endif
+    std::cout << "## Compute phi and psi               took "<<timer.diff()<<"s\t A: "<<accu<<"s\n";
+    timer.tic();
+
     // Transform n-1 to n and n to logn
     dg::blas1::subroutine( routines::ComputeLogN(), y[0], m_fields[0], m_logn);
 
@@ -883,9 +896,15 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
 
 #endif
 
-    // Add parallel dynamics --- needs m_logn
-    dg::Timer timer;
+    timer.toc();
+    accu += timer.diff();
+    #ifdef MPI_VERSION
+        if(rank==0)
+    #endif
+    std::cout << "## Compute Apar and perp dynamics    took "<<timer.diff()<<"s\t A: "<<accu<<"s\n";
     timer.tic();
+
+    // Add parallel dynamics --- needs m_logn
 #if FELTORPARALLEL == 1
     compute_parallel( t, y, m_fields, yp);
 #endif
@@ -965,11 +984,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
 #endif //DG_MANUFACTURED
     timer.toc();
+    accu += timer.diff();
     #ifdef MPI_VERSION
-        int rank;
-        MPI_Comm_rank( MPI_COMM_WORLD, &rank);
         if(rank==0)
     #endif
-    std::cout << "#Add parallel dynamics took "<<timer.diff()<<"s\n";
+    std::cout << "## Add parallel dynamics and sources took "<<timer.diff()<<"s\t A: "<<accu<<"\n";
 }
 } //namespace feltor
-- 
GitLab


From 311b2bc745b8d1dacd1caeb720b9ea0de6d56a36 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 26 May 2020 20:38:11 +0200
Subject: [PATCH 244/540] Fix minor bugs in feltor on catch and Abort

---
 src/feltor/feltor_hpc.cu        | 15 +++++++++++----
 src/feltor/feltordiag.cu        |  4 ++--
 src/feltor/interpolate_in_3d.cu |  2 +-
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index a3c2188f9..a80f22ff1 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -70,6 +70,7 @@ int main( int argc, char* argv[])
     cudaGetDeviceCount(&num_devices);
     if(num_devices==0){
         std::cerr << "No CUDA capable devices found"<<std::endl;
+        MPI_Abort(MPI_COMM_WORLD, -1);
         return -1;
     }
     int device = rank % num_devices; //assume # of gpus/node is fixed
@@ -92,7 +93,9 @@ int main( int argc, char* argv[])
         if( size != np[0]*np[1]*np[2])
         {
             std::cerr << "ERROR: Process partition needs to match total number of processes!"<<std::endl;
+#ifdef FELTOR_MPI
             MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
             return -1;
         }
     }
@@ -109,7 +112,8 @@ int main( int argc, char* argv[])
     {
         MPI_OUT std::cerr << "ERROR: Wrong number of arguments!\nUsage: "
                 << argv[0]<<" [input.json] [geometry.json] [output.nc]\n OR \n"
-                << argv[0]<<" [input.json] [geometry.json] [output.nc] [initial.nc] \n";
+                << argv[0]<<" [input.json] [geometry.json] [output.nc] [initial.nc] "<<std::endl;
+        MPI_Abort(MPI_COMM_WORLD, -1);
         return -1;
     }
     else
@@ -227,7 +231,7 @@ int main( int argc, char* argv[])
         try{
             y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
         }catch ( std::out_of_range& error){
-            MPI_OUT std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
+            MPI_OUT std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)" << std::endl;
 #ifdef FELTOR_MPI
             MPI_Abort(MPI_COMM_WORLD, -1);
 #endif //FELTOR_MPI
@@ -258,7 +262,7 @@ int main( int argc, char* argv[])
             p.damping_rate, dg::construct<DVec>(damping_profile)
         );
     }catch ( std::out_of_range& error){
-        std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
+        std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)"<<std::endl;
 #ifdef FELTOR_MPI
         MPI_Abort(MPI_COMM_WORLD, -1);
 #endif //FELTOR_MPI
@@ -393,8 +397,11 @@ int main( int argc, char* argv[])
             feltor( time, y0, y1);
         } catch( dg::Fail& fail) {
             MPI_OUT std::cerr << "CG failed to converge in first step to "
-                              <<fail.epsilon()<<"\n";
+                              <<fail.epsilon()<<std::endl;
             MPI_OUT err = nc_close(ncid);
+#ifdef FELTOR_MPI
+            MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
             return -1;
         }
     }
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index a7413ebab..95c75a65f 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -316,7 +316,7 @@ int main( int argc, char* argv[])
                 bool available = true;
                 try{
                     err = nc_inq_varid(ncid, (record.name+"_ta2d").data(), &dataID);
-                } catch ( file::NC_Error error)
+                } catch ( file::NC_Error& error)
                 {
                     if(  i == 0)
                     {
@@ -354,7 +354,7 @@ int main( int argc, char* argv[])
                 available = true;
                 try{
                     err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
-                } catch ( file::NC_Error error)
+                } catch ( file::NC_Error& error)
                 {
                     if(  i == 0)
                     {
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index 9de52c5f1..880e77ecf 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -209,7 +209,7 @@ int main( int argc, char* argv[])
             bool available = true;
             try{
                 err = nc_inq_varid(ncid_in, record.name.data(), &dataID);
-            } catch ( file::NC_Error error)
+            } catch ( file::NC_Error& error)
             {
                 if(  i == 0)
                 {
-- 
GitLab


From ac18d3d1d65b5c2526a521cbc3adf2ba08f12664 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 26 May 2020 20:46:19 +0200
Subject: [PATCH 245/540] Change dz behaviour in feltor

---
 inc/dg/elliptic.h       | 38 ++++++++++++----------
 inc/dg/elliptic2d_b.cu  |  3 +-
 inc/dg/enums.h          |  6 ++++
 inc/dg/helmholtz.h      | 24 +++++++++++---
 src/feltor/feltor.h     | 71 +++++++++++++++++++----------------------
 src/feltor/feltordiag.h | 20 ++++++------
 src/feltor/implicit.h   | 18 +++++------
 7 files changed, 99 insertions(+), 81 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 4a90a35bf..93a9dd577 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -365,10 +365,16 @@ class Elliptic3d
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * the direction of the z derivative is always \c dg::centered
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
+     * @param comp if compute::in_2d, the dz derivatives are not constructed and
+     * the symv function avoids the derivative in z,
+     * compute::in_3d represents the original behaviour.
+     * Restrict the problem to the first 2 dimensions.
+     * This effectively makes the behaviour of \c dg::Elliptic3d
+     * identical to the \c dg::Elliptic class.
      * @note chi is assumed 1 per default
      */
-    Elliptic3d( const Geometry& g, norm no = not_normed, direction dir = forward, value_type jfactor=1.):
-        Elliptic3d( g, g.bcx(), g.bcy(), g.bcz(), no, dir, jfactor)
+    Elliptic3d( const Geometry& g, norm no = not_normed, direction dir = forward, value_type jfactor=1., enum compute comp = compute::in_3d ):
+        Elliptic3d( g, g.bcx(), g.bcy(), g.bcz(), no, dir, jfactor, comp)
     {
     }
 
@@ -384,17 +390,28 @@ class Elliptic3d
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * the direction of the z derivative is always \c dg::centered
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
+     * @param comp if compute::in_2d, the dz derivatives are not constructed and
+     * the symv function avoids the derivative in z,
+     * compute::in_3d represents the original behaviour.
+     * Restrict the problem to the first 2 dimensions.
+     * This effectively makes the behaviour of \c dg::Elliptic3d
+     * identical to the \c dg::Elliptic class.
      * @note chi is the metric tensor multiplied by the volume element per default
      */
-    Elliptic3d( const Geometry& g, bc bcx, bc bcy, bc bcz, norm no = not_normed, direction dir = forward, value_type jfactor = 1.)
+    Elliptic3d( const Geometry& g, bc bcx, bc bcy, bc bcz, norm no = not_normed, direction dir = forward, value_type jfactor = 1., enum compute comp = compute::in_3d )
     {
         m_no=no, m_jfactor=jfactor;
         dg::blas2::transfer( dg::create::dx( g, inverse( bcx), inverse(dir)), m_leftx);
         dg::blas2::transfer( dg::create::dy( g, inverse( bcy), inverse(dir)), m_lefty);
-        dg::blas2::transfer( dg::create::dz( g, inverse( bcz), inverse(dg::centered)), m_leftz);
+        m_multiplyZ = false;
+        if(comp == compute::in_3d)
+            m_multiplyZ = true;
+        if(m_multiplyZ)
+            dg::blas2::transfer( dg::create::dz( g, inverse( bcz), inverse(dg::centered)), m_leftz);
         dg::blas2::transfer( dg::create::dx( g, bcx, dir), m_rightx);
         dg::blas2::transfer( dg::create::dy( g, bcy, dir), m_righty);
-        dg::blas2::transfer( dg::create::dz( g, bcz, dg::centered), m_rightz);
+        if(m_multiplyZ)
+            dg::blas2::transfer( dg::create::dz( g, bcz, dg::centered), m_rightz);
         dg::blas2::transfer( dg::create::jumpX( g, bcx),   m_jumpX);
         dg::blas2::transfer( dg::create::jumpY( g, bcy),   m_jumpY);
 
@@ -472,17 +489,6 @@ class Elliptic3d
     ///@copydoc Elliptic::get_jfactor()
     value_type get_jfactor() const {return m_jfactor;}
 
-    /**
-     * @brief Restrict the problem to the first 2 dimensions
-     *
-     * This effectively makes the behaviour of dg::Elliptic3d
-     * identical to the dg::Elliptic class.
-     * @param compute_in_2d if true, the symv function avoids the derivative in z, false reverts to the original behaviour.
-     */
-    void set_compute_in_2d( bool compute_in_2d ) {
-        m_multiplyZ = !compute_in_2d;
-    }
-
     ///@copydoc Elliptic::symv(const ContainerType0&,ContainerType1&)
     template<class ContainerType0, class ContainerType1>
     void symv( const ContainerType0& x, ContainerType1& y){
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 2a1eaa893..25f864085 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -137,8 +137,7 @@ int main()
     {
         //try the compute_in_2d handle of Elliptic3d
 	    dg::CartesianGrid3d grid( 0, lx, 0, ly, 0,1,n, Nx, Ny, 1, bcx, bcy, dg::PER);
-		dg::Elliptic3d<dg::CartesianGrid3d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
-        pol_backward.set_compute_in_2d(true);
+		dg::Elliptic3d<dg::CartesianGrid3d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor, dg::compute::in_2d);
 		pol_backward.set_chi( chi);
 		x = temp;
 		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index 7371ac869..a8376db36 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -143,5 +143,11 @@ enum class coo3d : char
     xz = 'c', //!< xz plane
 };
 
+///@brief compute mode in Elliptic3d
+enum class compute{
+    in_2d, //!< compute in 2d
+    in_3d //!< compute in 3d
+};
+
 ///@}
 }//namespace dg
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index c4c213f1e..e6542b75b 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -178,9 +178,20 @@ struct Helmholtz3d
         m_chi( m_laplaceM.weights())
     {
     }
-    ///@copydoc Helmholtz::Helmholtz(const Geometry&,value_type,direction,value_type)
-    Helmholtz3d( const Geometry& g, value_type alpha = 1., direction dir = dg::forward, value_type jfactor=1.):
-        Helmholtz3d( g, g.bcx(), g.bcy(), g.bcz(), alpha, dir, jfactor)
+    /**
+     * @brief Construct
+     *
+     * @param g The grid to use (boundary conditions are taken from there)
+     * @param alpha Scalar in the above formula
+     * @param dir Direction of the Laplace operator
+     * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
+     * @param comp if compute::in_2d, the dz derivatives are not constructed and
+     * the symv function avoids the derivative in z and uses a 2d Laplacian;
+     * compute::in_3d represents the original behaviour.
+     * @note The default value of \f$\chi\f$ is one. \c Helmholtz is never normed
+     */
+    Helmholtz3d( const Geometry& g, value_type alpha = 1., direction dir = dg::forward, value_type jfactor=1., enum compute comp = compute::in_3d):
+        Helmholtz3d( g, g.bcx(), g.bcy(), g.bcz(), alpha, dir, jfactor, comp)
     {
     }
     /**
@@ -193,10 +204,13 @@ struct Helmholtz3d
      * @param alpha Scalar in the above formula
      * @param dir Direction of the Laplace operator
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
+     * @param comp if compute::in_2d, the dz derivatives are not constructed and
+     * the symv function avoids the derivative in z and uses a 2d Laplacian;
+     * compute::in_3d represents the original behaviour.
      * @note The default value of \f$\chi\f$ is one. \c Helmholtz is never normed
      */
-    Helmholtz3d( const Geometry& g, bc bcx, bc bcy, bc bcz, value_type alpha = 1., direction dir = dg::forward, value_type jfactor=1.):
-        m_laplaceM( g, bcx, bcy, bcz, dg::not_normed, dir, jfactor),
+    Helmholtz3d( const Geometry& g, bc bcx, bc bcy, bc bcz, value_type alpha = 1., direction dir = dg::forward, value_type jfactor=1., enum compute comp = compute::in_3d):
+        m_laplaceM( g, bcx, bcy, bcz, dg::not_normed, dir, jfactor, comp),
         m_chi( m_laplaceM.weights()),
         m_alpha(alpha)
     {
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 03ac6a37f..0d58246af 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -13,6 +13,7 @@
 
 namespace feltor
 {
+//the memory access to explicit zeros in the case of non-true curvature mode is negligible in the following
 
 namespace routines{
 struct ComputePerpDrifts{
@@ -218,6 +219,7 @@ struct Explicit
         return m_apar;
     }
     const std::array<Container, 3> & gradN (int i) const {
+        //note that gradN[2] is zero if curvmore != "true"
         return m_dN[i];
     }
     const std::array<Container, 3> & gradU (int i) const {
@@ -247,22 +249,18 @@ struct Explicit
         // grad S_ne and grad S_ni
         dg::blas2::symv( m_dx_N, m_s[0][i], gradS[0]);
         dg::blas2::symv( m_dy_N, m_s[0][i], gradS[1]);
-        if(!m_p.symmetric)dg::blas2::symv( m_dz, m_s[0][i], gradS[2]);
-    }
-    const Container & compute_dppN(int i) { //2nd varphi derivative
-        dg::blas2::symv( m_dz, m_fields[0][i], m_temp0);
-        dg::blas2::symv( m_dz, m_temp0, m_temp1);
-        return m_temp1;
-    }
-    const Container & compute_dppP(int i) {
-        dg::blas2::symv( m_dz, m_phi[i], m_temp0);
-        dg::blas2::symv( m_dz, m_temp0, m_temp1);
-        return m_temp1;
+        if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, m_s[0][i], gradS[2]);
+        else dg::blas1::copy( 0., gradS[2]);
     }
     const Container & compute_dppU(int i) {
-        dg::blas2::symv( m_dz, m_fields[1][i], m_temp0);
-        dg::blas2::symv( m_dz, m_temp0, m_temp1);
-        return m_temp1;
+        if( "true" == m_p.curvmode)
+        {
+            dg::blas2::symv( m_dz, m_fields[1][i], m_temp0);
+            dg::blas2::symv( m_dz, m_temp0, m_temp1);
+            return m_temp1;
+        }
+        else
+            return m_fields[1][i];
     }
     const dg::SparseTensor<Container>& projection() const{
         return m_hh;
@@ -482,18 +480,15 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
     for( int i=0; i<3; i++)
         dg::blas1::pointwiseDot( vol, m_b[i], m_b[i]); //b_i/vol/B
     m_hh = dg::geo::createProjectionTensor( bhat, g);
-    m_lapperpN.construct ( g, p.bcxN, p.bcyN, dg::PER, dg::normed, dg::centered),
-    m_lapperpU.construct ( g, p.bcxU, p.bcyU, dg::PER, dg::normed, dg::centered),
-    m_lapperpP.construct ( g, p.bcxP, p.bcyP, dg::PER, dg::normed, dg::centered),
+    dg::compute comp = dg::compute::in_2d;
+    if( p.curvmode == "true")
+        comp = dg::compute::in_3d;
+    m_lapperpN.construct ( g, p.bcxN, p.bcyN, dg::PER, dg::normed, dg::centered, 1., comp),
+    m_lapperpU.construct ( g, p.bcxU, p.bcyU, dg::PER, dg::normed, dg::centered, 1., comp),
+    m_lapperpP.construct ( g, p.bcxP, p.bcyP, dg::PER, dg::normed, dg::centered, 1., comp),
     m_lapperpN.set_chi( m_hh);
     m_lapperpU.set_chi( m_hh);
     m_lapperpP.set_chi( m_hh);
-    if( p.curvmode != "true")
-    {
-        m_lapperpN.set_compute_in_2d(true);
-        m_lapperpU.set_compute_in_2d(true);
-        m_lapperpP.set_compute_in_2d(true);
-    }
     m_lapperpP.set_jfactor(0); //we don't want jump terms in source
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
@@ -511,17 +506,20 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
     m_multi_invgammaP.resize(p.stages);
     m_multi_invgammaN.resize(p.stages);
     m_multi_induction.resize(p.stages);
+    dg::compute comp = dg::compute::in_2d;
+    if( p.curvmode == "true")
+        comp = dg::compute::in_3d;
     for( unsigned u=0; u<p.stages; u++)
     {
         m_multi_pol[u].construct( m_multigrid.grid(u),
             p.bcxP, p.bcyP, dg::PER, dg::not_normed,
-            dg::centered, p.jfactor);
+            dg::centered, p.jfactor, comp);
         m_multi_invgammaP[u].construct(  m_multigrid.grid(u),
-            p.bcxP, p.bcyP, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered);
+            p.bcxP, p.bcyP, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered, 1., comp);
         m_multi_invgammaN[u].construct(  m_multigrid.grid(u),
-            p.bcxN, p.bcyN, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered);
+            p.bcxN, p.bcyN, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered, 1., comp);
         m_multi_induction[u].construct(  m_multigrid.grid(u),
-            p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
+            p.bcxU, p.bcyU, dg::PER, -1., dg::centered, 1., comp);
 
         dg::SparseTensor<Container> hh = dg::geo::createProjectionTensor(
             bhat, m_multigrid.grid(u));
@@ -529,12 +527,6 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
         m_multi_invgammaP[u].elliptic().set_chi( hh);
         m_multi_invgammaN[u].elliptic().set_chi( hh);
         m_multi_induction[u].elliptic().set_chi( hh);
-        if(p.curvmode != "true"){
-            m_multi_pol[u].set_compute_in_2d( true);
-            m_multi_invgammaP[u].elliptic().set_compute_in_2d( true);
-            m_multi_invgammaN[u].elliptic().set_compute_in_2d( true);
-            m_multi_induction[u].elliptic().set_compute_in_2d( true);
-        }
     }
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
@@ -551,12 +543,13 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     m_dy_N( dg::create::dy( g, p.bcyN) ),
     m_dy_U( dg::create::dy( g, p.bcyU) ),
     m_dy_P( dg::create::dy( g, p.bcyP) ),
-    m_dz( dg::create::dz( g, dg::PER) ),
     m_multigrid( g, p.stages),
     m_old_phi( 2, dg::evaluate( dg::zero, g)),
     m_old_psi( m_old_phi), m_old_gammaN( m_old_phi), m_old_apar( m_old_phi),
     m_p(p), m_full_system(full_system)
 {
+    if( "true" == m_p.curvmode)
+        m_dz = dg::create::dz( g, dg::PER);
     //--------------------------init vectors to 0-----------------//
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
     m_UE2 = m_temp2 = m_temp1 = m_temp0;
@@ -702,7 +695,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
     //-------Compute Psi and derivatives
     dg::blas2::symv( m_dx_P, m_phi[0], m_dP[0][0]);
     dg::blas2::symv( m_dy_P, m_phi[0], m_dP[0][1]);
-    if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[0], m_dP[0][2]);
+    if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, m_phi[0], m_dP[0][2]);
     dg::tensor::multiply3d( m_hh, //grad_perp
         m_dP[0][0], m_dP[0][1], m_dP[0][2], m_UE2, m_temp0, m_temp1);
     dg::blas1::subroutine( routines::ComputePsi(), m_phi[1],
@@ -715,7 +708,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
     //m_UE2 now contains u_E^2; also update derivatives
     dg::blas2::symv( m_dx_P, m_phi[1], m_dP[1][0]);
     dg::blas2::symv( m_dy_P, m_phi[1], m_dP[1][1]);
-    if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[1], m_dP[1][2]);
+    if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, m_phi[1], m_dP[1][2]);
 }
 template<class Geometry, class IMatrix, class Matrix, class Container>
 void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
@@ -750,7 +743,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
     //----------Compute Derivatives----------------------------//
     dg::blas2::symv( m_dx_U, m_apar, m_dA[0]);
     dg::blas2::symv( m_dy_U, m_apar, m_dA[1]);
-    if(!m_p.symmetric) dg::blas2::symv( m_dz, m_apar, m_dA[2]);
+    if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, m_apar, m_dA[2]);
 
     //----------Compute Velocities-----------------------------//
     dg::blas1::axpby( 1., fields[1][0], -1./m_p.mu[0], m_apar, fields[1][0]);
@@ -770,10 +763,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
         ////////////////////perpendicular dynamics////////////////////////
         dg::blas2::symv( m_dx_N, y[0][i], m_dN[i][0]);
         dg::blas2::symv( m_dy_N, y[0][i], m_dN[i][1]);
-        if(!m_p.symmetric) dg::blas2::symv( m_dz, y[0][i], m_dN[i][2]);
+        if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, y[0][i], m_dN[i][2]);
         dg::blas2::symv( m_dx_U, fields[1][i], m_dU[i][0]);
         dg::blas2::symv( m_dy_U, fields[1][i], m_dU[i][1]);
-        if(!m_p.symmetric) dg::blas2::symv( m_dz, fields[1][i], m_dU[i][2]);
+        if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, fields[1][i], m_dU[i][2]);
         if( m_p.beta == 0){
             dg::blas1::subroutine( routines::ComputePerpDrifts(
                 m_p.mu[i], m_p.tau[i]),
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 3d289ce23..f9ffc0df7 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -404,16 +404,16 @@ std::vector<Record> diagnostics2d_list = {
              dg::blas1::copy(v.f.dssU(0), result);
         }
     },
-    {"dppue", "2nd varphi derivative of electron velocity", false,
-        []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.compute_dppU(0), result);
-        }
-    },
-    {"dpue2", "1st varphi derivative squared of electron velocity", false,
-        []( DVec& result, Variables& v ) {
-             dg::blas1::pointwiseDot(v.f.gradU(0)[2], v.f.gradU(0)[2], result);
-        }
-    },
+    //{"dppue", "2nd varphi derivative of electron velocity", false,
+    //    []( DVec& result, Variables& v ) {
+    //         dg::blas1::copy(v.f.compute_dppU(0), result);
+    //    }
+    //},
+    //{"dpue2", "1st varphi derivative squared of electron velocity", false,
+    //    []( DVec& result, Variables& v ) {
+    //         dg::blas1::pointwiseDot(v.f.gradU(0)[2], v.f.gradU(0)[2], result);
+    //    }
+    //},
     {"lperpinv", "Perpendicular density gradient length scale", false,
         []( DVec& result, Variables& v ) {
             const std::array<DVec, 3>& dN = v.f.gradN(0);
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 3d2a971c9..537ca5b6c 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -25,7 +25,10 @@ struct ImplicitDensity
         dg::geo::TokamakMagneticField mag)
     {
         m_p = p;
-        m_lapM_perpN.construct( g, p.bcxN, p.bcyN,dg::PER, dg::normed, dg::centered);
+        dg::compute comp = dg::compute::in_2d;
+        if( p.curvmode == "true")
+            comp = dg::compute::in_3d;
+        m_lapM_perpN.construct( g, p.bcxN, p.bcyN,dg::PER, dg::normed, dg::centered, 1., comp);
         dg::assign( dg::evaluate( dg::zero, g), m_temp);
         auto bhat = dg::geo::createEPhi(+1); //bhat = ephi except when "true"
         if( p.curvmode == "true")
@@ -36,8 +39,6 @@ struct ImplicitDensity
             = dg::geo::createProjectionTensor( bhat, g);
         //set perpendicular projection tensor h
         m_lapM_perpN.set_chi( hh);
-        if( p.curvmode != "true")
-            m_lapM_perpN.set_compute_in_2d( true);
     }
 
     void operator()( double t, const std::array<Container,2>& y,
@@ -90,9 +91,12 @@ struct ImplicitVelocity
     void construct( const Geometry& g, feltor::Parameters p,
             dg::geo::TokamakMagneticField mag)
     {
+        dg::compute comp = dg::compute::in_2d;
+        if( p.curvmode == "true")
+            comp = dg::compute::in_3d;
         m_p=p;
         m_lapM_perpU.construct( g, p.bcxU,p.bcyU,dg::PER,
-            dg::normed, dg::centered);
+            dg::normed, dg::centered, 1., comp);
         if( !(p.perp_diff == "viscous" || p.perp_diff == "hyperviscous") )
             throw dg::Error(dg::Message(_ping_)<<"Warning! perp_diff value '"<<p.perp_diff<<"' not recognized!! I do not know how to proceed! Exit now!");
         dg::assign( dg::evaluate( dg::zero, g), m_temp);
@@ -108,8 +112,6 @@ struct ImplicitVelocity
             = dg::geo::createProjectionTensor( bhat, g);
         //set perpendicular projection tensor h
         m_lapM_perpU.set_chi( hh);
-        if( p.curvmode != "true")
-            m_lapM_perpU.set_compute_in_2d(true);
         //m_induction.construct(  g,
         //    p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
         //m_induction.elliptic().set_chi( hh);
@@ -122,10 +124,8 @@ struct ImplicitVelocity
             dg::SparseTensor<Container> hh = dg::geo::createProjectionTensor(
                 bhat, m_multigrid.grid(u));
             m_multi_induction[u].construct(  m_multigrid.grid(u),
-                p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
+                p.bcxU, p.bcyU, dg::PER, -1., dg::centered, 1., comp);
             m_multi_induction[u].elliptic().set_chi( hh);
-            if( p.curvmode != "true")
-                m_multi_induction[u].elliptic().set_compute_in_2d(true);
         }
         m_multi_chi = m_multigrid.project( m_temp);
         m_old_apar = dg::Extrapolation<Container>( 1, dg::evaluate( dg::zero, g));
-- 
GitLab


From e0b2e48046547b97e539b43de4346a5ef4eccc9d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 May 2020 18:57:47 +0200
Subject: [PATCH 246/540] Change json_utilities parameters to enum class

- for safety reasons: the compiler detects spelling errors
and prohibits implicit cast to int
---
 diag/crosscoherencdiag.cpp          |   2 +-
 diag/feltorSHdiag.cpp               |   2 +-
 diag/feltorSHdiag.cu                |   2 +-
 diag/feltorSHvmaxdiag.cu            |   2 +-
 diag/feltorSesoldiag.cpp            |   2 +-
 diag/feltorShwdiag.cpp              |   2 +-
 diag/feltorShwmerger.cpp            |   2 +-
 diag/feltorShwradstat.cpp           |   2 +-
 diag/feltorShwstat.cpp              |   2 +-
 diag/fftwdiag.cpp                   |   2 +-
 diag/growthrate.cpp                 |   2 +-
 diag/impRdiag.cu                    |   2 +-
 diag/normdiag.cu                    |   2 +-
 diag/reco2Ddiag.cu                  |   2 +-
 diag/toeflEPdiag.cu                 |   2 +-
 diag/toeflRdiag.cu                  |   2 +-
 inc/file/json_utilities.h           | 125 ++++++++++++++++++----------
 inc/geometries/geometry_diag.cu     |   8 +-
 inc/geometries/solovev_parameters.h |   3 +-
 src/ep/toeflR.cu                    |   6 +-
 src/ep/toefl_mpi.cu                 |   2 +-
 src/feltor/feltor.cu                |  10 +--
 src/feltor/feltor_hpc.cu            |  10 ++-
 src/feltor/feltordiag.cu            |   6 +-
 src/feltor/init_from_file.h         |   2 +-
 src/feltor/interpolate_in_3d.cu     |   6 +-
 src/feltor/manufactured.cu          |   6 +-
 src/feltor/parameters.h             |   2 +-
 src/feltorSH/feltor.cu              |   6 +-
 src/feltorSH/feltor_hpc.cu          |   2 +-
 src/feltorSH/feltor_mpi.cu          |   2 +-
 src/feltorSHp/feltor.cu             |   6 +-
 src/feltorSHp/feltor_hpc.cu         |   2 +-
 src/feltorSHp/feltor_mpi.cu         |   2 +-
 src/feltorSesol/feltor.cu           |   6 +-
 src/feltorSesol/feltor_hpc.cu       |   4 +-
 src/feltorSesol/feltor_mpi.cu       |   4 +-
 src/feltorShw/feltor.cu             |   6 +-
 src/feltorShw/feltor_hpc.cu         |   4 +-
 src/feltorShw/feltor_mpi.cu         |   4 +-
 src/hasegawa/hw.cu                  |   6 +-
 src/hasegawa/mima.cu                |   6 +-
 src/heat/heat.cu                    |  10 +--
 src/heat/heat_hpc.cu                |   8 +-
 src/impurities/toeflI.cu            |   6 +-
 src/impurities/toefl_hpc.cu         |   2 +-
 src/impurities/toefl_mpi.cu         |   2 +-
 src/lamb_dipole/shu_b.cu            |   2 +-
 src/polar/polar.cu                  |   4 +-
 src/polar/polar_mpi.cu              |   4 +-
 src/reco2D/reconnection.cu          |   6 +-
 src/reco2D/reconnection_hpc.cu      |   2 +-
 src/reco2D/reconnection_mpi.cu      |   2 +-
 src/toefl/toeflR.cu                 |   6 +-
 src/toefl/toefl_hpc.cu              |   2 +-
 55 files changed, 186 insertions(+), 148 deletions(-)

diff --git a/diag/crosscoherencdiag.cpp b/diag/crosscoherencdiag.cpp
index 297ed39b6..b136bc84f 100644
--- a/diag/crosscoherencdiag.cpp
+++ b/diag/crosscoherencdiag.cpp
@@ -52,7 +52,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
 
diff --git a/diag/feltorSHdiag.cpp b/diag/feltorSHdiag.cpp
index 23d7a11c5..e562eed88 100644
--- a/diag/feltorSHdiag.cpp
+++ b/diag/feltorSHdiag.cpp
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/feltorSHdiag.cu b/diag/feltorSHdiag.cu
index f0641aca4..c51365099 100644
--- a/diag/feltorSHdiag.cu
+++ b/diag/feltorSHdiag.cu
@@ -54,7 +54,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/feltorSHvmaxdiag.cu b/diag/feltorSHvmaxdiag.cu
index 674002331..8679dc936 100644
--- a/diag/feltorSHvmaxdiag.cu
+++ b/diag/feltorSHvmaxdiag.cu
@@ -34,7 +34,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
 //         std::cout << "input "<<input<<std::endl;
         Json::Value js;
-        file::string2Json( input, js, "strict");
+        file::string2Json( input, js, file::comments::are_forbidden);
         const eule::Parameters p(js);
         err = nc_inq_dimid( ncid, "time", &timeID);
         err = nc_inq_dimlen( ncid, timeID, &numOut);
diff --git a/diag/feltorSesoldiag.cpp b/diag/feltorSesoldiag.cpp
index e063bd412..34b3310b8 100644
--- a/diag/feltorSesoldiag.cpp
+++ b/diag/feltorSesoldiag.cpp
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
 
diff --git a/diag/feltorShwdiag.cpp b/diag/feltorShwdiag.cpp
index 7b74dd263..ffa41ff31 100644
--- a/diag/feltorShwdiag.cpp
+++ b/diag/feltorShwdiag.cpp
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     ///////////////////////////////////////////////////////////////////////////
diff --git a/diag/feltorShwmerger.cpp b/diag/feltorShwmerger.cpp
index 443b17f94..8c2a11aa2 100644
--- a/diag/feltorShwmerger.cpp
+++ b/diag/feltorShwmerger.cpp
@@ -44,7 +44,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        file::string2Json( input, js, "strict");
+        file::string2Json( input, js, file::comments::are_forbidden);
         const eule::Parameters p(js);
         
         dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
diff --git a/diag/feltorShwradstat.cpp b/diag/feltorShwradstat.cpp
index e524ee24e..72c08e6cc 100644
--- a/diag/feltorShwradstat.cpp
+++ b/diag/feltorShwradstat.cpp
@@ -35,7 +35,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        file::string2Json( input, js, "strict");
+        file::string2Json( input, js, file::comments::are_forbidden);
         const eule::Parameters p(js);
         
         dg::Grid1d g1d( 0., p.lx,p.n_out, p.Nx_out, p.bc_x);
diff --git a/diag/feltorShwstat.cpp b/diag/feltorShwstat.cpp
index d382670a9..907aef4e3 100644
--- a/diag/feltorShwstat.cpp
+++ b/diag/feltorShwstat.cpp
@@ -49,7 +49,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        file::string2Json( input, js, "strict");
+        file::string2Json( input, js, file::comments::are_forbidden);
         const eule::Parameters p(js);
         
 	size_t start0d  = 0;    
diff --git a/diag/fftwdiag.cpp b/diag/fftwdiag.cpp
index 75310399c..f8bbb76c1 100644
--- a/diag/fftwdiag.cpp
+++ b/diag/fftwdiag.cpp
@@ -33,7 +33,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/growthrate.cpp b/diag/growthrate.cpp
index a9d922291..35bd1259e 100644
--- a/diag/growthrate.cpp
+++ b/diag/growthrate.cpp
@@ -30,7 +30,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/impRdiag.cu b/diag/impRdiag.cu
index ce814c5ba..b0e7bba24 100644
--- a/diag/impRdiag.cu
+++ b/diag/impRdiag.cu
@@ -45,7 +45,7 @@ int main( int argc, char* argv[])
     std::cout << "input "<< input << std::endl;
     //parse: parameter string--json-->p.xxx
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
 
   const imp::Parameters p(js);
   p.display(std::cout);
diff --git a/diag/normdiag.cu b/diag/normdiag.cu
index 53dcd792f..557d4bb0b 100644
--- a/diag/normdiag.cu
+++ b/diag/normdiag.cu
@@ -33,7 +33,7 @@ int main( int argc, char* argv[])
     err = nc_close(ncid); 
 
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const eule::Parameters p(js);
     
     //////////////////////////////Grids//////////////////////////////////////
diff --git a/diag/reco2Ddiag.cu b/diag/reco2Ddiag.cu
index 5c9b0c6e4..f6f8b00ae 100644
--- a/diag/reco2Ddiag.cu
+++ b/diag/reco2Ddiag.cu
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const asela::Parameters p(js);
 
     //////////////////////////////Grids//////////////////////////////////////
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index c624bd87d..6e0b54685 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -72,7 +72,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index d39fed8a4..3853fddeb 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -67,7 +67,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, "strict");
+    file::string2Json( input, js, file::comments::are_forbidden);
     const Parameters p(js);
     p.display(std::cout);
     
diff --git a/inc/file/json_utilities.h b/inc/file/json_utilities.h
index 40989b0eb..7e778dd60 100644
--- a/inc/file/json_utilities.h
+++ b/inc/file/json_utilities.h
@@ -12,17 +12,22 @@
  */
 
 //Note that the json utilities are separate from netcdf utilities because
-//of the different dependencies that they incur
+//of the different library dependencies that they incur
 namespace file
 {
 
-/**
- * @brief Switch between modes how to handle missing keys in a Json file
- */
-enum ErrorMode{
-    throwOnError, //!< If a key is missing throw
-    warning, //!< If a key is missing, write a warning to std::cerr and continue
-    silent //!< If a key is missing, continue
+///@brief Switch between how to handle errors in a Json utitlity functions
+enum class error{
+    is_throw, //!< throw an error
+    is_warning, //!< Handle the error by writing a warning to \c std::cerr
+    is_silent //!< Ignore the error and silently continue execution
+};
+
+///@brief Switch how comments are treated in a json string or file
+enum class comments{
+    are_kept, //!< Keep comments in the Json value
+    are_discarded, //!< Allow comments but discard them in the Json value
+    are_forbidden //!< Treat comments as invalid Json
 };
 
 
@@ -30,7 +35,7 @@ enum ErrorMode{
  * @brief Wrapper around Json::Value::get function that handles missing keys
  *
  * @tparam T value type
- * @param mode determines what to do when a key is missing
+ * @param err determines what to do when a key is missing
  * @param js the input Json value
  * @param key the key to look for in js
  * @param value the value to take if key is missing
@@ -38,7 +43,7 @@ enum ErrorMode{
  * @return js[key] if key is present, else value
  */
 template<class T>
-Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, T value)
+Json::Value get( enum error err, const Json::Value& js, std::string key, T value)
 {
     if( js.isMember(key))
         return js[key];
@@ -46,9 +51,9 @@ Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, T
     {
         std::stringstream message;
         message <<"*** "<<key<<" not found.";
-        if( throwOnError == mode)
+        if( error::is_throw == err)
             throw std::runtime_error( message.str());
-        else if (warning == mode)
+        else if ( error::is_warning == err)
             std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
         else
             ;
@@ -60,7 +65,7 @@ Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, T
  * @brief Wrapper around Json::Value::get function that handles missing keys
  *
  * @tparam T value type
- * @param mode determines what to do when a key or index is missing
+ * @param err determines what to do when a key or index is missing
  * @param js the input Json value
  * @param key the key to look for in js
  * @param idx the idx within key to look for in js
@@ -69,7 +74,7 @@ Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, T
  * @return js[key][idx] if key is present, else value
  */
 template<class T>
-Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key, unsigned idx, T value)
+Json::Value get_idx( enum error err, const Json::Value& js, std::string key, unsigned idx, T value)
 {
     if( js.isMember(key))
     {
@@ -79,9 +84,9 @@ Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key
         {
             std::stringstream message;
             message << "*** Index "<<idx<<" not present in "<<key;
-            if( throwOnError == mode)
+            if( error::is_throw == err)
                 throw std::runtime_error( message.str());
-            else if (warning == mode)
+            else if (error::is_warning == err)
                 std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
             else
                 ;
@@ -92,9 +97,9 @@ Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key
     {
         std::stringstream message;
         message << "*** "<<key<<"["<<idx<<"] not found.";
-        if( throwOnError == mode)
+        if( error::is_throw == err)
             throw std::runtime_error( message.str());
-        else if (warning == mode)
+        else if (error::is_warning == err)
             std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
         else
             ;
@@ -105,7 +110,7 @@ Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key
  * @brief Wrapper around Json::Value::get function that handles missing keys
  *
  * @tparam T value type
- * @param mode determines what to do when a key is missing
+ * @param err determines what to do when a key is missing
  * @param js the input Json value
  * @param key the key to look for in js
  * @param key2 the key to look for in \c key
@@ -114,7 +119,7 @@ Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key
  * @return js[key][key2] if key is present, else value
  */
 template<class T>
-Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, std::string key2, T value)
+Json::Value get( enum error err, const Json::Value& js, std::string key, std::string key2, T value)
 {
     if( js.isMember(key))
     {
@@ -124,9 +129,9 @@ Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, st
         {
             std::stringstream message;
             message << "*** "<<key2<<" not found in "<<key;
-            if( throwOnError == mode)
+            if( error::is_throw == err)
                 throw std::runtime_error( message.str());
-            else if (warning == mode)
+            else if (error::is_warning == err)
                 std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
             else
                 ;
@@ -137,9 +142,9 @@ Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, st
     {
         std::stringstream message;
         message << "*** "<<key<<" : "<<key2<<" not found.";
-        if( throwOnError == mode)
+        if( error::is_throw == err)
             throw std::runtime_error( message.str());
-        else if (warning == mode)
+        else if (error::is_warning == err)
             std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
         else
             ;
@@ -150,7 +155,7 @@ Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, st
  * @brief Wrapper around Json::Value::get function that handles missing keys
  *
  * @tparam T value type
- * @param mode determines what to do when a key or index is missing
+ * @param err determines what to do when a key or index is missing
  * @param js the input Json value
  * @param key the key to look for in js
  * @param key2 the key to look for in \c key
@@ -160,7 +165,7 @@ Json::Value get( enum ErrorMode mode, const Json::Value& js, std::string key, st
  * @return js[key][key2][idx] if key is present, else value
  */
 template<class T>
-Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key, std::string key2, unsigned idx, T value)
+Json::Value get_idx( enum error err, const Json::Value& js, std::string key, std::string key2, unsigned idx, T value)
 {
     if( js.isMember(key))
     {
@@ -172,9 +177,9 @@ Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key
             {
                 std::stringstream message;
                 message << "*** Index "<<idx<<" not present in "<<key<<" : "<<key2;
-                if( throwOnError == mode)
+                if( error::is_throw == err)
                     throw std::runtime_error( message.str());
-                else if (warning == mode)
+                else if (error::is_warning == err)
                     std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
                 else
                     ;
@@ -185,9 +190,9 @@ Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key
         {
             std::stringstream message;
             message << "*** "<<key2<<"["<<idx<<"] not found in "<<key;
-            if( throwOnError == mode)
+            if( error::is_throw == err)
                 throw std::runtime_error( message.str());
-            else if (warning == mode)
+            else if (error::is_warning == err)
                 std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
             else
                 ;
@@ -198,9 +203,9 @@ Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key
     {
         std::stringstream message;
         message << "*** "<<key<<" : "<<key2<<"["<<idx<<"] not found.";
-        if( throwOnError == mode)
+        if( error::is_throw == err)
             throw std::runtime_error( message.str());
-        else if (warning == mode)
+        else if (error::is_warning == err)
             std::cerr <<"WARNING "<< message.str()<<" Using default value "<<value<<"\n";
         else
             ;
@@ -211,20 +216,21 @@ Json::Value get_idx( enum ErrorMode mode, const Json::Value& js, std::string key
 /**
  * @brief Convenience wrapper to open a file and parse it into a Json::Value
  *
- * @attention This function will throw a \c std::runtime_error containing an error message on any error that occurs on parsing.
  * @note included in \c json_utilities.h
  * @param filename Name of the JSON file to parse
  * @param js Contains all the found Json variables on output
- * @param mode "default": comments are allowed and read;
- * "discardComments": comments are allowed but ignored;
- * "strict": comments are not allowed;
+ * @param comm determines the handling of comments in the Json file
+ * @param err determines how parser errors are handled by the function
+ * \c error::is_throw:  throw a \c std::runtime_error containing an error message on any error that occurs on parsing;
+ * \c error::is_warning: write the error message to std::cerr and return;
+ * \c error::is_silent: silently return
  */
-static inline void file2Json( std::string filename, Json::Value& js, std::string mode = "discardComments")
+static inline void file2Json(std::string filename, Json::Value& js, enum comments comm = file::comments::are_discarded, enum error err = file::error::is_throw)
 {
     Json::CharReaderBuilder parser;
-    if( "strict" == mode )
+    if( comments::are_forbidden == comm )
         Json::CharReaderBuilder::strictMode( &parser.settings_);
-    else if( "discardComments" == mode )
+    else if( comments::are_discarded == comm )
     {
         Json::CharReaderBuilder::strictMode( &parser.settings_);
         parser.settings_["allowComments"] = true;
@@ -239,12 +245,23 @@ static inline void file2Json( std::string filename, Json::Value& js, std::string
         std::string message = "\nAn error occured while parsing "+filename+"\n";
         message +=  "*** File does not exist! *** \n\n";
         throw std::runtime_error( message);
+        if( err == error::is_throw)
+            throw std::runtime_error( message);
+        else if (err == error::is_warning)
+            std::cerr << "WARNING: "<<message<<std::endl;
+        else
+            return;
     }
     std::string errs;
-    if( !parseFromStream( parser, isI, &js, &errs))
+    if( !parseFromStream( parser, isI, &js, &errs) )
     {
         std::string message = "An error occured while parsing "+filename+"\n"+errs;
-        throw std::runtime_error( message);
+        if( err == error::is_throw)
+            throw std::runtime_error( message);
+        else if (err == error::is_warning)
+            std::cerr << "WARNING: "<<message<<std::endl;
+        else
+            return;
     }
 }
 /**
@@ -255,19 +272,37 @@ static inline void file2Json( std::string filename, Json::Value& js, std::string
  * @note included in \c json_utilities.h
  * @param input The string to interpret as a Json string
  * @param js Contains all the found Json variables on output
- * @param mode Either "default" in which case comments are allowed or "strict" in which case they are not
+ * @param comm determines the handling of comments in the Json string
+ * @param err determines how parser errors are handled by the function
+ * \c error::is_throw:  throw a \c std::runtime_error containing an error message on any error that occurs on parsing;
+ * \c error::is_warning: write the error message to std::cerr and return;
+ * \c error::is_silent: silently return
  */
-static inline void string2Json( std::string input, Json::Value& js, std::string mode = "default")
+static inline void string2Json(std::string input, Json::Value& js, enum comments comm = file::comments::are_discarded, enum error err = file::error::is_throw)
 {
     Json::CharReaderBuilder parser;
-    if( "strict" == mode )
+    if( comments::are_forbidden == comm )
+        Json::CharReaderBuilder::strictMode( &parser.settings_);
+    else if( comments::are_discarded == comm )
+    {
         Json::CharReaderBuilder::strictMode( &parser.settings_);
+        parser.settings_["allowComments"] = true;
+        parser.settings_["collectComments"] = false;
+    }
     else
         Json::CharReaderBuilder::setDefaults( &parser.settings_);
+
     std::string errs;
     std::stringstream ss(input);
     if( !parseFromStream( parser, ss, &js, &errs) )
-        throw std::runtime_error( errs);
+    {
+        if( err == error::is_throw)
+            throw std::runtime_error( errs);
+        else if (err == error::is_warning)
+            std::cerr << "WARNING: "<<errs<<std::endl;
+        else
+            return;
+    }
 }
 
 }//namespace file
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 2b88fac8d..105ddf326 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -87,8 +87,8 @@ int main( int argc, char* argv[])
     {
         newfilename = argv[3];
         std::cout << argv[0]<< " "<<argv[1]<<" & "<<argv[2]<<" -> " <<argv[3]<<std::endl;
-        file::file2Json( argv[1], input_js, "strict");
-        file::file2Json( argv[2], geom_js, "strict");
+        file::file2Json( argv[1], input_js, file::comments::are_discarded);
+        file::file2Json( argv[2], geom_js, file::comments::are_discarded);
     }
     else if( argc == 3)
     {
@@ -106,8 +106,8 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geomfile[0]);
         err = nc_close( ncid_in);
         Json::Value js,gs;
-        file::string2Json(inputfile, input_js, "strict");
-        file::string2Json(geomfile, geom_js, "strict");
+        file::string2Json(inputfile, input_js, file::comments::are_discarded);
+        file::string2Json(geomfile, geom_js, file::comments::are_discarded);
     }
     else
     {
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index 7f7d771ba..551748652 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -36,10 +36,11 @@ struct Parameters
      * @param js Can contain the variables "A" (0), "c" (0), "PP" (1.), "PI"
      * (1.), "R_0" , "inverseaspectratio" , "elongation" (1), "triangularity"
      * (0), "equilibrium" ("solovev")
+     * @param mode determine what happens when a key is missing
      * @note the default values in brackets are taken if the variables are not found in the input file
      * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
      */
-    Parameters( const Json::Value& js, file::ErrorMode mode = file::silent) {
+    Parameters( const Json::Value& js, file::error mode = file::error::is_silent) {
         A  = file::get( mode, js, "A", 0).asDouble();
         pp  = file::get( mode, js, "PP", 1).asDouble();
         pi  = file::get( mode, js, "PI", 1).asDouble();
diff --git a/src/ep/toeflR.cu b/src/ep/toeflR.cu
index a0f1fce0d..e696978d7 100644
--- a/src/ep/toeflR.cu
+++ b/src/ep/toeflR.cu
@@ -18,9 +18,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json("input.json", js, "default");
+        file::file2Json("input.json", js, file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json(argv[1], js, "default");
+        file::file2Json(argv[1], js, file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -29,7 +29,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json("window_params.json", js, "default");
+    file::file2Json("window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/ep/toefl_mpi.cu b/src/ep/toefl_mpi.cu
index 08f3234fb..1a1295ee7 100644
--- a/src/ep/toefl_mpi.cu
+++ b/src/ep/toefl_mpi.cu
@@ -56,7 +56,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const Parameters p( js);
     if(rank==0)p.display( std::cout);
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 5428e824a..6b1b76638 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -37,8 +37,8 @@ int main( int argc, char* argv[])
         return -1;
     }
     try{
-        file::file2Json( inputfile, js, "strict");
-        feltor::Parameters(js, file::throwOnError);
+        file::file2Json( inputfile, js, file::comments::are_forbidden);
+        feltor::Parameters(js, file::error::is_throw);
     }catch(std::runtime_error& e)
     {
 
@@ -47,8 +47,8 @@ int main( int argc, char* argv[])
         return -1;
     }
     try{
-        file::file2Json( geomfile, gs, "default");
-        dg::geo::solovev::Parameters(gs, file::throwOnError);
+        file::file2Json( geomfile, gs, file::comments::are_discarded);
+        dg::geo::solovev::Parameters(gs, file::error::is_throw);
     }catch(std::runtime_error& e)
     {
 
@@ -171,7 +171,7 @@ int main( int argc, char* argv[])
     /////////glfw initialisation ////////////////////////////////////////////
     //
     std::stringstream title;
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     unsigned red = js.get("reduction", 1).asUInt();
     double rows = js["rows"].asDouble(), cols = p.Nz/red+1,
            width = js["width"].asDouble(), height = js["height"].asDouble();
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index a80f22ff1..a79787b94 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -113,14 +113,16 @@ int main( int argc, char* argv[])
         MPI_OUT std::cerr << "ERROR: Wrong number of arguments!\nUsage: "
                 << argv[0]<<" [input.json] [geometry.json] [output.nc]\n OR \n"
                 << argv[0]<<" [input.json] [geometry.json] [output.nc] [initial.nc] "<<std::endl;
+#ifdef FELTOR_MPI
         MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
         return -1;
     }
     else
     {
         try{
-            file::file2Json( argv[1], js, "discardComments");
-            feltor::Parameters( js, file::throwOnError);
+            file::file2Json( argv[1], js, file::comments::are_discarded);
+            feltor::Parameters( js, file::error::is_throw);
         } catch( std::exception& e) {
             MPI_OUT std::cerr << "ERROR in input parameter file "<<argv[1]<<std::endl;
             MPI_OUT std::cerr << e.what()<<std::endl;
@@ -130,8 +132,8 @@ int main( int argc, char* argv[])
             return -1;
         }
         try{
-            file::file2Json( argv[2], gs, "discardComments");
-            dg::geo::solovev::Parameters( gs, file::throwOnError);
+            file::file2Json( argv[2], gs, file::comments::are_discarded);
+            dg::geo::solovev::Parameters( gs, file::error::is_throw);
         } catch( std::exception& e) {
             MPI_OUT std::cerr << "ERROR in geometry file "<<argv[2]<<std::endl;
             MPI_OUT std::cerr << e.what()<<std::endl;
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 95c75a65f..b18f3ad58 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -42,9 +42,9 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geomfile[0]);
     err = nc_close( ncid_in);
     Json::Value js,gs;
-    file::string2Json(inputfile, js, "strict");
-    file::string2Json(geomfile, gs, "strict");
-    const feltor::Parameters p(js, file::warning);
+    file::string2Json(inputfile, js, file::comments::are_forbidden);
+    file::string2Json(geomfile, gs, file::comments::are_forbidden);
+    const feltor::Parameters p(js, file::error::is_warning);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index 789a9b723..8dffbb4f8 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -24,7 +24,7 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &length);
     std::string input(length, 'x');
     errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &input[0]);
-    file::string2Json( input, jsIN, "strict");
+    file::string2Json( input, jsIN, file::comments::are_forbidden);
     unsigned  pINn  = jsIN["n"].asUInt();
     unsigned  pINNx = jsIN["Nx"].asUInt();
     unsigned  pINNy = jsIN["Ny"].asUInt();
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index 880e77ecf..6364aa1a5 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -53,9 +53,9 @@ int main( int argc, char* argv[])
     std::string geomfile(length, 'x');
     err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geomfile[0]);
     Json::Value js,gs;
-    file::string2Json(inputfile, js, "strict");
-    file::string2Json(geomfile, gs, "strict");
-    const feltor::Parameters p(js, file::warning);
+    file::string2Json(inputfile, js, file::comments::are_forbidden);
+    file::string2Json(geomfile, gs, file::comments::are_forbidden);
+    const feltor::Parameters p(js, file::error::is_warning);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index b59024e77..819aa31f4 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -20,15 +20,15 @@ int main( int argc, char* argv[])
 {
     Json::Value js, gs;
     if( argc == 1)
-        file::file2Json( "input.json", js, "strict");
+        file::file2Json( "input.json", js, file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile]\n";
         return -1;
     }
-    const feltor::Parameters p( js, file::throwOnError);// p.display( std::cout);
+    const feltor::Parameters p( js, file::error::is_throw);// p.display( std::cout);
     std::cout << "# "<<p.n<<" x "<<p.Nx<<" x "<<p.Ny<<" x "<<p.Nz<<"\n";
     const double R_0 = 10;
     const double I_0 = 20; //q factor at r=1 is I_0/R_0
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index aa63476fd..9adfb579a 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -51,7 +51,7 @@ struct Parameters
     std::string source_type;
     bool symmetric, periodify;
     Parameters() = default;
-    Parameters( const Json::Value& js, enum file::ErrorMode mode = file::silent ) {
+    Parameters( const Json::Value& js, enum file::error mode = file::error::is_warning ) {
         //We need to check if a member is present
         n       = file::get(mode, js,"n", 3).asUInt();
         Nx      = file::get(mode, js,"Nx", 0).asUInt();
diff --git a/src/feltorSH/feltor.cu b/src/feltorSH/feltor.cu
index e7342f5fe..62ffb7e14 100644
--- a/src/feltorSH/feltor.cu
+++ b/src/feltorSH/feltor.cu
@@ -17,9 +17,9 @@ int main( int argc, char* argv[])
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "strict");
+        file::file2Json( "input.json", js, file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -28,7 +28,7 @@ int main( int argc, char* argv[])
     const eule::Parameters p(  js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     std::stringstream title;
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
diff --git a/src/feltorSH/feltor_hpc.cu b/src/feltorSH/feltor_hpc.cu
index 25f1e859f..5c0b5ea2e 100644
--- a/src/feltorSH/feltor_hpc.cu
+++ b/src/feltorSH/feltor_hpc.cu
@@ -21,7 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     p.display( std::cout);
diff --git a/src/feltorSH/feltor_mpi.cu b/src/feltorSH/feltor_mpi.cu
index a3fd4795e..5f49667a2 100644
--- a/src/feltorSH/feltor_mpi.cu
+++ b/src/feltorSH/feltor_mpi.cu
@@ -40,7 +40,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     if(rank==0)p.display( std::cout);
diff --git a/src/feltorSHp/feltor.cu b/src/feltorSHp/feltor.cu
index ef6cfbb3c..f24eb18a6 100644
--- a/src/feltorSHp/feltor.cu
+++ b/src/feltorSHp/feltor.cu
@@ -17,9 +17,9 @@ int main( int argc, char* argv[])
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "strict");
+        file::file2Json( "input.json", js, file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -29,7 +29,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorSHp/feltor_hpc.cu b/src/feltorSHp/feltor_hpc.cu
index 18fe39c8a..66e510e68 100644
--- a/src/feltorSHp/feltor_hpc.cu
+++ b/src/feltorSHp/feltor_hpc.cu
@@ -21,7 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     p.display( std::cout);
diff --git a/src/feltorSHp/feltor_mpi.cu b/src/feltorSHp/feltor_mpi.cu
index 6ea2e317d..a66edd018 100644
--- a/src/feltorSHp/feltor_mpi.cu
+++ b/src/feltorSHp/feltor_mpi.cu
@@ -40,7 +40,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     if(rank==0)p.display( std::cout);
diff --git a/src/feltorSesol/feltor.cu b/src/feltorSesol/feltor.cu
index 015b8b980..611afbda0 100644
--- a/src/feltorSesol/feltor.cu
+++ b/src/feltorSesol/feltor.cu
@@ -18,9 +18,9 @@ int main( int argc, char* argv[])
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json("input.json", js, "default");
+        file::file2Json("input.json", js, file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json(argv[1], js, "default");
+        file::file2Json(argv[1], js, file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -30,7 +30,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorSesol/feltor_hpc.cu b/src/feltorSesol/feltor_hpc.cu
index d4f11e172..80c1f788b 100644
--- a/src/feltorSesol/feltor_hpc.cu
+++ b/src/feltorSesol/feltor_hpc.cu
@@ -21,7 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     p.display( std::cout);
@@ -83,7 +83,7 @@ int main( int argc, char* argv[])
         std::string inputIN(length, 'x');
         errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json(inputIN, jsIN, "strict");
+        file::string2Json(inputIN, jsIN, file::comments::are_forbidden);
 
         const eule::Parameters pIN(  jsIN);
         std::cout << "[input.nc] file parameters" << std::endl;
diff --git a/src/feltorSesol/feltor_mpi.cu b/src/feltorSesol/feltor_mpi.cu
index a57736f16..beb8cf455 100644
--- a/src/feltorSesol/feltor_mpi.cu
+++ b/src/feltorSesol/feltor_mpi.cu
@@ -38,7 +38,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     if(rank==0)p.display( std::cout);
@@ -112,7 +112,7 @@ int main( int argc, char* argv[])
         std::string inputIN(length, 'x');
         errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json(inputIN, jsIN, "strict");
+        file::string2Json(inputIN, jsIN, file::comments::are_forbidden);
         const eule::Parameters pIN(  jsIN);    
         std::cout << "[input.nc] file parameters" << std::endl;
         pIN.display( std::cout);   
diff --git a/src/feltorShw/feltor.cu b/src/feltorShw/feltor.cu
index bcad81c3b..61f94e94d 100644
--- a/src/feltorShw/feltor.cu
+++ b/src/feltorShw/feltor.cu
@@ -19,9 +19,9 @@ int main( int argc, char* argv[])
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "strict");
+        file::file2Json( "input.json", js, file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorShw/feltor_hpc.cu b/src/feltorShw/feltor_hpc.cu
index 0c7bdb027..b8989b28d 100644
--- a/src/feltorShw/feltor_hpc.cu
+++ b/src/feltorShw/feltor_hpc.cu
@@ -21,7 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     p.display( std::cout);
@@ -107,7 +107,7 @@ int main( int argc, char* argv[])
         std::string inputIN(length, 'x');
         errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json(inputIN, jsIN, "strict");
+        file::string2Json(inputIN, jsIN, file::comments::are_forbidden);
 
         const eule::Parameters pIN(  jsIN);    
         std::cout << "[input.nc] file parameters" << std::endl;
diff --git a/src/feltorShw/feltor_mpi.cu b/src/feltorShw/feltor_mpi.cu
index 2d7b2c0cf..6d0badd89 100644
--- a/src/feltorShw/feltor_mpi.cu
+++ b/src/feltorShw/feltor_mpi.cu
@@ -61,7 +61,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     if(rank==0) p.display( std::cout);
@@ -160,7 +160,7 @@ int main( int argc, char* argv[])
         std::string inputIN(length, 'x');
         errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json(inputIN, jsIN, "strict");
+        file::string2Json(inputIN, jsIN, file::comments::are_forbidden);
         const eule::Parameters pIN(  jsIN);    
         if(rank==0) std::cout << "[input.nc] file parameters" << std::endl;
         if(rank==0) pIN.display( std::cout);   
diff --git a/src/hasegawa/hw.cu b/src/hasegawa/hw.cu
index 0c5452fcb..1a0a11cf8 100644
--- a/src/hasegawa/hw.cu
+++ b/src/hasegawa/hw.cu
@@ -17,9 +17,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "default");
+        file::file2Json( "input.json", js, file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "default");
+        file::file2Json( argv[1], js, file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -28,7 +28,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/hasegawa/mima.cu b/src/hasegawa/mima.cu
index ef3ede527..a29cdc594 100644
--- a/src/hasegawa/mima.cu
+++ b/src/hasegawa/mima.cu
@@ -16,9 +16,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "default");
+        file::file2Json( "input.json", js, file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "default");
+        file::file2Json( argv[1], js, file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -27,7 +27,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/heat/heat.cu b/src/heat/heat.cu
index 4a81c691e..eef8bbf20 100644
--- a/src/heat/heat.cu
+++ b/src/heat/heat.cu
@@ -22,13 +22,13 @@ int main( int argc, char* argv[])
     Json::Value js, gs;
     if( argc == 1)
     {
-        file::file2Json("input/default.json", js, "default");
-        file::file2Json("geometry/geometry_params.json", gs, "default");
+        file::file2Json("input/default.json", js, file::comments::are_discarded);
+        file::file2Json("geometry/geometry_params.json", gs, file::comments::are_discarded);
     }
     else if( argc == 3)
     {
-        file::file2Json(argv[1], js, "strict");
-        file::file2Json(argv[2], gs, "strict");
+        file::file2Json(argv[1], js, file::comments::are_forbidden);
+        file::file2Json(argv[2], gs, file::comments::are_forbidden);
     }
     else
     {
@@ -53,7 +53,7 @@ int main( int argc, char* argv[])
     std::cout << "Initialize implicit" << std::endl;
     heat::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
     /////////glfw initialisation ////////////////////////////////////////
-    file::file2Json("window_params.json", js, "default");
+    file::file2Json("window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     //////////////////////////////////////////////////////////////////////
diff --git a/src/heat/heat_hpc.cu b/src/heat/heat_hpc.cu
index 01e1c88d8..14bff4b44 100644
--- a/src/heat/heat_hpc.cu
+++ b/src/heat/heat_hpc.cu
@@ -32,8 +32,8 @@ int main( int argc, char* argv[])
     }
     else
     {
-        file::file2Json(argv[1], js, "strict");
-        file::file2Json(argv[2], gs, "strict");
+        file::file2Json(argv[1], js, file::comments::are_forbidden);
+        file::file2Json(argv[2], gs, file::comments::are_forbidden);
     }
     const heat::Parameters p( js); p.display( std::cout);
     const dg::geo::solovev::Parameters gp(gs); gp.display( std::cout);
@@ -70,8 +70,8 @@ int main( int argc, char* argv[])
         std::string geomin(length, 'x');
         errin = nc_get_att_text( ncidin, NC_GLOBAL, "geomfile", &geomin[0]);
         Json::Value js,gs;
-        file::string2Json(inputin, js, "strict");
-        file::string2Json(geomin, gs, "strict");
+        file::string2Json(inputin, js, file::comments::are_forbidden);
+        file::string2Json(geomin, gs, file::comments::are_forbidden);
         std::cout << "input in"<<inputin<<std::endl;
         std::cout << "geome in"<<geomin <<std::endl;
         const heat::Parameters pin(js);
diff --git a/src/impurities/toeflI.cu b/src/impurities/toeflI.cu
index e3a851caf..3deddbf12 100644
--- a/src/impurities/toeflI.cu
+++ b/src/impurities/toeflI.cu
@@ -17,9 +17,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "default");
+        file::file2Json( "input.json", js, file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "default");
+        file::file2Json( argv[1], js, file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -28,7 +28,7 @@ int main( int argc, char* argv[])
     const imp::Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/impurities/toefl_hpc.cu b/src/impurities/toefl_hpc.cu
index ee9d8d74f..72ccf6ee1 100644
--- a/src/impurities/toefl_hpc.cu
+++ b/src/impurities/toefl_hpc.cu
@@ -20,7 +20,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     std::cout << js<<std::endl;
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const imp::Parameters p( js);
diff --git a/src/impurities/toefl_mpi.cu b/src/impurities/toefl_mpi.cu
index 0d7d4ccc0..32ef33c1a 100644
--- a/src/impurities/toefl_mpi.cu
+++ b/src/impurities/toefl_mpi.cu
@@ -48,7 +48,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else
-        file::file2Json(argv[1], js, "strict");
+        file::file2Json(argv[1], js, file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const imp::Parameters p( js);
     if(rank==0)p.display( std::cout);
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 3dd6948ee..0f4b59b7b 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -29,7 +29,7 @@ int main( int argc, char* argv[])
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input/default.json", js, "default");
+        file::file2Json( "input/default.json", js, file::comments::are_discarded);
     else if( argc == 2)
         file::file2Json( argv[1], js);
     else
diff --git a/src/polar/polar.cu b/src/polar/polar.cu
index b6531d6f9..fbc088fd1 100644
--- a/src/polar/polar.cu
+++ b/src/polar/polar.cu
@@ -98,9 +98,9 @@ int main(int argc, char* argv[])
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "default");
+        file::file2Json( "input.json", js, file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "default");
+        file::file2Json( argv[1], js, file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
diff --git a/src/polar/polar_mpi.cu b/src/polar/polar_mpi.cu
index 1f250da99..a556eb90d 100644
--- a/src/polar/polar_mpi.cu
+++ b/src/polar/polar_mpi.cu
@@ -38,9 +38,9 @@ int main(int argc, char* argv[])
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "default");
+        file::file2Json( "input.json", js, file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "default");
+        file::file2Json( argv[1], js, file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
diff --git a/src/reco2D/reconnection.cu b/src/reco2D/reconnection.cu
index 69743fe01..6a897ca62 100644
--- a/src/reco2D/reconnection.cu
+++ b/src/reco2D/reconnection.cu
@@ -19,9 +19,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "default");
+        file::file2Json( "input.json", js, file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "default");
+        file::file2Json( argv[1], js, file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -30,7 +30,7 @@ int main( int argc, char* argv[])
     const asela::Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/reco2D/reconnection_hpc.cu b/src/reco2D/reconnection_hpc.cu
index 66e71ec7d..98189f386 100644
--- a/src/reco2D/reconnection_hpc.cu
+++ b/src/reco2D/reconnection_hpc.cu
@@ -19,7 +19,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     const asela::Parameters p( js);
     p.display( std::cout);
     std::string input = js.toStyledString();
diff --git a/src/reco2D/reconnection_mpi.cu b/src/reco2D/reconnection_mpi.cu
index 3562f3062..aee79ec94 100644
--- a/src/reco2D/reconnection_mpi.cu
+++ b/src/reco2D/reconnection_mpi.cu
@@ -52,7 +52,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     const asela::Parameters p( js);
     if(rank==0)p.display( std::cout);
     std::string input = js.toStyledString();
diff --git a/src/toefl/toeflR.cu b/src/toefl/toeflR.cu
index 2ec250ee7..537f17a06 100644
--- a/src/toefl/toeflR.cu
+++ b/src/toefl/toeflR.cu
@@ -24,9 +24,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, "strict");
+        file::file2Json( "input.json", js, file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -35,7 +35,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, "default");
+    file::file2Json( "window_params.json", js, file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/toefl/toefl_hpc.cu b/src/toefl/toefl_hpc.cu
index 7362dfaae..88ec3ddf5 100644
--- a/src/toefl/toefl_hpc.cu
+++ b/src/toefl/toefl_hpc.cu
@@ -73,7 +73,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else
-        file::file2Json( argv[1], js, "strict");
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
     MPI_OUT std::cout << js<<std::endl;
     const Parameters p( js);
     MPI_OUT p.display( std::cout);
-- 
GitLab


From 56736bb850e95f3a66166e42535f45cd286a2407 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 May 2020 19:01:36 +0200
Subject: [PATCH 247/540] Revert "Change dz behaviour in feltor"

This reverts commit ac18d3d1d65b5c2526a521cbc3adf2ba08f12664.
The reason is that it did not really help speedwise
---
 inc/dg/elliptic.h       | 38 ++++++++++------------
 inc/dg/elliptic2d_b.cu  |  3 +-
 inc/dg/enums.h          |  6 ----
 inc/dg/helmholtz.h      | 24 +++-----------
 src/feltor/feltor.h     | 71 ++++++++++++++++++++++-------------------
 src/feltor/feltordiag.h | 20 ++++++------
 src/feltor/implicit.h   | 18 +++++------
 7 files changed, 81 insertions(+), 99 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 93a9dd577..4a90a35bf 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -365,16 +365,10 @@ class Elliptic3d
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * the direction of the z derivative is always \c dg::centered
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
-     * @param comp if compute::in_2d, the dz derivatives are not constructed and
-     * the symv function avoids the derivative in z,
-     * compute::in_3d represents the original behaviour.
-     * Restrict the problem to the first 2 dimensions.
-     * This effectively makes the behaviour of \c dg::Elliptic3d
-     * identical to the \c dg::Elliptic class.
      * @note chi is assumed 1 per default
      */
-    Elliptic3d( const Geometry& g, norm no = not_normed, direction dir = forward, value_type jfactor=1., enum compute comp = compute::in_3d ):
-        Elliptic3d( g, g.bcx(), g.bcy(), g.bcz(), no, dir, jfactor, comp)
+    Elliptic3d( const Geometry& g, norm no = not_normed, direction dir = forward, value_type jfactor=1.):
+        Elliptic3d( g, g.bcx(), g.bcy(), g.bcz(), no, dir, jfactor)
     {
     }
 
@@ -390,28 +384,17 @@ class Elliptic3d
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * the direction of the z derivative is always \c dg::centered
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
-     * @param comp if compute::in_2d, the dz derivatives are not constructed and
-     * the symv function avoids the derivative in z,
-     * compute::in_3d represents the original behaviour.
-     * Restrict the problem to the first 2 dimensions.
-     * This effectively makes the behaviour of \c dg::Elliptic3d
-     * identical to the \c dg::Elliptic class.
      * @note chi is the metric tensor multiplied by the volume element per default
      */
-    Elliptic3d( const Geometry& g, bc bcx, bc bcy, bc bcz, norm no = not_normed, direction dir = forward, value_type jfactor = 1., enum compute comp = compute::in_3d )
+    Elliptic3d( const Geometry& g, bc bcx, bc bcy, bc bcz, norm no = not_normed, direction dir = forward, value_type jfactor = 1.)
     {
         m_no=no, m_jfactor=jfactor;
         dg::blas2::transfer( dg::create::dx( g, inverse( bcx), inverse(dir)), m_leftx);
         dg::blas2::transfer( dg::create::dy( g, inverse( bcy), inverse(dir)), m_lefty);
-        m_multiplyZ = false;
-        if(comp == compute::in_3d)
-            m_multiplyZ = true;
-        if(m_multiplyZ)
-            dg::blas2::transfer( dg::create::dz( g, inverse( bcz), inverse(dg::centered)), m_leftz);
+        dg::blas2::transfer( dg::create::dz( g, inverse( bcz), inverse(dg::centered)), m_leftz);
         dg::blas2::transfer( dg::create::dx( g, bcx, dir), m_rightx);
         dg::blas2::transfer( dg::create::dy( g, bcy, dir), m_righty);
-        if(m_multiplyZ)
-            dg::blas2::transfer( dg::create::dz( g, bcz, dg::centered), m_rightz);
+        dg::blas2::transfer( dg::create::dz( g, bcz, dg::centered), m_rightz);
         dg::blas2::transfer( dg::create::jumpX( g, bcx),   m_jumpX);
         dg::blas2::transfer( dg::create::jumpY( g, bcy),   m_jumpY);
 
@@ -489,6 +472,17 @@ class Elliptic3d
     ///@copydoc Elliptic::get_jfactor()
     value_type get_jfactor() const {return m_jfactor;}
 
+    /**
+     * @brief Restrict the problem to the first 2 dimensions
+     *
+     * This effectively makes the behaviour of dg::Elliptic3d
+     * identical to the dg::Elliptic class.
+     * @param compute_in_2d if true, the symv function avoids the derivative in z, false reverts to the original behaviour.
+     */
+    void set_compute_in_2d( bool compute_in_2d ) {
+        m_multiplyZ = !compute_in_2d;
+    }
+
     ///@copydoc Elliptic::symv(const ContainerType0&,ContainerType1&)
     template<class ContainerType0, class ContainerType1>
     void symv( const ContainerType0& x, ContainerType1& y){
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 25f864085..2a1eaa893 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -137,7 +137,8 @@ int main()
     {
         //try the compute_in_2d handle of Elliptic3d
 	    dg::CartesianGrid3d grid( 0, lx, 0, ly, 0,1,n, Nx, Ny, 1, bcx, bcy, dg::PER);
-		dg::Elliptic3d<dg::CartesianGrid3d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor, dg::compute::in_2d);
+		dg::Elliptic3d<dg::CartesianGrid3d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
+        pol_backward.set_compute_in_2d(true);
 		pol_backward.set_chi( chi);
 		x = temp;
 		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index a8376db36..7371ac869 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -143,11 +143,5 @@ enum class coo3d : char
     xz = 'c', //!< xz plane
 };
 
-///@brief compute mode in Elliptic3d
-enum class compute{
-    in_2d, //!< compute in 2d
-    in_3d //!< compute in 3d
-};
-
 ///@}
 }//namespace dg
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index e6542b75b..c4c213f1e 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -178,20 +178,9 @@ struct Helmholtz3d
         m_chi( m_laplaceM.weights())
     {
     }
-    /**
-     * @brief Construct
-     *
-     * @param g The grid to use (boundary conditions are taken from there)
-     * @param alpha Scalar in the above formula
-     * @param dir Direction of the Laplace operator
-     * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
-     * @param comp if compute::in_2d, the dz derivatives are not constructed and
-     * the symv function avoids the derivative in z and uses a 2d Laplacian;
-     * compute::in_3d represents the original behaviour.
-     * @note The default value of \f$\chi\f$ is one. \c Helmholtz is never normed
-     */
-    Helmholtz3d( const Geometry& g, value_type alpha = 1., direction dir = dg::forward, value_type jfactor=1., enum compute comp = compute::in_3d):
-        Helmholtz3d( g, g.bcx(), g.bcy(), g.bcz(), alpha, dir, jfactor, comp)
+    ///@copydoc Helmholtz::Helmholtz(const Geometry&,value_type,direction,value_type)
+    Helmholtz3d( const Geometry& g, value_type alpha = 1., direction dir = dg::forward, value_type jfactor=1.):
+        Helmholtz3d( g, g.bcx(), g.bcy(), g.bcz(), alpha, dir, jfactor)
     {
     }
     /**
@@ -204,13 +193,10 @@ struct Helmholtz3d
      * @param alpha Scalar in the above formula
      * @param dir Direction of the Laplace operator
      * @param jfactor The jfactor used in the Laplace operator (probably 1 is always the best factor but one never knows...)
-     * @param comp if compute::in_2d, the dz derivatives are not constructed and
-     * the symv function avoids the derivative in z and uses a 2d Laplacian;
-     * compute::in_3d represents the original behaviour.
      * @note The default value of \f$\chi\f$ is one. \c Helmholtz is never normed
      */
-    Helmholtz3d( const Geometry& g, bc bcx, bc bcy, bc bcz, value_type alpha = 1., direction dir = dg::forward, value_type jfactor=1., enum compute comp = compute::in_3d):
-        m_laplaceM( g, bcx, bcy, bcz, dg::not_normed, dir, jfactor, comp),
+    Helmholtz3d( const Geometry& g, bc bcx, bc bcy, bc bcz, value_type alpha = 1., direction dir = dg::forward, value_type jfactor=1.):
+        m_laplaceM( g, bcx, bcy, bcz, dg::not_normed, dir, jfactor),
         m_chi( m_laplaceM.weights()),
         m_alpha(alpha)
     {
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 0d58246af..03ac6a37f 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -13,7 +13,6 @@
 
 namespace feltor
 {
-//the memory access to explicit zeros in the case of non-true curvature mode is negligible in the following
 
 namespace routines{
 struct ComputePerpDrifts{
@@ -219,7 +218,6 @@ struct Explicit
         return m_apar;
     }
     const std::array<Container, 3> & gradN (int i) const {
-        //note that gradN[2] is zero if curvmore != "true"
         return m_dN[i];
     }
     const std::array<Container, 3> & gradU (int i) const {
@@ -249,18 +247,22 @@ struct Explicit
         // grad S_ne and grad S_ni
         dg::blas2::symv( m_dx_N, m_s[0][i], gradS[0]);
         dg::blas2::symv( m_dy_N, m_s[0][i], gradS[1]);
-        if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, m_s[0][i], gradS[2]);
-        else dg::blas1::copy( 0., gradS[2]);
+        if(!m_p.symmetric)dg::blas2::symv( m_dz, m_s[0][i], gradS[2]);
+    }
+    const Container & compute_dppN(int i) { //2nd varphi derivative
+        dg::blas2::symv( m_dz, m_fields[0][i], m_temp0);
+        dg::blas2::symv( m_dz, m_temp0, m_temp1);
+        return m_temp1;
+    }
+    const Container & compute_dppP(int i) {
+        dg::blas2::symv( m_dz, m_phi[i], m_temp0);
+        dg::blas2::symv( m_dz, m_temp0, m_temp1);
+        return m_temp1;
     }
     const Container & compute_dppU(int i) {
-        if( "true" == m_p.curvmode)
-        {
-            dg::blas2::symv( m_dz, m_fields[1][i], m_temp0);
-            dg::blas2::symv( m_dz, m_temp0, m_temp1);
-            return m_temp1;
-        }
-        else
-            return m_fields[1][i];
+        dg::blas2::symv( m_dz, m_fields[1][i], m_temp0);
+        dg::blas2::symv( m_dz, m_temp0, m_temp1);
+        return m_temp1;
     }
     const dg::SparseTensor<Container>& projection() const{
         return m_hh;
@@ -480,15 +482,18 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
     for( int i=0; i<3; i++)
         dg::blas1::pointwiseDot( vol, m_b[i], m_b[i]); //b_i/vol/B
     m_hh = dg::geo::createProjectionTensor( bhat, g);
-    dg::compute comp = dg::compute::in_2d;
-    if( p.curvmode == "true")
-        comp = dg::compute::in_3d;
-    m_lapperpN.construct ( g, p.bcxN, p.bcyN, dg::PER, dg::normed, dg::centered, 1., comp),
-    m_lapperpU.construct ( g, p.bcxU, p.bcyU, dg::PER, dg::normed, dg::centered, 1., comp),
-    m_lapperpP.construct ( g, p.bcxP, p.bcyP, dg::PER, dg::normed, dg::centered, 1., comp),
+    m_lapperpN.construct ( g, p.bcxN, p.bcyN, dg::PER, dg::normed, dg::centered),
+    m_lapperpU.construct ( g, p.bcxU, p.bcyU, dg::PER, dg::normed, dg::centered),
+    m_lapperpP.construct ( g, p.bcxP, p.bcyP, dg::PER, dg::normed, dg::centered),
     m_lapperpN.set_chi( m_hh);
     m_lapperpU.set_chi( m_hh);
     m_lapperpP.set_chi( m_hh);
+    if( p.curvmode != "true")
+    {
+        m_lapperpN.set_compute_in_2d(true);
+        m_lapperpU.set_compute_in_2d(true);
+        m_lapperpP.set_compute_in_2d(true);
+    }
     m_lapperpP.set_jfactor(0); //we don't want jump terms in source
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
@@ -506,20 +511,17 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
     m_multi_invgammaP.resize(p.stages);
     m_multi_invgammaN.resize(p.stages);
     m_multi_induction.resize(p.stages);
-    dg::compute comp = dg::compute::in_2d;
-    if( p.curvmode == "true")
-        comp = dg::compute::in_3d;
     for( unsigned u=0; u<p.stages; u++)
     {
         m_multi_pol[u].construct( m_multigrid.grid(u),
             p.bcxP, p.bcyP, dg::PER, dg::not_normed,
-            dg::centered, p.jfactor, comp);
+            dg::centered, p.jfactor);
         m_multi_invgammaP[u].construct(  m_multigrid.grid(u),
-            p.bcxP, p.bcyP, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered, 1., comp);
+            p.bcxP, p.bcyP, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered);
         m_multi_invgammaN[u].construct(  m_multigrid.grid(u),
-            p.bcxN, p.bcyN, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered, 1., comp);
+            p.bcxN, p.bcyN, dg::PER, -0.5*p.tau[1]*p.mu[1], dg::centered);
         m_multi_induction[u].construct(  m_multigrid.grid(u),
-            p.bcxU, p.bcyU, dg::PER, -1., dg::centered, 1., comp);
+            p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
 
         dg::SparseTensor<Container> hh = dg::geo::createProjectionTensor(
             bhat, m_multigrid.grid(u));
@@ -527,6 +529,12 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
         m_multi_invgammaP[u].elliptic().set_chi( hh);
         m_multi_invgammaN[u].elliptic().set_chi( hh);
         m_multi_induction[u].elliptic().set_chi( hh);
+        if(p.curvmode != "true"){
+            m_multi_pol[u].set_compute_in_2d( true);
+            m_multi_invgammaP[u].elliptic().set_compute_in_2d( true);
+            m_multi_invgammaN[u].elliptic().set_compute_in_2d( true);
+            m_multi_induction[u].elliptic().set_compute_in_2d( true);
+        }
     }
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
@@ -543,13 +551,12 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     m_dy_N( dg::create::dy( g, p.bcyN) ),
     m_dy_U( dg::create::dy( g, p.bcyU) ),
     m_dy_P( dg::create::dy( g, p.bcyP) ),
+    m_dz( dg::create::dz( g, dg::PER) ),
     m_multigrid( g, p.stages),
     m_old_phi( 2, dg::evaluate( dg::zero, g)),
     m_old_psi( m_old_phi), m_old_gammaN( m_old_phi), m_old_apar( m_old_phi),
     m_p(p), m_full_system(full_system)
 {
-    if( "true" == m_p.curvmode)
-        m_dz = dg::create::dz( g, dg::PER);
     //--------------------------init vectors to 0-----------------//
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
     m_UE2 = m_temp2 = m_temp1 = m_temp0;
@@ -695,7 +702,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
     //-------Compute Psi and derivatives
     dg::blas2::symv( m_dx_P, m_phi[0], m_dP[0][0]);
     dg::blas2::symv( m_dy_P, m_phi[0], m_dP[0][1]);
-    if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, m_phi[0], m_dP[0][2]);
+    if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[0], m_dP[0][2]);
     dg::tensor::multiply3d( m_hh, //grad_perp
         m_dP[0][0], m_dP[0][1], m_dP[0][2], m_UE2, m_temp0, m_temp1);
     dg::blas1::subroutine( routines::ComputePsi(), m_phi[1],
@@ -708,7 +715,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
     //m_UE2 now contains u_E^2; also update derivatives
     dg::blas2::symv( m_dx_P, m_phi[1], m_dP[1][0]);
     dg::blas2::symv( m_dy_P, m_phi[1], m_dP[1][1]);
-    if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, m_phi[1], m_dP[1][2]);
+    if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[1], m_dP[1][2]);
 }
 template<class Geometry, class IMatrix, class Matrix, class Container>
 void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
@@ -743,7 +750,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
     //----------Compute Derivatives----------------------------//
     dg::blas2::symv( m_dx_U, m_apar, m_dA[0]);
     dg::blas2::symv( m_dy_U, m_apar, m_dA[1]);
-    if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, m_apar, m_dA[2]);
+    if(!m_p.symmetric) dg::blas2::symv( m_dz, m_apar, m_dA[2]);
 
     //----------Compute Velocities-----------------------------//
     dg::blas1::axpby( 1., fields[1][0], -1./m_p.mu[0], m_apar, fields[1][0]);
@@ -763,10 +770,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
         ////////////////////perpendicular dynamics////////////////////////
         dg::blas2::symv( m_dx_N, y[0][i], m_dN[i][0]);
         dg::blas2::symv( m_dy_N, y[0][i], m_dN[i][1]);
-        if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, y[0][i], m_dN[i][2]);
+        if(!m_p.symmetric) dg::blas2::symv( m_dz, y[0][i], m_dN[i][2]);
         dg::blas2::symv( m_dx_U, fields[1][i], m_dU[i][0]);
         dg::blas2::symv( m_dy_U, fields[1][i], m_dU[i][1]);
-        if( "true" == m_p.curvmode) dg::blas2::symv( m_dz, fields[1][i], m_dU[i][2]);
+        if(!m_p.symmetric) dg::blas2::symv( m_dz, fields[1][i], m_dU[i][2]);
         if( m_p.beta == 0){
             dg::blas1::subroutine( routines::ComputePerpDrifts(
                 m_p.mu[i], m_p.tau[i]),
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index f9ffc0df7..3d289ce23 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -404,16 +404,16 @@ std::vector<Record> diagnostics2d_list = {
              dg::blas1::copy(v.f.dssU(0), result);
         }
     },
-    //{"dppue", "2nd varphi derivative of electron velocity", false,
-    //    []( DVec& result, Variables& v ) {
-    //         dg::blas1::copy(v.f.compute_dppU(0), result);
-    //    }
-    //},
-    //{"dpue2", "1st varphi derivative squared of electron velocity", false,
-    //    []( DVec& result, Variables& v ) {
-    //         dg::blas1::pointwiseDot(v.f.gradU(0)[2], v.f.gradU(0)[2], result);
-    //    }
-    //},
+    {"dppue", "2nd varphi derivative of electron velocity", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::copy(v.f.compute_dppU(0), result);
+        }
+    },
+    {"dpue2", "1st varphi derivative squared of electron velocity", false,
+        []( DVec& result, Variables& v ) {
+             dg::blas1::pointwiseDot(v.f.gradU(0)[2], v.f.gradU(0)[2], result);
+        }
+    },
     {"lperpinv", "Perpendicular density gradient length scale", false,
         []( DVec& result, Variables& v ) {
             const std::array<DVec, 3>& dN = v.f.gradN(0);
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 537ca5b6c..3d2a971c9 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -25,10 +25,7 @@ struct ImplicitDensity
         dg::geo::TokamakMagneticField mag)
     {
         m_p = p;
-        dg::compute comp = dg::compute::in_2d;
-        if( p.curvmode == "true")
-            comp = dg::compute::in_3d;
-        m_lapM_perpN.construct( g, p.bcxN, p.bcyN,dg::PER, dg::normed, dg::centered, 1., comp);
+        m_lapM_perpN.construct( g, p.bcxN, p.bcyN,dg::PER, dg::normed, dg::centered);
         dg::assign( dg::evaluate( dg::zero, g), m_temp);
         auto bhat = dg::geo::createEPhi(+1); //bhat = ephi except when "true"
         if( p.curvmode == "true")
@@ -39,6 +36,8 @@ struct ImplicitDensity
             = dg::geo::createProjectionTensor( bhat, g);
         //set perpendicular projection tensor h
         m_lapM_perpN.set_chi( hh);
+        if( p.curvmode != "true")
+            m_lapM_perpN.set_compute_in_2d( true);
     }
 
     void operator()( double t, const std::array<Container,2>& y,
@@ -91,12 +90,9 @@ struct ImplicitVelocity
     void construct( const Geometry& g, feltor::Parameters p,
             dg::geo::TokamakMagneticField mag)
     {
-        dg::compute comp = dg::compute::in_2d;
-        if( p.curvmode == "true")
-            comp = dg::compute::in_3d;
         m_p=p;
         m_lapM_perpU.construct( g, p.bcxU,p.bcyU,dg::PER,
-            dg::normed, dg::centered, 1., comp);
+            dg::normed, dg::centered);
         if( !(p.perp_diff == "viscous" || p.perp_diff == "hyperviscous") )
             throw dg::Error(dg::Message(_ping_)<<"Warning! perp_diff value '"<<p.perp_diff<<"' not recognized!! I do not know how to proceed! Exit now!");
         dg::assign( dg::evaluate( dg::zero, g), m_temp);
@@ -112,6 +108,8 @@ struct ImplicitVelocity
             = dg::geo::createProjectionTensor( bhat, g);
         //set perpendicular projection tensor h
         m_lapM_perpU.set_chi( hh);
+        if( p.curvmode != "true")
+            m_lapM_perpU.set_compute_in_2d(true);
         //m_induction.construct(  g,
         //    p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
         //m_induction.elliptic().set_chi( hh);
@@ -124,8 +122,10 @@ struct ImplicitVelocity
             dg::SparseTensor<Container> hh = dg::geo::createProjectionTensor(
                 bhat, m_multigrid.grid(u));
             m_multi_induction[u].construct(  m_multigrid.grid(u),
-                p.bcxU, p.bcyU, dg::PER, -1., dg::centered, 1., comp);
+                p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
             m_multi_induction[u].elliptic().set_chi( hh);
+            if( p.curvmode != "true")
+                m_multi_induction[u].elliptic().set_compute_in_2d(true);
         }
         m_multi_chi = m_multigrid.project( m_temp);
         m_old_apar = dg::Extrapolation<Container>( 1, dg::evaluate( dg::zero, g));
-- 
GitLab


From 4637ca12f9c5bc9f42a062d9f4799d3c70353e4f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 May 2020 19:49:05 +0200
Subject: [PATCH 248/540] Insert two more error conditions in feltor

- about the partition of Nz in MPI
---
 src/feltor/feltor.tex    | 26 +++++++++++++++++++++-----
 src/feltor/feltor_hpc.cu |  8 ++++++++
 src/feltor/feltordiag.cu | 10 +++++++++-
 3 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index cd6b1137a..df4af5f38 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1254,7 +1254,7 @@ Compilation:\\
 \texttt{make feltor\_mpi device=\{gpu,omp,skl,knl\}} Compile \texttt{feltor\_hpc.cu} for distributed memory systems. Also needs {\it serial netcdf}\\
 Usage:\\
 \texttt{./feltor\_hpc input.json geometry.json output.nc [initial.nc]} \\
-\texttt{./feltor\_mpi input.json geometry.json output.nc [initial.nc]} \\
+\texttt{echo npx npy npz | mpirun -n np ./feltor\_mpi input.json geometry.json output.nc [initial.nc]} \\
 \texttt{./feltor input.json geometry.json } \\
 
 The programs \texttt{feltor\_hpc.cu} and \texttt{feltor.cu} expect two input
@@ -1270,6 +1270,14 @@ the output file \texttt{output.nc}.
  Both programs write unstructured human readable performance information of the running simulation
  to \texttt{std::cout}.
 
+Note that when compiled for mpi, the program \texttt{feltor\_hpc.cu} expects the
+partition of the total number of processes np into the three directions x, y and z
+as an input from the command line. Make sure that \texttt{npx*npy*npz==np} and that
+they evenly divide the number of grid points in the respective direction! The
+number of stages in the multigrid algorithm and the compression parameters further
+restrict this choice. Also note that the number of processes in a direction must
+not equal the number of grid points in that direction!
+
 
 \subsection{Input file structure} \label{sec:input_file}
 Input file format: json
@@ -1599,7 +1607,7 @@ termination
 \rowcolor{gray!50}\textbf{Error condition} &  \textbf{Handling} \\ \midrule
 An input file does not exist or is otherwise invalid
 &
-Program terminates with an error message to \texttt{std::cerr}
+Program terminates with an error message to \texttt{std::cerr}. \texttt{feltordiag.cu} writes an error to \texttt{std::cerr} and continues with the next input file.
     \\
 An input netcdf file misses a required field
 &
@@ -1618,16 +1626,24 @@ valuable computing time on the cluster due to a typo is bigger than the
 added convenience. We want to be sure that the program
 does what the user wants).
 The other programs just issue warnings
-if a key is not found and just use a default value
+if a key is not found and use a default value
 which is $0$ if not otherwise specified.
     \\
     An input Json file has an invalid value, e.g. a typo in a string value
 &
 Invalid values lead to termination with an error message to \texttt{std::cerr}, once and if program tries to use the value
     \\
-Number of processes in $x$, $y$ and $z$ direction does not match total number of Processes
+    Number of processes in $x$, $y$ and $z$ direction does not match total number of Processes
 &
-Program terminates with an error message to \texttt{std::cerr}
+Program terminates with an error message to \texttt{std::cerr}.
+    \\
+    $2^{s-1}$ or $c_x$ or $c_y$ does not evenly divide $N_x$ and $N_y$, where $s$ is the number of stages in the multigrid algorithm.
+&
+Program terminates on thrown error. Make sure the numbers add up.
+    \\
+    Number of processes in $x$, $y$ and $z$ direction does not evenly divide or is greater or equal $N_x/2^{s-1}$, $N_y/2^{s-1}$ and $N_z$, where $s$ is the number of stages in the multigrid algorithm.
+&
+Program terminates on failed assert
     \\
 An MPI error occurs
 &
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index a79787b94..0440b9da5 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -148,6 +148,14 @@ int main( int argc, char* argv[])
     MPI_OUT p.display( std::cout);
     MPI_OUT gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
+#ifdef FELTOR_MPI
+    if( np[2] >= (int)p.Nz)
+    {
+        MPI_OUT std::cerr << "ERROR: Number of processes in z "<<np[2]<<" may not be larger or equal Nz "<<p.Nz<<std::endl;
+        MPI_Finalize();
+        return -1;
+    }
+#endif //FELTOR_MPI
     ////////////////////////////////set up computations///////////////////////////
     double Rmin=gp.R_0-p.boxscaleRm*gp.a;
     double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index b18f3ad58..8b8cd550c 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -289,7 +289,15 @@ int main( int argc, char* argv[])
 
         size_t steps;
         std::cout << "Opening file "<<argv[j]<<"\n";
-        err = nc_open( argv[j], NC_NOWRITE, &ncid); //open 3d file
+        try{
+            err = nc_open( argv[j], NC_NOWRITE, &ncid); //open 3d file
+        } catch ( file::NC_Error& error)
+        {
+            std::cerr << "An error occurded opening file "<<argv[j]<<"\n";
+            std::cerr << error.what()<<std::endl;
+            std::cerr << "Continue with next file\n";
+            continue;
+        }
         err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
         err = nc_inq_dimlen( ncid, timeID, &steps);
         //steps = 3;
-- 
GitLab


From 65fffd2e93a66723ae9211cede242a68de962e5d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 May 2020 21:48:08 +0200
Subject: [PATCH 249/540] Add first and second order Karniadakis coeffs

Hopefully we can achieve a larger timestep with it
---
 inc/dg/multistep.h    | 55 +++++++++++++++++++++++++++++++++++--------
 inc/dg/multistep_t.cu | 15 ++++++++++--
 2 files changed, 58 insertions(+), 12 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 07120811a..18b592cc2 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -172,9 +172,23 @@ void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType&
     \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11} \\
     \gamma_0 = \frac{6}{11}
 \f]
+    for the default third order method and
+    \f[
+    \alpha_0 = \frac{4}{3}\ \alpha_1 = -\frac{1}{3}\ \alpha_2 = 0 \\
+    \beta_0  = \frac{4}{3}\ \beta_1 = -\frac{2}{3}\ \beta_2 = 0 \\
+    \gamma_0 = \frac{2}{3}
+\f]
+    for a second order method and
+    \f[
+    \alpha_0 = 1\ \alpha_1 = 0\ \alpha_2 = 0\\
+    \beta_0  = 1\ \beta_1 = 0\ \beta_2 = 0\\
+    \gamma_0 = 1
+\f]
+for a semi-implicit (first order) Euler method
 *
 * The necessary Inversion in the imlicit part is provided by the \c SolverType class.
 * Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$).
+* @note This scheme implements <a href = "https://dx.doi.org/10.1016/0021-9991(91)90007-8"> Karniadakis, et al. J. Comput. Phys. 97 (1991)</a>
 * @note The implicit part equals a third order backward differentiation formula (BDF) https://en.wikipedia.org/wiki/Backward_differentiation_formula
 *
 The following code example demonstrates how to implement the method of manufactured solutions on a 2d partial differential equation with the dg library:
@@ -201,7 +215,7 @@ struct Karniadakis
     template<class ...SolverParams>
     Karniadakis( SolverParams&& ...ps):m_solver( std::forward<SolverParams>(ps)...){
         m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
-        init_coeffs();
+        set_coefficients(3);
     }
     /**
      * @brief Reserve memory for the integration
@@ -213,7 +227,7 @@ struct Karniadakis
     void construct( SolverParams&& ...ps){
         m_solver = Solver( std::forward<SolverParams>(ps)...);
         m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
-        init_coeffs();
+        set_coefficients(3);
     }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
@@ -248,15 +262,36 @@ struct Karniadakis
     template< class Explicit, class Implicit>
     void step( Explicit& ex, Implicit& im, value_type& t, ContainerType& u);
 
-  private:
-    void init_coeffs(){
-        //a[0] =  1.908535476882378;  b[0] =  1.502575553858997;
-        //a[1] = -1.334951446162515;  b[1] = -1.654746338401493;
-        //a[2] =  0.426415969280137;  b[2] =  0.670051276940255;
-        a[0] =  18./11.;    b[0] =  18./11.;
-        a[1] = -9./11.;     b[1] = -18./11.;
-        a[2] = 2./11.;      b[2] = 6./11.;   //Karniadakis !!!
+    /**
+     * @brief Set the order of the method
+     *
+     * Change the coefficients \f$ \alpha_i,\ \beta_i,\ \gamma_0\f$ to a first,
+     * second or third order set.
+     * @param order 1 (Euler), 2 or 3 (default)
+     */
+    void set_coefficients(unsigned order){
+        switch( order){
+            case 1:
+                a[0] = 1.;  b[0] = 1.;
+                a[1] = 0.;  b[1] = 0.;
+                a[2] = 0.;  b[2] = 0.;   //Euler
+                g0 = 1.;
+                break;
+            case 2:
+                a[0] =  4./3.;  b[0] = 4./3.;
+                a[1] = -1./3.;  b[1] = -2./3.;
+                a[2] = 0.;      b[2] = 0.;   //2nd Karniadakis
+                g0 = 2./3.;
+                break;
+            default:
+                a[0] =  18./11.;    b[0] =  18./11.;
+                a[1] = -9./11.;     b[1] = -18./11.;
+                a[2] = 2./11.;      b[2] = 6./11.;   //Karniadakis
+                g0 = 6./11.;
+                break;
+        }
     }
+  private:
     SolverType m_solver;
     std::array<ContainerType,3> m_u, m_f;
     value_type m_t, m_dt;
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index c98e7d6fc..83e020735 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -154,7 +154,7 @@ int main()
     }
     Explicit<dg::DVec> ex( grid, nu);
     Implicit<dg::DMatrix, dg::DVec> im( grid, nu);
-    std::cout << "### Test semi-implicit Karniadakis methods with "<<NT<<"steps\n";
+    std::cout << "### Test semi-implicit Karniadakis methods with "<<NT<<" steps\n";
     //![karniadakis]
     //construct time stepper
     dg::Karniadakis< dg::DVec > karniadakis( y0, y0.size(), eps);
@@ -167,7 +167,18 @@ int main()
     //![karniadakis]
     dg::blas1::axpby( -1., sol, 1., y0);
     res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
-    std::cout << "Relative error Karniadakis is "<< res.d<<"\t"<<res.i<<std::endl;
+    std::cout << "Relative error Karniadakis 3 is "<< res.d<<"\t"<<res.i<<std::endl;
+    for( unsigned i=2; i>0; i--)
+    {
+        time = 0., y0 = init;
+        karniadakis.set_coefficients(i);
+        karniadakis.init( ex, im, time, y0, dt);
+        for( unsigned i=0; i<NT; i++)
+            karniadakis.step( ex, im, time, y0);
+        dg::blas1::axpby( -1., sol, 1., y0);
+        res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
+        std::cout << "Relative error Karniadakis "<<i<<" is "<< res.d<<"\t"<<res.i<<std::endl;
+    }
 
 
     std::cout << "### Test semi-implicit ARK methods\n";
-- 
GitLab


From 5d48194754e6c72b35a16fe74adb6c3b20173fee Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 May 2020 22:06:16 +0200
Subject: [PATCH 250/540] Add subroutine benchmarks to blas_b and blas_mpib

---
 inc/dg/blas_b.cu    | 33 +++++++++++++++++++++++++++++++--
 inc/dg/blas_mpib.cu | 33 +++++++++++++++++++++++++++++++--
 2 files changed, 62 insertions(+), 4 deletions(-)

diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index e88b64e5e..6c5ad2910 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -20,6 +20,25 @@ struct Expression{
        u = param*u*v + w;
    }
 };
+struct test_routine{
+    test_routine( double mu, double alpha):m_mu(mu), m_alpha(alpha){}
+    DG_DEVICE
+    void operator()( double g11, double g12, double g22, double in1, double in2, double& out1, double& out2){
+        out1 = (g11)*(in1) + (g12)*(in2) + m_mu;
+        out2 = (g12)*(in1) + (g22)*(in2) + m_alpha;
+    }
+private:
+    double m_mu, m_alpha;
+};
+
+struct test_inplace{
+    DG_DEVICE
+    void operator()( double g11, double g12, double g22, double& inout1, double& inout2){
+        double t = g11*inout1 + g12*inout2;
+        inout2 = g12*inout1 + g22*inout2;
+        inout1 = t;
+    }
+};
 
 using value_type= double;
 using Vector    = thrust::device_vector<double>;
@@ -60,7 +79,7 @@ int main()
     int multi=100;
     //t.tic();
     std::cout<<"\nNo communication\n";
-    ArrayVec y(x), z(x), u(x), v(x);
+    ArrayVec y(x), z(x), u(x), v(x), w(x), h(x);
     t.tic();
     for( int i=0; i<multi; i++)
         dg::blas1::axpby( 1., y, -1., x);
@@ -97,7 +116,17 @@ int main()
     for( int i=0; i<multi; i++)
         dg::blas1::subroutine( Expression(), u, v, x, array_p);
     t.toc();
-    std::cout<<"SUBroutine (p*yx+w)              "<<t.diff()/multi<<"s\t" <<4*gbytes*multi/t.diff()<<"GB/s\n";
+    std::cout<<"Subroutine (p*yx+w)              "<<t.diff()/multi<<"s\t" <<4*gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas1::subroutine( test_routine(2.,4.), x, y, z, u, v, w, h);
+    t.toc();
+    std::cout<<"Subroutine ( G Cdot x = y)       "<<t.diff()/multi<<"s\t"<<9*gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas1::subroutine( test_inplace(), x, y, z, u, v);
+    t.toc();
+    std::cout<<"Subroutine ( G Cdot x = x)       "<<t.diff()/multi<<"s\t"<<7*gbytes*multi/t.diff()<<"GB/s\n";
     /////////////////////SYMV////////////////////////////////
     std::cout<<"\nLocal communication\n";
     Matrix M;
diff --git a/inc/dg/blas_mpib.cu b/inc/dg/blas_mpib.cu
index fb1094280..024019475 100644
--- a/inc/dg/blas_mpib.cu
+++ b/inc/dg/blas_mpib.cu
@@ -22,6 +22,25 @@ struct Expression{
        u = param*u*v + w;
    }
 };
+struct test_routine{
+    test_routine( double mu, double alpha):m_mu(mu), m_alpha(alpha){}
+    DG_DEVICE
+    void operator()( double g11, double g12, double g22, double in1, double in2, double& out1, double& out2){
+        out1 = (g11)*(in1) + (g12)*(in2) + m_mu;
+        out2 = (g12)*(in1) + (g22)*(in2) + m_alpha;
+    }
+private:
+    double m_mu, m_alpha;
+};
+
+struct test_inplace{
+    DG_DEVICE
+    void operator()( double g11, double g12, double g22, double& inout1, double& inout2){
+        double t = g11*inout1 + g12*inout2;
+        inout2 = g12*inout1 + g22*inout2;
+        inout1 = t;
+    }
+};
 
 using value_type= double;
 using Vector    = dg::MDVec;
@@ -66,7 +85,7 @@ int main( int argc, char* argv[])
 
     int multi=100;
     if(rank==0)std::cout<<"\nNo communication\n";
-    ArrayVec y(x), z(x), u(x), v(x);
+    ArrayVec y(x), z(x), u(x), v(x), w(x), h(x);
     t.tic();
     for( int i=0; i<multi; i++)
         dg::blas1::axpby( 1., y, -1., x);
@@ -103,7 +122,17 @@ int main( int argc, char* argv[])
     for( int i=0; i<multi; i++)
         dg::blas1::subroutine( Expression(), u, v, x, array_p);
     t.toc();
-    if(rank==0)std::cout<<"SUBroutine (p*yx+w)              "<<t.diff()/multi<<"s\t" <<4*gbytes*multi/t.diff()<<"GB/s\n";
+    if(rank==0)std::cout<<"Subroutine (p*yx+w)              "<<t.diff()/multi<<"s\t" <<4*gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas1::subroutine( test_routine(2.,4.), x, y, z, u, v, w, h);
+    t.toc();
+    if(rank==0)std::cout<<"Subroutine ( G Cdot x = y)       "<<t.diff()/multi<<"s\t"<<9*gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        dg::blas1::subroutine( test_inplace(), x, y, z, u, v);
+    t.toc();
+    if(rank==0)std::cout<<"Subroutine ( G Cdot x = x)       "<<t.diff()/multi<<"s\t"<<7*gbytes*multi/t.diff()<<"GB/s\n";
     /////////////////////SYMV////////////////////////////////
     if(rank==0)std::cout<<"\nLocal communication\n";
     Matrix M;
-- 
GitLab


From 676777823d2c19b0e975661cac0bd27a9e9f2faa Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 27 May 2020 23:25:37 +0200
Subject: [PATCH 251/540] rename set_coefficients to set_order

Is more recognizable
---
 inc/dg/multistep.h    | 6 +++---
 inc/dg/multistep_t.cu | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 18b592cc2..9843d655a 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -215,7 +215,7 @@ struct Karniadakis
     template<class ...SolverParams>
     Karniadakis( SolverParams&& ...ps):m_solver( std::forward<SolverParams>(ps)...){
         m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
-        set_coefficients(3);
+        set_order(3);
     }
     /**
      * @brief Reserve memory for the integration
@@ -227,7 +227,7 @@ struct Karniadakis
     void construct( SolverParams&& ...ps){
         m_solver = Solver( std::forward<SolverParams>(ps)...);
         m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
-        set_coefficients(3);
+        set_order(3);
     }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
@@ -269,7 +269,7 @@ struct Karniadakis
      * second or third order set.
      * @param order 1 (Euler), 2 or 3 (default)
      */
-    void set_coefficients(unsigned order){
+    void set_order(unsigned order){
         switch( order){
             case 1:
                 a[0] = 1.;  b[0] = 1.;
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 83e020735..f810b1d3d 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -168,16 +168,16 @@ int main()
     dg::blas1::axpby( -1., sol, 1., y0);
     res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
     std::cout << "Relative error Karniadakis 3 is "<< res.d<<"\t"<<res.i<<std::endl;
-    for( unsigned i=2; i>0; i--)
+    for( unsigned s=2; s>0; s--)
     {
         time = 0., y0 = init;
-        karniadakis.set_coefficients(i);
+        karniadakis.set_order(s);
         karniadakis.init( ex, im, time, y0, dt);
         for( unsigned i=0; i<NT; i++)
             karniadakis.step( ex, im, time, y0);
         dg::blas1::axpby( -1., sol, 1., y0);
         res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
-        std::cout << "Relative error Karniadakis "<<i<<" is "<< res.d<<"\t"<<res.i<<std::endl;
+        std::cout << "Relative error Karniadakis "<<s<<" is "<< res.d<<"\t"<<res.i<<std::endl;
     }
 
 
-- 
GitLab


From 0bc7022e7e1a9d28e67e2901839a69f831245ff0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 28 May 2020 15:50:11 +0200
Subject: [PATCH 252/540] Fix bug in json_utilities check statements

---
 inc/file/json_utilities.h | 18 ++++++++++--------
 src/feltor/feltordiag.cu  |  1 +
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/inc/file/json_utilities.h b/inc/file/json_utilities.h
index 7e778dd60..b5e8389a8 100644
--- a/inc/file/json_utilities.h
+++ b/inc/file/json_utilities.h
@@ -78,7 +78,7 @@ Json::Value get_idx( enum error err, const Json::Value& js, std::string key, uns
 {
     if( js.isMember(key))
     {
-        if( js[key].isValidIndex(idx))
+        if( js[key].isArray() && js[key].isValidIndex(idx))
             return js[key][idx];
         else
         {
@@ -123,7 +123,7 @@ Json::Value get( enum error err, const Json::Value& js, std::string key, std::st
 {
     if( js.isMember(key))
     {
-        if( js[key].isMember(key2))
+        if( js[key].isObject() && js[key].isMember(key2))
             return js[key][key2];
         else
         {
@@ -169,9 +169,9 @@ Json::Value get_idx( enum error err, const Json::Value& js, std::string key, std
 {
     if( js.isMember(key))
     {
-        if( js[key].isMember(key2))
+        if( js[key].isObject() && js[key].isMember(key2))
         {
-            if( js[key][key2].isValidIndex(idx))
+            if( js[key][key2].isArray() && js[key][key2].isValidIndex(idx))
                 return js[key][key2][idx];
             else
             {
@@ -244,13 +244,13 @@ static inline void file2Json(std::string filename, Json::Value& js, enum comment
     {
         std::string message = "\nAn error occured while parsing "+filename+"\n";
         message +=  "*** File does not exist! *** \n\n";
-        throw std::runtime_error( message);
         if( err == error::is_throw)
             throw std::runtime_error( message);
         else if (err == error::is_warning)
             std::cerr << "WARNING: "<<message<<std::endl;
         else
-            return;
+            ;
+        return;
     }
     std::string errs;
     if( !parseFromStream( parser, isI, &js, &errs) )
@@ -261,7 +261,8 @@ static inline void file2Json(std::string filename, Json::Value& js, enum comment
         else if (err == error::is_warning)
             std::cerr << "WARNING: "<<message<<std::endl;
         else
-            return;
+            ;
+        return;
     }
 }
 /**
@@ -301,7 +302,8 @@ static inline void string2Json(std::string input, Json::Value& js, enum comments
         else if (err == error::is_warning)
             std::cerr << "WARNING: "<<errs<<std::endl;
         else
-            return;
+            ;
+        return;
     }
 }
 
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 8b8cd550c..86436496c 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -44,6 +44,7 @@ int main( int argc, char* argv[])
     Json::Value js,gs;
     file::string2Json(inputfile, js, file::comments::are_forbidden);
     file::string2Json(geomfile, gs, file::comments::are_forbidden);
+    //we only need some parameters from p, not all
     const feltor::Parameters p(js, file::error::is_warning);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
-- 
GitLab


From 57512dfd9aee1a566b8bb352f824acd336a00f12 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mwiesenb@login02.m100.cineca.it>
Date: Sat, 30 May 2020 14:20:48 +0200
Subject: [PATCH 253/540] Begin experimenting with single precision

- fix some value_type bugs
- axpby and symv are approx twice as fast as expected
- blas1::dot surprisingly works for float on GPUs
(it just converts double to float I guess)
- normal cg seems to work fine but nested_iterations do not
- more intensive testing required
---
 inc/dg/blas_mpib.cu              | 40 ++++++++---------
 inc/dg/elliptic2d_mpib.cu        | 73 +++++++++++++++++---------------
 inc/dg/elliptic_mpib.cu          | 72 +++++++++++++++++--------------
 inc/dg/multigrid.h               | 24 +++++------
 inc/dg/topology/mpi_grid.h       |  2 +-
 inc/dg/topology/split_and_join.h | 22 +++++-----
 6 files changed, 124 insertions(+), 109 deletions(-)

diff --git a/inc/dg/blas_mpib.cu b/inc/dg/blas_mpib.cu
index 024019475..cdf97bad6 100644
--- a/inc/dg/blas_mpib.cu
+++ b/inc/dg/blas_mpib.cu
@@ -12,41 +12,41 @@
 #include "topology/mpi_weights.h"
 #include "topology/fast_interpolation.h"
 
-const double lx = 2.*M_PI;
-const double ly = 2.*M_PI;
-double left( double x, double y, double z) {return sin(x)*cos(y)*z;}
-double right( double x, double y, double z) {return cos(x)*sin(y)*z;}
+using value_type= double;
+using Vector    = dg::MDVec;
+using Matrix    = dg::MDMatrix;
+using ArrayVec  = std::array<Vector, 3>;
+
+const value_type lx = 2.*M_PI;
+const value_type ly = 2.*M_PI;
+value_type left( value_type x, value_type y, value_type z) {return sin(x)*cos(y)*z;}
+value_type right( value_type x, value_type y, value_type z) {return cos(x)*sin(y)*z;}
+
 struct Expression{
    DG_DEVICE
-   void operator() ( double& u, double v, double w, double param){
+   void operator() ( value_type& u, value_type v, value_type w, value_type param){
        u = param*u*v + w;
    }
 };
 struct test_routine{
-    test_routine( double mu, double alpha):m_mu(mu), m_alpha(alpha){}
+    test_routine( value_type mu, value_type alpha):m_mu(mu), m_alpha(alpha){}
     DG_DEVICE
-    void operator()( double g11, double g12, double g22, double in1, double in2, double& out1, double& out2){
+    void operator()( value_type g11, value_type g12, value_type g22, value_type in1, value_type in2, value_type& out1, value_type& out2){
         out1 = (g11)*(in1) + (g12)*(in2) + m_mu;
         out2 = (g12)*(in1) + (g22)*(in2) + m_alpha;
     }
 private:
-    double m_mu, m_alpha;
+    value_type m_mu, m_alpha;
 };
-
 struct test_inplace{
     DG_DEVICE
-    void operator()( double g11, double g12, double g22, double& inout1, double& inout2){
-        double t = g11*inout1 + g12*inout2;
+    void operator()( value_type g11, value_type g12, value_type g22, value_type& inout1, value_type& inout2){
+        value_type t = g11*inout1 + g12*inout2;
         inout2 = g12*inout1 + g22*inout2;
         inout1 = t;
     }
 };
 
-using value_type= double;
-using Vector    = dg::MDVec;
-using Matrix    = dg::MDMatrix;
-using ArrayVec  = std::array<Vector, 3>;
-
 int main( int argc, char* argv[])
 {
 #ifdef _OPENMP
@@ -68,8 +68,8 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "    npz: # of processes in z (must divide Nz and total # of processes!\n";
     dg::mpi_init3d( dg::PER, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
 
-    dg::MPIGrid3d grid( 0., lx, 0, ly,0., ly, n, Nx, Ny, Nz, comm);
-    dg::MPIGrid3d grid_half = grid; grid_half.multiplyCellNumbers(0.5, 0.5);
+    dg::RealMPIGrid3d<value_type> grid( 0., lx, 0, ly,0., ly, n, Nx, Ny, Nz, comm);
+    dg::RealMPIGrid3d<value_type> grid_half = grid; grid_half.multiplyCellNumbers(0.5, 0.5);
     Vector w2d;
     dg::assign( dg::create::weights(grid), w2d);
     dg::Timer t;
@@ -77,7 +77,7 @@ int main( int argc, char* argv[])
     ArrayVec x;
     dg::assign( dg::evaluate( left, grid), x);
     t.toc();
-    double gbytes=(double)x.size()*grid.size()*sizeof(double)/1e9;
+    value_type gbytes=(value_type)x.size()*grid.size()*sizeof(value_type)/1e9;
     if(rank==0)std::cout << "Sizeof vectors is "<<gbytes<<" GB\n";
     dg::MultiMatrix<Matrix, ArrayVec> inter, project;
     dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2), inter);
@@ -117,7 +117,7 @@ int main( int argc, char* argv[])
     t.toc();
     if(rank==0)std::cout<<"pointwiseDot (1*yx+2*uv=v) (A)   "<<t.diff()/multi<<"s\t" <<5*gbytes*multi/t.diff()<<"GB/s\n";
     //Test new evaluate
-    std::array<double, 3> array_p{ 1,2,3};
+    std::array<value_type, 3> array_p{ 1,2,3};
     t.tic();
     for( int i=0; i<multi; i++)
         dg::blas1::subroutine( Expression(), u, v, x, array_p);
diff --git a/inc/dg/elliptic2d_mpib.cu b/inc/dg/elliptic2d_mpib.cu
index ace08e494..783ec7aed 100644
--- a/inc/dg/elliptic2d_mpib.cu
+++ b/inc/dg/elliptic2d_mpib.cu
@@ -13,23 +13,29 @@
 //
 //global relative error in L2 norm is O(h^P)
 //as a rule of thumb with n=4 the true error is err = 1e-3 * eps as long as eps > 1e3*err
-
-const double lx = M_PI;
-const double ly = 2.*M_PI;
+//using value_type = float;
+//using Vector = dg::fMDVec;
+//using Matrix = dg::fMDMatrix;
+using value_type = double;
+using Vector = dg::MDVec;
+using Matrix = dg::MDMatrix;
+
+const value_type lx = M_PI;
+const value_type ly = 2.*M_PI;
 dg::bc bcx = dg::DIR;
 dg::bc bcy = dg::PER;
 
-double initial( double x, double y) {return 0.;}
-double amp = 0.9999;
-double pol( double x, double y) {return 1. + amp*sin(x)*sin(y); } //must be strictly positive
-//double pol( double x, double y) {return 1.; }
-//double pol( double x, double y) {return 1. + sin(x)*sin(y) + x; } //must be strictly positive
+value_type initial( value_type x, value_type y) {return 0.;}
+value_type amp = 0.9999;
+value_type pol( value_type x, value_type y) {return 1. + amp*sin(x)*sin(y); } //must be strictly positive
+//value_type pol( value_type x, value_type y) {return 1.; }
+//value_type pol( value_type x, value_type y) {return 1. + sin(x)*sin(y) + x; } //must be strictly positive
 
-double rhs( double x, double y) { return 2.*sin(x)*sin(y)*(amp*sin(x)*sin(y)+1)-amp*sin(x)*sin(x)*cos(y)*cos(y)-amp*cos(x)*cos(x)*sin(y)*sin(y);}
-//double rhs( double x, double y) { return 2.*sin( x)*sin(y);}
-//double rhs( double x, double y) { return 2.*sin(x)*sin(y)*(sin(x)*sin(y)+1)-sin(x)*sin(x)*cos(y)*cos(y)-cos(x)*cos(x)*sin(y)*sin(y)+(x*sin(x)-cos(x))*sin(y) + x*sin(x)*sin(y);}
-double sol(double x, double y)  { return sin( x)*sin(y);}
-double der(double x, double y)  { return cos( x)*sin(y);}
+value_type rhs( value_type x, value_type y) { return 2.*sin(x)*sin(y)*(amp*sin(x)*sin(y)+1)-amp*sin(x)*sin(x)*cos(y)*cos(y)-amp*cos(x)*cos(x)*sin(y)*sin(y);}
+//value_type rhs( value_type x, value_type y) { return 2.*sin( x)*sin(y);}
+//value_type rhs( value_type x, value_type y) { return 2.*sin(x)*sin(y)*(sin(x)*sin(y)+1)-sin(x)*sin(x)*cos(y)*cos(y)-cos(x)*cos(x)*sin(y)*sin(y)+(x*sin(x)-cos(x))*sin(y) + x*sin(x)*sin(y);}
+value_type sol(value_type x, value_type y)  { return sin( x)*sin(y);}
+value_type der(value_type x, value_type y)  { return cos( x)*sin(y);}
 
 
 int main(int argc, char* argv[] )
@@ -42,31 +48,32 @@ int main(int argc, char* argv[] )
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     dg::Timer t;
-    double eps = 1e-6;
+    value_type eps = 1e-4;
+
     //if(rank==0)std::cout << "Type epsilon! \n";
     //if(rank==0)std::cin >> eps;
-    MPI_Bcast(  &eps,1 , MPI_DOUBLE, 0, comm);
+    MPI_Bcast(  &eps,1 , dg::getMPIDataType<value_type>(), 0, comm);
     //////////////////////begin program///////////////////////
     //create functions A(chi) x = b
-    dg::CartesianMPIGrid2d grid( 0., lx, 0, ly, n, Nx, Ny, bcx, bcy, comm);
-    const dg::MDVec w2d = dg::create::weights( grid);
-    const dg::MDVec v2d = dg::create::inv_weights( grid);
-    dg::MDVec x =    dg::evaluate( initial, grid);
-    dg::MDVec b =    dg::evaluate( rhs, grid);
-    dg::MDVec chi =  dg::evaluate( pol, grid);
+    dg::RealCartesianMPIGrid2d<value_type> grid( 0., lx, 0, ly, n, Nx, Ny, bcx, bcy, comm);
+    const Vector w2d = dg::create::weights( grid);
+    const Vector v2d = dg::create::inv_weights( grid);
+    Vector x =    dg::evaluate( initial, grid);
+    Vector b =    dg::evaluate( rhs, grid);
+    Vector chi =  dg::evaluate( pol, grid);
 
 
 
-    if(rank==0)std::cout << "Create Polarisation object and set chi!\n";
+    if(rank==0)std::cout <<rank<< "Create Polarisation object and set chi!\n";
     t.tic();
-    //dg::Elliptic<dg::CartesianMPIGrid2d, dg::MDMatrix, dg::MDVec> pol( grid, dg::not_normed, dg::centered);
+    //dg::Elliptic<dg::RealCartesianMPIGrid2d<value_type>, Matrix, Vector> pol( grid, dg::not_normed, dg::centered);
     //pol.set_chi( chi);
     unsigned stages = 3;
 
-    dg::MultigridCG2d<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec > multigrid( grid, stages, 0);
+    dg::MultigridCG2d<dg::aRealMPIGeometry2d<value_type>, Matrix, Vector > multigrid( grid, stages, 0);
 
-    std::vector<dg::MDVec> chi_ = multigrid.project( chi);
-    std::vector<dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec> > multi_pol( stages);
+    std::vector<Vector> chi_ = multigrid.project( chi);
+    std::vector<dg::Elliptic<dg::aRealMPIGeometry2d<value_type>, Matrix, Vector> > multi_pol( stages);
 
     for(unsigned u=0; u<stages; u++)
     {
@@ -76,7 +83,7 @@ int main(int argc, char* argv[] )
     t.toc();
     if(rank==0)std::cout << "Creation of polarisation object took: "<<t.diff()<<"s\n";
 
-    //dg::Invert<dg::MDVec > invert( x, n*n*Nx*Ny, eps);
+    //dg::Invert<Vector > invert( x, n*n*Nx*Ny, eps);
     t.tic();
     //unsigned number = invert( pol, x, b);
     std::vector<unsigned> number = multigrid.direct_solve( multi_pol, x, b, eps);
@@ -87,15 +94,15 @@ int main(int argc, char* argv[] )
     if(rank==0)std::cout << " took "<<t.diff()<<"s\n";
 
     //compute error
-    const dg::MDVec solution = dg::evaluate( sol, grid);
-    const dg::MDVec derivati = dg::evaluate( der, grid);
-    dg::MDVec error( solution);
+    const Vector solution = dg::evaluate( sol, grid);
+    const Vector derivati = dg::evaluate( der, grid);
+    Vector error( solution);
     dg::blas1::axpby( 1.,x,-1., error);
 
-    double err = dg::blas2::dot( w2d, error);
-    double norm = dg::blas2::dot( w2d, solution);
+    value_type err = dg::blas2::dot( w2d, error);
+    value_type norm = dg::blas2::dot( w2d, solution);
     if(rank==0)std::cout << "L2 Norm of relative error is               "<<sqrt( err/norm)<<std::endl;
-    dg::MDMatrix DX = dg::create::dx( grid);
+    Matrix DX = dg::create::dx( grid);
     dg::blas2::gemv( DX, x, error);
     dg::blas1::axpby( 1.,derivati,-1., error);
     err = dg::blas2::dot( w2d, error);
diff --git a/inc/dg/elliptic_mpib.cu b/inc/dg/elliptic_mpib.cu
index eb7ec79b4..1b10ce0c0 100644
--- a/inc/dg/elliptic_mpib.cu
+++ b/inc/dg/elliptic_mpib.cu
@@ -10,19 +10,27 @@
 #include "backend/mpi_init.h"
 #include "topology/split_and_join.h"
 
+//using value_type = float;
+//using Matrix = dg::fMDMatrix;
+//using Vector = dg::fMDVec;
+//using LVector = dg::fDVec;
+using value_type = double;
+using Matrix = dg::MDMatrix;
+using Vector = dg::MDVec;
+using LVector = dg::DVec;
 
-const double R_0 = 10;
-const double lx = 2.*M_PI;
-const double ly = 2.*M_PI;
-const double lz = 2.*M_PI;
-double fct(double x, double y, double z){ return sin(x-R_0)*sin(y)*sin(z);}
-double derivative( double x, double y, double z){return cos(x-R_0)*sin(y)*sin(z);}
-double laplace2d_fct( double x, double y, double z) { return -1./x*cos(x-R_0)*sin(y)*sin(z) + 2.*fct(x,y,z);}
-double laplace3d_fct( double x, double y, double z) { return -1./x*cos(x-R_0)*sin(y)*sin(z) + 2.*fct(x,y,z) + 1./x/x*fct(x,y,z);}
+const value_type R_0 = 10;
+const value_type lx = 2.*M_PI;
+const value_type ly = 2.*M_PI;
+const value_type lz = 2.*M_PI;
+value_type fct(value_type x, value_type y, value_type z){ return sin(x-R_0)*sin(y)*sin(z);}
+value_type derivative( value_type x, value_type y, value_type z){return cos(x-R_0)*sin(y)*sin(z);}
+value_type laplace2d_fct( value_type x, value_type y, value_type z) { return -1./x*cos(x-R_0)*sin(y)*sin(z) + 2.*fct(x,y,z);}
+value_type laplace3d_fct( value_type x, value_type y, value_type z) { return -1./x*cos(x-R_0)*sin(y)*sin(z) + 2.*fct(x,y,z) + 1./x/x*fct(x,y,z);}
 dg::bc bcx = dg::DIR;
 dg::bc bcy = dg::DIR;
 dg::bc bcz = dg::PER;
-double initial( double x, double y, double z) {return sin(0);}
+value_type initial( value_type x, value_type y, value_type z) {return sin(0);}
 
 
 int main( int argc, char* argv[])
@@ -32,33 +40,33 @@ int main( int argc, char* argv[])
     MPI_Comm comm;
     dg::mpi_init3d( bcx, bcy, dg::PER, n, Nx, Ny, Nz, comm);
 
-    dg::CylindricalMPIGrid3d grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, bcy, dg::PER, comm);
-    const dg::MDVec w3d = dg::create::volume( grid);
-    const dg::MDVec v3d = dg::create::inv_volume( grid);
+    dg::RealCylindricalMPIGrid3d<value_type> grid( R_0, R_0+lx, 0, ly, 0,lz, n, Nx, Ny,Nz, bcx, bcy, dg::PER, comm);
+    const Vector w3d = dg::create::volume( grid);
+    const Vector v3d = dg::create::inv_volume( grid);
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
-    double eps=1e-6;
+    value_type eps=1e-6;
     if(rank==0)std::cout << "Type epsilon! \n";
     if(rank==0)std::cin >> eps;
-    MPI_Bcast(  &eps,1 , MPI_DOUBLE, 0, comm);
+    MPI_Bcast(  &eps,1 , dg::getMPIDataType<value_type>(), 0, comm);
     /////////////////////////////////////////////////////////////////
     if(rank==0)std::cout<<"TEST CYLINDRIAL LAPLACIAN!\n";
     dg::Timer t;
-    dg::MDVec x = dg::evaluate( initial, grid);
+    Vector x = dg::evaluate( initial, grid);
 
     if(rank==0)std::cout << "Create Laplacian\n";
     t.tic();
-    dg::Elliptic3d<dg::aMPIGeometry3d, dg::MDMatrix, dg::MDVec> laplace(grid, dg::not_normed, dg::centered);
-    dg::MDMatrix DX = dg::create::dx( grid);
+    dg::Elliptic3d<dg::aRealMPIGeometry3d<value_type>, Matrix, Vector> laplace(grid, dg::not_normed, dg::centered);
+    Matrix DX = dg::create::dx( grid);
     t.toc();
     if(rank==0)std::cout<< "Creation took "<<t.diff()<<"s\n";
 
-    dg::CG< dg::MDVec > pcg( x, n*n*Nx*Ny);
+    dg::CG< Vector > pcg( x, n*n*Nx*Ny);
 
     if(rank==0)std::cout<<"Expand right hand side\n";
-    const dg::MDVec solution = dg::evaluate ( fct, grid);
-    const dg::MDVec deriv = dg::evaluate( derivative, grid);
-    dg::MDVec b = dg::evaluate ( laplace3d_fct, grid);
+    const Vector solution = dg::evaluate ( fct, grid);
+    const Vector deriv = dg::evaluate( derivative, grid);
+    Vector b = dg::evaluate ( laplace3d_fct, grid);
     //compute W b
     dg::blas2::symv( w3d, b, b);
 
@@ -68,11 +76,11 @@ int main( int argc, char* argv[])
     t.toc();
     if(rank==0)std::cout << "Number of pcg iterations "<< num<<std::endl;
     if(rank==0)std::cout << "... took                 "<< t.diff()<<"s\n";
-    dg::MDVec  error(  solution);
+    Vector  error(  solution);
     dg::blas1::axpby( 1., x,-1., error);
 
-    double normerr = dg::blas2::dot( w3d, error);
-    double norm = dg::blas2::dot( w3d, solution);
+    value_type normerr = dg::blas2::dot( w3d, error);
+    value_type norm = dg::blas2::dot( w3d, solution);
     exblas::udouble res;
     norm = sqrt(normerr/norm); res.d = norm;
     if(rank==0)std::cout << "L2 Norm of relative error is:               " <<res.d<<"\t"<<res.i<<std::endl;
@@ -86,18 +94,18 @@ int main( int argc, char* argv[])
     x = dg::evaluate( initial, grid);
     b = dg::evaluate ( laplace2d_fct, grid);
     //create grid and perp and parallel volume
-    dg::ClonePtr<dg::aMPIGeometry2d> grid_perp = grid.perp_grid();
-    dg::MDVec v2d = dg::create::inv_volume( *grid_perp);
-    dg::MDVec w2d = dg::create::volume( *grid_perp);
-    dg::MDVec g_parallel = grid.metric().value(2,2);
+    dg::ClonePtr<dg::aRealMPIGeometry2d<value_type>> grid_perp = grid.perp_grid();
+    Vector v2d = dg::create::inv_volume( *grid_perp);
+    Vector w2d = dg::create::volume( *grid_perp);
+    Vector g_parallel = grid.metric().value(2,2);
     dg::blas1::transform( g_parallel, g_parallel, dg::SQRT<>());
-    dg::MDVec chi = dg::evaluate( dg::one, grid);
+    Vector chi = dg::evaluate( dg::one, grid);
     dg::blas1::pointwiseDivide( chi, g_parallel, chi);
     //create split Laplacian
-    std::vector< dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec> > laplace_split(
-            grid.local().Nz(), dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec>(*grid_perp, dg::not_normed, dg::centered));
+    std::vector< dg::Elliptic<dg::aRealMPIGeometry2d<value_type>, Matrix, Vector> > laplace_split(
+            grid.local().Nz(), dg::Elliptic<dg::aRealMPIGeometry2d<value_type>, Matrix, Vector>(*grid_perp, dg::not_normed, dg::centered));
     // create split  vectors and solve
-    std::vector<dg::MPI_Vector<dg::View<dg::DVec>>> b_split, x_split, chi_split;
+    std::vector<dg::MPI_Vector<dg::View<LVector>>> b_split, x_split, chi_split;
     pcg.construct( w2d, w2d.size());
     std::vector<unsigned>  number(grid.local().Nz());
     t.tic();
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 1b6aac040..2c8b757eb 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -175,7 +175,7 @@ struct MultigridCG2d
      * @note If the Macro \c DG_BENCHMARK is defined this function will write timings to \c std::cout
     */
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
-    std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, ContainerType0&  x, const ContainerType1& b, double eps)
+    std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, ContainerType0&  x, const ContainerType1& b, value_type eps)
     {
         dg::blas2::symv(op[0].weights(), b, m_b[0]);
         // compute residual r = Wb - A x
@@ -256,8 +256,8 @@ struct MultigridCG2d
     */
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
     void fmg_solve( std::vector<SymmetricOp>& op,
-    ContainerType0& x, const ContainerType1& b, std::vector<double> ev, unsigned nu_pre, unsigned
-    nu_post, unsigned gamma, double eps)
+    ContainerType0& x, const ContainerType1& b, std::vector<value_type> ev, unsigned nu_pre, unsigned
+    nu_post, unsigned gamma, value_type eps)
     {
         //FULL MULTIGRID
         //solve for residuum ( if not we always get the same solution)
@@ -316,8 +316,8 @@ struct MultigridCG2d
     */
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
     void pcg_solve( std::vector<SymmetricOp>& op,
-    ContainerType0& x, const ContainerType1& b, std::vector<double> ev, unsigned nu_pre, unsigned
-    nu_post, unsigned gamma, double eps)
+    ContainerType0& x, const ContainerType1& b, std::vector<value_type> ev, unsigned nu_pre, unsigned
+    nu_post, unsigned gamma, value_type eps)
     {
 
         dg::blas2::symv(op[0].weights(), b, m_b[0]);
@@ -368,8 +368,8 @@ struct MultigridCG2d
     template<class SymmetricOp>
     void multigrid_cycle( std::vector<SymmetricOp>& op,
     std::vector<Container>& x, std::vector<Container>& b,
-    std::vector<double> ev,
-        unsigned nu1, unsigned nu2, unsigned gamma, unsigned p, double eps)
+    std::vector<value_type> ev,
+        unsigned nu1, unsigned nu2, unsigned gamma, unsigned p, value_type eps)
     {
         // 1 multigrid cycle beginning on grid p
         // p < m_stages-1
@@ -386,7 +386,7 @@ struct MultigridCG2d
         //std::cout << "STAGE "<<p<<"\n";
         //dg::blas2::symv( op[p], x[p], m_r[p]);
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
-        //double norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
+        //value_type norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
 //#ifdef DG_BENCHMARK
 //        Timer t;
@@ -423,7 +423,7 @@ struct MultigridCG2d
 //#endif //DG_BENCHMARK
             //dg::blas2::symv( op[p+1], x[p+1], m_r[p+1]);
             //dg::blas1::axpby( 1., b[p+1], -1., m_r[p+1]);
-            //double norm_res = sqrt(dg::blas1::dot( m_r[p+1], m_r[p+1]));
+            //value_type norm_res = sqrt(dg::blas1::dot( m_r[p+1], m_r[p+1]));
             //std::cout<< " Exact solution "<<norm_res<<"\n";
         }
         else
@@ -444,15 +444,15 @@ struct MultigridCG2d
         //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2, op[p].inv_weights());
         //dg::blas2::symv( op[p], x[p], m_r[p]);
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
-        //double norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
+        //value_type norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum after "<<norm_res<<"\n";
     }
 
 
 	template<class SymmetricOp>
     void full_multigrid( std::vector<SymmetricOp>& op,
-        std::vector<Container>& x, std::vector<Container>& b, std::vector<double> ev,
-        unsigned nu1, unsigned nu2, unsigned gamma, unsigned mu, double eps)
+        std::vector<Container>& x, std::vector<Container>& b, std::vector<value_type> ev,
+        unsigned nu1, unsigned nu2, unsigned gamma, unsigned mu, value_type eps)
     {
         for( unsigned u=0; u<m_stages-1; u++)
         {
diff --git a/inc/dg/topology/mpi_grid.h b/inc/dg/topology/mpi_grid.h
index 7f8ea29d6..d872e2084 100644
--- a/inc/dg/topology/mpi_grid.h
+++ b/inc/dg/topology/mpi_grid.h
@@ -621,7 +621,7 @@ struct aRealMPITopology3d
         unsigned Ny = g.Ny()/dims[1];
         unsigned Nz = g.Nz()/dims[2];
 
-        l = Grid3d(x0, x1, y0, y1, z0, z1, g.n(), Nx, Ny, Nz, g.bcx(), g.bcy(), g.bcz());
+        l = RealGrid3d<real_type>(x0, x1, y0, y1, z0, z1, g.n(), Nx, Ny, Nz, g.bcx(), g.bcy(), g.bcz());
     }
     RealGrid3d<real_type> g, l; //global grid
     MPI_Comm comm, planeComm; //just an integer...
diff --git a/inc/dg/topology/split_and_join.h b/inc/dg/topology/split_and_join.h
index 99ecb4d04..6923162dd 100644
--- a/inc/dg/topology/split_and_join.h
+++ b/inc/dg/topology/split_and_join.h
@@ -24,8 +24,8 @@ namespace dg
 * @tparam SharedContainer \c TensorTraits exists for this class and the
 *   \c tensor_category derives from \c SharedVectorTag
 */
-template<class SharedContainer>
-void split( SharedContainer& in, std::vector<View<SharedContainer>>& out, const aTopology3d& grid)
+template<class SharedContainer, class real_type>
+void split( SharedContainer& in, std::vector<View<SharedContainer>>& out, const aRealTopology3d<real_type>& grid)
 {
     assert( out.size() == grid.Nz());
     unsigned size2d=grid.n()*grid.n()*grid.Nx()*grid.Ny();
@@ -40,11 +40,11 @@ void split( SharedContainer& in, std::vector<View<SharedContainer>>& out, const
 * @tparam SharedContainer \c TensorTraits exists for this class and the
 *   \c tensor_category derives from \c SharedVectorTag
 */
-template<class SharedContainer>
-std::vector<View<SharedContainer>> split( SharedContainer& in, const aTopology3d& grid)
+template<class SharedContainer, class real_type>
+std::vector<View<SharedContainer>> split( SharedContainer& in, const aRealTopology3d<real_type>& grid)
 {
     std::vector<View<SharedContainer>> out;
-    Grid3d l( grid);
+    RealGrid3d<real_type> l( grid);
     unsigned size2d=l.n()*l.n()*l.Nx()*l.Ny();
     out.resize( l.Nz());
     for(unsigned i=0; i<l.Nz(); i++)
@@ -72,12 +72,12 @@ using get_mpi_view_type = typename
  * @param grid provide dimensions in 3rd and first two dimensions
  * @tparam MPIContainer An MPI_Vector of a \c SharedContainer
 */
-template<class MPIContainer>
+template<class MPIContainer, class real_type>
 void split( MPIContainer& in, std::vector<get_mpi_view_type<MPIContainer> >&
-    out, const aMPITopology3d& grid)
+    out, const aRealMPITopology3d<real_type>& grid)
 {
     //local size2d
-    Grid3d l = grid.local();
+    RealGrid3d<real_type> l = grid.local();
     unsigned size2d=l.n()*l.n()*l.Nx()*l.Ny();
     for(unsigned i=0; i<l.Nz(); i++)
     {
@@ -94,9 +94,9 @@ void split( MPIContainer& in, std::vector<get_mpi_view_type<MPIContainer> >&
 * @note two seperately split vectors have congruent (not identical) MPI_Communicators Note here the MPI concept of congruent (same process group, different contexts) vs. identical (same process group, same context) communicators.
 * @tparam MPIContainer An MPI_Vector of a \c SharedContainer
 */
-template< class MPIContainer>
+template< class MPIContainer, class real_type>
 std::vector<get_mpi_view_type<MPIContainer> > split(
-    MPIContainer& in, const aMPITopology3d& grid)
+    MPIContainer& in, const aRealMPITopology3d<real_type>& grid)
 {
     std::vector<get_mpi_view_type<MPIContainer>> out;
     int result;
@@ -105,7 +105,7 @@ std::vector<get_mpi_view_type<MPIContainer> > split(
     MPI_Comm planeComm = grid.get_perp_comm(), comm_mod, comm_mod_reduce;
     exblas::mpi_reduce_communicator( planeComm, &comm_mod, &comm_mod_reduce);
     //local size2d
-    Grid3d l = grid.local();
+    RealGrid3d<real_type> l = grid.local();
     unsigned size2d=l.n()*l.n()*l.Nx()*l.Ny();
     out.resize( l.Nz());
     for(unsigned i=0; i<l.Nz(); i++)
-- 
GitLab


From c63492decef2c794d72594dd17862532fe1b132e Mon Sep 17 00:00:00 2001
From: mattwi <mattwi@fysik.dtu.dk>
Date: Sun, 31 May 2020 13:36:59 +0200
Subject: [PATCH 254/540] Add float test to evaluation

- test good without VCL (with VCL we need to test still)
---
 inc/dg/topology/evaluation_t.cu | 47 +++++++++++++++++++++------------
 1 file changed, 30 insertions(+), 17 deletions(-)

diff --git a/inc/dg/topology/evaluation_t.cu b/inc/dg/topology/evaluation_t.cu
index 7918afcc3..a2e7d1966 100644
--- a/inc/dg/topology/evaluation_t.cu
+++ b/inc/dg/topology/evaluation_t.cu
@@ -26,18 +26,16 @@ double operator()( double x)
 }
 };
 
-double function( double x, double y)
+template<class T>
+T function( T x, T y)
 {
         return exp(x)*exp(y);
 }
-double function( double x, double y, double z)
+double function3d( double x, double y, double z)
 {
         return exp(x)*exp(y)*exp(z);
 }
 
-typedef thrust::device_vector< double>   DVec;
-typedef thrust::host_vector< double>     HVec;
-
 int main()
 {
     std::cout << "This program tests the exblas::dot function. The tests succeed only if the evaluation and grid functions but also the weights and especially the exblas::dot function are correctly implemented and compiled. Furthermore, the compiler implementation of the exp function in the math library must be consistent across platforms to get reproducible results\n";
@@ -47,34 +45,43 @@ int main()
 
     dg::Grid1d g1d( 1, 2, n, Nx);
     dg::Grid2d g2d( 1, 2, 3, 4, n, Nx, Ny);
+    dg::RealGrid2d<float> gf2d( 1, 2, 3, 4, n, Nx, Ny);
     dg::Grid3d g3d( 1, 2, 3, 4, 5, 6, n, Nx, Ny, Nz,dg::PER,dg::PER,dg::PER);
 
     //test evaluation functions
-    const DVec func1d = dg::construct<DVec>( dg::evaluate( exp, g1d));
-    const DVec func2d = dg::construct<DVec>( dg::evaluate( function, g2d));
-    const DVec func3d = dg::construct<DVec>( dg::evaluate( function, g3d));
-    const DVec w1d = dg::construct<DVec>( dg::create::weights( g1d));
-    const DVec w2d = dg::construct<DVec>( dg::create::weights( g2d));
-    const DVec w3d = dg::construct<DVec>( dg::create::weights( g3d));
+    const dg::DVec func1d = dg::construct<dg::DVec>( dg::evaluate( exp, g1d));
+    const dg::DVec func2d = dg::construct<dg::DVec>( dg::evaluate( function<double>, g2d));
+    const dg::fDVec funcf2d = dg::construct<dg::fDVec>( dg::evaluate( function<float>, gf2d));
+    const dg::DVec func3d = dg::construct<dg::DVec>( dg::evaluate( function3d, g3d));
+    const dg::DVec w1d = dg::construct<dg::DVec>( dg::create::weights( g1d));
+    const dg::DVec w2d = dg::construct<dg::DVec>( dg::create::weights( g2d));
+    const dg::fDVec wf2d = dg::construct<dg::fDVec>( dg::create::weights( gf2d));
+    const dg::DVec w3d = dg::construct<dg::DVec>( dg::create::weights( g3d));
     exblas::udouble res;
 
     double integral = dg::blas1::dot( w1d, func1d); res.d = integral;
     std::cout << "1D integral               "<<std::setw(6)<<integral <<"\t" << res.i - 4616944842743393935  << "\n";
     double sol = (exp(2.) -exp(1));
     std::cout << "Correct integral is       "<<std::setw(6)<<sol<<std::endl;
-    std::cout << "Absolute 1d error is      "<<(integral-sol)<<"\n\n";
+    std::cout << "Relative 1d error is      "<<(integral-sol)/sol<<"\n\n";
 
     double integral2d = dg::blas1::dot( w2d, func2d); res.d = integral2d;
     std::cout << "2D integral               "<<std::setw(6)<<integral2d <<"\t" << res.i - 4639875759346476257<< "\n";
     double sol2d = (exp(2.)-exp(1))*(exp(4.)-exp(3));
     std::cout << "Correct integral is       "<<std::setw(6)<<sol2d<<std::endl;
-    std::cout << "Absolute 2d error is      "<<(integral2d-sol2d)<<"\n\n";
+    std::cout << "Relative 2d error is      "<<(integral2d-sol2d)/sol2d<<"\n\n";
+
+    float integralf2d = dg::blas1::dot( wf2d, func2d); res.d = integralf2d;
+    std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\t" << res.i - 4639875760323035136<< "\n";
+    float solf2d = (exp(2.)-exp(1))*(exp(4.)-exp(3));
+    std::cout << "Correct integral is       "<<std::setw(6)<<solf2d<<std::endl;
+    std::cout << "Relative 2d error (float) "<<(integralf2d-solf2d)/solf2d<<"\n\n";
 
     double integral3d = dg::blas1::dot( w3d, func3d); res.d = integral3d;
     std::cout << "3D integral               "<<std::setw(6)<<integral3d <<"\t" << res.i - 4675882723962622631<< "\n";
     double sol3d = sol2d*(exp(6.)-exp(5.));
     std::cout << "Correct integral is       "<<std::setw(6)<<sol3d<<std::endl;
-    std::cout << "Absolute 3d error is      "<<(integral3d-sol3d)<<"\n\n";
+    std::cout << "Relative 3d error is      "<<(integral3d-sol3d)/sol3d<<"\n\n";
 
     double norm = dg::blas2::dot( func1d, w1d, func1d); res.d = norm;
     std::cout << "Square normalized 1D norm "<<std::setw(6)<<norm<<"\t" << res.i - 4627337306989890294 <<"\n";
@@ -95,12 +102,12 @@ int main()
     std::cout << "Relative 3d error is      "<<(norm3d-solution3d)/solution3d<<"\n\n";
 
     std::cout << "TEST result of a sin and exp function to compare compiler specific math libraries:\n";
-    DVec x(1, 6.12610567450009658);
+    dg::DVec x(1, 6.12610567450009658);
     dg::blas1::transform( x, x, sin_function() );
     res.d = x[0];
     std::cout << "Result of sin:    "<<res.i<<"\n"
               << "          GCC:    -4628567870976535683 (correct)"<<std::endl;
-    DVec y(1, 5.9126151457310376);
+    dg::DVec y(1, 5.9126151457310376);
     dg::blas1::transform( y, y, exp_function() );
     res.d = y[0];
     std::cout << "Result of exp:     "<<res.i<<"\n"
@@ -115,7 +122,13 @@ int main()
     std::cout << " Error norm of  1d integral function "<<norm<<"\n";
     // TEST if dot throws on NaN
     dg::blas1::transform( x,x, dg::LN<double>());
-    dg::blas1::dot( x,x);
+    try{
+        dg::blas1::dot( x,x);
+    }catch ( std::exception& e)
+    {
+        std::cerr << "Error thrown as expected\n";
+        std::cerr << e.what() << std::endl;
+    }
 
     std::cout << "\nFINISHED! Continue with topology/derivatives_t.cu !\n\n";
     return 0;
-- 
GitLab


From dd1cd7d8697d57fdf0dc05fe5ec6b33f7a9f7513 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Jun 2020 11:20:26 +0200
Subject: [PATCH 255/540] Fix BUG in accumulate for float

---
 inc/dg/backend/exblas/accumulate.h | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/inc/dg/backend/exblas/accumulate.h b/inc/dg/backend/exblas/accumulate.h
index 1cbfb747a..ddf262ff3 100644
--- a/inc/dg/backend/exblas/accumulate.h
+++ b/inc/dg/backend/exblas/accumulate.h
@@ -43,18 +43,15 @@ static inline vcl::Vec8d make_vcl_vec8d( float x, int i){
     return vcl::Vec8d((double)x);
 }
 static inline vcl::Vec8d make_vcl_vec8d( const float* x, int i){
-    double tmp[8];
-    for(int i=0; i<8; i++)
-        tmp[i] = (double)x[i];
-    return vcl::Vec8d().load( tmp);
+    return vcl::Vec8d( x[i], x[i+1], x[i+2], x[i+3], x[i+4], x[i+5], x[i+6], x[i+7]);
 }
 static inline vcl::Vec8d make_vcl_vec8d( float x, int i, int num){
     return vcl::Vec8d((double)x);
 }
 static inline vcl::Vec8d make_vcl_vec8d( const float* x, int i, int num){
     double tmp[8];
-    for(int i=0; i<num; i++)
-        tmp[i] = (double)x[i];
+    for(int j=0; j<num; j++)
+        tmp[j] = (double)x[i+j];
     return vcl::Vec8d().load_partial( num, tmp);
 }
 #endif//_WITHOUT_VCL
-- 
GitLab


From feceef679652e9e2825f6d61c768048ce104d2aa Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Jun 2020 11:52:36 +0200
Subject: [PATCH 256/540] Add more float tests and benchmarks

---
 inc/dg/blas1.h                     |  3 +-
 inc/dg/blas2.h                     |  3 +-
 inc/dg/blas_b.cu                   | 48 ++++++++++++++++--------------
 inc/dg/blas_mpib.cu                | 12 +++++---
 inc/dg/elliptic2d_mpib.cu          | 12 ++++----
 inc/dg/topology/derivatives_t.cu   | 31 +++++++++----------
 inc/dg/topology/evaluation_mpit.cu | 26 +++++++++-------
 7 files changed, 76 insertions(+), 59 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index a6352d302..4f78c5376 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -59,7 +59,8 @@ double result = dg::blas1::dot( two, three); // result = 600 (100*(2*3))
  * @note Our implementation guarantees binary reproducible results.
  * The sum is computed with infinite precision and the result is rounded
  * to the nearest double precision number.
- * This is possible with the help of an adapted version of the \c ::exblas library.
+ * This is possible with the help of an adapted version of the \c ::exblas library and
+* works for single and double precision.
 
  * @param x Left Container
  * @param y Right Container may alias x
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index 9734ccc84..76097fd94 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -65,7 +65,8 @@ inline std::vector<int64_t> doDot_superacc( const ContainerType1& x, const Matri
  * Furthermore, the sum is computed with infinite precision and the result is then rounded
  * to the nearest double precision number. Although the products are not computed with
  * infinite precision, the order of multiplication is guaranteed.
- * This is possible with the help of an adapted version of the \c ::exblas library.
+ * This is possible with the help of an adapted version of the \c ::exblas library and
+* works for single and double precision.
  *
  * @param x Left input
  * @param m The diagonal Matrix.
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 6c5ad2910..788b490b7 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -10,41 +10,45 @@
 #include "topology/evaluation.h"
 #include "topology/fast_interpolation.h"
 
-const double lx = 2.*M_PI;
-const double ly = 2.*M_PI;
-double left( double x, double y, double z) {return sin(x)*cos(y)*z;}
-double right( double x, double y, double z) {return cos(x)*sin(y)*z;}
+using value_type = double;
+using Vector     = dg::DVec;
+using Matrix     = dg::DMatrix;
+//using value_type = float;
+//using Vector     = dg::fDVec;
+//using Matrix     = dg::fDMatrix;
+
+using ArrayVec   = std::array<Vector, 3>;
+
+const value_type lx = 2.*M_PI;
+const value_type ly = 2.*M_PI;
+value_type left( value_type x, value_type y, value_type z) {return sin(x)*cos(y)*z;}
+value_type right( value_type x, value_type y, value_type z) {return cos(x)*sin(y)*z;}
+
 struct Expression{
    DG_DEVICE
-   void operator() ( double& u, double v, double w, double param){
+   void operator() ( value_type& u, value_type v, value_type w, value_type param){
        u = param*u*v + w;
    }
 };
 struct test_routine{
-    test_routine( double mu, double alpha):m_mu(mu), m_alpha(alpha){}
+    test_routine( value_type mu, value_type alpha):m_mu(mu), m_alpha(alpha){}
     DG_DEVICE
-    void operator()( double g11, double g12, double g22, double in1, double in2, double& out1, double& out2){
+    void operator()( value_type g11, value_type g12, value_type g22, value_type in1, value_type in2, value_type& out1, value_type& out2){
         out1 = (g11)*(in1) + (g12)*(in2) + m_mu;
         out2 = (g12)*(in1) + (g22)*(in2) + m_alpha;
     }
 private:
-    double m_mu, m_alpha;
+    value_type m_mu, m_alpha;
 };
-
 struct test_inplace{
     DG_DEVICE
-    void operator()( double g11, double g12, double g22, double& inout1, double& inout2){
-        double t = g11*inout1 + g12*inout2;
+    void operator()( value_type g11, value_type g12, value_type g22, value_type& inout1, value_type& inout2){
+        value_type t = g11*inout1 + g12*inout2;
         inout2 = g12*inout1 + g22*inout2;
         inout1 = t;
     }
 };
 
-using value_type= double;
-using Vector    = thrust::device_vector<double>;
-using Matrix    = dg::DMatrix;
-using ArrayVec  = std::array<Vector, 3>;
-
 int main()
 {
     dg::Timer t;
@@ -59,8 +63,8 @@ int main()
     std::cout << "   Nz: # of cells in z\n";
     std::cout << "Type n (3), Nx (512) , Ny (512) and Nz (10) \n";
     std::cin >> n >> Nx >> Ny >> Nz;
-    dg::Grid3d grid(      0., lx, 0, ly, 0, ly, n, Nx, Ny, Nz);
-    dg::Grid3d grid_half = grid; grid_half.multiplyCellNumbers(0.5, 0.5);
+    dg::RealGrid3d<value_type> grid(      0., lx, 0, ly, 0, ly, n, Nx, Ny, Nz);
+    dg::RealGrid3d<value_type> grid_half = grid; grid_half.multiplyCellNumbers(0.5, 0.5);
     Vector w2d = dg::construct<Vector>( dg::create::weights(grid));
 
     //std::cout<<"Evaluate a function on the grid\n";
@@ -111,7 +115,7 @@ int main()
     t.toc();
     std::cout<<"pointwiseDot (1*yx+2*uv=v) (A)   "<<t.diff()/multi<<"s\t" <<5*gbytes*multi/t.diff()<<"GB/s\n";
     //Test new evaluate
-    std::array<double, 3> array_p{ 1,2,3};
+    std::array<value_type, 3> array_p{ 1,2,3};
     t.tic();
     for( int i=0; i<multi; i++)
         dg::blas1::subroutine( Expression(), u, v, x, array_p);
@@ -219,8 +223,8 @@ int main()
 
     std::cout << "\nSequential recursive calls";
     unsigned size_rec = 1e4;
-    std::vector<double> test_recursive(size_rec, 0.1);
-    gbytes=(double)size_rec*sizeof(double)/1e9;
+    std::vector<value_type> test_recursive(size_rec, 0.1);
+    gbytes=(value_type)size_rec*sizeof(value_type)/1e9;
     std::cout << " with size "<<gbytes<<"GB\n";
     norm += dg::blas1::dot( 1., test_recursive);//warm up
     t.tic();
@@ -228,7 +232,7 @@ int main()
         norm += dg::blas1::dot( 1., test_recursive);//warm up
     t.toc();
     std::cout<<"recursive dot took               " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
-    thrust::host_vector<double> test_serial((int)size_rec, (double)0.1);
+    thrust::host_vector<value_type> test_serial((int)size_rec, (value_type)0.1);
     norm += dg::blas1::dot( 1., test_serial);//warm up
     t.tic();
     for( int i=0; i<multi; i++)
diff --git a/inc/dg/blas_mpib.cu b/inc/dg/blas_mpib.cu
index cdf97bad6..8df427f20 100644
--- a/inc/dg/blas_mpib.cu
+++ b/inc/dg/blas_mpib.cu
@@ -12,10 +12,14 @@
 #include "topology/mpi_weights.h"
 #include "topology/fast_interpolation.h"
 
-using value_type= double;
-using Vector    = dg::MDVec;
-using Matrix    = dg::MDMatrix;
-using ArrayVec  = std::array<Vector, 3>;
+//using value_type = float;
+//using Vector     = dg::fMDVec;
+//using Matrix     = dg::fMDMatrix;
+using value_type = double;
+using Vector     = dg::MDVec;
+using Matrix     = dg::MDMatrix;
+
+using ArrayVec   = std::array<Vector, 3>;
 
 const value_type lx = 2.*M_PI;
 const value_type ly = 2.*M_PI;
diff --git a/inc/dg/elliptic2d_mpib.cu b/inc/dg/elliptic2d_mpib.cu
index 783ec7aed..871213512 100644
--- a/inc/dg/elliptic2d_mpib.cu
+++ b/inc/dg/elliptic2d_mpib.cu
@@ -13,12 +13,12 @@
 //
 //global relative error in L2 norm is O(h^P)
 //as a rule of thumb with n=4 the true error is err = 1e-3 * eps as long as eps > 1e3*err
-//using value_type = float;
-//using Vector = dg::fMDVec;
-//using Matrix = dg::fMDMatrix;
-using value_type = double;
-using Vector = dg::MDVec;
-using Matrix = dg::MDMatrix;
+using value_type = float;
+using Vector = dg::fMDVec;
+using Matrix = dg::fMDMatrix;
+//using value_type = double;
+//using Vector = dg::MDVec;
+//using Matrix = dg::MDMatrix;
 
 const value_type lx = M_PI;
 const value_type ly = 2.*M_PI;
diff --git a/inc/dg/topology/derivatives_t.cu b/inc/dg/topology/derivatives_t.cu
index a3311f8db..e02652d93 100644
--- a/inc/dg/topology/derivatives_t.cu
+++ b/inc/dg/topology/derivatives_t.cu
@@ -4,18 +4,19 @@
 #include "derivatives.h"
 #include "evaluation.h"
 
-double zero( double x, double y) { return 0;}
-double sine( double x, double y) { return sin(x)*sin(y);}
-double cosx( double x, double y) { return cos(x)*sin(y);}
-double cosy( double x, double y) { return cos(y)*sin(x);}
-double zero( double x, double y, double z) { return 0;}
-double sine( double x, double y, double z) { return sin(x)*sin(y)*sin(z);}
-double cosx( double x, double y, double z) { return cos(x)*sin(y)*sin(z);}
-double cosy( double x, double y, double z) { return cos(y)*sin(x)*sin(z);}
-double cosz( double x, double y, double z) { return cos(z)*sin(x)*sin(y);}
+using Matrix = dg::DMatrix;
+using Vector = dg::DVec;
+using value_t = double;
 
-typedef dg::DMatrix Matrix;
-typedef dg::DVec Vector;
+value_t zero( value_t x, value_t y) { return 0;}
+value_t sine( value_t x, value_t y) { return sin(x)*sin(y);}
+value_t cosx( value_t x, value_t y) { return cos(x)*sin(y);}
+value_t cosy( value_t x, value_t y) { return cos(y)*sin(x);}
+value_t zero( value_t x, value_t y, value_t z) { return 0;}
+value_t sine( value_t x, value_t y, value_t z) { return sin(x)*sin(y)*sin(z);}
+value_t cosx( value_t x, value_t y, value_t z) { return cos(x)*sin(y)*sin(z);}
+value_t cosy( value_t x, value_t y, value_t z) { return cos(y)*sin(x)*sin(z);}
+value_t cosz( value_t x, value_t y, value_t z) { return cos(z)*sin(x)*sin(y);}
 
 int main()
 {
@@ -24,7 +25,7 @@ int main()
     unsigned n = 3, Nx = 24, Ny = 28, Nz = 100;
     std::cout << "On Grid "<<n<<" x "<<Nx<<" x "<<Ny<<" x "<<Nz<<"\n";
     dg::bc bcx=dg::DIR, bcy=dg::PER, bcz=dg::NEU_DIR;
-    dg::Grid2d g2d( 0, M_PI, 0.1, 2*M_PI+0.1, n, Nx, Ny, bcx, bcy);
+    dg::RealGrid2d<value_t> g2d( 0, M_PI, 0.1, 2*M_PI+0.1, n, Nx, Ny, bcx, bcy);
     const Vector w2d = dg::create::weights( g2d);
 
     Matrix dx2 = dg::create::dx( g2d, dg::forward);
@@ -46,10 +47,10 @@ int main()
         Vector error = sol2[i];
         dg::blas2::symv( -1., m2[i], f2d, 1., error);
         dg::blas1::pointwiseDot( error, error, error);
-        double norm = sqrt(dg::blas1::dot( w2d, error)); res.d = norm;
+        value_t norm = sqrt(dg::blas1::dot( w2d, error)); res.d = norm;
         std::cout << "Distance to true solution: "<<norm<<"\t"<<res.i-binary2[i]<<"\n";
     }
-    dg::Grid3d g3d( 0,M_PI, 0.1, 2.*M_PI+0.1, M_PI/2.,M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz);
+    dg::RealGrid3d<value_t> g3d( 0,M_PI, 0.1, 2.*M_PI+0.1, M_PI/2.,M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz);
     const Vector w3d = dg::create::weights( g3d);
     Matrix dx3 = dg::create::dx( g3d, dg::forward);
     Matrix dy3 = dg::create::dy( g3d, dg::centered);
@@ -71,7 +72,7 @@ int main()
     {
         Vector error = sol3[i];
         dg::blas2::symv( -1., m3[i], f3d, 1., error);
-        double norm = sqrt(dg::blas2::dot( error, w3d, error)); res.d = norm;
+        value_t norm = sqrt(dg::blas2::dot( error, w3d, error)); res.d = norm;
         std::cout << "Distance to true solution: "<<norm<<"\t"<<res.i-binary3[i]<<"\n";
     }
     std::cout << "\nFINISHED! Continue with arakawa_t.cu !\n\n";
diff --git a/inc/dg/topology/evaluation_mpit.cu b/inc/dg/topology/evaluation_mpit.cu
index e6ed3217b..85a91645f 100644
--- a/inc/dg/topology/evaluation_mpit.cu
+++ b/inc/dg/topology/evaluation_mpit.cu
@@ -14,7 +14,8 @@ double function( double x)
     return exp(x);
 }
 
-double function( double x, double y)
+template<class T>
+T function( T x, T y)
 {
         return exp(x)*exp(y);
 }
@@ -23,9 +24,6 @@ double function( double x, double y, double z)
         return exp(x)*exp(y)*exp(z);
 }
 
-typedef dg::MPI_Vector<thrust::host_vector<double> > MHVec;
-typedef dg::MPI_Vector<thrust::device_vector<double> > MDVec;
-
 int main(int argc, char** argv)
 {
     MPI_Init( &argc, &argv);
@@ -38,28 +36,36 @@ int main(int argc, char** argv)
     MPI_Comm comm2d, comm3d;
     mpi_init2d( dg::PER, dg::PER, comm2d);
     dg::MPIGrid2d g2d( 1, 2., 3, 4, n, Nx, Ny, dg::PER, dg::PER, comm2d);
+    dg::RealMPIGrid2d<float> gf2d( 1, 2., 3, 4, n, Nx, Ny, dg::PER, dg::PER, comm2d);
     mpi_init3d( dg::PER, dg::PER, dg::PER, comm3d);
     dg::MPIGrid3d g3d( 1, 2, 3, 4, 5, 6, n, Nx, Ny, Nz, dg::PER, dg::PER, dg::PER, comm3d);
 
     //test evaluation and expand functions
-    MDVec func2d = dg::construct<MDVec>(dg::evaluate( function, g2d));
-    MDVec func3d = dg::construct<MDVec>(dg::evaluate( function, g3d));
+    dg::MDVec func2d = dg::construct<dg::MDVec>(dg::evaluate( function<double>, g2d));
+    dg::MDVec funcf2d = dg::construct<dg::MDVec>(dg::evaluate( function<float>, g2d));
+    dg::MDVec func3d = dg::construct<dg::MDVec>(dg::evaluate( function, g3d));
     //test weights
-    const MDVec w2d = dg::construct<MDVec>(dg::create::weights(g2d));
-    const MDVec w3d = dg::construct<MDVec>(dg::create::weights(g3d));
+    const dg::MDVec w2d = dg::construct<dg::MDVec>(dg::create::weights(g2d));
+    const dg::fMDVec wf2d = dg::construct<dg::fMDVec>(dg::create::weights(gf2d));
+    const dg::MDVec w3d = dg::construct<dg::MDVec>(dg::create::weights(g3d));
     exblas::udouble res;
 
     double integral2d = dg::blas1::dot( w2d, func2d); res.d = integral2d;
     if(rank==0)std::cout << "2D integral               "<<std::setw(6)<<integral2d <<"\t" << res.i - 4639875759346476257 << "\n";
     double sol2d = (exp(2.)-exp(1))*(exp(4.)-exp(3));
     if(rank==0)std::cout << "Correct integral is       "<<std::setw(6)<<sol2d<<std::endl;
-    if(rank==0)std::cout << "Absolute 2d error is      "<<(integral2d-sol2d)<<"\n\n";
+    if(rank==0)std::cout << "Relative 2d error is      "<<(integral2d-sol2d)/sol2d<<"\n\n";
+    float integralf2d = dg::blas1::dot( wf2d, funcf2d); res.d = integralf2d;
+    if(rank==0)std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\t" << res.i - 4639875760323035136<< "\n";
+    float solf2d = (exp(2.)-exp(1))*(exp(4.)-exp(3));
+    if(rank==0)std::cout << "Correct integral is       "<<std::setw(6)<<solf2d<<std::endl;
+    if(rank==0)std::cout << "Relative 2d error (float) "<<(integralf2d-solf2d)/solf2d<<"\n\n";
 
     double integral3d = dg::blas1::dot( w3d, func3d); res.d = integral3d;
     if(rank==0)std::cout << "3D integral               "<<std::setw(6)<<integral3d <<"\t" << res.i - 4675882723962622631<< "\n";
     double sol3d = sol2d*(exp(6.)-exp(5));
     if(rank==0)std::cout << "Correct integral is       "<<std::setw(6)<<sol3d<<std::endl;
-    if(rank==0)std::cout << "Absolute 3d error is      "<<(integral3d-sol3d)<<"\n\n";
+    if(rank==0)std::cout << "Relative 3d error is      "<<(integral3d-sol3d)/sol3d<<"\n\n";
 
     double norm2d = dg::blas2::dot( w2d, func2d); res.d = norm2d;
     if(rank==0)std::cout << "Square normalized 2D norm "<<std::setw(6)<<norm2d<<"\t" << res.i - 4674091193523851724<<"\n";
-- 
GitLab


From a0ad3638954bb9e8e01db00cbef1cec2f2fcea94 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 9 Jun 2020 14:15:50 +0200
Subject: [PATCH 257/540] Introduce save on dots parameter in CG

in order to improve performance of direct_solve
we need to focus on improving performance of dot for small vector
sizes
---
 inc/dg/cg.h           | 36 ++++++++++++++++++++----------------
 inc/dg/multigrid.h    |  2 +-
 src/feltor/feltor.tex |  2 --
 3 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 6c3c3f318..442fefeb3 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -98,6 +98,7 @@ class CG
      * @param S (Inverse) Weights used to compute the norm for the error condition
      * @param eps The relative error to be respected
      * @param nrmb_correction the absolute error \c C in units of \c eps to be respected
+     * @param test_frequency if set to 1 then the norm of the error is computed in every iteration to test if the loop can be terminated. Sometimes, especially for small sizes the dot product is expensive to compute, then it is beneficial to set this parameter to e.g. 10, which means that the errror condition is only evaluated every 10th iteration.
      *
      * @return Number of iterations used to achieve desired precision
      * @note Required memops per iteration (\c P and \c S are assumed vectors):
@@ -109,7 +110,7 @@ class CG
      * @tparam SquareNorm A type for which the blas2::dot( const SquareNorm&, const ContainerType&) function is callable. This can e.g. be one of the ContainerType types.
      */
     template< class MatrixType, class ContainerType0, class ContainerType1, class Preconditioner, class SquareNorm >
-    unsigned operator()( MatrixType& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1);
+    unsigned operator()( MatrixType& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, SquareNorm& S, value_type eps = 1e-12, value_type nrmb_correction = 1, int test_frequency = 1);
   private:
     ContainerType r, p, ap;
     unsigned max_iter;
@@ -178,13 +179,13 @@ unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const Con
         nrm2r_new = blas2::dot( P, r);
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
-    if(rank==0)
+        if(rank==0)
 #endif //MPI
-    {
-        std::cout << "# Absolute "<<sqrt( nrm2r_new) <<"\t ";
-        std::cout << "#  < Critical "<<eps*nrmb + eps <<"\t ";
-        std::cout << "# (Relative "<<sqrt( nrm2r_new)/nrmb << ")\n";
-    }
+        {
+            std::cout << "# Absolute "<<sqrt( nrm2r_new) <<"\t ";
+            std::cout << "#  < Critical "<<eps*nrmb + eps <<"\t ";
+            std::cout << "# (Relative "<<sqrt( nrm2r_new)/nrmb << ")\n";
+        }
 #endif //DG_DEBUG
         if( sqrt( nrm2r_new) < eps*(nrmb + nrmb_correction))
             return i;
@@ -197,7 +198,7 @@ unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const Con
 
 template< class ContainerType>
 template< class Matrix, class ContainerType0, class ContainerType1, class Preconditioner, class SquareNorm>
-unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction)
+unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, SquareNorm& S, value_type eps, value_type nrmb_correction, int save_on_dots )
 {
     value_type nrmb = sqrt( blas2::dot( S, b));
 #ifdef DG_DEBUG
@@ -230,18 +231,21 @@ unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const Con
         alpha =  nrmzr_old/blas1::dot( p, ap);
         blas1::axpby( alpha, p, 1.,x);
         blas1::axpby( -alpha, ap, 1., r);
+        if( 0 == i%save_on_dots )
+        {
 #ifdef DG_DEBUG
 #ifdef MPI_VERSION
-    if(rank==0)
+            if(rank==0)
 #endif //MPI
-    {
-        std::cout << "# Absolute r*S*r "<<sqrt( blas2::dot(S,r)) <<"\t ";
-        std::cout << "#  < Critical "<<eps*nrmb + eps <<"\t ";
-        std::cout << "# (Relative "<<sqrt( blas2::dot(S,r) )/nrmb << ")\n";
-    }
+            {
+                std::cout << "# Absolute r*S*r "<<sqrt( blas2::dot(S,r)) <<"\t ";
+                std::cout << "#  < Critical "<<eps*nrmb + eps <<"\t ";
+                std::cout << "# (Relative "<<sqrt( blas2::dot(S,r) )/nrmb << ")\n";
+            }
 #endif //DG_DEBUG
-        if( sqrt( blas2::dot(S,r)) < eps*(nrmb + nrmb_correction))
-            return i;
+                if( sqrt( blas2::dot(S,r)) < eps*(nrmb + nrmb_correction))
+                    return i;
+        }
         blas2::symv(P,r,ap);
         nrmzr_new = blas1::dot( ap, r);
         blas1::axpby(1.,ap, nrmzr_new/nrmzr_old, p );
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 2c8b757eb..856d8039d 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -197,7 +197,7 @@ struct MultigridCG2d
             t.tic();
 #endif //DG_BENCHMARK
             number[u] = m_cg[u]( op[u], m_x[u], m_r[u], op[u].precond(),
-                op[u].inv_weights(), eps/2, 1.);
+                op[u].inv_weights(), eps*1.5, 1., 10);
             dg::blas2::symv( m_inter[u-1], m_x[u], m_x[u-1]);
 #ifdef DG_BENCHMARK
             t.toc();
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index df4af5f38..3fcbab46e 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1161,8 +1161,6 @@ The relevant terms in the output file are (the Lorentz force term is described i
     jsparbphiexbi\_tt   & $\mu_i N_iU_ib_\varphi(\bhat\times\vn\phi)\cn \psi_p/B$ \\
     sparmirrore\_tt & $-z_e\tau_en_e\npar \ln B$ &
     sparmirrori\_tt & $-z_i\tau_iN_i\npar \ln B$ \\
-    jsparexbi\_tt       & $\mu_i N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ &
-    jsparbphiexbi\_tt   & $\mu_i N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ \\
     sparsni\_tt & $\mu_i S_{N_i} U_i$ &
     sparsnibphi\_tt & $\mu_i S_{N_i} U_ib_\varphi $ \\
 \bottomrule
-- 
GitLab


From d487236253254602ca7495cef28f4c587819ce09 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 9 Jun 2020 16:32:53 +0200
Subject: [PATCH 258/540] Do not accumulate rest of multiplication in exblas

now consistently in both blas1 and blas2 dot functions
hopefully gives a boost of gpu performance
---
 inc/dg/backend/exblas/exdot_cuda.cuh | 45 ++++++++++++++--------------
 inc/dg/backend/exblas/exdot_omp.h    | 33 ++++++++++----------
 inc/dg/backend/exblas/exdot_serial.h | 23 +++++++-------
 3 files changed, 51 insertions(+), 50 deletions(-)

diff --git a/inc/dg/backend/exblas/exdot_cuda.cuh b/inc/dg/backend/exblas/exdot_cuda.cuh
index ef7351c50..28d456434 100644
--- a/inc/dg/backend/exblas/exdot_cuda.cuh
+++ b/inc/dg/backend/exblas/exdot_cuda.cuh
@@ -62,9 +62,10 @@ __global__ void ExDOT(
     //Read data from global memory and scatter it to sub-superaccs
     double a[NBFPE] = {0.0};
     for(uint pos = blockIdx.x*blockDim.x+threadIdx.x; pos < NbElements; pos += gridDim.x*blockDim.x) {
-        double r = 0.0;
-        double x = TwoProductFMA(get_element(d_a,pos), get_element(d_b,pos), &r);
-        //double x = d_a[pos]*d_b[pos];//ATTENTION: if we write it like this, cpu compiler might generate an fma from this while nvcc does not...
+        //double r = 0.0;
+        //double x = TwoProductFMA(get_element(d_a,pos), get_element(d_b,pos), &r);
+        double x = d_a[pos]*d_b[pos];
+        //we do not accumulate the rest of this multiplication
 
         //Check if the input is sane
         if( !isfinite(x) ) *error = true;
@@ -85,23 +86,23 @@ __global__ void ExDOT(
             }
         }
 
-        if (r != 0.0) {//add the rest r in the same manner
-            #pragma unroll
-            for(uint i = 0; i != NBFPE; ++i) {
-                double s;
-                a[i] = KnuthTwoSum(a[i], r, &s);
-                r = s;
-            }
-            if (r != 0.0) {
-                Accumulate(l_workingBase, r, WARP_COUNT);
-                // Flush FPEs to superaccs
-                #pragma unroll
-                for(uint i = 0; i != NBFPE; ++i) {
-                    Accumulate(l_workingBase, a[i], WARP_COUNT);
-                    a[i] = 0.0;
-                }
-            }
-        }
+        //if (r != 0.0) {//add the rest r in the same manner
+        //    #pragma unroll
+        //    for(uint i = 0; i != NBFPE; ++i) {
+        //        double s;
+        //        a[i] = KnuthTwoSum(a[i], r, &s);
+        //        r = s;
+        //    }
+        //    if (r != 0.0) {
+        //        Accumulate(l_workingBase, r, WARP_COUNT);
+        //        // Flush FPEs to superaccs
+        //        #pragma unroll
+        //        for(uint i = 0; i != NBFPE; ++i) {
+        //            Accumulate(l_workingBase, a[i], WARP_COUNT);
+        //            a[i] = 0.0;
+        //        }
+        //    }
+        //}
     }
     //Flush FPEs to superaccs
     #pragma unroll
@@ -156,8 +157,8 @@ __global__ void ExDOT(
         //double r  = 0.0, r2 = 0.0;
         //double x  = TwoProductFMA(d_a[pos], d_b[pos], &r);
         //double x2 = TwoProductFMA(x , d_c[pos], &r2);
-        double x1 = __fma_rn( get_element(d_a,pos), get_element(d_b,pos), 0);
-        double x2 = __fma_rn( x1                  , get_element(d_c,pos), 0);
+        double x1 = get_element(d_a,pos)*get_element(d_b,pos);
+        double x2 = x1                  *get_element(d_c,pos);
 
         //Check if the input is sane
         if( !isfinite(x2) ) *error = true;
diff --git a/inc/dg/backend/exblas/exdot_omp.h b/inc/dg/backend/exblas/exdot_omp.h
index fa62f0967..3e47e47e2 100644
--- a/inc/dg/backend/exblas/exdot_omp.h
+++ b/inc/dg/backend/exblas/exdot_omp.h
@@ -114,39 +114,38 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* h_superacc,
 #ifndef _MSC_VER
             asm ("# myloop");
 #endif
-            vcl::Vec8d r1 ;
-            vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,i), make_vcl_vec8d(b,i), r1);
-            //vcl::Vec8d x  = TwoProductFMA(vcl::Vec8d().load(a+i), vcl::Vec8d().load(b+i), r1);
-            //vcl::Vec8d x  = vcl::mul_add( vcl::Vec8d().load(a+i),vcl::Vec8d().load(b+i),0);
+            //vcl::Vec8d r1 ;
+            //vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,i), make_vcl_vec8d(b,i), r1);
+            vcl::Vec8d x  = make_vcl_vec8d(a,i)*make_vcl_vec8d(b,i);
             //MW: check sanity of input
             vcl::Vec8db finite = vcl::is_finite( x);
             if( !vcl::horizontal_and( finite) ) error[tid] = true;
 
             cache.Accumulate(x);
-            cache.Accumulate(r1); //MW: exact product but halfs the speed
+            //cache.Accumulate(r1); //MW: exact product but halfs the speed
         }
         if( tid+1==tnum && r != N-1) {
             r+=1;
             //accumulate remainder
-            vcl::Vec8d r1;
-            vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,r,N-r), make_vcl_vec8d(b,r,N-r), r1);
-            //vcl::Vec8d x  = TwoProductFMA(vcl::Vec8d().load_partial(N-r, a+r), vcl::Vec8d().load_partial(N-r,b+r), r1);
-            //vcl::Vec8d x  = vcl::mul_add( vcl::Vec8d().load_partial(N-r,a+r),vcl::Vec8d().load_partial(N-r,b+r),0);
+            //vcl::Vec8d r1;
+            //vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,r,N-r), make_vcl_vec8d(b,r,N-r), r1);
+            vcl::Vec8d x  = make_vcl_vec8d(a,r,N-r)*make_vcl_vec8d(b,r,N-r);
 
             //MW: check sanity of input
             vcl::Vec8db finite = vcl::is_finite( x);
             if( !vcl::horizontal_and( finite) ) error[tid] = true;
             cache.Accumulate(x);
-            cache.Accumulate(r1);
+            //cache.Accumulate(r1);
         }
 #else// _WITHOUT_VCL
         int l = ((tid * int64_t(N)) / tnum);
         int r = ((((tid+1) * int64_t(N)) / tnum) ) - 1;
         for(int i = l; i <= r; i++) {
-            double r1;
-            double x = TwoProductFMA(get_element(a,i),get_element(b,i),r1);
+            //double r1;
+            //double x = TwoProductFMA(get_element(a,i),get_element(b,i),r1);
+            double x = get_element(a,i)*get_element(b,i);
             cache.Accumulate(x);
-            cache.Accumulate(r1);
+            //cache.Accumulate(r1);
         }
 #endif// _WITHOUT_VCL
         cache.Flush();
@@ -191,8 +190,8 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, in
             //vcl::Vec8d x2 = TwoProductFMA(x , cvec, r2);
             //vcl::Vec8d x1  = vcl::mul_add(vcl::Vec8d().load(a+i),vcl::Vec8d().load(b+i), 0);
             //vcl::Vec8d x2  = vcl::mul_add( x1                   ,vcl::Vec8d().load(c+i), 0);
-            vcl::Vec8d x1  = vcl::mul_add(make_vcl_vec8d(a,i),make_vcl_vec8d(b,i), 0);
-            vcl::Vec8d x2  = vcl::mul_add( x1                ,make_vcl_vec8d(c,i), 0);
+            vcl::Vec8d x1  = make_vcl_vec8d(a,i)*make_vcl_vec8d(b,i);
+            vcl::Vec8d x2  =  x1                *make_vcl_vec8d(c,i);
             vcl::Vec8db finite = vcl::is_finite( x2);
             if( !vcl::horizontal_and( finite) ) error[tid] = true;
             cache.Accumulate(x2);
@@ -209,8 +208,8 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, in
             //vcl::Vec8d x2 = TwoProductFMA(x , cvec, r2);
             //vcl::Vec8d x1  = vcl::mul_add(vcl::Vec8d().load_partial(N-r, a+r),vcl::Vec8d().load_partial(N-r,b+r), 0);
             //vcl::Vec8d x2  = vcl::mul_add( x1                   ,vcl::Vec8d().load_partial(N-r,c+r), 0);
-            vcl::Vec8d x1  = vcl::mul_add(make_vcl_vec8d(a,r,N-r),make_vcl_vec8d(b,r,N-r), 0);
-            vcl::Vec8d x2  = vcl::mul_add( x1                    ,make_vcl_vec8d(c,r,N-r), 0);
+            vcl::Vec8d x1  = make_vcl_vec8d(a,r,N-r)*make_vcl_vec8d(b,r,N-r);
+            vcl::Vec8d x2  =  x1                    *make_vcl_vec8d(c,r,N-r);
             vcl::Vec8db finite = vcl::is_finite( x2);
             if( !vcl::horizontal_and( finite) ) error[tid] = true;
             cache.Accumulate(x2);
diff --git a/inc/dg/backend/exblas/exdot_serial.h b/inc/dg/backend/exblas/exdot_serial.h
index bcb3b6afb..5616ca4c2 100644
--- a/inc/dg/backend/exblas/exdot_serial.h
+++ b/inc/dg/backend/exblas/exdot_serial.h
@@ -38,33 +38,34 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, int64_t* acc, boo
 #ifndef _MSC_VER
         asm ("# myloop");
 #endif
-        vcl::Vec8d r1 ;
-        vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,i), make_vcl_vec8d(b,i), r1);
+        //vcl::Vec8d r1 ;
+        //vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,i), make_vcl_vec8d(b,i), r1);
         //vcl::Vec8d x  = TwoProductFMA(vcl::Vec8d().load(a+i), vcl::Vec8d().load(b+i), r1);
-        //vcl::Vec8d x  = vcl::Vec8d().load(a+i)*vcl::Vec8d().load(b+i);
+        vcl::Vec8d x  = make_vcl_vec8d(a,i)* make_vcl_vec8d(b,i);
         vcl::Vec8db finite = vcl::is_finite( x);
         if( !vcl::horizontal_and( finite) ) *error = true;
         cache.Accumulate(x);
-        cache.Accumulate(r1);
+        //cache.Accumulate(r1);
     }
     if( r != N) {
         //accumulate remainder
-        vcl::Vec8d r1;
-        vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,r,N-r), make_vcl_vec8d(b,r,N-r), r1);
+        //vcl::Vec8d r1;
+        //vcl::Vec8d x  = TwoProductFMA(make_vcl_vec8d(a,r,N-r), make_vcl_vec8d(b,r,N-r), r1);
         //vcl::Vec8d x  = TwoProductFMA(vcl::Vec8d().load_partial(N-r, a+r), vcl::Vec8d().load_partial(N-r,b+r), r1);
-        //vcl::Vec8d x  = vcl::Vec8d().load_partial(N-r, a+r)*vcl::Vec8d().load_partial(N-r,b+r);
+        vcl::Vec8d x  = make_vcl_vec8d(a,r,N-r)*make_vcl_vec8d(b,r,N-r);
         vcl::Vec8db finite = vcl::is_finite( x);
         if( !vcl::horizontal_and( finite) ) *error = true;
         cache.Accumulate(x);
-        cache.Accumulate(r1);
+        //cache.Accumulate(r1);
     }
 #else// _WITHOUT_VCL
     for(int i = 0; i < N; i++) {
-        double r1;
-        double x = TwoProductFMA(get_element(a,i),get_element(b,i),r1);
+        //double r1;
+        //double x = TwoProductFMA(get_element(a,i),get_element(b,i),r1);
+        double x = get_element(a,i)*get_element(b,i);
         if( !std::isfinite(x) ) *error = true;
         cache.Accumulate(x);
-        cache.Accumulate(r1);
+        //cache.Accumulate(r1);
     }
 #endif// _WITHOUT_VCL
     cache.Flush();
-- 
GitLab


From 2326940acbaac21b617c6dd751ba433813655097 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Jun 2020 19:37:00 +0200
Subject: [PATCH 259/540] Insert warning in Chebyshev documentation

---
 inc/dg/chebyshev.h | 2 ++
 inc/dg/multigrid.h | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index 32f313919..ac55cbf81 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -26,6 +26,8 @@ namespace dg
 * For more information see the book "Iteratvie Methods for Sparse
 * Linear Systems" 2nd edition by Yousef Saad
 *
+* @attention Chebyshev iteration may diverge if the elliptical bound of the Eigenvaleus is not accurate or if an ellipsis is not a good fit for the spectrum of the matrix
+*
 * @ingroup invert
 *
 * @copydoc hide_ContainerType
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 856d8039d..547e3955a 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -412,6 +412,7 @@ struct MultigridCG2d
 //#endif //DG_BENCHMARK
             int number = m_cg[p+1]( op[p+1], x[p+1], b[p+1], op[p+1].precond(),
                 op[p+1].inv_weights(), eps/2.);
+            number++;//avoid compiler warning
 //#ifdef DG_BENCHMARK
 //            t.toc();
 //#ifdef MPI_VERSION
@@ -468,6 +469,7 @@ struct MultigridCG2d
 #endif //DG_BENCHMARK
         int number = m_cg[s]( op[s], x[s], b[s], op[s].precond(),
             op[s].inv_weights(), eps/2.);
+        number++;//avoid compiler warning
 #ifdef DG_BENCHMARK
         t.toc();
 #ifdef MPI_VERSION
-- 
GitLab


From ab97e84676219b81708da4f75544ed11a0f3f9a9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 16 Jun 2020 14:51:19 +0200
Subject: [PATCH 260/540] FIX bug on 3dstatic output in feltor

We will need to fix static outputs in all files produced up to
today
---
 src/feltor/feltor_hpc.cu | 10 ++++++----
 src/feltor/feltordiag.h  |  6 +++---
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 0440b9da5..f7b8340e2 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -337,8 +337,9 @@ int main( int argc, char* argv[])
             "long_name", record.long_name.size(), record.long_name.data());
         MPI_OUT err = nc_enddef( ncid);
         MPI_OUT std::cout << "Computing "<<record.name<<"\n";
-        record.function( resultH, var, grid);
-        dg::blas2::symv( projectH, resultH, transferH);
+        record.function( transferH, var, g3d_out);
+        //record.function( resultH, var, grid);
+        //dg::blas2::symv( projectH, resultH, transferH);
         file::put_var_double( ncid, vecID, g3d_out, transferH);
         MPI_OUT err = nc_redef(ncid);
     }
@@ -352,8 +353,9 @@ int main( int argc, char* argv[])
             "long_name", record.long_name.size(), record.long_name.data());
         MPI_OUT err = nc_enddef( ncid);
         MPI_OUT std::cout << "Computing2d "<<record.name<<"\n";
-        record.function( resultH, var, grid);
-        dg::blas2::symv( projectH, resultH, transferH);
+        record.function( transferH, var, g3d_out);
+        //record.function( resultH, var, grid);
+        //dg::blas2::symv( projectH, resultH, transferH);
         if(write2d)file::put_var_double( ncid, vecID, *g2d_out_ptr, transferH);
         MPI_OUT err = nc_redef(ncid);
     }
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 3d289ce23..c8b9fa1ea 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -218,17 +218,17 @@ std::vector<Record_static> diagnostics3d_static_list = {
     },
     { "xc", "x-coordinate in Cartesian coordinate system",
         []( HVec& result, Variables& v, Geometry& grid ){
-            return dg::evaluate( dg::cooRZP2X, grid);
+            result = dg::evaluate( dg::cooRZP2X, grid);
         }
     },
     { "yc", "y-coordinate in Cartesian coordinate system",
         []( HVec& result, Variables& v, Geometry& grid ){
-            return dg::evaluate( dg::cooRZP2Y, grid);
+            result = dg::evaluate( dg::cooRZP2Y, grid);
         }
     },
     { "zc", "z-coordinate in Cartesian coordinate system",
         []( HVec& result, Variables& v, Geometry& grid ){
-            return dg::evaluate( dg::cooRZP2Z, grid);
+            result = dg::evaluate( dg::cooRZP2Z, grid);
         }
     },
 };
-- 
GitLab


From ec0463a6b8a3740d070ce2ee72a96be98ea250f6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Jun 2020 23:41:30 +0200
Subject: [PATCH 261/540] Add preconditioned EVE

- it is still uncertain how reliable this is
- specific tests are needed
---
 inc/dg/eve.h | 93 +++++++++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 84 insertions(+), 9 deletions(-)

diff --git a/inc/dg/eve.h b/inc/dg/eve.h
index 0f855519e..e90a4fdd7 100644
--- a/inc/dg/eve.h
+++ b/inc/dg/eve.h
@@ -6,8 +6,7 @@
 #include "functors.h"
 
 /*! @file
- * EVE adds an estimator for the largest Eigenvalue
- *  to not-yet-preconditioned CG.
+ * EVE adds an estimator for the largest Eigenvalue of the generalized Eigenvalue problem
  *  @author Eduard Reiter and Matthias Wiesenberger
  */
 
@@ -15,7 +14,21 @@
 namespace dg
 {
 
-/*! @brief (EigenValueEstimator) estimate largest Eigenvalue using conjugate gradient method
+/*! @brief The Eigen-Value-Estimator (EVE) finds largest Eigenvalue of \f[ M^{-1}Ax = \lambda_\max x\f]
+ *
+ * Estimate largest Eigenvalue
+ * of a symmetric positive definite matrix \f$ A\f$ with possible preconditioner \f$ M^{-1}\f$ using the conjugate gradient (CG) method.
+ * The unpreconditioned version is the algorithm suggested in <a href="http://www.iam.fmph.uniba.sk/amuc/ojs/index.php/algoritmy/article/view/421">Tichy, On error estimation in the conjugate gradient method: Normwise backward error, Proceedings of the Conference Algoritmy, 323-332, 2016 </a>.
+ * The reason this works also for the preconditioned CG method is because
+ * preconditioned CG is equivalent to applying the
+ * unpreconditioned CG method to \f$ \bar A\bar x = \bar b\f$ with \f$ \bar A
+ * := {E^{-1}}^\mathrm{T} A E^{-1} \f$, \f$ \bar x := Ex\f$ and \f$ \bar b :=
+ * {E^{-1}}^\mathrm{T}\f$, where \f$ M^{-1} = {E^{-1}}^\mathrm{T} E^{-1}\f$ is
+ * the preconditioner.
+ * The maximum Eigenvalue of \f$ M^{-1} A\f$ is the same as the
+ * maximum EV of \f$ {E^{-1}}^\mathrm{T} A E^{-1} \f$
+ *
+* @attention beware the sign: a negative definite matrix does @b not work in Conjugate gradient
 * @copydoc hide_ContainerType
  * @ingroup invert
 */
@@ -46,19 +59,40 @@ class EVE
     /// Get maximum number of iterations
     unsigned get_max() const {   return m_max_iter; }
     /**
-     * @brief Unpreconditioned CG to estimate maximum Eigenvalue
+     * @brief Unpreconditioned CG to estimate maximum Eigenvalue of \f$ A x = \lambda x\f$
      *
+     * This implements the original algorithm suggested in <a href="http://www.iam.fmph.uniba.sk/amuc/ojs/index.php/algoritmy/article/view/421">Tichy, On error estimation in the conjugate gradient method: Normwise backward error, Proceedings of the Conference Algorithmy, 323-332, 2016 </a>
+     * @note This is just a regular CG algorithm which updates an estimate for the largest Eigenvalue in each iteration and returns once the change is marginal.
+     * This means on output \c x is the same as after the same number of iterations of a regular CG method.
      * @param A A symmetric, positive definit matrix
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
      * @param ev_max (output) maximum Eigenvalue on output
+     * @param eps_ev The desired relative accuracy of the largest Eigenvalue
+     *
+     * @return Number of iterations used to achieve desired precision or max_iterations
+     * @copydoc hide_matrix
+     */
+    template< class MatrixType, class ContainerType0, class ContainerType1>
+    unsigned operator()( MatrixType& A, ContainerType0& x, const ContainerType1& b, value_type& ev_max, value_type eps_ev=1e-16);
+    /**
+     * @brief Preconditioned CG to estimate maximum Eigenvalue of the generalized problem \f$ Ax = \lambda M x\f$
+     *
+     * where \f$ M^{-1}\f$ is the preconditioner.
+     * @note This is just a regular PCG algorithm which updates an estimate for the largest Eigenvalue in each iteration and returns once the change is marginal.
+     * This means on output \c x is the same as after the same number of iterations of a regular PCG method.
+     * @param A A symmetric, positive definit matrix
+     * @param x Contains an initial value on input and the solution on output.
+     * @param b The right hand side vector. x and b may be the same vector.
+     * @param P The preconditioner (\f$ M^{-1}\f$  in the above notation)
+     * @param ev_max (output) maximum Eigenvalue on output
      * @param eps_ev The desired accuracy of the largest Eigenvalue
      *
      * @return Number of iterations used to achieve desired precision or max_iterations
      * @copydoc hide_matrix
      */
-    template< class MatrixType>
-    unsigned operator()( MatrixType& A, ContainerType& x, const ContainerType& b, value_type& ev_max, value_type eps_ev=1e-16);
+    template< class MatrixType, class ContainerType0, class ContainerType1, class Preconditioner>
+    unsigned operator()( MatrixType& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, value_type& ev_max, value_type eps_ev = 1e-12);
   private:
     ContainerType r, p, ap;
     unsigned m_max_iter;
@@ -66,8 +100,8 @@ class EVE
 
 ///@cond
 template< class ContainerType>
-template< class MatrixType>
-unsigned EVE< ContainerType>::operator()( MatrixType& A, ContainerType& x, const ContainerType&
+template< class MatrixType, class ContainerType0, class ContainerType1>
+unsigned EVE< ContainerType>::operator()( MatrixType& A, ContainerType0& x, const ContainerType1&
 b, value_type& ev_max, value_type eps_ev)
 {
     blas2::symv( A, x, r);
@@ -90,21 +124,62 @@ b, value_type& ev_max, value_type eps_ev)
         blas1::axpby( alpha, p, 1., x);
         blas1::axpby( -alpha, ap, 1., r);
         nrm2r_new = blas1::dot( r, r);
+
         delta = nrm2r_new /nrm2r_old;                  // EVE!
         evdash = ev_est -lambda;                       // EVE!
         omega = sqrt( evdash*evdash +4.*beta*gamma);   // EVE!
         gamma = 0.5 *(1. -evdash /omega);              // EVE!
         ev_max += omega*gamma;                         // EVE!
+        beta = delta*alpha_inv*alpha_inv;              // EVE!
         if( fabs(ev_est-ev_max) < eps_ev*ev_max) {
             return i;
         }
-        beta = delta*alpha_inv*alpha_inv;              // EVE!
         blas1::axpby(1., r, delta, p);
         nrm2r_old=nrm2r_new;
         ev_est = ev_max;
     }
     return m_max_iter;
 };
+
+template< class ContainerType>
+template< class Matrix, class ContainerType0, class ContainerType1, class Preconditioner>
+unsigned EVE< ContainerType>::operator()( Matrix& A, ContainerType0& x, const ContainerType1& b, Preconditioner& P, value_type& ev_max, value_type eps_ev )
+{
+    blas2::symv( A,x,r);
+    blas1::axpby( 1., b, -1., r);
+    blas2::symv( P, r, p );
+    value_type nrmzr_old = blas1::dot( p,r);
+    value_type nrmzr_new, nrmAp;
+    value_type alpha = 1., alpha_inv = 1., delta = 0.;
+    value_type evdash, gamma = 0., lambda, omega, beta = 0.;
+    value_type ev_est = 0.;
+    ev_max = 0.;
+    for( unsigned i=1; i<m_max_iter; i++)
+    {
+        lambda = delta*alpha_inv;    // EVE!
+        blas2::symv( A, p, ap);
+        nrmAp = blas1::dot( p, ap);
+        alpha =  nrmzr_old/nrmAp;
+        alpha_inv = nrmAp/nrmzr_old; //EVE!
+        lambda+= alpha_inv;          //EVE!
+        blas1::axpby( alpha, p, 1.,x);
+        blas1::axpby( -alpha, ap, 1., r);
+        blas2::symv(P,r,ap);
+        nrmzr_new = blas1::dot( ap, r);
+        delta = nrmzr_new /nrmzr_old;                  // EVE!
+        evdash = ev_est -lambda;                       // EVE!
+        omega = sqrt( evdash*evdash +4.*beta*gamma);   // EVE!
+        gamma = 0.5 *(1. -evdash /omega);              // EVE!
+        ev_max += omega*gamma;                         // EVE!
+        beta = delta*alpha_inv*alpha_inv;              // EVE!
+        if( fabs( ev_est - ev_max) < eps_ev*ev_max)
+            return i;
+        blas1::axpby(1.,ap, delta, p );
+        nrmzr_old=nrmzr_new;
+        ev_est = ev_max;
+    }
+    return m_max_iter;
+}
 ///@endcond
 
 } //namespace dg
-- 
GitLab


From 0b0d830ec2c2bd54571419ac4c274085082936b3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Jun 2020 23:47:57 +0200
Subject: [PATCH 262/540] Add Preconditioned Chebyshev and LeastSquares

- polynomial preconditioners hopefully accelerate cases where
scalar product dominates runtime
---
 inc/dg/cg.h        |   5 +-
 inc/dg/chebyshev.h | 308 +++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 289 insertions(+), 24 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 442fefeb3..d069534bf 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -20,10 +20,13 @@ namespace dg{
 
 /**
 * @brief Preconditioned conjugate gradient method to solve
-* \f[ Ax=b\f]
+* \f[ M^{-1}Ax=M^{-1}b\f]
 *
 * @ingroup invert
 *
+* @sa This implements the PCG algorithm as given in https://en.wikipedia.org/wiki/Conjugate_gradient_method
+or the book
+* <a href="https://www-users.cs.umn.edu/~saad/IterMethBook_2ndEd.pdf">Iteratvie Methods for Sparse Linear Systems" 2nd edition by Yousef Saad </a>
 * @note Conjugate gradients might become unstable for positive semidefinite
 * matrices arising e.g. in the discretization of the periodic laplacian
 * @attention beware the sign: a negative definite matrix does @b not work in Conjugate gradient
diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index ac55cbf81..723c0c85e 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -6,43 +6,63 @@
 #include "blas.h"
 
 /*!@file
- * Chebyshev solver
+ * Polynomial Preconditioners and solvers
  */
 
 namespace dg
 {
 
 /**
-* @brief Three-term recursion of the Chebyshev iteration for solving
-* \f[ Ax=b\f]
+* @brief Preconditioned Chebyshev iteration for solving
+* \f[ M^{-1}Ax=M^{-1}b\f]
+*
+* Chebyshev iteration is not well-suited for solving matrix equations
+* on its own. Rather, it is very well-suited as a smoother for a multigrid algorithm
+* and also as a Preconditioner for the Conjugate Gradient method.
+* In both cases its appeal stems from not having any scalar products,
+* which makes it appaeling for both small and highly parallelized systems.
 *
 * Given the minimum and maximum Eigenvalue of the matrix A we define
 * \f[ \theta = (\lambda_\min+\lambda_\max)/2 \quad \delta = (\lambda_\max - \lambda_\min)/2 \\
 *     \rho_0 := \frac{\delta}{\theta},\ x_0 := x, \ x_{1} = x_0+\frac{1}{\theta} (b-Ax_0) \\
 *     \rho_{k}:=\left(\frac{2\theta}{\delta}-\rho_{k-1}\right)^{-1} \\
 *     x_{k+1} := x_k + \rho_k\left( \rho_{k-1}(x_k - x_{k-1})
-*     + \frac{2}{\delta} ( b - Ax_k) \right)
+*     + \frac{2}{\delta} M^{-1}( b - Ax_k) \right)
 * \f]
-* For more information see the book "Iteratvie Methods for Sparse
-* Linear Systems" 2nd edition by Yousef Saad
+* The preconditioned version is obtained by applying the regular version to
+ * \f$ \bar A\bar x = \bar b\f$ with \f$ \bar A
+ * := {E^{-1}}^\mathrm{T} A E^{-1} \f$, \f$ \bar x := Ex\f$ and \f$ \bar b :=
+ * {E^{-1}}^\mathrm{T}\f$, where \f$ M^{-1} = {E^{-1}}^\mathrm{T} E^{-1}\f$ is
+ * the preconditioner. The bounds on the spectrum then need to be on the \f$M^{-1}A\f$ matrix.
+* @note The maximum Eigenvalue of \f$ A\f$ and \f$ M^{-1} A\f$ can be estimated
+* using the \c EVE class.
+* \sa For more information see the book
+* <a href="https://www-users.cs.umn.edu/~saad/IterMethBook_2ndEd.pdf">Iteratvie Methods for Sparse Linear Systems" 2nd edition by Yousef Saad </a>
+* @note If the initial vector is zero Chebyshev iteration will produce the
+* Chebyshev polynomial \f$ C_k( A) b\f$ applied to the right hand side
+* and the preconditioned version produces
+* \f$ C_{k-1}(M^{-1}A)M^{-1}b = E^{-1} C_{k-1}(
+* {E^{-1}}^\mathrm{T} A E^{-1}){E^{-1}}^\mathrm{T}\f$
 *
-* @attention Chebyshev iteration may diverge if the elliptical bound of the Eigenvaleus is not accurate or if an ellipsis is not a good fit for the spectrum of the matrix
+* @attention Chebyshev iteration may diverge if the elliptical bound of the
+* Eigenvalues is not accurate or if an ellipsis is not a good fit for the
+* spectrum of the matrix
 *
 * @ingroup invert
 *
 * @copydoc hide_ContainerType
 */
 template< class ContainerType>
-class Chebyshev
+class ChebyshevIteration
 {
   public:
     using container_type = ContainerType;
     using value_type = get_value_type<ContainerType>; //!< value type of the ContainerType class
     ///@brief Allocate nothing, Call \c construct method before usage
-    Chebyshev(){}
+    ChebyshevIteration(){}
     ///@copydoc construct()
-    Chebyshev( const ContainerType& copyable):
-        m_ax(copyable), m_xm1(m_ax){}
+    ChebyshevIteration( const ContainerType& copyable):
+        m_ax(copyable), m_z( m_ax), m_xm1(m_ax){}
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_ax;}
@@ -53,34 +73,50 @@ class Chebyshev
      * @param copyable A ContainerType must be copy-constructible from this
      */
     void construct( const ContainerType& copyable) {
-        m_xm1 = m_ax = copyable;
+        m_xm1 = m_z = m_ax = copyable;
     }
     /**
-     * @brief Solve the system A*x = b using Chebyshev iteration
+     * @brief Solve the system \f$ Ax = b\f$ using Chebyshev iteration
      *
      * The iteration stops when the maximum number of iterations is reached
      * @param A A symmetric, positive definit matrix
      * @param x Contains an initial value on input and the solution on output.
      * @param b The right hand side vector. x and b may be the same vector.
-     * @param min_ev the minimum Eigenvalue
-     * @param max_ev the maximum Eigenvalue (must be larger than \c min_ev)
-     * @param num_iter the number of iterations k (equals the number of times A is applied)
+     * @param min_ev an estimate of the minimum Eigenvalue
+     * @param max_ev an estimate of the maximum Eigenvalue of \f$ A\f$ (must be larger than \c min_ev)
+     * Use \c EVE to get this value
+     * @param num_iter the number of iterations \c k (equals the number of times A is applied)
+     * If 0 the function returns immediately
+     * @param x_is_zero If true, the first matrix-vector multiplication is avoided
+     * by assuming x is zero. (This works even if x is not actually 0)
+     * This is in particular in the case when Chebyshev Iteration is used as a Preconditioner
+     * @note In the \c x_is_zero mode \c k iterations  will produce the \c k-1
+     * Chebyshev polynomial applied to
+     * the right hand side \f$ x = C_{k-1}(A)b\f$
      *
      * @copydoc hide_matrix
      * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
      */
     template< class MatrixType, class ContainerType0, class ContainerType1>
     void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b,
-        double min_ev, double max_ev, unsigned num_iter)
+        value_type min_ev, value_type max_ev, unsigned num_iter, bool x_is_zero = false)
     {
         if( num_iter == 0)
             return;
         assert ( min_ev < max_ev);
-        double theta = (min_ev+max_ev)/2., delta = (max_ev-min_ev)/2.;
-        double rhokm1 = delta/theta, rhok=0;
-        dg::blas1::copy( x, m_xm1); //x0
-        dg::blas2::symv( A, x, m_ax);
-        dg::blas1::axpbypgz( 1./theta, b, -1./theta, m_ax, 1., x); //x1
+        value_type theta = (min_ev+max_ev)/2., delta = (max_ev-min_ev)/2.;
+        value_type rhokm1 = delta/theta, rhok=0;
+        if( !x_is_zero)
+        {
+            dg::blas1::copy( x, m_xm1); //x_{k-1}
+            dg::blas2::symv( A, x, m_ax);
+            dg::blas1::axpbypgz( 1./theta, b, -1./theta, m_ax, 1., x); //x_1
+        }
+        else
+        {
+            dg::blas1::copy( 0., m_xm1); //x_{k-1}
+            dg::blas1::axpby( 1./theta, b, 0., x); //x_1
+        }
         for ( unsigned k=1; k<num_iter; k++)
         {
             rhok = 1./(2.*theta/delta - rhokm1);
@@ -95,10 +131,236 @@ class Chebyshev
             rhokm1 = rhok;
         }
     }
+    /**
+     * @brief Solve the system \f$ M^{-1}Ax = M^{-1}b \f$ using Preconditioned Chebyshev iteration
+     *
+     * The iteration stops when the maximum number of iterations is reached
+     * @param A A symmetric, positive definit matrix
+     * @param x Contains an initial value on input and the solution on output.
+     * @param b The right hand side vector. x and b may be the same vector.
+     * @param P the Preconditioner (\f$ M^{-1}\f$ in the above notation
+     * @param min_ev an estimate of the minimum Eigenvalue
+     * @param max_ev an estimate of the maximum Eigenvalue of \f$ M^{-1} A\f$ (must be larger than \c min_ev)
+     * Use \c EVE to get this value
+     * @param num_iter the number of iterations k (equals the number of times \c A is applied)
+     * If 0 the function returns immediately
+     * @param x_is_zero If true, the first matrix-vector multiplication is avoided
+     * by assuming x is zero. (This works even if x is not actually 0)
+     * This is in particular in the case when Chebyshev Iteration is used as a Preconditioner
+     * @note In the \c x_is_zero mode \c k iterations  will produce the \c k-1 Chebyshev polynomial applied to
+     * the right hand side \f$ x = C_{k-1}(M^{-1}A)M^{-1}b = E^{-1} C_{k-1}(
+     * {E^{-1}}^\mathrm{T} A E^{-1}){E^{-1}}^\mathrm{T}\f$
+     *
+     * @copydoc hide_matrix
+     * @tparam ContainerTypes must be usable with \c MatrixType and \c ContainerType in \ref dispatch
+     */
+    template< class MatrixType, class Preconditioner, class ContainerType0, class ContainerType1>
+    void solve( MatrixType& A, ContainerType0& x, const ContainerType1& b,
+            Preconditioner& P, value_type min_ev, value_type max_ev, unsigned num_iter,
+            bool x_is_zero = false)
+    {
+        if( num_iter == 0)
+            return;
+        assert ( min_ev < max_ev);
+        value_type theta = (min_ev+max_ev)/2., delta = (max_ev-min_ev)/2.;
+        value_type rhokm1 = delta/theta, rhok=0;
+        if( !x_is_zero)
+        {
+            dg::blas1::copy( x, m_xm1); //x_{k-1}
+            dg::blas2::symv( A, x, m_ax);
+            dg::blas1::axpby( 1., b, -1., m_ax); //r_0
+            dg::blas2::symv( P, m_ax, m_z);
+            dg::blas1::axpby( 1./theta, m_z, 1., x); //x_{k-1}
+        }
+        else
+        {
+            dg::blas1::scal( m_xm1, 0.);
+            dg::blas1::copy( b, m_ax); //r_0
+            dg::blas2::symv( P, m_ax, m_z);
+            dg::blas1::axpby( 1./theta, m_z, 0., x);
+        }
+        for ( unsigned k=1; k<num_iter; k++)
+        {
+            rhok = 1./(2.*theta/delta - rhokm1);
+            dg::blas2::symv( A, x, m_ax);
+            dg::blas1::axpby( 1., b, -1., m_ax); //r_k
+            dg::blas2::symv( P, m_ax, m_z);
+            dg::blas1::axpbypgz(
+                             1.+rhok*rhokm1, x,
+                             2.*rhok/delta,  m_z,
+                            -rhok*rhokm1,    m_xm1
+                            );
+            x.swap(m_xm1);
+            rhokm1 = rhok;
+        }
+    }
   private:
-    ContainerType m_ax, m_xm1;
+    ContainerType m_ax, m_z, m_xm1;
 };
 
+/**
+ * @brief Chebyshev Polynomial Preconditioner \f[ C( A)\f]
+ *
+ * This class can be used as a Preconditioner in the CG algorithm
+ * @sa ChebyshevIteration
+ * @tparam Matrix Preferably a reference type
+ * @tparam ContainerType
+ */
+template<class Matrix, class ContainerType>
+struct ChebyshevPreconditioner
+{
+    using container_type = ContainerType;
+    using value_type = get_value_type<ContainerType>; //!< value type of the ContainerType class
+    /**
+     * @brief  Construct the k-th Chebyshev Polynomial
+     *
+     * @param op The Matrix (copied, so maybe choose a reference type for shallow copying) will be called as \c dg::blas2::symv( op, x, y)
+     * @param P The inner Preconditioner (copied, so maybe choose a reference type for shallow copying) will be called as \c dg::blas2::symv( op, x, y)
+     * @param copyable A ContainerType must be copy-constructible from this
+     * @param min_ev an estimate of the minimum Eigenvalue (It is important to get a good value here. Unfortunately, we currently have no perfect way of getting this value, as a suggestion use \c 0.01*max_ev)
+     * @param max_ev an estimate of the maximum Eigenvalue of \f$ A\f$ (must be larger than \c min_ev)
+     * Use \c EVE to get this value
+     * @param degree degree k of the Polynomial (5 should be a good number - only up to degree 10 polynomials are implemented at the moment)
+     */
+    ChebyshevPreconditioner( Matrix op, const ContainerType& copyable, value_type ev_min,
+            value_type ev_max, unsigned degree):
+        m_op(op), m_ch( copyable),
+        m_ev_min(ev_min), m_ev_max(ev_max), m_degree(degree){}
+
+    template<class ContainerType0, class ContainerType1>
+    void symv( const ContainerType0& x, ContainerType1& y)
+    {
+        //m_ch.solve( m_op, y, x, m_p, m_ev_min, m_ev_max, m_degree+1, true);
+        m_ch.solve( m_op, y, x, m_ev_min, m_ev_max, m_degree+1, true);
+    }
+    private:
+    Matrix m_op;
+    ChebyshevIteration<ContainerType> m_ch;
+    value_type m_ev_min, m_ev_max;
+    unsigned m_degree;
+};
+
+/**
+ * @brief Least Squares Polynomial Preconditioner \f[ M^{-1} s( AM^{-1})\f]
+ *
+ * This class can be used as a Preconditioner in the CG algorithm.
+ *
+ * Implements the least squares polynomial preconditioner as suggested by
+ * <a href= "https://doi.org/10.1137/0906059"> Youcef Saad, Practical Use of Polynomial Preconditionings for the Conjugate Gradient Method,SIAM J. Sci. and Stat. Comput., 6(4), 865–881 (1985) </a>
+ * @note The least squares polynomial might perform better than Chebyshev Polynomials
+ * and they do not need an estimate of the lowest Eigenvalue
+ *
+ * \sa For more information see the book
+ * <a href="https://www-users.cs.umn.edu/~saad/IterMethBook_2ndEd.pdf">Iteratvie Methods for Sparse Linear Systems" 2nd edition by Yousef Saad </a>
+ * @tparam Matrix Preferably a reference type
+ * @tparam InnerPreconditioner Preferably a reference type
+ * @tparam ContainerType
+ */
+template<class Matrix, class InnerPreconditioner, class ContainerType>
+struct LeastSquaresPreconditioner
+{
+    using container_type = ContainerType;
+    using value_type = get_value_type<ContainerType>; //!< value type of the ContainerType class
+    /**
+     * @brief  Construct k-th Least Squares Polynomial
+     *
+     * @param op The Matrix (copied, so maybe choose a reference type for shallow copying) will be called as \c dg::blas2::symv( op, x, y)
+     * @param P The inner Preconditioner (copied, so maybe choose a reference type for shallow copying) will be called as \c dg::blas2::symv( op, x, y)
+     * @param copyable A ContainerType must be copy-constructible from this
+     * @param ev_max An estimate of the largest Eigenvalue of \f$ M^{-1} A\f$. Use \c EVE to get this value
+     * @param degree degree k of the Polynomial (5 should be a good number - only up to degree 10 polynomials are implemented at the moment)
+     */
+    LeastSquaresPreconditioner( Matrix op, InnerPreconditioner P, const ContainerType& copyable, value_type ev_max, unsigned degree):
+        m_op(op), m_p(P), m_z(copyable),
+        m_ev_max( ev_max), m_degree(degree){
+            m_c = coeffs(degree);
+    }
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_z;}
+
+    template<class ContainerType0, class ContainerType1>
+    void symv( const ContainerType0& x, ContainerType1& y)
+    {
+        //Horner scheme
+        dg::blas1::axpby( m_c[m_degree],x, 0., m_z);
+        for( int i=m_degree-1; i>=0; i--)
+        {
+            //dg::blas1::copy( m_z, y);
+            dg::blas2::symv( m_p, m_z, y);
+            dg::blas2::symv( m_op, y, m_z);
+            dg::blas1::axpby( m_c[i], x, +4./m_ev_max, m_z);
+        }
+        //dg::blas1::copy( m_z, y);
+        dg::blas2::symv( m_p, m_z, y);
+    }
+    private:
+    std::vector<value_type> coeffs( unsigned degree){
+        switch( degree){
+            case 0: return {1.};
+            case 1: return {5., -1.};
+            case 2: return { 14., -7., 1.};
+            case 3: return {30., -27., 9., -1.};
+            case 4: return {55., -77., 44., -11., 1.};
+            case 5: return {91., -182., 156., -65., 13., -1. };
+            case 6: return {140., -378., 450., -275., 90., -15., 1. };
+            case 7: return {204., -714.,1122., -935., 442., -119., 17., -1.};
+            case 8: return {285.,-1254., 2508., -2717., 1729., -665., 152., -19., 1.};
+            case 9: return {385., -2079., 5148.,-7007.,5733.,-2940.,952.,-189.,21.,-1.};
+            default:
+                if (degree > 10)
+                    std::cerr << "WARNING: LeastSquares Case "<<degree<<" not implemented. Taking 10 instead!\n";
+                return {506., -3289., 9867.,-16445.,16744.,-10948.,4692.,-1311.,230.,-23.,1. };
+        };
+    }
+    std::vector<value_type> m_c;
+    Matrix m_op;
+    InnerPreconditioner m_p;
+    ContainerType m_z;
+    value_type m_ev_max;
+    unsigned m_degree;
+};
+
+///@cond
+template<class M, class P, class V>
+struct TensorTraits<ChebyshevPreconditioner<M,P,V>>
+{
+    using value_type      = get_value_type<V>;
+    using tensor_category = SelfMadeMatrixTag;
+};
+template<class M, class P, class V>
+struct TensorTraits<LeastSquaresPreconditioner<M,P,V>>
+{
+    using value_type      = get_value_type<V>;
+    using tensor_category = SelfMadeMatrixTag;
+};
+
+///@endcond
+
+//template<class Matrix, class Container>
+//struct WrapperSpectralShift
+//{
+//    WrapperSpectralShift( Matrix& op, value_type ev_max):
+//        m_op(op), m_ev_max(ev_max){}
+//    template<class ContainerType0, class ContainerType1>
+//    void symv( const ContainerType0& x, ContainerType1& y)
+//    {
+//        dg::blas1::axpby( m_ev_max, x, 0., y);
+//        dg::blas2::symv( -1., m_op, x, 1., y);
+//    }
+//
+//    private:
+//    Matrix& m_op;
+//    value_type m_ev_max;
+//
+//};
+//template<class M, class V>
+//struct TensorTraits<detail::WrapperSpectralShift<M,V>>
+//{
+//    using value_type      = get_value_type<V>;
+//    using tensor_category = SelfMadeMatrixTag;
+//};
+
 } //namespace dg
 
 #endif // _DG_CHEB_
-- 
GitLab


From 8987a536555b535177adf38564433bf793b72d7e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Jun 2020 23:50:05 +0200
Subject: [PATCH 263/540] Add direct_solve_with_chebyshev to multigrid

- let us see how well that performs
---
 inc/dg/cg2d_t.cu      |   2 +-
 inc/dg/chebyshev.h    |   4 +-
 inc/dg/multigrid.h    | 106 ++++++++++++++++++++++++++++++++++++++++--
 inc/dg/multigrid_b.cu |  22 +++++++--
 4 files changed, 124 insertions(+), 10 deletions(-)

diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index d06d5c9b3..9802af855 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -22,7 +22,7 @@ void solve( std::string solver, Matrix& A, Container& x, const Container& b, con
     if( "cheby" == solver)
     {
         std::cout <<" CHEBYSHEV SOLVER:\n";
-        dg::Chebyshev<Container> cheby( x);
+        dg::ChebyshevIteration<Container> cheby( x);
         double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
         double hxhy = lx*ly/(n*n*Nx*Ny);
         lmin *= hxhy, lmax *= hxhy; //we multiplied the matrix by w2d
diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index 723c0c85e..d9ffc01e3 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -322,8 +322,8 @@ struct LeastSquaresPreconditioner
 };
 
 ///@cond
-template<class M, class P, class V>
-struct TensorTraits<ChebyshevPreconditioner<M,P,V>>
+template<class M, class V>
+struct TensorTraits<ChebyshevPreconditioner<M,V>>
 {
     using value_type      = get_value_type<V>;
     using tensor_category = SelfMadeMatrixTag;
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 547e3955a..f80d32606 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -231,6 +231,104 @@ struct MultigridCG2d
         return number;
     }
 
+    /**
+     * @brief Nested iterations with Chebyshev as preconditioner for CG
+     *
+     * - Compute residual with given initial guess.
+     * - Project residual down to the coarsest grid.
+     * - Solve equation on the coarse grid
+     * - interpolate solution up to next finer grid and repeat until the original grid is reached.
+     * @note The preconditioner for the CG solver on the coarse grids is a Chebyshev polynomial preconditioner
+     * @copydoc hide_symmetric_op
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     * @param op Index 0 is the \c SymmetricOp on the original grid, 1 on the half grid, 2 on the quarter grid, ...
+     * @param x (read/write) contains initial guess on input and the solution on output
+     * @param b The right hand side (will be multiplied by \c weights)
+     * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$
+     * @return the number of iterations in each of the stages beginning with the finest grid
+     * @note If the Macro \c DG_BENCHMARK is defined this function will write timings to \c std::cout
+    */
+	template<class SymmetricOp, class ContainerType0, class ContainerType1>
+    std::vector<unsigned> direct_solve_with_chebyshev( std::vector<SymmetricOp>& op, ContainerType0&  x, const ContainerType1& b, value_type eps, unsigned num_cheby)
+    {
+#ifdef DG_BENCHMARK
+        Timer t;
+        t.tic();
+#endif //DG_BENCHMARK
+        dg::blas2::symv(op[0].weights(), b, m_b[0]);
+        // compute residual r = Wb - A x
+        dg::blas2::symv(op[0], x, m_r[0]);
+        dg::blas1::axpby(-1.0, m_r[0], 1.0, m_b[0], m_r[0]);
+        // project residual down to coarse grid
+        for( unsigned u=0; u<m_stages-1; u++)
+            dg::blas2::gemv( m_interT[u], m_r[u], m_r[u+1]);
+        std::vector<unsigned> number(m_stages);
+
+        dg::blas1::scal( m_x[m_stages-1], 0.0);
+        unsigned lowest = m_stages-1;
+        dg::EVE<Container> eve( m_x[lowest]);
+        double evu_max;
+        Container tmp = m_x[lowest];
+        dg::blas1::scal( tmp, 0.);
+        //unsigned counter = eve( op[u], tmp, m_b[u], op[u].precond(), evu_max, 1e-16);
+        unsigned counter = eve( op[lowest], tmp, m_r[lowest], evu_max, 1e-10);
+        std::cout << "# MAX EV is "<<evu_max<<" in "<<counter<<" iterations\t";
+            t.toc();
+            std::cout << " took "<<t.diff()<<"s\n";
+        //now solve residual equations
+		for( unsigned u=m_stages-1; u>0; u--)
+        {
+#ifdef DG_BENCHMARK
+            t.tic();
+#endif //DG_BENCHMARK
+
+            //double evu_min;
+            //dg::detail::WrapperSpectralShift<SymmetricOp, Container> shift(
+            //        op[u], evu_max);
+            //counter = eve( shift, m_x[u], m_r[u], evu_min, eps);
+            //evu_min = evu_max - evu_min;
+            //std::cout << "# MIN EV is "<<evu_min<<" in "<<counter<<"iterations\n";
+            dg::ChebyshevPreconditioner<SymmetricOp&, Container> precond(
+                    op[u], m_x[u], 0.01*evu_max, 1.1*evu_max, num_cheby );
+            //dg::LeastSquaresPreconditioner<SymmetricOp&, const Container&, Container> precond(
+            //        op[u], op[u].precond(), m_x[u], evu_max, num_cheby );
+            number[u] = m_cg[u]( op[u], m_x[u], m_r[u], precond,
+                op[u].inv_weights(), eps, 1., 3);
+            dg::blas2::symv( m_inter[u-1], m_x[u], m_x[u-1]);
+#ifdef DG_BENCHMARK
+            t.toc();
+#ifdef MPI_VERSION
+            int rank;
+            MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+            if(rank==0)
+#endif //MPI
+            std::cout << "# Nested iterations stage: " << u << ", iter: " << number[u] << ", took "<<t.diff()<<"s\n";
+#endif //DG_BENCHMARK
+
+        }
+#ifdef DG_BENCHMARK
+        t.tic();
+#endif //DG_BENCHMARK
+
+        dg::ChebyshevPreconditioner<SymmetricOp&, Container> precond(
+                op[0], m_x[0], 0.01*evu_max, 1.1*evu_max, num_cheby );
+        //update initial guess
+        dg::blas1::axpby( 1., m_x[0], 1., x);
+        number[0] = m_cg[0]( op[0], x, m_b[0],// op[0].precond(),
+                precond,
+            op[0].inv_weights(), eps);
+#ifdef DG_BENCHMARK
+        t.toc();
+#ifdef MPI_VERSION
+        int rank;
+        MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+        if(rank==0)
+#endif //MPI
+        std::cout << "# Nested iterations stage: " << 0 << ", iter: " << number[0] << ", took "<<t.diff()<<"s\n";
+#endif //DG_BENCHMARK
+
+        return number;
+    }
 
     /**
      * @brief Full multigrid cycles (experimental, use at own risk)
@@ -252,7 +350,7 @@ struct MultigridCG2d
      * @attention This method is rather unreliable, it only converges if the
      * parameters are chosen correctly ( there need to be enough smooting steps
      * for instance, and a large jump  factor in the Elliptic class also seems
-     * to help) and otherwise just iterates to infinity
+     * to help) and otherwise just iterates to infinity. This behaviour is probably related to the use of the Chebyshev solver as a smoother
     */
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
     void fmg_solve( std::vector<SymmetricOp>& op,
@@ -394,7 +492,7 @@ struct MultigridCG2d
 
         //std::vector<Container> out( x);
 
-        m_cheby[p].solve( op[p], x[p], b[p], 1e-2*ev[p], 1.1*ev[p], nu1);
+        m_cheby[p].solve( op[p], x[p], b[p], op[p].precond(), 1e-2*ev[p], 1.1*ev[p], nu1);
         //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1, op[p].inv_weights());
         // 2. Residuum
         dg::blas2::symv( op[p], x[p], m_r[p]);
@@ -441,7 +539,7 @@ struct MultigridCG2d
         //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
         // 6. Post-Smooth nu2 times
-        m_cheby[p].solve( op[p], x[p], b[p], 1e-2*ev[p], 1.1*ev[p], nu2);
+        m_cheby[p].solve( op[p], x[p], b[p], op[p].precond(), 1e-2*ev[p], 1.1*ev[p], nu2);
         //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2, op[p].inv_weights());
         //dg::blas2::symv( op[p], x[p], m_r[p]);
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
@@ -493,7 +591,7 @@ struct MultigridCG2d
     std::vector< MultiMatrix<Matrix, Container> >  m_interT;
     std::vector< MultiMatrix<Matrix, Container> >  m_project;
     std::vector< CG<Container> > m_cg;
-    std::vector< Chebyshev<Container>> m_cheby;
+    std::vector< ChebyshevIteration<Container>> m_cheby;
     std::vector< Container> m_x, m_r, m_b;
     Container  m_p, m_cgr;
 
diff --git a/inc/dg/multigrid_b.cu b/inc/dg/multigrid_b.cu
index 86b407879..898bf8799 100644
--- a/inc/dg/multigrid_b.cu
+++ b/inc/dg/multigrid_b.cu
@@ -60,16 +60,17 @@ int main()
     unsigned stages = 3;
     std::cout<< "Type number of stages (3) and jfactor (10) !\n";
     std::cin >> stages >> jfactor;
+    std::cout << stages << " "<<jfactor<<std::endl;
     dg::MultigridCG2d<dg::aGeometry2d, dg::DMatrix, dg::DVec > multigrid(
         grid, stages);
     const std::vector<dg::DVec> multi_chi = multigrid.project( chi);
 
     std::vector<dg::DVec> multi_x = multigrid.project( x);
-    const std::vector<dg::DVec> multi_b = multigrid.project( b);
+    std::vector<dg::DVec> multi_b = multigrid.project( b);
     std::vector<dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> > multi_pol( stages);
     std::vector<dg::EVE<dg::DVec> > multi_eve(stages);
     std::vector<double> multi_ev(stages);
-    double eps_ev = 1e-4;
+    double eps_ev = 1e-10;
     double hxhy = lx*ly/(n*n*Nx*Ny);
     unsigned counter;
     std::cout << "\nPrecision EVE is "<<eps_ev<<"\n";
@@ -80,7 +81,9 @@ int main()
         multi_pol[u].set_chi( multi_chi[u]);
         //estimate EVs
         multi_eve[u].construct( multi_chi[u]);
+        dg::blas2::symv(multi_pol[u].weights(), multi_b[u], multi_b[u]);
         counter = multi_eve[u]( multi_pol[u], multi_x[u], multi_b[u],
+                multi_pol[u].precond(),
             multi_ev[u], eps_ev);
         //multi_ev[u]/=hxhy;
         std::cout << "Eigenvalue estimate eve: "<<multi_ev[u]<<"\n";
@@ -92,13 +95,14 @@ int main()
     std::cout << "Type nu1 (20), nu2 (20) gamma (1) \n";
     unsigned nu1, nu2, gamma;
     std::cin >> nu1 >> nu2 >> gamma;
+    std::cout << nu1 << " "<<nu2<<" "<<gamma<<std::endl;
     dg::Timer t;
     std::cout << "MULTIGRID NESTED ITERATIONS SOLVE:\n";
     x = dg::evaluate( initial, grid);
     t.tic();
     multigrid.direct_solve(multi_pol, x, b, eps);
     t.toc();
-    const double norm = dg::blas2::dot( w2d, solution);
+    double norm = dg::blas2::dot( w2d, solution);
     dg::DVec error( solution);
     dg::blas1::axpby( 1.,x,-1., solution, error);
     double err = dg::blas2::dot( w2d, error);
@@ -106,6 +110,18 @@ int main()
     std::cout << " Error of nested iterations "<<err<<"\n";
     std::cout << "Took "<<t.diff()<<"s\n\n";
     ////////////////////////////////////////////////////
+    std::cout << "MULTIGRID NESTED ITERATIONS WITH CHEBYSHEV SOLVE:\n";
+    x = dg::evaluate( initial, grid);
+    t.tic();
+    multigrid.direct_solve_with_chebyshev(multi_pol, x, b, eps, nu1);
+    t.toc();
+    norm = dg::blas2::dot( w2d, solution);
+    error= solution;
+    dg::blas1::axpby( 1.,x,-1., solution, error);
+    err = dg::blas2::dot( w2d, error);
+    err = sqrt( err/norm);
+    std::cout << " Error of nested iterations "<<err<<"\n";
+    std::cout << "Took "<<t.diff()<<"s\n\n";
     {
         std::cout << "MULTIGRID PCG SOLVE:\n";
         x = dg::evaluate( initial, grid);
-- 
GitLab


From 002cab3ffce40e20276569e151b8c6c9b7e52b17 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Jun 2020 23:53:49 +0200
Subject: [PATCH 264/540] Experimental addition of direct chebyshev solve

to felto3d
---
 src/feltor/feltor.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 03ac6a37f..2c37966a2 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -668,8 +668,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_phi(
 #endif //DG_MANUFACTURED
     //----------Invert polarisation----------------------------//
     m_old_phi.extrapolate( time, m_phi[0]);
-    std::vector<unsigned> number = m_multigrid.direct_solve(
-        m_multi_pol, m_phi[0], m_temp0, m_p.eps_pol);
+    std::vector<unsigned> number = m_multigrid.direct_solve_with_chebyshev(
+        m_multi_pol, m_phi[0], m_temp0, m_p.eps_pol, 5);
     m_old_phi.update( time, m_phi[0]);
     if(  number[0] == m_multigrid.max_iter())
         throw dg::Fail( m_p.eps_pol);
-- 
GitLab


From 7fda007e07b111cfc95bbf6dc27ba7078c3125d4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Jun 2020 15:43:06 +0200
Subject: [PATCH 265/540] More tests

- remove use of direct_solve_cheby from feltor
---
 inc/dg/chebyshev.h  |  5 ++---
 inc/dg/multigrid.h  | 37 +++++++++++++++++++------------------
 src/feltor/feltor.h |  4 ++--
 3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index d9ffc01e3..23f0e96aa 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -175,8 +175,7 @@ class ChebyshevIteration
         else
         {
             dg::blas1::scal( m_xm1, 0.);
-            dg::blas1::copy( b, m_ax); //r_0
-            dg::blas2::symv( P, m_ax, m_z);
+            dg::blas2::symv( P, b, m_z);
             dg::blas1::axpby( 1./theta, m_z, 0., x);
         }
         for ( unsigned k=1; k<num_iter; k++)
@@ -230,7 +229,7 @@ struct ChebyshevPreconditioner
     template<class ContainerType0, class ContainerType1>
     void symv( const ContainerType0& x, ContainerType1& y)
     {
-        //m_ch.solve( m_op, y, x, m_p, m_ev_min, m_ev_max, m_degree+1, true);
+        //m_ch.solve( m_op, y, x, m_op.precond(), m_ev_min, m_ev_max, m_degree+1, true);
         m_ch.solve( m_op, y, x, m_ev_min, m_ev_max, m_degree+1, true);
     }
     private:
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index f80d32606..85c576026 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -265,22 +265,23 @@ struct MultigridCG2d
         std::vector<unsigned> number(m_stages);
 
         dg::blas1::scal( m_x[m_stages-1], 0.0);
-        unsigned lowest = m_stages-1;
-        dg::EVE<Container> eve( m_x[lowest]);
-        double evu_max;
-        Container tmp = m_x[lowest];
-        dg::blas1::scal( tmp, 0.);
-        //unsigned counter = eve( op[u], tmp, m_b[u], op[u].precond(), evu_max, 1e-16);
-        unsigned counter = eve( op[lowest], tmp, m_r[lowest], evu_max, 1e-10);
-        std::cout << "# MAX EV is "<<evu_max<<" in "<<counter<<" iterations\t";
-            t.toc();
-            std::cout << " took "<<t.diff()<<"s\n";
         //now solve residual equations
 		for( unsigned u=m_stages-1; u>0; u--)
         {
 #ifdef DG_BENCHMARK
             t.tic();
 #endif //DG_BENCHMARK
+        unsigned lowest = u;
+        dg::EVE<Container> eve( m_x[lowest]);
+        double evu_max;
+        Container tmp = m_x[lowest];
+        dg::blas1::scal( tmp, 0.);
+        //unsigned counter = eve( op[lowest], tmp, m_r[lowest], op[u].precond(), evu_max, 1e-10);
+        unsigned counter = eve( op[lowest], tmp, m_r[lowest], evu_max, 1e-10);
+        //std::cout << "# MAX EV is "<<evu_max<<" in "<<counter<<" iterations\t";
+        //    t.toc();
+        //    std::cout << " took "<<t.diff()<<"s\n";
+        //    t.tic();
 
             //double evu_min;
             //dg::detail::WrapperSpectralShift<SymmetricOp, Container> shift(
@@ -293,7 +294,7 @@ struct MultigridCG2d
             //dg::LeastSquaresPreconditioner<SymmetricOp&, const Container&, Container> precond(
             //        op[u], op[u].precond(), m_x[u], evu_max, num_cheby );
             number[u] = m_cg[u]( op[u], m_x[u], m_r[u], precond,
-                op[u].inv_weights(), eps, 1., 3);
+                op[u].inv_weights(), eps, 1., 10);
             dg::blas2::symv( m_inter[u-1], m_x[u], m_x[u-1]);
 #ifdef DG_BENCHMARK
             t.toc();
@@ -310,12 +311,12 @@ struct MultigridCG2d
         t.tic();
 #endif //DG_BENCHMARK
 
-        dg::ChebyshevPreconditioner<SymmetricOp&, Container> precond(
-                op[0], m_x[0], 0.01*evu_max, 1.1*evu_max, num_cheby );
+        //dg::ChebyshevPreconditioner<SymmetricOp&, Container> precond(
+        //        op[0], m_x[0], 0.01*evu_max, 1.1*evu_max, num_cheby );
         //update initial guess
         dg::blas1::axpby( 1., m_x[0], 1., x);
-        number[0] = m_cg[0]( op[0], x, m_b[0],// op[0].precond(),
-                precond,
+        number[0] = m_cg[0]( op[0], x, m_b[0], op[0].precond(),
+                //precond,
             op[0].inv_weights(), eps);
 #ifdef DG_BENCHMARK
         t.toc();
@@ -492,7 +493,7 @@ struct MultigridCG2d
 
         //std::vector<Container> out( x);
 
-        m_cheby[p].solve( op[p], x[p], b[p], op[p].precond(), 1e-2*ev[p], 1.1*ev[p], nu1);
+        m_cheby[p].solve( op[p], x[p], b[p], 1e-2*ev[p], 1.1*ev[p], nu1);
         //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu1, op[p].inv_weights());
         // 2. Residuum
         dg::blas2::symv( op[p], x[p], m_r[p]);
@@ -539,7 +540,7 @@ struct MultigridCG2d
         //norm_res = sqrt(dg::blas1::dot( m_r[p], m_r[p]));
         //std::cout<< " Norm residuum befor "<<norm_res<<"\n";
         // 6. Post-Smooth nu2 times
-        m_cheby[p].solve( op[p], x[p], b[p], op[p].precond(), 1e-2*ev[p], 1.1*ev[p], nu2);
+        m_cheby[p].solve( op[p], x[p], b[p], 1e-2*ev[p], 1.1*ev[p], nu2);
         //m_cheby[p].solve( op[p], x[p], b[p], 0.1*ev[p], 1.1*ev[p], nu2, op[p].inv_weights());
         //dg::blas2::symv( op[p], x[p], m_r[p]);
         //dg::blas1::axpby( 1., b[p], -1., m_r[p]);
@@ -575,7 +576,7 @@ struct MultigridCG2d
         MPI_Comm_rank(MPI_COMM_WORLD, &rank);
         if(rank==0)
 #endif //MPI
-        //std::cout << "# Multigrid stage: " << s << ", iter: " << number << ", took "<<t.diff()<<"s\n";
+        std::cout << "# Multigrid stage: " << s << ", iter: " << number << ", took "<<t.diff()<<"s\n";
 #endif //DG_BENCHMARK
 
 		for( int p=m_stages-2; p>=0; p--)
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 2c37966a2..03ac6a37f 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -668,8 +668,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_phi(
 #endif //DG_MANUFACTURED
     //----------Invert polarisation----------------------------//
     m_old_phi.extrapolate( time, m_phi[0]);
-    std::vector<unsigned> number = m_multigrid.direct_solve_with_chebyshev(
-        m_multi_pol, m_phi[0], m_temp0, m_p.eps_pol, 5);
+    std::vector<unsigned> number = m_multigrid.direct_solve(
+        m_multi_pol, m_phi[0], m_temp0, m_p.eps_pol);
     m_old_phi.update( time, m_phi[0]);
     if(  number[0] == m_multigrid.max_iter())
         throw dg::Fail( m_p.eps_pol);
-- 
GitLab


From 86f796614f1be056a7757ea562441f189e8ae48e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Jun 2020 17:47:10 +0200
Subject: [PATCH 266/540] Add missing Apar terms in vorticity equation

---
 src/feltor/feltor.tex   | 12 ++++---
 src/feltor/feltordiag.h | 73 +++++++++++++++++++++++++++++++++++++++--
 2 files changed, 78 insertions(+), 7 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 3fcbab46e..5468a50fc 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1096,7 +1096,7 @@ Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continui
 &\partial_t \RA{\Omega_E} + \frac{\partial}{\partial v} \frac{\d v}{\d \psi_p}\RA{ \vec j_{\Omega_E}\cn\psi_p} = -\RA{F_{L,\varphi}}+ \RA{\mathcal S_{\Omega_E}} + \RA{\Lambda_{\Omega_E}} \label{eq:exb_average} \\
 \Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \equiv \mu_i N_i u_{E,\varphi} \\
 \vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
-    - \frac{1}{\beta} \vn A_\parallel\cn\psi_p \left(\frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
+    - \vn A_\parallel\cn\psi_p \left(\frac{1}{\beta} \frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
     \mathcal S_{\Omega_E} &:= \mu_i S_{n_e} \frac{\vn\psi_p\cn\phi}{B^2} \quad
     \Lambda_{\Omega_E} := \mu_i \Lambda_{n_e}\frac{\vn\psi_p\cn\phi}{B^2}
 \end{align}
@@ -1117,7 +1117,9 @@ In the output file we have
     jsodiaeUE\_tt &$\mu_i \tau_i\vn\psi_p\cn n_e \frac{\bhat\times\vn\phi\cn \psi_p}{B}$ \\
     jsoexbiUD\_tt &$\mu_i\tau_i \frac{\vn\psi_p\cn\phi}{B^2} \frac{\bhat\times\vn N_i\cn \psi_p}{B}$ &
     jsoexbeUD\_tt &$\mu_i\tau_i \frac{\vn\psi_p\cn\phi}{B^2} \frac{\bhat\times\vn n_e\cn \psi_p}{B}$ \\
-    jsoapar\_tt &$ \vn\psi_p\cn A_\parallel \frac{\bhat\times\vn A_\parallel\cn \psi_p}{B\beta}$ &
+    jsoapar\_tt &$ -\vn\psi_p\cn A_\parallel \frac{\bhat\times\vn A_\parallel\cn \psi_p}{B\beta}$ &
+    jsodiaApar\_tt & $ -\frac{1}{2}\tau_i \vn\psi_p\cn  (N_iU_i)\frac{\bhat\times\vn A_\parallel}{B}\cn\psi_p$ \\
+    jsoexbApar\_tt & $ -\frac{1}{2}\tau_i \bhat\times\vn  (N_iU_i)\cn\psi_p \vn A_\parallel\cn\psi_p$ &
     socurve\_tt &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
     socurvi\_tt &$z_i\tau_i N_i \mathcal K(\psi_p)$ &
     socurvkappae\_tt &$z_e\mu_e n_eu_e^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
@@ -1134,7 +1136,7 @@ The flux surface average over the parallel momentum equation under species summa
 \begin{align}
   \frac{\partial}{\partial t}\RA{\mu_iN_iU_{i} }
     % \nonumber\\
-    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i \frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_sN_s + \mu_s N_sU_s^2) b_{\perp}^{\;v}  }
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i \frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_sN_s + z_s\mu_s N_sU_s^2) b_{\perp}^{\;v}  }
     \nonumber\\
    = \sum_s\RA{-z_s\tau_s N_s\npar \ln B} + \mu_i \RA{ S_{N_i} U_i}
    \label{eq:parallel_momentum}
@@ -1142,7 +1144,7 @@ The flux surface average over the parallel momentum equation under species summa
 while the toroidal parallel angular momentum contribution reads
 \begin{align}\label{eq:parallel_momentum_direction}
     \frac{\partial}{\partial t}  \RA{\mu_iN_iU_i b_\varphi}
-    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i b_\varphi\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_s N_s + \mu_sN_sU_s^2) b_\varphi b_{\perp}^{\;v} }
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i b_\varphi\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_s N_s + z_s\mu_sN_sU_s^2) b_\varphi b_{\perp}^{\;v} }
     \nonumber\\
    = \RA{F_{L,\varphi}} + \mu_i \RA{ S_{N_i} U_i b_\varphi}
 \end{align}
@@ -1159,6 +1161,8 @@ The relevant terms in the output file are (the Lorentz force term is described i
     niuibphi &$\mu_i N_iU_ib_\varphi$ \\
     jsparexbi\_tt       & $\mu_i N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ &
     jsparbphiexbi\_tt   & $\mu_i N_iU_ib_\varphi(\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    jsparApar\_tt       & $\sum_s (z_s \tau_s N_s + z_s \mu_s N_s U_s^2)b_\perp^v$ &
+    jsparbphiApar\_tt   & $\sum_s (z_s \tau_s N_s + z_s \mu_s N_s U_s^2)b_\varphi b_\perp^v$ \\
     sparmirrore\_tt & $-z_e\tau_en_e\npar \ln B$ &
     sparmirrori\_tt & $-z_i\tau_iN_i\npar \ln B$ \\
     sparsni\_tt & $\mu_i S_{N_i} U_i$ &
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index c8b9fa1ea..94a92d6fc 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -819,10 +819,38 @@ std::vector<Record> diagnostics2d_list = {
                 dg::blas1::scal( result, 0.);
             else
             {
-                // - AxB Dot GradPsi
-                routines::jacobian( v.f.bhatgB(), v.gradPsip, v.f.gradA(), result);
+                routines::jacobian( v.f.bhatgB(), v.f.gradA(), v.gradPsip, result);
                 routines::dot( v.f.gradA(), v.gradPsip, v.tmp[0]);
-                dg::blas1::pointwiseDot( 1./v.p.beta, result, v.tmp[0], 0., result);
+                dg::blas1::pointwiseDot( -1./v.p.beta, result, v.tmp[0], 0., result);
+            }
+        }
+    },
+    {"jsodiaApar_tt", "A parallel diamagnetic vorticity flux term (magnetization stress) (Time average)", true,
+        []( DVec& result, Variables& v){
+            if( v.p.beta == 0)
+                dg::blas1::scal( result, 0.);
+            else
+            {
+                routines::dot( v.gradPsip, v.f.gradU(1), v.tmp[0]);
+                routines::dot( v.gradPsip, v.f.gradN(1), v.tmp[1]);
+                dg::blas1::pointwiseDot( 1., v.tmp[0], v.f.density(1), 1., v.tmp[0], v.f.velocity(1), 0., result);
+
+                routines::jacobian( v.f.bhatgB(), v.f.gradA(), v.gradPsip, result);
+                dg::blas1::pointwiseDot( -1./2.*v.p.tau[1], result, v.tmp[0], 0., result);
+            }
+        }
+    },
+    {"jsoexbApar_tt", "A parallel ExB vorticity flux term (magnetization stress) (Time average)", true,
+        []( DVec& result, Variables& v){
+            if( v.p.beta == 0)
+                dg::blas1::scal( result, 0.);
+            else
+            {
+                routines::jacobian( v.f.bhatgB(), v.f.gradU(1), v.gradPsip, v.tmp[0]);
+                routines::jacobian( v.f.bhatgB(), v.f.gradN(1), v.gradPsip, v.tmp[1]);
+                dg::blas1::pointwiseDot( 1., v.tmp[0], v.f.density(1), 1., v.tmp[1], v.f.velocity(1), 0., result);
+                routines::dot( v.f.gradA(), v.gradPsip, v.tmp[2]);
+                dg::blas1::pointwiseDot( -1./2.*v.p.tau[1], result, v.tmp[2], 0., result);
             }
         }
     },
@@ -906,6 +934,45 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( 1., result, v.tmp[0],v.f.bphi(), 0., result);
         }
     },
+    {"jsparApar_tt", "Parallel momentum radial flux by magnetic flutter (Time average)", true,
+        []( DVec& result, Variables& v){
+            if( v.p.beta == 0)
+            {
+                dg::blas1::scal( result, 0.);
+            }
+            else
+            {
+                //b_\perp^v
+                routines::jacobian( v.f.gradA() , v.f.bhatgB(), v.gradPsip, v.tmp[2]);
+                dg::blas1::pointwiseDot( -v.p.mu[0], v.f.velocity(0), v.f.velocity(0), v.f.density(0),  0., v.tmp[0]);
+                dg::blas1::pointwiseDot( +v.p.mu[1], v.f.velocity(1), v.f.velocity(1), v.f.density(1),  0., v.tmp[1]);
+                dg::blas1::pointwiseDot( -v.p.tau[0], v.f.density(0), v.tmp[2], 0., result);
+                dg::blas1::pointwiseDot( +v.p.tau[1], v.f.density(1), v.tmp[2], 1., result);
+                dg::blas1::pointwiseDot( 1., v.tmp[0], v.tmp[2], 1., result);
+                dg::blas1::pointwiseDot( 1., v.tmp[1], v.tmp[2], 1., result);
+            }
+        }
+    },
+    {"jsparbphiApar_tt", "Parallel angular momentum radial flux by magnetic flutter (Time average)", true,
+        []( DVec& result, Variables& v){
+            if( v.p.beta == 0)
+            {
+                dg::blas1::scal( result, 0.);
+            }
+            else
+            {
+                //b_\perp^v
+                routines::jacobian( v.f.gradA() , v.f.bhatgB(), v.gradPsip, v.tmp[2]);
+                dg::blas1::pointwiseDot( v.tmp[2], v.f.bphi(), v.tmp[2]);
+                dg::blas1::pointwiseDot( -v.p.mu[0], v.f.velocity(0), v.f.velocity(0), v.f.density(0),  0., v.tmp[0]);
+                dg::blas1::pointwiseDot( +v.p.mu[1], v.f.velocity(1), v.f.velocity(1), v.f.density(1),  0., v.tmp[1]);
+                dg::blas1::pointwiseDot( -v.p.tau[0], v.f.density(0), v.tmp[2], 0., result);
+                dg::blas1::pointwiseDot( +v.p.tau[1], v.f.density(1), v.tmp[2], 1., result);
+                dg::blas1::pointwiseDot( 1., v.tmp[0], v.tmp[2], 1., result);
+                dg::blas1::pointwiseDot( 1., v.tmp[1], v.tmp[2], 1., result);
+            }
+        }
+    },
     /// --------------------- Parallel momentum source terms ---------------------//
     {"sparsni_tt", "Parallel momentum source by density source", true,
         []( DVec& result, Variables& v ) {
-- 
GitLab


From dbc559cb06eac8c6e520a966de15b7a40131892b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 20 Jun 2020 18:25:56 +0200
Subject: [PATCH 267/540] Provide accuracy at all stages in multigrid solve

---
 inc/dg/chebyshev.h | 13 +++++++------
 inc/dg/multigrid.h | 42 ++++++++++++++++++++++++++++++++++++------
 2 files changed, 43 insertions(+), 12 deletions(-)

diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index 23f0e96aa..2beb3d1ab 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -24,7 +24,7 @@ namespace dg
 *
 * Given the minimum and maximum Eigenvalue of the matrix A we define
 * \f[ \theta = (\lambda_\min+\lambda_\max)/2 \quad \delta = (\lambda_\max - \lambda_\min)/2 \\
-*     \rho_0 := \frac{\delta}{\theta},\ x_0 := x, \ x_{1} = x_0+\frac{1}{\theta} (b-Ax_0) \\
+*     \rho_0 := \frac{\delta}{\theta},\ x_0 := x, \ x_{1} = x_0+\frac{1}{\theta} M^{-1}(b-Ax_0) \\
 *     \rho_{k}:=\left(\frac{2\theta}{\delta}-\rho_{k-1}\right)^{-1} \\
 *     x_{k+1} := x_k + \rho_k\left( \rho_{k-1}(x_k - x_{k-1})
 *     + \frac{2}{\delta} M^{-1}( b - Ax_k) \right)
@@ -45,7 +45,7 @@ namespace dg
 * {E^{-1}}^\mathrm{T} A E^{-1}){E^{-1}}^\mathrm{T}\f$
 *
 * @attention Chebyshev iteration may diverge if the elliptical bound of the
-* Eigenvalues is not accurate or if an ellipsis is not a good fit for the
+* Eigenvalues is not accurate (in particular if \f$\lambda_\max\f$ is underestimated) or if an ellipsis is not a good fit for the
 * spectrum of the matrix
 *
 * @ingroup invert
@@ -204,6 +204,7 @@ class ChebyshevIteration
  * @sa ChebyshevIteration
  * @tparam Matrix Preferably a reference type
  * @tparam ContainerType
+ * @ingroup invert
  */
 template<class Matrix, class ContainerType>
 struct ChebyshevPreconditioner
@@ -214,12 +215,11 @@ struct ChebyshevPreconditioner
      * @brief  Construct the k-th Chebyshev Polynomial
      *
      * @param op The Matrix (copied, so maybe choose a reference type for shallow copying) will be called as \c dg::blas2::symv( op, x, y)
-     * @param P The inner Preconditioner (copied, so maybe choose a reference type for shallow copying) will be called as \c dg::blas2::symv( op, x, y)
      * @param copyable A ContainerType must be copy-constructible from this
-     * @param min_ev an estimate of the minimum Eigenvalue (It is important to get a good value here. Unfortunately, we currently have no perfect way of getting this value, as a suggestion use \c 0.01*max_ev)
-     * @param max_ev an estimate of the maximum Eigenvalue of \f$ A\f$ (must be larger than \c min_ev)
+     * @param ev_min an estimate of the minimum Eigenvalue (It is important to get a good value here. Unfortunately, we currently have no perfect way of getting this value, as a suggestion use \c 0.01*max_ev)
+     * @param ev_max an estimate of the maximum Eigenvalue of \f$ A\f$ (must be larger than \c min_ev)
      * Use \c EVE to get this value
-     * @param degree degree k of the Polynomial (5 should be a good number - only up to degree 10 polynomials are implemented at the moment)
+     * @param degree degree k of the Polynomial (5 should be a good number)
      */
     ChebyshevPreconditioner( Matrix op, const ContainerType& copyable, value_type ev_min,
             value_type ev_max, unsigned degree):
@@ -254,6 +254,7 @@ struct ChebyshevPreconditioner
  * @tparam Matrix Preferably a reference type
  * @tparam InnerPreconditioner Preferably a reference type
  * @tparam ContainerType
+ * @ingroup invert
  */
 template<class Matrix, class InnerPreconditioner, class ContainerType>
 struct LeastSquaresPreconditioner
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 85c576026..dbe144ddf 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -176,6 +176,15 @@ struct MultigridCG2d
     */
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
     std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, ContainerType0&  x, const ContainerType1& b, value_type eps)
+    {
+        std::vector<value_type> v_eps( m_stages, eps);
+		for( unsigned u=m_stages-1; u>0; u--)
+            v_eps[u] = 1.5*eps;
+        return direct_solve( op, x, b, v_eps);
+    }
+    ///@copydoc direct_solve()
+	template<class SymmetricOp, class ContainerType0, class ContainerType1>
+    std::vector<unsigned> direct_solve( std::vector<SymmetricOp>& op, ContainerType0&  x, const ContainerType1& b, std::vector<value_type> eps)
     {
         dg::blas2::symv(op[0].weights(), b, m_b[0]);
         // compute residual r = Wb - A x
@@ -197,7 +206,7 @@ struct MultigridCG2d
             t.tic();
 #endif //DG_BENCHMARK
             number[u] = m_cg[u]( op[u], m_x[u], m_r[u], op[u].precond(),
-                op[u].inv_weights(), eps*1.5, 1., 10);
+                op[u].inv_weights(), eps[u], 1., 10);
             dg::blas2::symv( m_inter[u-1], m_x[u], m_x[u-1]);
 #ifdef DG_BENCHMARK
             t.toc();
@@ -217,7 +226,7 @@ struct MultigridCG2d
         //update initial guess
         dg::blas1::axpby( 1., m_x[0], 1., x);
         number[0] = m_cg[0]( op[0], x, m_b[0], op[0].precond(),
-            op[0].inv_weights(), eps);
+            op[0].inv_weights(), eps[0]);
 #ifdef DG_BENCHMARK
         t.toc();
 #ifdef MPI_VERSION
@@ -245,11 +254,23 @@ struct MultigridCG2d
      * @param x (read/write) contains initial guess on input and the solution on output
      * @param b The right hand side (will be multiplied by \c weights)
      * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$
+     * @param num_cheby Number of chebyshev iterations
      * @return the number of iterations in each of the stages beginning with the finest grid
      * @note If the Macro \c DG_BENCHMARK is defined this function will write timings to \c std::cout
     */
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
     std::vector<unsigned> direct_solve_with_chebyshev( std::vector<SymmetricOp>& op, ContainerType0&  x, const ContainerType1& b, value_type eps, unsigned num_cheby)
+    {
+        std::vector<value_type> v_eps( m_stages, eps);
+        std::vector<unsigned> v_num_cheby( m_stages, num_cheby);
+        v_num_cheby[0] = 0;
+		for( unsigned u=m_stages-1; u>0; u--)
+            v_eps[u] = 1.5*eps;
+        return direct_solve_with_chebyshev( op, x, b, v_eps, v_num_cheby);
+    }
+    ///@copydoc direct_solve_with_chebyshev()
+	template<class SymmetricOp, class ContainerType0, class ContainerType1>
+    std::vector<unsigned> direct_solve_with_chebyshev( std::vector<SymmetricOp>& op, ContainerType0&  x, const ContainerType1& b, std::vector<value_type> eps, std::vector<unsigned> num_cheby)
     {
 #ifdef DG_BENCHMARK
         Timer t;
@@ -278,6 +299,7 @@ struct MultigridCG2d
         dg::blas1::scal( tmp, 0.);
         //unsigned counter = eve( op[lowest], tmp, m_r[lowest], op[u].precond(), evu_max, 1e-10);
         unsigned counter = eve( op[lowest], tmp, m_r[lowest], evu_max, 1e-10);
+        counter++;
         //std::cout << "# MAX EV is "<<evu_max<<" in "<<counter<<" iterations\t";
         //    t.toc();
         //    std::cout << " took "<<t.diff()<<"s\n";
@@ -290,11 +312,11 @@ struct MultigridCG2d
             //evu_min = evu_max - evu_min;
             //std::cout << "# MIN EV is "<<evu_min<<" in "<<counter<<"iterations\n";
             dg::ChebyshevPreconditioner<SymmetricOp&, Container> precond(
-                    op[u], m_x[u], 0.01*evu_max, 1.1*evu_max, num_cheby );
+                    op[u], m_x[u], 0.01*evu_max, 1.1*evu_max, num_cheby[u] );
             //dg::LeastSquaresPreconditioner<SymmetricOp&, const Container&, Container> precond(
             //        op[u], op[u].precond(), m_x[u], evu_max, num_cheby );
             number[u] = m_cg[u]( op[u], m_x[u], m_r[u], precond,
-                op[u].inv_weights(), eps, 1., 10);
+                op[u].inv_weights(), eps[u], 1., 10);
             dg::blas2::symv( m_inter[u-1], m_x[u], m_x[u-1]);
 #ifdef DG_BENCHMARK
             t.toc();
@@ -310,14 +332,22 @@ struct MultigridCG2d
 #ifdef DG_BENCHMARK
         t.tic();
 #endif //DG_BENCHMARK
+        //unsigned lowest = 0;
+        //dg::EVE<Container> eve( m_x[lowest]);
+        //double evu_max;
+        //Container tmp = m_x[lowest];
+        //dg::blas1::scal( tmp, 0.);
+        ////unsigned counter = eve( op[lowest], tmp, m_r[lowest], op[u].precond(), evu_max, 1e-10);
+        //unsigned counter = eve( op[lowest], tmp, m_r[lowest], evu_max, 1e-10);
+        //counter++;
 
         //dg::ChebyshevPreconditioner<SymmetricOp&, Container> precond(
-        //        op[0], m_x[0], 0.01*evu_max, 1.1*evu_max, num_cheby );
+        //        op[0], m_x[0], 0.01*evu_max, 1.1*evu_max, num_cheby[0] );
         //update initial guess
         dg::blas1::axpby( 1., m_x[0], 1., x);
         number[0] = m_cg[0]( op[0], x, m_b[0], op[0].precond(),
                 //precond,
-            op[0].inv_weights(), eps);
+            op[0].inv_weights(), eps[0]);
 #ifdef DG_BENCHMARK
         t.toc();
 #ifdef MPI_VERSION
-- 
GitLab


From c7ea9b3da2d9e3a587b06dc8dba219b51eb63ba6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 20 Jun 2020 23:15:46 +0200
Subject: [PATCH 268/540] Add EVE and Cheby tests to cg2d_t.cu

---
 inc/dg/cg2d_t.cu | 38 ++++++++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index 9802af855..bd16ec1de 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -2,6 +2,7 @@
 #include <iomanip>
 
 #include "cg.h"
+#include "eve.h"
 #include "bicgstabl.h"
 #include "lgmres.h"
 #include "elliptic.h"
@@ -19,6 +20,28 @@ template<class Matrix, class Container>
 void solve( std::string solver, Matrix& A, Container& x, const Container& b, const dg::Grid2d& grid)
 {
     unsigned n = grid.n(), Nx = grid.Nx(), Ny = grid.Ny();
+    if( "eve cg" == solver)
+    {
+        std::cout <<" EVE SOLVER:\n";
+        dg::EVE<Container> eve( x);
+        double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
+        double hxhy = lx*ly/(n*n*Nx*Ny);
+        lmin *= hxhy, lmax *= hxhy; //we multiplied the matrix by w2d
+        std::cout << "L_min     "<<lmin<<" L_max     "<<lmax<<"\n";
+        double eve_max;
+        unsigned counter = eve( A, x, b, eve_max, 1e-10);
+        std::cout << "Maximum EV mod "<<eve_max<<" after "<<counter<<" EVE iterations\n";
+    }
+    if( "eve pcg" == solver)
+    {
+        std::cout <<" PRECONDITIONED EVE SOLVER:\n";
+        dg::EVE<Container> eve( x);
+        double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
+        std::cout << "L_min     "<<lmin<<" L_max     "<<lmax<<"\n";
+        double eve_max;
+        unsigned counter = eve( A, x, b, A.inv_weights(), eve_max, 1e-10);
+        std::cout << "Maximum EV     "<<eve_max<<" after "<<counter<<" EVE iterations\n";
+    }
     if( "cheby" == solver)
     {
         std::cout <<" CHEBYSHEV SOLVER:\n";
@@ -26,10 +49,17 @@ void solve( std::string solver, Matrix& A, Container& x, const Container& b, con
         double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
         double hxhy = lx*ly/(n*n*Nx*Ny);
         lmin *= hxhy, lmax *= hxhy; //we multiplied the matrix by w2d
-        //std::cout << "Type number of Chebyshev iterations\n";
         unsigned num_iter =200;
-        //std::cin >> num_iter;
-        cheby.solve( A, x, b, lmin, lmax, num_iter);
+        cheby.solve( A, x, b, lmin, lmax/2., num_iter);
+        std::cout << "After "<<num_iter<<" Chebyshev iterations we have:\n";
+    }
+    if( "P cheby" == solver)
+    {
+        std::cout <<" PRECONDITIONED CHEBYSHEV SOLVER:\n";
+        dg::ChebyshevIteration<Container> cheby( x);
+        double lmin = 1+1, lmax = n*n*Nx*Nx + n*n*Ny*Ny; //Eigenvalues of Laplace
+        unsigned num_iter =200;
+        cheby.solve( A, x, b, A.inv_weights(), lmin, lmax/2., num_iter);
         std::cout << "After "<<num_iter<<" Chebyshev iterations we have:\n";
     }
     if( "bicgstabl" == solver)
@@ -107,7 +137,7 @@ int main()
     std::cout << "L2 Norm of Residuum is        " << res.d<<"\t"<<res.i << std::endl<<std::endl;
     //Fehler der Integration des Sinus ist vernachlässigbar (vgl. evaluation_t)
 
-    std::vector<std::string> solvers{ "cheby", "bicgstabl", "lgmres"};
+    std::vector<std::string> solvers{ "eve cg", "eve pcg", "cheby", "P cheby", "bicgstabl", "lgmres"};
     for(auto solver : solvers)
     {
         dg::blas1::copy( 0., x);
-- 
GitLab


From b05c00bd18726e0222d20a69fc3958ae2c97bafd Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 20 Jun 2020 23:20:25 +0200
Subject: [PATCH 269/540] Add Modified Chebyshev Preconditioner

---
 inc/dg/chebyshev.h | 101 +++++++++++++++++++++++++++++++++++++++++----
 inc/dg/multigrid.h |  31 +++++++-------
 2 files changed, 107 insertions(+), 25 deletions(-)

diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index 2beb3d1ab..10dbe0ad5 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -17,9 +17,9 @@ namespace dg
 * \f[ M^{-1}Ax=M^{-1}b\f]
 *
 * Chebyshev iteration is not well-suited for solving matrix equations
-* on its own. Rather, it is very well-suited as a smoother for a multigrid algorithm
+* on its own. Rather, it is suited as a smoother for a multigrid algorithm
 * and also as a Preconditioner for the Conjugate Gradient method.
-* In both cases its appeal stems from not having any scalar products,
+* It does not contain scalar products,
 * which makes it appaeling for both small and highly parallelized systems.
 *
 * Given the minimum and maximum Eigenvalue of the matrix A we define
@@ -174,9 +174,10 @@ class ChebyshevIteration
         }
         else
         {
+            dg::blas2::symv( P, b, x);
+            if( num_iter == 1) return;
             dg::blas1::scal( m_xm1, 0.);
-            dg::blas2::symv( P, b, m_z);
-            dg::blas1::axpby( 1./theta, m_z, 0., x);
+            dg::blas1::scal( x, 1./theta);
         }
         for ( unsigned k=1; k<num_iter; k++)
         {
@@ -197,10 +198,21 @@ class ChebyshevIteration
     ContainerType m_ax, m_z, m_xm1;
 };
 
+ /** @class hide_polynomial
+ *
+ * @note This class can be used as a Preconditioner in the CG algorithm. The CG
+ * algorithm forms an approximation to the solution in the form \f$ x_{k+1} =
+ * x_0 + P_k(A) r_0\f$ where \f$ P_k\f$ is a polynomial of degree \c k, which
+ * is optimal in minimizing the A-norm. Thus a polynomial preconditioner cannot
+ * decrease the number of matrix-vector multiplications needed to achieve a
+ * certain accuracy.  However, since polynomial preconditioners do not use scalar products they may
+ * offset the increased overhead if the dot product becomes a bottleneck for performance or scalability.
+  */
+
 /**
  * @brief Chebyshev Polynomial Preconditioner \f[ C( A)\f]
  *
- * This class can be used as a Preconditioner in the CG algorithm
+ * @copydoc hide_polynomial
  * @sa ChebyshevIteration
  * @tparam Matrix Preferably a reference type
  * @tparam ContainerType
@@ -240,14 +252,79 @@ struct ChebyshevPreconditioner
 };
 
 /**
- * @brief Least Squares Polynomial Preconditioner \f[ M^{-1} s( AM^{-1})\f]
+ * @brief Approximate inverse Chebyshev Polynomial Preconditioner \f[ A^{-1} = \frac{c_0}{2} I + \sum_{k=1}^{r}c_kT_k( Z)\f]
  *
- * This class can be used as a Preconditioner in the CG algorithm.
+ * This is the polynomial preconditioner as proposed by <a href="https://ieeexplore.ieee.org/document/1245544">Dag and Semlyen, A New Preconditioned Conjugate Gradient Power Flow, IEEE Transactions on power Systems, 18, (2003)</a>
+ * We have \f$ c_k = \sqrt{\lambda_\min\lambda_\max}^{-1} (\sqrt{\lambda_\min/\lambda_\max}-1)^k / (\sqrt{\lambda_\min/\lambda_\max }+ 1)^k\f$ and \f$ Z = 2 ( A - (\lambda_\max + \lambda_\min)I/2)/(\lambda_\max-\lambda_\min)\f$
+ *
+ * They propose to use \f$ \lambda_\min = \lambda_\max / (5r)\f$ where r is the degree
+ * of the polynomial
+ * @copydoc hide_polynomial
+ * @tparam Matrix Preferably a reference type
+ * @tparam ContainerType
+ * @ingroup invert
+ */
+template<class Matrix, class ContainerType>
+struct ModifiedChebyshevPreconditioner
+{
+    using container_type = ContainerType;
+    using value_type = get_value_type<ContainerType>; //!< value type of the ContainerType class
+    /**
+     * @brief  Construct the k-th Chebyshev Polynomial approximate
+     *
+     * @param op The Matrix (copied, so maybe choose a reference type for shallow copying) will be called as \c dg::blas2::symv( op, x, y)
+     * @param copyable A ContainerType must be copy-constructible from this
+     * @param ev_min an estimate of the minimum Eigenvalue (It is important to
+     * get a good value here. The authors propose to use
+     * \f$ \lambda_\min = \lambda_\max / (5r)\f$ where \c r is the \c degree
+     * @param ev_max an estimate of the maximum Eigenvalue of \f$ A\f$ (must be larger than \c min_ev)
+     * Use \c EVE to get this value
+     * @param degree degree k of the Polynomial (5 should be a good number)
+     */
+    ModifiedChebyshevPreconditioner( Matrix op, const ContainerType& copyable, value_type ev_min,
+            value_type ev_max, unsigned degree):
+        m_op(op), m_ax(copyable), m_z1(m_ax), m_z2(m_ax),
+        m_ev_min(ev_min), m_ev_max(ev_max), m_degree(degree){}
+
+    template<class ContainerType0, class ContainerType1>
+    void symv( const ContainerType0& x, ContainerType1& y)
+    {
+        value_type theta = (m_ev_min+m_ev_max)/2., delta = (m_ev_max-m_ev_min)/2.;
+        value_type c_k = 1./sqrt(m_ev_min*m_ev_max);
+        dg::blas1::axpby( c_k/2., x, 0., y);
+        if( m_degree == 0) return;
+        dg::blas2::symv( m_op, x, m_ax);
+        dg::blas1::axpby( 1./delta, m_ax, -theta/delta, x, m_z1); //T_{k-1} x
+        c_k *= (sqrt( m_ev_min/m_ev_max) - 1.)/(sqrt(m_ev_min/m_ev_max)+1);
+        dg::blas1::axpby( c_k, m_z1, 1., y);
+        if( m_degree == 1) return;
+        dg::blas1::copy( x, m_z2); //T_{k-2} x
+        for( unsigned i=1; i<m_degree; i++)
+        {
+            dg::blas2::symv( m_op, m_z1, m_ax);
+            dg::blas1::axpby( 1./delta, m_ax, -theta/delta, m_z1, m_ax); //Z T_{k-1}
+            dg::blas1::axpby( 2., m_ax, -1., m_z2, m_z2); //T_k
+            c_k *= (sqrt( m_ev_min/m_ev_max) - 1.)/(sqrt(m_ev_min/m_ev_max)+1);
+            dg::blas1::axpby( c_k, m_z2, 1., y);
+            m_z1.swap(m_z2);
+        }
+    }
+    private:
+    Matrix m_op;
+    ContainerType m_ax, m_z1, m_z2;
+    value_type m_ev_min, m_ev_max;
+    unsigned m_degree;
+};
+
+/**
+ * @brief Least Squares Polynomial Preconditioner \f[ M^{-1} s( AM^{-1})\f]
  *
  * Implements the least squares polynomial preconditioner as suggested by
  * <a href= "https://doi.org/10.1137/0906059"> Youcef Saad, Practical Use of Polynomial Preconditionings for the Conjugate Gradient Method,SIAM J. Sci. and Stat. Comput., 6(4), 865–881 (1985) </a>
- * @note The least squares polynomial might perform better than Chebyshev Polynomials
- * and they do not need an estimate of the lowest Eigenvalue
+ * @note The least squares polynomial might (or might not) perform
+ * better than Chebyshev Polynomials and does not need an estimate of the
+ * lowest Eigenvalue
+ * @copydoc hide_polynomial
  *
  * \sa For more information see the book
  * <a href="https://www-users.cs.umn.edu/~saad/IterMethBook_2ndEd.pdf">Iteratvie Methods for Sparse Linear Systems" 2nd edition by Yousef Saad </a>
@@ -328,6 +405,12 @@ struct TensorTraits<ChebyshevPreconditioner<M,V>>
     using value_type      = get_value_type<V>;
     using tensor_category = SelfMadeMatrixTag;
 };
+template<class M, class V>
+struct TensorTraits<ModifiedChebyshevPreconditioner<M,V>>
+{
+    using value_type      = get_value_type<V>;
+    using tensor_category = SelfMadeMatrixTag;
+};
 template<class M, class P, class V>
 struct TensorTraits<LeastSquaresPreconditioner<M,P,V>>
 {
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index dbe144ddf..238d903aa 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -170,7 +170,10 @@ struct MultigridCG2d
      * @param op Index 0 is the \c SymmetricOp on the original grid, 1 on the half grid, 2 on the quarter grid, ...
      * @param x (read/write) contains initial guess on input and the solution on output
      * @param b The right hand side (will be multiplied by \c weights)
-     * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$
+     * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon(
+     * ||b|| + 1) \f$. If needed (and it is recommended to tune these values)
+     * the accuracy can be set for each stage separately. Per default the same
+     * accuracy is used at all stages.
      * @return the number of iterations in each of the stages beginning with the finest grid
      * @note If the Macro \c DG_BENCHMARK is defined this function will write timings to \c std::cout
     */
@@ -241,22 +244,16 @@ struct MultigridCG2d
     }
 
     /**
-     * @brief Nested iterations with Chebyshev as preconditioner for CG
+     * @brief Nested iterations with Chebyshev as preconditioner for CG (experimental)
      *
-     * - Compute residual with given initial guess.
-     * - Project residual down to the coarsest grid.
-     * - Solve equation on the coarse grid
-     * - interpolate solution up to next finer grid and repeat until the original grid is reached.
-     * @note The preconditioner for the CG solver on the coarse grids is a Chebyshev polynomial preconditioner
-     * @copydoc hide_symmetric_op
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     * @param op Index 0 is the \c SymmetricOp on the original grid, 1 on the half grid, 2 on the quarter grid, ...
-     * @param x (read/write) contains initial guess on input and the solution on output
-     * @param b The right hand side (will be multiplied by \c weights)
-     * @param eps the accuracy: iteration stops if \f$ ||b - Ax|| < \epsilon( ||b|| + 1) \f$
-     * @param num_cheby Number of chebyshev iterations
-     * @return the number of iterations in each of the stages beginning with the finest grid
-     * @note If the Macro \c DG_BENCHMARK is defined this function will write timings to \c std::cout
+     * @note This function does the same as direct_solve but uses a
+     * ChebyshevPreconditioner (with EVE to estimate the largest EV) at the coarse
+     * grid levels (but not the fine level).
+     * @copydetails direct_solve()
+     * @param num_cheby Number of chebyshev iterations. If needed can be set
+     * for each stage separately. Per default it is the same for all stages.
+     * The 0 stage does not use the Chebyshev preconditioner, therefore
+     * num_cheby[0] will be ignored.
     */
 	template<class SymmetricOp, class ContainerType0, class ContainerType1>
     std::vector<unsigned> direct_solve_with_chebyshev( std::vector<SymmetricOp>& op, ContainerType0&  x, const ContainerType1& b, value_type eps, unsigned num_cheby)
@@ -313,6 +310,8 @@ struct MultigridCG2d
             //std::cout << "# MIN EV is "<<evu_min<<" in "<<counter<<"iterations\n";
             dg::ChebyshevPreconditioner<SymmetricOp&, Container> precond(
                     op[u], m_x[u], 0.01*evu_max, 1.1*evu_max, num_cheby[u] );
+            //dg::ModifiedChebyshevPreconditioner<SymmetricOp&, Container> precond(
+            //        op[u], m_x[u], evu_max/5./(num_cheby[u]), evu_max, num_cheby[u] );
             //dg::LeastSquaresPreconditioner<SymmetricOp&, const Container&, Container> precond(
             //        op[u], op[u].precond(), m_x[u], evu_max, num_cheby );
             number[u] = m_cg[u]( op[u], m_x[u], m_r[u], precond,
-- 
GitLab


From f28509211aa5421b3500607cc52d6b44031a6047 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mwiesenb@login01.m100.cineca.it>
Date: Sat, 20 Jun 2020 23:24:22 +0200
Subject: [PATCH 270/540] Adapt direct_solve call in feltor

---
 src/feltor/feltor.h   | 2 +-
 src/feltor/implicit.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 03ac6a37f..b6071fca4 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -669,7 +669,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_phi(
     //----------Invert polarisation----------------------------//
     m_old_phi.extrapolate( time, m_phi[0]);
     std::vector<unsigned> number = m_multigrid.direct_solve(
-        m_multi_pol, m_phi[0], m_temp0, m_p.eps_pol);
+        m_multi_pol, m_phi[0], m_temp0, {m_p.eps_pol, 2000*m_p.eps_pol, 100*m_p.eps_pol});
     m_old_phi.update( time, m_phi[0]);
     if(  number[0] == m_multigrid.max_iter())
         throw dg::Fail( m_p.eps_pol);
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 3d2a971c9..ae85135f9 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -159,7 +159,7 @@ struct ImplicitVelocity
             //    inv_weights(), precond());
             m_old_apar.extrapolate( m_apar);
             std::vector<unsigned> number = m_multigrid.direct_solve(
-                m_multi_induction, m_apar, m_temp, m_p.eps_pol);
+                m_multi_induction, m_apar, m_temp, {m_p.eps_pol,m_p.eps_pol,m_p.eps_pol});
             m_old_apar.update( m_apar);
             if(  number[0] == m_multigrid.max_iter())
                 throw dg::Fail( m_p.eps_pol);
-- 
GitLab


From 3ea5f84937d902d9f0a94d20ede96524eb361ba6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 22 Jun 2020 19:35:33 +0200
Subject: [PATCH 271/540] Add p_0 to background pressure profile

---
 src/feltor/feltor.tex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 5468a50fc..5968320a8 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -172,7 +172,7 @@ with $\Delta^*_\perp \psi_p = R\partial_R (R^{-1}\psi_R) + \psi_{ZZ}$.
 The Solov'ev assumptions consist of \(A/R_0 = -I \frac{d I}{d  \psi_p }\) and \((1-A)/R_0 = -\frac{d p}{d  \psi_p }\), where \(A\) is a constant~\cite{Cerfon2010,Cerfon2014}.
 By integration over \(\psi_p\) we find
 $
- p(\psi_p) = (A-1)\psi_p/R_0/\beta$,
+p(\psi_p) = (A-1)\psi_p/R_0/\beta + p(0)$,
  $I(\psi_p) = \sqrt{-2 A \psi_p/R_0 + 1}$,
  and
     $j_{\hat\varphi} = \left[(A-1)R^2/R_0^2 - A \right]/R/\beta $.
@@ -194,7 +194,7 @@ and thus represent a general solution to Equation~\eqref{eq:GSEdimless} as~\cite
 with $\mathcal P_\psi$ a free constant, $\mathcal P_I = \pm \mathcal P_\psi$ for $A\neq 0$ and $\mathcal P_I$ arbitrary for $A=0$ (purely toroidal equilibrium current).
 We have
 \begin{align}
-    p(\psi_p) = \mathcal P_\psi \frac{( A-1)\psi_p}{\beta R_0 } \qquad
+    p(\psi_p) = \mathcal P_\psi \frac{( A-1)\psi_p}{\beta R_0 } + p(0) \qquad
     j_{\hat\varphi} = \frac{\mathcal P_\psi}{\beta } \left[\frac{(A-1)R}{R_0^2} - \frac{A}{R}\right]
 \end{align}
 \rowcolors{2}{gray!25}{white}
-- 
GitLab


From 4a3774b2136f6ae2549741dda3396b0a91dac619 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 23 Jun 2020 16:10:34 +0200
Subject: [PATCH 272/540] Add curvne_tt to feltordiag

as a consistency test
---
 src/feltor/Makefile     | 4 ++--
 src/feltor/feltordiag.h | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index 0a93b91da..6d6d091fb 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -21,10 +21,10 @@ interpolate_in_3d: interpolate_in_3d.cu feltordiag.h
 feltor: feltor.cu feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
-feltor_hpc: feltor_hpc.cu feltor.h implicit.h init.h parameters.h init_from_file.h
+feltor_hpc: feltor_hpc.cu feltordiag.h feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(CC) -g $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
 
-feltor_mpi: feltor_hpc.cu feltor.h implicit.h init.h parameters.h init_from_file.h
+feltor_mpi: feltor_hpc.cu feltordiag.h feltor.h implicit.h init.h parameters.h init_from_file.h
 	$(MPICC) $(OPT) $(MPICFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DFELTOR_MPI -DDG_BENCHMARK
 
 .PHONY: clean
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 94a92d6fc..733efa052 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -448,6 +448,11 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
+    {"curvne_tt", "Curvature operator applied to electron density", true,
+        []( DVec& result, Variables& v ) {
+            routines::dot( v.f.curv(), v.f.gradN(0), result);
+        }
+    },
     /// ------------------ Correlation terms --------------------//
     {"ne2", "Square of electron density", false,
         []( DVec& result, Variables& v ) {
-- 
GitLab


From 14c69d6c22554ca6ea205acb2c5f821962d239b6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 31 Jul 2020 15:08:21 +0200
Subject: [PATCH 273/540] Add get_element in exdot_cuda version

---
 inc/dg/backend/exblas/exdot_cuda.cuh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inc/dg/backend/exblas/exdot_cuda.cuh b/inc/dg/backend/exblas/exdot_cuda.cuh
index 28d456434..7695be63f 100644
--- a/inc/dg/backend/exblas/exdot_cuda.cuh
+++ b/inc/dg/backend/exblas/exdot_cuda.cuh
@@ -64,7 +64,7 @@ __global__ void ExDOT(
     for(uint pos = blockIdx.x*blockDim.x+threadIdx.x; pos < NbElements; pos += gridDim.x*blockDim.x) {
         //double r = 0.0;
         //double x = TwoProductFMA(get_element(d_a,pos), get_element(d_b,pos), &r);
-        double x = d_a[pos]*d_b[pos];
+        double x = get_element(d_a,pos)*get_element(d_b,pos);
         //we do not accumulate the rest of this multiplication
 
         //Check if the input is sane
-- 
GitLab


From 2efaedf52a46780476c1c827d148ecac15c3c112 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 31 Jul 2020 15:19:04 +0200
Subject: [PATCH 274/540] remove const from get_element

- since the compiler will automatically add it for const variables
---
 inc/dg/backend/exblas/accumulate.h   | 2 +-
 inc/dg/backend/exblas/exdot_cuda.cuh | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/inc/dg/backend/exblas/accumulate.h b/inc/dg/backend/exblas/accumulate.h
index ddf262ff3..3efa315a3 100644
--- a/inc/dg/backend/exblas/accumulate.h
+++ b/inc/dg/backend/exblas/accumulate.h
@@ -60,7 +60,7 @@ inline double get_element( T x, int i){
 	return (double)x;
 }
 template<class T>
-inline double get_element( const T* x, int i){
+inline double get_element( T* x, int i){
 	return (double)(*(x+i));
 }
 
diff --git a/inc/dg/backend/exblas/exdot_cuda.cuh b/inc/dg/backend/exblas/exdot_cuda.cuh
index 7695be63f..cb124a8fa 100644
--- a/inc/dg/backend/exblas/exdot_cuda.cuh
+++ b/inc/dg/backend/exblas/exdot_cuda.cuh
@@ -43,6 +43,11 @@ static inline double KnuthTwoSum(double a, double b, double *s) {
     return r;
 }
 
+////////////////////////////////////////////////////////////////////////////////
+// Main Kernels
+////////////////////////////////////////////////////////////////////////////////
+//In the first kernel each block produces exactly one superacc (because threads within a block can be synchronized and shared memory lives for a block )
+//the second kernel reduces all superaccs from the first kernel (because we need a global synchronization, which is induced by separate kernel launches)
 
 template<uint NBFPE, uint WARP_COUNT, class PointerOrValue1, class PointerOrValue2>
 __global__ void ExDOT(
@@ -302,6 +307,7 @@ void ExDOTComplete(
         Normalize(&d_PartialSuperaccs[gid * BIN_COUNT * MERGE_SIZE], imin, imax);
     }
 
+    //MW: don't we need a global synchronization here??
     __syncthreads();
     if ((lid < BIN_COUNT) && (gid == 0)) {
         int64_t sum = 0;
-- 
GitLab


From 99103af83fcd12aa8ec746bf7ba53f841f8ca54e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 31 Jul 2020 15:25:23 +0200
Subject: [PATCH 275/540] Add 1d linear interpolation function to functors

---
 inc/dg/functors.h | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 7ebbb190f..cbea4ac11 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -650,6 +650,21 @@ struct ExpProfX
   private:
     double m_amp, m_bamp, m_ln;
 };
+
+/**
+ * @brief The linear interpolation polynomial
+ * \f[ f(x) = y_1\frac{x-x_0}{x_1-x_0} + y_0\frac{x-x_1}{x_0-x_1}\f]
+ */
+struct Line{
+    Line(double x0, double y0, double x1, double y1) :
+        m_x0(x0), m_y0(y0), m_x1(x1), m_y1(y1){}
+    double operator()(double x){
+        return m_y1*(x-m_x0)/(m_x1-m_x0) + m_y0*(x-m_x1)/(m_x0-m_x1);
+    }
+    private:
+    double m_x0, m_y0, m_x1, m_y1;
+};
+
 /**
  * @brief A linear function in x-direction
  * \f[ f(x) = f(x,y) = f(x,y,z) = ax+b \f]
@@ -1873,8 +1888,10 @@ struct Compose
  * @tparam G Inner functor
  * @param f outer functor
  * @param g inner functor
- * @note only works for host functions. If a version for device functions is ever needed
- * it can be easily provided
+ * @note only works for host functions. The rationale is that this function is
+ * intended to work with lambda functions and is to be used in the \c dg::evaluate function.
+ * If a version for device functions is ever needed
+ * it can be easily provided but the lambda support for CUDA is rather poor.
  *
  * @return a function object that forwards all parameters to g and returns the
  * return value of f, which is \f$ f(g(x_0,x_1,...)) \f$
-- 
GitLab


From fdd42fbf1a7d75f6b569765332d031045ce77d82 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 31 Jul 2020 15:25:54 +0200
Subject: [PATCH 276/540] Add y-values to NoRoot exception class

---
 inc/dg/nullstelle.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index 77fa25758..4105af40a 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -19,19 +19,19 @@ namespace dg{
 class NoRoot1d: public std::exception
 {
   private:
-    double x1, x2;
+    double x1, x2, wert_min, wert_max;
   public:
     /*! @brief construct
      *
      * \param x_min left boundary
      * \param x_max right boundary
      */
-    NoRoot1d(double x_min, double x_max): x1(x_min), x2(x_max){}
+    NoRoot1d(double x_min, double x_max, double wert_min, double wert_max): x1(x_min), x2(x_max), wert_min(wert_min), wert_max(wert_max){}
     /*! @brief display left and right boundary on std::cerr
      */
     void display() const
     {
-      std::cerr << "Between " <<x1 << " and "<<x2<<" is no root!\n";
+      std::cerr << "Between " <<x1<<"/"<<wert_min << " and "<<x2<<"/"<<wert_max<<" is no root!\n";
     }
     /**
      * @brief what string
@@ -66,7 +66,7 @@ int bisection1d (UnaryOp& op, double& x_min, double& x_max, const double eps)
     wert_min=op(x_min);
 
     if(wert_max*wert_min>=0)
-        throw NoRoot1d(x_min, x_max);
+        throw NoRoot1d(x_min, x_max, wert_min, wert_max);
 
     int j_max = 60;
     for(int j=0; j<j_max; j++)
-- 
GitLab


From 6416be676e34a1768e89037a4c257a8c37e8000c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 31 Jul 2020 15:33:46 +0200
Subject: [PATCH 277/540] small additions to doc in geometries

---
 inc/geometries/curvilinearX.h         | 2 +-
 inc/geometries/refined_curvilinearX.h | 2 +-
 inc/geometries/solovev.h              | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/inc/geometries/curvilinearX.h b/inc/geometries/curvilinearX.h
index dae331ca4..164090b02 100644
--- a/inc/geometries/curvilinearX.h
+++ b/inc/geometries/curvilinearX.h
@@ -104,7 +104,7 @@ struct RealCurvilinearGridX2d : public dg::aRealGeometryX2d<real_type>
 {
     /*!@brief Constructor
 
-     * @param generator must generate an orthogonal grid (class takes ownership of the pointer)
+     * @param generator must generate an orthogonal grid
      * @param fx a rational number indicating partition of the x - direction
      * @param fy a rational number indicating partition of the y - direction
      * @param n number of polynomial coefficients
diff --git a/inc/geometries/refined_curvilinearX.h b/inc/geometries/refined_curvilinearX.h
index 06020d8c1..4e85533d1 100644
--- a/inc/geometries/refined_curvilinearX.h
+++ b/inc/geometries/refined_curvilinearX.h
@@ -118,7 +118,7 @@ struct RealCurvilinearRefinedGridX2d : public dg::aRealGeometryX2d<real_type>
     /*!@brief Constructor
 
      * @param ref a X-point refinement
-     * @param generator must generate an orthogonal grid (class takes ownership of the pointer)
+     * @param generator must generate an orthogonal grid
      * @param fx a rational number indicating partition of the x - direction
      * @param fy a rational number indicating partition of the y - direction
      * @param n number of polynomial coefficients
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 3c2ab0171..f70cbbe77 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -561,7 +561,7 @@ static inline dg::geo::TokamakMagneticField createSolovevField(
  * @param gp Solovev parameters
  * @param psi0 boundary value where psi is modified to a constant psi0
  * @param alpha radius of the transition region where the modification acts (smaller is quicker)
- * @param sign determines which side of Psi to dampen (negative or positive)
+ * @param sign determines which side of Psi to dampen (negative or positive, forwarded to \c dg::IPolynomialHeaviside)
  * @return A magnetic field object
  * @ingroup geom
  */
-- 
GitLab


From 8ec817c75312534b41cc762b0bdf2e3ae9ecd02c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 31 Jul 2020 15:35:20 +0200
Subject: [PATCH 278/540] Make Fpsi more resilient to nan

---
 inc/geometries/flux.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/inc/geometries/flux.h b/inc/geometries/flux.h
index e2fdb9c92..2be419704 100644
--- a/inc/geometries/flux.h
+++ b/inc/geometries/flux.h
@@ -69,13 +69,14 @@ struct Fpsi
         begin[0] = R_0, begin[1] = Z_0;
         double eps = 1e10, eps_old = 2e10;
         unsigned N = 50;
+        unsigned nan_counter = 0;
         while( (eps < eps_old || eps > 1e-7)&& eps > 1e-14)
         {
             eps_old = eps, end_old = end; N*=2;
             dg::stepperRK( "Feagin-17-8-10",  fieldRZYT_, 0., begin, 2*M_PI, end, N);
             eps = sqrt( (end[0]-begin[0])*(end[0]-begin[0]) + (end[1]-begin[1])*(end[1]-begin[1]));
             if(m_verbose)std::cout << "\t error "<<eps<<" with "<<N<<" steps\t";
-            //Attention: if this succeeds on the first attempt end_old won't be updated
+            if( std::isnan( eps) && nan_counter < 4) eps = 1e10, end = end_old, nan_counter++;
         }
         if(m_verbose)std::cout << "\t error "<<eps<<" with "<<N<<" steps\t";
         if(m_verbose)std::cout <<end_old[2] << " "<<end[2] <<"\n";
-- 
GitLab


From 802ede55ffa84a6b1e5bbf63e7d07090041797e7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 31 Jul 2020 17:05:26 +0200
Subject: [PATCH 279/540] Add utitlity program normalize_solovev_t.cu

The argument is that a change in the parameter file itself to get Psi_p
to zero on the separatrix is more consistent than writing a function that
has to be called every time the parameter file is used.
---
 inc/geometries/normalize_solovev_t.cu | 45 +++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 inc/geometries/normalize_solovev_t.cu

diff --git a/inc/geometries/normalize_solovev_t.cu b/inc/geometries/normalize_solovev_t.cu
new file mode 100644
index 000000000..d5d1c73a2
--- /dev/null
+++ b/inc/geometries/normalize_solovev_t.cu
@@ -0,0 +1,45 @@
+#include <iostream>
+#include <fstream>
+#include "dg/algorithm.h"
+#include "dg/file/file.h"
+#include "solovev.h"
+#include "fluxfunctions.h"
+
+int main( int argc, char* argv[])
+{
+    Json::Value geom_js;
+    if( argc == 3)
+    {
+        std::cout << argv[0]<< " "<<argv[1]<<" -> " <<argv[2]<<std::endl;
+        file::file2Json( argv[1], geom_js, file::comments::are_discarded);
+    }
+    else
+    {
+        std::cerr << "This program reads solovev parameters from an input json file and subtracts a constant such that the resulting Psi_p is zero on the X-point. The resulting parameters are written into an output file, which may overwrite the input file. This assumes that there is an X-point!\n";
+        std::cerr << " Usage: "<< argv[0]<<" [input.json] [normalized.json]\n";
+        return -1;
+    }
+    dg::geo::solovev::Parameters gp(geom_js);
+    std::cout << "Input file: \n"<< geom_js.toStyledString();
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    double RX = gp.R_0-1.1*gp.triangularity*gp.a;
+    double ZX = -1.1*gp.elongation*gp.a;
+    if( gp.hasXpoint())
+        dg::geo::findXpoint( mag.get_psip(), RX, ZX);
+    else
+    {
+        std::cerr << "Parameters have no X-point!\n";
+        return -1;
+    }
+    const double psipX = mag.psip()( RX, ZX);
+    std::cout << "X-point found at "<<RX<<" "<<ZX<<" with Psip = "<<psipX<<std::endl;
+    gp.c[0] = gp.c[0] - psipX/gp.pp/gp.R_0;
+    Json::Value output = gp.dump();
+    std::cout << "Output file "<<argv[2]<<": \n"<< output.toStyledString();
+    std::fstream file( argv[2], std::fstream::out | std::fstream::trunc);
+    file << output.toStyledString();
+    file << std::endl;
+    file.close();
+    return 0;
+}
+
-- 
GitLab


From 28ca607325ed00affbea0b19197775dd9e76692c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 31 Jul 2020 17:28:39 +0200
Subject: [PATCH 280/540] Output all separatrix points in X grid generator

---
 inc/geometries/utilitiesX.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/inc/geometries/utilitiesX.h b/inc/geometries/utilitiesX.h
index a3ddf8827..ec3d33446 100644
--- a/inc/geometries/utilitiesX.h
+++ b/inc/geometries/utilitiesX.h
@@ -375,20 +375,23 @@ struct SeparatriX
         double R_X = xX; double Z_X = yX;
         PsipSep psip_sep( psi.f());
         psip_sep.set_Z( Z_X + 1.);
-        double R_min = R_X, R_max = R_X + 10;
+        double R_min = R_X, R_max = R_X + 10.;
         dg::bisection1d( psip_sep, R_min, R_max, 1e-13);
         R_i[0] = (R_min+R_max)/2., Z_i[0] = Z_X+1.;
+        if(m_verbose)std::cout << "Found 1st point "<<R_i[0]<<" "<<Z_i[0]<<"\n";
         R_min = R_X-10, R_max = R_X;
         dg::bisection1d( psip_sep, R_min, R_max, 1e-13);
         R_i[1] = (R_min+R_max)/2., Z_i[1] = Z_X+1.;
+        if(m_verbose)std::cout << "Found 2nd point "<<R_i[1]<<" "<<Z_i[1]<<"\n";
         psip_sep.set_Z( Z_X - 1.);
         R_min = R_X-10, R_max = R_X;
         dg::bisection1d( psip_sep, R_min, R_max, 1e-13);
         R_i[2] = (R_min+R_max)/2., Z_i[2] = Z_X-1.;
+        if(m_verbose)std::cout << "Found 3rd point "<<R_i[2]<<" "<<Z_i[2]<<"\n";
         R_min = R_X, R_max = R_X+10;
         dg::bisection1d( psip_sep, R_min, R_max, 1e-13);
         R_i[3] = (R_min+R_max)/2., Z_i[3] = Z_X-1.;
-        if(m_verbose)std::cout << "Found 3rd point "<<R_i[3]<<" "<<Z_i[3]<<"\n";
+        if(m_verbose)std::cout << "Found 4th point "<<R_i[3]<<" "<<Z_i[3]<<"\n";
         //now measure y distance to X-point
         std::array<double, 3> begin2d{ {0,0,0} }, end2d(begin2d);
         for( int i=0; i<4; i++)
-- 
GitLab


From 346b508a66bf8df4e042823149d9b89d005414f2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 3 Aug 2020 16:50:01 +0200
Subject: [PATCH 281/540] Add findCriticalPoint function

with the aim to reveal more information about the critical points
---
 inc/geometries/fluxfunctions.h            | 72 +++++++++++++++++++----
 inc/geometries/normalize_solovev_t.cu     | 10 ++--
 inc/geometries/separatrix_orthogonal.h    | 22 ++++---
 inc/geometries/separatrix_orthogonal_t.cu | 22 +++----
 4 files changed, 90 insertions(+), 36 deletions(-)

diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 6ad95edc2..d9e5b18c5 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -262,37 +262,83 @@ struct CylindricalFunctorsLvl2
  * @brief This function finds critical points of psi (any point with vanishing gradient, including the X-point or O-point) via Newton iteration applied to the gradient of psi
  *
  * The inverse of the Hessian matrix is computed analytically
- * @param psi \f$ \psi(R,Z)\f$, where R, Z are cylindrical coordinates
- * @param R_X start value on input, critical point on output
- * @param Z_X start value on input, critical point on output
+ * @param psi \f$ \psi(R,Z)\f$
+ * @param RC start value on input, critical point on output
+ * @param ZC start value on input, critical point on output
+ * @return 0 if no critical point or Hessian is 0,
+ * 1 if local minimum,
+ * 2 if local maximum,
+ * 3 if saddle point
  * @ingroup misc_geo
  */
-static inline void findOpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
+static inline int findCriticalPoint( const CylindricalFunctorsLvl2& psi, double& RC, double& ZC)
 {
     std::array<double, 2> X{ {0,0} }, XN(X), X_OLD(X);
-    X[0] = R_X, X[1] = Z_X;
+    X[0] = RC, X[1] = ZC;
     double eps = 1e10, eps_old= 2e10;
-    while( (eps < eps_old || eps > 1e-7) && eps > 1e-10)
+    unsigned counter = 0; //safety measure to avoid deadlock
+    double Dinv = 0., psipRR  = 0., psipZZ = 0., psipRZ = 0.;
+    while( (eps < eps_old || eps > 1e-7) && eps > 1e-10 && counter < 100)
     {
         X_OLD = X; eps= eps_old;
-        double psipRZ = psi.dfxy()(X[0], X[1]);
-        double psipRR = psi.dfxx()(X[0], X[1]), psipZZ = psi.dfyy()(X[0],X[1]);
+        psipRZ = psi.dfxy()(X[0], X[1]);
+        psipRR = psi.dfxx()(X[0], X[1]), psipZZ = psi.dfyy()(X[0],X[1]);
         double psipR  = psi.dfx()(X[0], X[1]), psipZ = psi.dfy()(X[0], X[1]);
-        double Dinv = 1./(psipZZ*psipRR - psipRZ*psipRZ);
+        Dinv = 1./(psipZZ*psipRR - psipRZ*psipRZ);
         XN[0] = X[0] - Dinv*(psipZZ*psipR - psipRZ*psipZ);
         XN[1] = X[1] - Dinv*(-psipRZ*psipR + psipRR*psipZ);
         XN.swap(X);
         eps = sqrt( (X[0]-X_OLD[0])*(X[0]-X_OLD[0]) + (X[1]-X_OLD[1])*(X[1]-X_OLD[1]));
+        counter++;
     }
-    R_X = X[0], Z_X = X[1];
+    if ( counter >= 100 || std::isnan( Dinv) )
+        return 0;
+    RC = X[0], ZC = X[1];
+    if( Dinv > 0 &&  psipRR > 0)
+        return 1; //local minimum
+    if( Dinv < 0 &&  psipRR < 0)
+        return 2; //local maximum
+    //if( Dinv < 0)
+    return 3; //saddle point
 }
 
-///@copydoc findOpoint()
-static inline void findXpoint( const CylindricalFunctorsLvl2& psi, double& R_X, double& Z_X)
+/**
+ * @brief This function finds O-points of psi
+ *
+ * Same as \c findCriticalPoint except that this function throws if it does
+ * not find a local minimum or a local maximum
+ * @param psi \f$ \psi(R,Z)\f$
+ * @param RC start value on input, O-point on output
+ * @param ZC start value on input, O-point on output
+ * @return 1 if local minimum, 2 if local maximum,
+ * @ingroup misc_geo
+ */
+static inline int findOpoint( const CylindricalFunctorsLvl2& psi, double& RC, double& ZC)
 {
-    findOpoint( psi, R_X, Z_X);
+    int point = findCriticalPoint( psi, RC, ZC);
+    if( point == 3 || point == 0 )
+        throw dg::Error(dg::Message(_ping_)<<"There is no O-point near "<<RC<<" "<<ZC);
+    return point;
 }
 
+/**
+ * @brief This function finds X-points of psi
+ *
+ * Same as \c findCriticalPoint except that this function throws if it does
+ * not find a saddle point
+ * @param psi \f$ \psi(R,Z)\f$
+ * @param RC start value on input, X-point on output
+ * @param ZC start value on input, X-point on output
+ * @ingroup misc_geo
+ */
+static inline void findXpoint( const CylindricalFunctorsLvl2& psi, double& RC, double& ZC)
+{
+    int point = findCriticalPoint( psi, RC, ZC);
+    if( point != 3)
+        throw dg::Error(dg::Message(_ping_)<<"There is no X-point near "<<RC<<" "<<ZC);
+}
+
+
 /// A symmetric 2d tensor field and its divergence
 ///@snippet hector_t.cu doxygen
 struct CylindricalSymmTensorLvl1
diff --git a/inc/geometries/normalize_solovev_t.cu b/inc/geometries/normalize_solovev_t.cu
index d5d1c73a2..faa232795 100644
--- a/inc/geometries/normalize_solovev_t.cu
+++ b/inc/geometries/normalize_solovev_t.cu
@@ -15,7 +15,7 @@ int main( int argc, char* argv[])
     }
     else
     {
-        std::cerr << "This program reads solovev parameters from an input json file and subtracts a constant such that the resulting Psi_p is zero on the X-point. The resulting parameters are written into an output file, which may overwrite the input file. This assumes that there is an X-point!\n";
+        std::cerr << "This program reads solovev parameters from an input json file and modifies c[0] such that the resulting Psi_p is zero on the X-point. The resulting parameters are written into an output file, which may overwrite the input file. The program aborts if it is unable to find an X-point\n";
         std::cerr << " Usage: "<< argv[0]<<" [input.json] [normalized.json]\n";
         return -1;
     }
@@ -24,12 +24,12 @@ int main( int argc, char* argv[])
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     double RX = gp.R_0-1.1*gp.triangularity*gp.a;
     double ZX = -1.1*gp.elongation*gp.a;
-    if( gp.hasXpoint())
+    try{
         dg::geo::findXpoint( mag.get_psip(), RX, ZX);
-    else
+    }catch ( std::exception& e)
     {
-        std::cerr << "Parameters have no X-point!\n";
-        return -1;
+        std::cerr << e.what() << std::endl;
+        return -1.;
     }
     const double psipX = mag.psip()( RX, ZX);
     std::cout << "X-point found at "<<RX<<" "<<ZX<<" with Psip = "<<psipX<<std::endl;
diff --git a/inc/geometries/separatrix_orthogonal.h b/inc/geometries/separatrix_orthogonal.h
index db123532a..87b627d0b 100644
--- a/inc/geometries/separatrix_orthogonal.h
+++ b/inc/geometries/separatrix_orthogonal.h
@@ -112,17 +112,16 @@ void computeX_rzy( const CylindricalFunctorsLvl1& psi,
 
 } //namespace detail
 }//namespace orthogonal
-///@endcond
 
 /**
  * @brief Choose points on inside or outside line
  *
+ * @attention Not consistent, do not use unless you know what you are doing
  * @ingroup generators_geo
  */
 struct SimpleOrthogonalX : public aGeneratorX2d
 {
     SimpleOrthogonalX(): f0_(1), firstline_(0){}
-    ///psi_0 must be the closed surface, 0 the separatrix
     SimpleOrthogonalX( const CylindricalFunctorsLvl2& psi, double psi_0,
             double xX, double yX, double x0, double y0, int firstline =0): psi_(psi)
     {
@@ -178,10 +177,17 @@ struct SimpleOrthogonalX : public aGeneratorX2d
     double zeta0_, f0_;
     int firstline_;
 };
+///@endcond
 
 /**
  * @brief Choose points on separatrix and construct grid from there
  *
+ * This is the algorithm described in:
+ * "M. Wiesenberger, M. Held, L. Einkemmer, A. Kendl Streamline integration as a method for structured grid generation in X-point geometry Journal of Computational Physics 373, 370-384 (2018)"
+ * @note The resulting coordinate transformation for \f$ \zeta\f$ will by linear in \f$ \psi\f$
+ * @attention Assumes that the separatrix is given by \f$ \psi = 0\f$. If this
+ * is not the case, then use the \c normalize_solovev_t program to change the parameters.
+ * Further, it is assumed that closed flux surfaces inside of the separatrix exist.
  * @ingroup generators_geo
  */
 struct SeparatrixOrthogonal : public aGeneratorX2d
@@ -189,13 +195,13 @@ struct SeparatrixOrthogonal : public aGeneratorX2d
     /**
      * @brief Construct
      *
-     * @param psi the flux function
-     * @param chi the monitor tensor
-     * @param psi_0 must be the closed flux surface inside domain boundary
-     * @param xX the X-point x - coordinate
+     * @param psi the flux function, the separatrix must be at \f$ \psi = 0\f$
+     * @param chi the monitor tensor, see \c dg::geo::make_Xconst_monitor or \c dg::geo::make_Xbump_monitor
+     * @param psi_0 The left boundary of the grid this generator will generate. Must be a closed flux surface.
+       @param xX the X-point x - coordinate
      * @param yX the X-point y - coordinate
-     * @param x0 a point in the inside of the domain bounded by \c psi_0 (shouldn't be the O-point)
-     * @param y0 a point in the inside of the domain bounded by \c psi_0 (shouldn't be the O-point)
+     * @param x0 a point in the inside of the separatrix (can be the O-point, defines the angle for initial separatrix integration)
+     * @param y0 a point in the inside of the separatrix (can be the O-point, defines the angle for initial separatrix integration)
      * @param firstline =0 means conformal, =1 means equalarc discretization of the separatrix
      * @param verbose if true the integrators will write additional information to \c std::cout
      */
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index c2bb15a72..baf29002a 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -114,16 +114,18 @@ int main( int argc, char* argv[])
     gp.display( std::cout);
     std::cout << "Constructing orthogonal grid ... \n";
     t.tic();
-    double R_X = gp.R_0-1.1*gp.triangularity*gp.a;
-    double Z_X = -1.1*gp.elongation*gp.a;
+    double RX = gp.R_0-1.1*gp.triangularity*gp.a;
+    double ZX = -1.1*gp.elongation*gp.a;
+    dg::geo::findXpoint( mag.get_psip(), RX, ZX);
+    const double psipX = mag.psip()( RX, ZX);
     //dg::geo::CylindricalSymmTensorLvl1 monitor_chi;
-    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), R_X, Z_X) ;
-    //dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xbump_monitor( mag.get_psip(), R_X, Z_X, 100, 100) ;
-    std::cout << "X-point set at "<<R_X<<" "<<Z_X<<"\n";
+    dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), RX, ZX) ;
+    //dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xbump_monitor( ag.get_psip(), RX, ZX, 100, 100) ;
+    std::cout << "X-point set at "<<RX<<" "<<ZX<<" with Psi_p = "<<psipX<<"\n";
 
-    double R0 = gp.R_0, Z0 = 0;
-    dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psi_0, R_X,Z_X, R0, Z0,0, true);
-    //dg::geo::SimpleOrthogonalX generator(mag.get_psip(), psi_0, R_X,Z_X, R0, Z0,0);
+    //dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psi_0, RX,ZX, mag.R0(), 0, 0, true);
+    dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psi_0, RX, ZX, mag.R0(), 0, 0, true);
+    //dg::geo::SimpleOrthogonalX generator(mag.get_psip(), psi_0, RX,ZX, mag.R0(), 0,0);
     dg::EquidistXRefinement equi(add_x, add_y, 1,1);
     dg::geo::CurvilinearRefinedProductGridX3d g3d(equi, generator, fx_0, fy_0, n, Nx, Ny,Nz, dg::DIR, dg::NEU);
     dg::geo::CurvilinearRefinedGridX2d g2d(equi, generator, fx_0, fy_0, n, Nx, Ny,dg::DIR, dg::NEU);
@@ -195,7 +197,7 @@ int main( int argc, char* argv[])
     std::cout << "FILE orthogonalX.nc CLOSED AND READY TO USE NOW!\n" <<std::endl;
 
     std::cout << "TEST VOLUME IS:\n";
-    dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, Z_X, 1.2*gp.a*gp.elongation, 1, 5e2, 5e2, dg::PER, dg::PER);
+    dg::CartesianGrid2d g2dC( gp.R_0 -1.2*gp.a, gp.R_0 + 1.2*gp.a, ZX, 1.2*gp.a*gp.elongation, 1, 5e2, 5e2, dg::PER, dg::PER);
     double psipmax = 0., psipmin_iris = psi_0;
     auto iris = dg::compose( dg::Iris( psipmin_iris, psipmax), mag.psip());
     dg::HVec vec  = dg::evaluate( iris, g2dC);
@@ -203,7 +205,7 @@ int main( int argc, char* argv[])
     double volumeRZP = dg::blas1::dot( vec, g2d_weights);
 
     dg::HVec cutter = dg::pullback( iris, g2d), vol_cut( cutter);
-    dg::geo::ZCutter cut(Z_X);
+    dg::geo::ZCutter cut(ZX);
     dg::HVec zcutter = dg::pullback( cut, g2d);
     w2d = dg::create::weights( g2d);//make weights w/o refined weights
     dg::blas1::pointwiseDot(cutter, w2d, vol_cut);
-- 
GitLab


From 19b811d3e0fb93689559528385ac6644a3dbecb7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 3 Aug 2020 21:33:58 +0200
Subject: [PATCH 282/540] Revert back to RhoP usage without psipX

---
 inc/geometries/geometry_diag.cu | 41 +++++++++++++++---------
 src/feltor/feltor.tex           | 56 ++++++++++++++++++++-------------
 2 files changed, 60 insertions(+), 37 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 105ddf326..509f85479 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -137,11 +137,15 @@ int main( int argc, char* argv[])
     dg::geo::TokamakMagneticField mag = mag_origin;
     //Find O-point
     double RO = gp.R_0, ZO = 0.;
+    int point = 1;
     if( !gp.isToroidal() )
-        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+        point = dg::geo::findOpoint( mag.get_psip(), RO, ZO);
     const double psipO = mag.psip()( RO, ZO);
-    const double psipmin = mag.psip()(RO, ZO);
-    std::cout << "O-point found at "<<RO<<" "<<ZO<<" with Psip "<<psipmin<<std::endl;
+    std::cout << "O-point found at "<<RO<<" "<<ZO<<" with Psip "<<psipO<<std::endl;
+    if( point == 1 )
+        std::cout << " (minimum)"<<std::endl;
+    if( point == 2 )
+        std::cout << " (maximum)"<<std::endl;
     const double psip0 = mag.psip()(gp.R_0, 0);
     std::cout << "psip( R_0, 0) = "<<psip0<<"\n";
     if( p.damping_alpha > 0.)
@@ -220,7 +224,7 @@ int main( int argc, char* argv[])
             p.damping_boundary+p.damping_alpha/2.,
             p.damping_alpha/2., +1 ), dg::geo::RhoP(mag_origin))},
         {"Nprofile", "A flux aligned profile", dg::compose( dg::LinearX( p.nprofileamp/mag.psip()(mag.R0(),0.), p.nprofileamp ), mag.psip())},
-        {"Delta", "A flux aligned Gaussian peak", dg::compose( dg::GaussianX( psipmin*0.2, 0.1, 1./(sqrt(2.*M_PI)*0.1)), mag.psip())},
+        {"Delta", "A flux aligned Gaussian peak", dg::compose( dg::GaussianX( psipO*0.2, 0.1, 1./(sqrt(2.*M_PI)*0.1)), mag.psip())},
         {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::compose( dg::TanhProfX( -3*p.source_alpha, p.source_alpha, -1), mag.psip())},
         ////
         {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
@@ -237,11 +241,18 @@ int main( int argc, char* argv[])
         std::cout << "Generate X-point flux-aligned grid ... \n";
         double RX = gp.R_0-1.1*gp.triangularity*gp.a;
         double ZX = -1.1*gp.elongation*gp.a;
+        double psipX = mag.psip()(RX, ZX);
+        std::cout << "Found X-point at "<<RX<<" "<<ZX<<" with Psip = "<<psipX<<std::endl;
+        if( fabs(psipX ) > 1e-10)
+        {
+            std::cerr << " Psip at X-point is not zero. Unable to construct grid\n";
+            return -1;
+        }
         dg::geo::findXpoint( mag.get_psip(), RX, ZX);
         dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), RX, ZX) ;
-        dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipmin, RX, ZX, mag.R0(), 0, 0, false);
+        dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipO, RX, ZX, mag.R0(), 0, 0, false);
         double fx_0 = 1./8.;
-        psipmax = -fx_0/(1.-fx_0)*psipmin;
+        psipmax = -fx_0/(1.-fx_0)*psipO;
         //std::cout << "psi 1 is          "<<psipmax<<"\n";
         gX2d = std::make_unique<dg::geo::CurvilinearGridX2d>(generator, fx_0, 0., npsi, Npsi, 640, dg::DIR, dg::NEU);
         std::cout << "DONE! \n";
@@ -254,8 +265,8 @@ int main( int argc, char* argv[])
         dg::HVec dvdpsip;
         avg_eta( volX2d, dvdpsip, false);
         dg::blas1::scal( dvdpsip, 4.*M_PI*M_PI*f0);
-        dg::Grid1d gX1d(psipmin<psipmax ? psipmin : psipmax,
-            psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::DIR_NEU); //inner value is always zero
+        dg::Grid1d gX1d(psipO<psipmax ? psipO : psipmax,
+            psipO<psipmax ? psipmax : psipO, npsi ,Npsi,dg::DIR_NEU); //inner value is always zero
         dg::HVec X_psi_vol = dg::integrate( dvdpsip, gX1d);
         map1d.emplace_back( "dvdpsip", dvdpsip,
             "Derivative of flux volume with respect to flux label psi");
@@ -291,12 +302,12 @@ int main( int argc, char* argv[])
         }
     }
     /// --------- More flux labels --------------------------------
-    dg::Grid1d grid1d(psipmin<psipmax ? psipmin : psipmax,
-            psipmin<psipmax ? psipmax : psipmin, npsi ,Npsi,dg::DIR_NEU); //inner value is always zero
+    dg::Grid1d grid1d(psipO<psipmax ? psipO : psipmax,
+            psipO<psipmax ? psipmax : psipO, npsi ,Npsi,dg::DIR_NEU); //inner value is always zero
     if( !gp.isToroidal())
     {
         dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
-        dg::blas1::axpby( -1./psipmin, rho, +1., 1., rho); //transform psi to rho
+        dg::blas1::axpby( -1./psipO, rho, +1., 1., rho); //transform psi to rho
         map1d.emplace_back("rho", rho,
             "Alternative flux label rho = -psi/psimin + 1");
         dg::blas1::transform( rho, rho, dg::SQRT<double>());
@@ -312,8 +323,8 @@ int main( int argc, char* argv[])
             map1d.emplace_back("psit1d", psit,
                 "Toroidal flux label psi_t integrated  on grid1d using direct q");
             //we need to avoid integrating >=0
-            dg::Grid1d g1d_fine(psipmin<0. ? psipmin : 0.,
-                    psipmin<0. ? 0. : psipmin, npsi, Npsi,dg::NEU);
+            dg::Grid1d g1d_fine(psipO<0. ? psipO : 0.,
+                    psipO<0. ? 0. : psipO, npsi, Npsi,dg::NEU);
             qprofile = dg::evaluate( qprof, g1d_fine);
             dg::HVec w1d = dg::create::weights( g1d_fine);
             double psit_tot = dg::blas1::dot( w1d, qprofile);
@@ -356,7 +367,7 @@ int main( int argc, char* argv[])
     if( gp.hasXpoint())
     {
         int dim_idsX[2] = {0,0};
-        err = file::define_dimensions( ncid, dim_idsX, gX2d->grid(), {"eta", "psi"} );
+        err = file::define_dimensions( ncid, dim_idsX, gX2d->grid(), {"eta", "zeta"} );
         std::string long_name = "Flux surface label";
         err = nc_put_att_text( ncid, dim_idsX[0], "long_name",
             long_name.size(), long_name.data());
@@ -380,7 +391,7 @@ int main( int argc, char* argv[])
     }
     else
     {
-        err = file::define_dimension( ncid, &dim1d_ids[0], grid1d, "psi");
+        err = file::define_dimension( ncid, &dim1d_ids[0], grid1d, "zeta");
         std::string psi_long_name = "Flux surface label";
         err = nc_put_att_text( ncid, dim1d_ids[0], "long_name",
             psi_long_name.size(), psi_long_name.data());
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 5968320a8..18c254aac 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -137,14 +137,20 @@ The cylindrical coordinate basis vectors are mutually orthogonal to each other.
 \subsection{Solov'ev equilbrium}\label{sec:solovev}
 In cylindrical coordinates the general axisymmetric  magnetic field can be written as (dimensionless)
 \begin{align}
- \vec{B} &= \frac{R_0}{R}\left[I(\psi) \ehat_{\varphi} + \frac{\partial
+ \vec{B} &= \frac{R_0}{R}\left[I(\psi_p) \ehat_{\varphi} + \frac{\partial
  \psi_p}{\partial Z} \ehat_R -  \frac{\partial \psi_p}{\partial R} \ehat_Z\right] ,
 \end{align}
 which can obviously not be manipulated to be in Clebsch form.
 Hence we are dealing with a non-flux aligned coordinate system.
-For the sake of clarity we define the poloidal magnetic field \( \vec{B}_p = \frac{R_0}{R}\left( \frac{\partial \psi}{\partial Z}\ehat_R - \frac{\partial \psi}{\partial R}\ehat_Z\right)
+For the sake of clarity we define the poloidal magnetic field \( \vec{B}_p = \frac{R_0}{R}\left( \frac{\partial \psi_p}{\partial Z}\ehat_R - \frac{\partial \psi_p}{\partial R}\ehat_Z\right)
 \) and the toroidal magnetic field \(\vec{B}_t =\frac{R_0I}{R} \ehat_{\varphi}\).
-Note that with a typically convex function $\psi$, $I(\psi)>0$ and the previously defined coordinate system the field line winding is a {\bf left handed screw} in the positive $\vec B$-direction.
+Note that with a typically convex function $\psi_p$ (second derivative is
+positive), $I(\psi_p)>0$ and the previously defined coordinate system the field
+line winding is a {\bf left handed screw} in the positive $\ehat_\varphi$-direction.
+Also note that then $\vec B\times\vn\vec B$ points {\bf down}, towards the magnetic X-point,
+and we have the {\bf favourable} drift direction (since in experiments H-mode
+is reached easier in this configuration).
+
 
 We scaled $R$, $Z$ and $R_0$ with $\rho_s = \sqrt{T_e m_i}/(eB_0)$, the
 magnetic field with $B_0$, the poloidal flux with $\psi_{p0} = B_0\rho_s \hat
@@ -172,7 +178,7 @@ with $\Delta^*_\perp \psi_p = R\partial_R (R^{-1}\psi_R) + \psi_{ZZ}$.
 The Solov'ev assumptions consist of \(A/R_0 = -I \frac{d I}{d  \psi_p }\) and \((1-A)/R_0 = -\frac{d p}{d  \psi_p }\), where \(A\) is a constant~\cite{Cerfon2010,Cerfon2014}.
 By integration over \(\psi_p\) we find
 $
-p(\psi_p) = (A-1)\psi_p/R_0/\beta + p(0)$,
+p(\psi_p) = (A-1)\psi_p/R_0/\beta + p(0)$, %Does that mean that psi_p has to be negative if A=0?
  $I(\psi_p) = \sqrt{-2 A \psi_p/R_0 + 1}$,
  and
     $j_{\hat\varphi} = \left[(A-1)R^2/R_0^2 - A \right]/R/\beta $.
@@ -228,6 +234,14 @@ $\bar{\psi}_{p11}=3 \bar{Z}\bar{R}^4 - 4\bar{Z}^3\bar{R}^2$\\
    & \\
 \bottomrule
 \end{longtable}
+Since Eq.~\eqref{eq:solovev} is given analytically we can numerically evaluate $\psi_p$ and $I$
+and all their derivatives
+at arbitrary points to machine precision, which is simple to implement and fast to execute.
+This translates to an exact representation of the magnetic field and related
+quantities like the curvature operators in code. In particular,
+the X-point and O-point can be determined to machine
+precision via a few Newton iterations.
+
 The choice of the coefficients \(c_{i}\) and \(A\) determines the actual form
 of the magnetic field.
 Eq.~\eqref{eq:solovev} can for example represent single and asymmetric double X-point configurations, force-free states,
@@ -236,11 +250,8 @@ $R_0$ appears as an artificial scaling factor
 (note here that a change in $\rho_s$ changes $R_0$ but not the form or size of
 the dimensional equilibrium magnetic field).
 The scaling factors $\mathcal P_\psi$ and $\mathcal P_I$ are mainly introduced to maximize the flexibility e.g. to adapt the solution to experimental equilibria or to reverse the sign of the magnetic field.
-
-Since Eq.~\eqref{eq:solovev} is given analytically we can numerically evaluate $\psi_p$ and $I$
-and all their derivatives
-at arbitrary points to machine precision, which is simple to implement and fast to execute.
-This translates to an exact representation of the magnetic field and related quantities like the curvature operators in code.
+If an X-point is present, we choose $c_1$ such that
+$\psi_p(R_X, Z_X) = 0$ that is the separatrix is given by $\psi_p(R,Z) = 0$.
 
 Note that
 \begin{align}
@@ -445,7 +456,7 @@ With the help of the volume
 flux label (notice that both the volume $v$ as well as the poloidal flux $\psi_p$ have physical
 meaning while the coordinate $\zeta(\psi_p)$ is an arbitrary choice) we define
 \begin{align} \label{eq:fsa_vol}
-v(\psi_p) :=& \int_{\psi_{p,\min}}^\psi \dV = \int^{\zeta(\psi_p)} \sqrt{g}\d\zeta\d\eta\d\varphi,
+v(\psi_p) :=& \int_{\psi_{p,O}}^\psi \dV = \int^{\zeta(\psi_p)} \sqrt{g}\d\zeta\d\eta\d\varphi,
 \\
 \frac{\d v}{\d\psi_p} =& \int\dA |\vn\psi_p|^{-1} = 2\pi f_0\oint_{\zeta(\psi_p)} \sqrt{g}\d\eta \\
 \RA{ f }_\psi :=& \frac{\partial}{\partial v} \int \dV f
@@ -536,17 +547,17 @@ We integrate this equation with the help of one of our ODE integrators, i.e. we
 and refine the stepsize until machine-precision is reached.
 Notice that the safety factor diverges on the last closed flux
 surface whereas Eq.~\eqref{eq:total_flux}
-remains finite due to the $\vn\psi$ factor.
+remains finite due to the $\vn\psi_p$ factor.
 \subsection{Alternative flux labels}
 We find the toroidal flux $\psi_t$ by integrating the q-profile $\psi_t = \int^{\psi_p} \d\psi_p q(\psi_p)$. Since $q$ diverges, $\psi_t$, in contrast to $\psi_p$,
 is not defined outside the last closed flux-surface (but has a finite value on the last closed flux surface). We now define the normalized poloidal and toroidal flux labels $\rho_p$ and $\rho_t$
 \begin{align}
-    \rho_p&:= \sqrt{1-\frac{\psi_p}{\psi_{p,\min}}} \ \leftrightarrow\ \psi_p = (1-\rho_p^2)\psi_{p,\min} \\
+    \rho_p&:= \sqrt{1-\frac{\psi_p }{\psi_{p,O}}} \ \leftrightarrow\ \psi_p = (1-\rho_p^2)\psi_{p,O} \\
     \rho_t&:= \sqrt{\frac{\psi_t}{\psi_{t,\mathrm{sep}}}},\\
-    \text{In general }\psi_{p,\min} &= \psi_p(R_O, Z_O) \neq\psi_{p}(R_0,0)
+    \text{with }\psi_{p,O} &= \psi_p(R_O, Z_O)% \text{ and } \psi_{p,X} = \psi_p(R_X, Z_X)
 \end{align}
 where $R_O$, $Z_O$ are the coordinates of the O-point.
-These labels are useful because
+The labels $\rho_t$ and $\rho_p$ are useful because
 equidistant $\rho_p$ and $\rho_t$ values tend to translate to equidistant flux-surfaces
 in configuration space.
 
@@ -597,9 +608,10 @@ instead of $\psi_p$ for the computation of the
 magnetic field, which introduces a shear layer between $\psi_{p,b}$ and $\psi_{p,b}+\alpha$ where the
 fieldlines are straightened to match $\ehat_\varphi$.
 In order to simplify the setup of this region we give $\psi_{p,b}$ and $\alpha$ in terms of
-$\rho_p$ and $\alpha_p$ via $\psi_{p,b} = (1-\rho_{p,b}^2)\psi_{p,\min}$ and $\alpha = -(2\rho_{p,b} \alpha_p + \alpha_p^2)\psi_{p,\min}$. In case we change the sign
-of $\psi_p$ vie $\mathcal P_\psi$ we need to adapt Eq.~\eqref{eq:modified_psip}
-accordingly.
+$\rho_p$ and $\alpha_p$ via $\psi_{p,b} = (1-\rho_{p,b}^2)\psi_{p,O}$ and $\alpha = -(2\rho_{p,b} \alpha_p + \alpha_p^2)\psi_{p,O}$. In case we change the sign
+of $\psi_p$ via $\mathcal P_\psi$ (to make it concave) note that $\alpha$ becomes
+negative and $\psi_{p,O}$ is positive).
+We then need to point mirror Eq.~\eqref{eq:modified_psip} at $\psi_{p,b}+\frac{\alpha}{2}$.
 
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -770,7 +782,7 @@ Note that we should take care to intitialize a smooth profile with ideally well-
 We define a flux-aligned density profile as
 \begin{align} \label{eq:density_profile}
   n_{\text{prof}}(R,Z)=
-  n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_{p,\min}}\Theta_{\alpha_p/2}\left(-\rho_p(R, Z)-\frac{\alpha_p}{2}\right) H(Z-Z_X)
+  n_0 + \triangle n_{peak}\frac{\psi_p(R,Z) }{\psi_{p,O}}\Theta_{\alpha_p/2}\left(1-\rho_p(R, Z)-\frac{\alpha_p}{2}\right) H(Z-Z_X)
 \end{align}
 The second Heaviside is multiplied only if the equilibrium $\psi_p$ has an
 X-point and avoids a profile in the private flux region. The factor $\alpha_p$ provides a smooth transition
@@ -1436,10 +1448,10 @@ small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes,
 \\
 damping & dict & & & magnetic and density damping region \\
 \qquad rate & float & 0    & 0   & Friction coefficient $\omega_d$ in density and velocity damping Eq.~\eqref{eq:velocity_source} \\
-\qquad boundary & float & 0.2  & 1.2 & Modification region boundary $\rho_{p,b}$: yields $\psi_0 = (1-\rho_{p,b}^2)\psi_{p,\min}$ in Eq.~\eqref{eq:modified_psip}.
+\qquad boundary & float & 0.2  & 1.2 & Modification region boundary $\rho_{p,b}$: yields $\psi_0 = (1-\rho_{p,b}^2)\psi_{p,O}$ in Eq.~\eqref{eq:modified_psip}.
 \\
 \qquad alpha   & float & 0.25 & 0 & Transition width $\alpha_p$: yields
-$\alpha=-2\rho_{p,b}\alpha_p+\alpha_p^2)\psi_{p,\min}$ for the Heaviside in the modified
+$\alpha=-2\rho_{p,b}\alpha_p+\alpha_p^2)\psi_{p,O}$ for the Heaviside in the modified
 $\psi_p$ function \eqref{eq:modified_psip}. If zero, then we do not modify the
 magnetic field and damping is ignored.\\
 \bottomrule
@@ -1573,8 +1585,8 @@ psi\_area        & Dataset & 1 (psi) & The area of the flux surfaces $A(\psi_p)
 q-profile        & Dataset & 1 (psi) & The safety factor $q(\psi_p)$ \eqref{eq:safety_factor} using direct integration ( accurate but unavailable outside separatrix) \\
 psi\_psi         & Dataset & 1 (psi) & explicit $\psi_p$ values; Same as psi \\
 psit1d           & Dataset & 1 (psi) & Toroidal flux (integrated q-profile) $\psi_t = \int^{\psi_p} \d\psi_p q(\psi_p)$ \\
-rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= 1 - \psi_p/\psi_{p,\min}$ \\
-rho\_p           & Dataset & 1 (psi) & poloidal flux label $\rho_p:= \sqrt{1 - \psi_p/\psi_{p,\min}}$ \\
+rho              & Dataset & 1 (psi) & Transformed flux label $\rho:= 1 - \psi_p/\psi_{p,O}$ \\
+rho\_p           & Dataset & 1 (psi) & poloidal flux label $\rho_p:= \sqrt{1 - \psi_p/\psi_{p,O}}$ \\
 rho\_t           & Dataset & 1 (psi) & Toroidal flux label $\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$ (is similar to $\rho$ in the edge but $\rho_t$ is nicer in the core domain, because equidistant $\rho_t$ make more equidistant flux-surfaces)\\
 Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \RA{ Z}(R,Z)$ \\
 Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\RA{ Z}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
-- 
GitLab


From 19635ebfa728a8e0a7364fdd6a2f81e403ffcc23 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 3 Aug 2020 22:11:07 +0200
Subject: [PATCH 283/540] Make code use -ephi automatically

- reduces the number of parameters to choose from for the user
---
 src/feltor/feltor.h   | 39 ++++++++++++++++++++-------------------
 src/feltor/feltor.tex | 13 ++++---------
 src/feltor/implicit.h |  4 ++--
 3 files changed, 26 insertions(+), 30 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index b6071fca4..185e399a8 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -412,27 +412,28 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_mag(
     }
     else if( p.curvmode == "low beta")
     {
-        curvNabla = curvKappa = dg::geo::createCurvatureNablaB(mag, +1);
+        if( mag.ipol()( g.x0(), g.y0()) < 0)
+            curvNabla = curvKappa = dg::geo::createCurvatureNablaB(mag, -1);
+        else
+            curvNabla = curvKappa = dg::geo::createCurvatureNablaB(mag, +1);
         dg::assign( dg::evaluate(dg::zero, g), m_divCurvKappa);
     }
     else if( p.curvmode == "toroidal")
     {
-        curvNabla = dg::geo::createCurvatureNablaB(mag, +1);
-        curvKappa = dg::geo::createCurvatureKappa(mag, +1);
-        dg::assign(  dg::pullback(dg::geo::DivCurvatureKappa(mag, +1), g),
-            m_divCurvKappa);
-    }
-    else if( p.curvmode == "low beta negative")
-    {
-        curvNabla = curvKappa = dg::geo::createCurvatureNablaB(mag, -1);
-        dg::assign( dg::evaluate(dg::zero, g), m_divCurvKappa);
-    }
-    else if( p.curvmode == "toroidal negative")
-    {
-        curvNabla = dg::geo::createCurvatureNablaB(mag, -1);
-        curvKappa = dg::geo::createCurvatureKappa(mag, -1);
-        dg::assign(  dg::pullback(dg::geo::DivCurvatureKappa(mag, -1), g),
-            m_divCurvKappa);
+        if( mag.ipol()( g.x0(), g.y0()) < 0)
+        {
+            curvNabla = dg::geo::createCurvatureNablaB(mag, -1);
+            curvKappa = dg::geo::createCurvatureKappa(mag, -1);
+            dg::assign(  dg::pullback(dg::geo::DivCurvatureKappa(mag, -1), g),
+                m_divCurvKappa);
+        }
+        else
+        {
+            curvNabla = dg::geo::createCurvatureNablaB(mag, +1);
+            curvKappa = dg::geo::createCurvatureKappa(mag, +1);
+            dg::assign(  dg::pullback(dg::geo::DivCurvatureKappa(mag, +1), g),
+                m_divCurvKappa);
+        }
     }
     else
         throw dg::Error(dg::Message(_ping_)<<"Warning! curvmode value '"<<p.curvmode<<"' not recognized!! I don't know what to do! I exit!\n");
@@ -470,7 +471,7 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
     bhat = dg::geo::createEPhi(+1);
     if( p.curvmode == "true")
         bhat = dg::geo::createBHat(mag);
-    else if ( p.curvmode == "toroidal negative" || p.curvmode == "low beta negative")
+    else if( mag.ipol()( g.x0(), g.y0()) < 0)
         bhat = dg::geo::createEPhi(-1);
     dg::pushForward(bhat.x(), bhat.y(), bhat.z(), m_b[0], m_b[1], m_b[2], g);
     dg::SparseTensor<Container> metric = g.metric();
@@ -504,7 +505,7 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
     auto bhat = dg::geo::createEPhi(+1); //bhat = ephi except when "true"
     if( p.curvmode == "true")
         bhat = dg::geo::createBHat( mag);
-    else if ( p.curvmode == "toroidal negative" || p.curvmode == "low beta negative")
+    else if( mag.ipol()( g.x0(), g.y0()) < 0)
         bhat = dg::geo::createEPhi(-1);
     m_multi_chi = m_multigrid.project( m_temp0);
     m_multi_pol.resize(p.stages);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 18c254aac..a15599d13 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -313,7 +313,7 @@ Note that in an actual toroidal field we have
   \vec B(R) := \pm \frac{R_0}{R} \ehat_\varphi
   \label{}
 \end{align}
-We then have $\bhat = \ehat_\varphi$ and the curvature operators further
+We then have $\bhat = \pm\ehat_\varphi$ and the curvature operators further
 simplify to
 \begin{align}
   \vec{ \mathcal K_{\vn\times\bhat}} = \vec{ \mathcal K_{\vn B}} = -\frac{\pm 1}{R_0} \ehat_Z =
@@ -322,6 +322,7 @@ simplify to
     \npar \ln B = 0
     \label{}
 \end{align}
+Note: the negative sign is automatically chosen in code if $I(R_0, 0)<0$.
 
 \subsubsection{Low beta approximation}\label{sec:lowbetaapprox}
 In this approximation we apply the toroidal field line approximation
@@ -678,14 +679,10 @@ If we change the direction of the magnetic field vector $\bhat$, we immediately
 drifts and $U\bhat$ change directions. On the other side, the diffusive and resistive terms remain unchanged.
 Without resistivity and diffusion a change in direction of the magnetic field thus corresponds to
 a time reversal $t\rightarrow t'=-t$.
+In the code $\bhat$ changes sign by using both $-\mathcal P_\psi$ and $-\mathcal P_I$.
 
 Also note that changing the sign of the magnetic field only in the parallel derivatives $\npar \rightarrow -\npar$ does not
 have any effect. This can be seen by simply renormalizing $U'=-U$. This reverts the equations back to the original equations.
-In the code this situation can happen when you change the sign of $\bhat$ using $\mathcal P_\psi$  and $\mathcal P_I$
-and use the toroidal field approximation at the same time, because $\ehat_\varphi$
-does not change sign automatically.
-To actually change the sign of the magnetic field in code you need to use the "toroidal
-negative" value for the \textit{curvmode} input parameter, which reverts $\ehat_\varphi$ in the other direction.
 \subsubsection{Scaling of density}
 If $N, U, \phi, A_\parallel$ are a solution to the model equations
 then so are $N'=\alpha N$, $U'=U$, $\phi'=\phi$ and $A_\parallel'=A_\parallel$ with the changed parameters $S_N' = \alpha S_N$, $\eta' = \eta/\alpha$ and $ \beta' = \beta/\alpha$. If $N$
@@ -1384,9 +1381,7 @@ curvature mode (
 "low beta",
 "true": no approximation - requires significantly more resolution in Nz,
 "toroidal": toroidal field approx - elliptic equation does not need
-communication in z,
-"low beta negative": reversed sign of low beta,
-"toroidal negative": reversed sign of toroidal
+communication in z
 )
 \\
 symmetric & bool & false & false & If true, initialize all quantities symmetric
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index ae85135f9..c75eb611c 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -30,7 +30,7 @@ struct ImplicitDensity
         auto bhat = dg::geo::createEPhi(+1); //bhat = ephi except when "true"
         if( p.curvmode == "true")
             bhat = dg::geo::createBHat(mag);
-        else if ( p.curvmode == "toroidal negative" || p.curvmode == "low beta negative")
+        else if( mag.ipol()( g.x0(), g.y0()) < 0)
             bhat = dg::geo::createEPhi(-1);
         dg::SparseTensor<Container> hh
             = dg::geo::createProjectionTensor( bhat, g);
@@ -102,7 +102,7 @@ struct ImplicitVelocity
         auto bhat = dg::geo::createEPhi(+1); //bhat = ephi except when "true"
         if( p.curvmode == "true")
             bhat = dg::geo::createBHat(mag);
-        else if ( p.curvmode == "toroidal negative" || p.curvmode == "low beta negative")
+        else if( mag.ipol()( g.x0(), g.y0()) < 0)
             bhat = dg::geo::createEPhi(-1);
         dg::SparseTensor<Container> hh
             = dg::geo::createProjectionTensor( bhat, g);
-- 
GitLab


From 9586289eb54b8dde953ad2837579ee99f7f03dac Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 3 Aug 2020 22:46:45 +0200
Subject: [PATCH 284/540] Update Doxyfiles and eliminate doc warnings

---
 inc/dg/Doxyfile                        | 234 +++++++++++++++--------
 inc/dg/adaptive_t.cu                   |   4 +-
 inc/dg/backend/exblas/Doxyfile         |  16 +-
 inc/dg/backend/exblas/mpi_accumulate.h |   2 +
 inc/dg/multigrid.h                     |   9 +-
 inc/dg/nullstelle.h                    |   2 +
 inc/file/Doxyfile                      | 244 ++++++++++++++++--------
 inc/geometries/Doxyfile                | 248 +++++++++++++++++--------
 inc/geometries/curvilinear.h           |   2 +-
 inc/geometries/ribeiro.h               |   1 -
 10 files changed, 526 insertions(+), 236 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 3be47544b..1653863ab 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -1,4 +1,4 @@
-# Doxyfile 1.8.13
+# Doxyfile 1.8.17
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project.
@@ -17,11 +17,11 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 
-# This tag specifies the encoding used for all characters in the config file
-# that follow. The default is UTF-8 which is also the encoding used for all text
-# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
-# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
-# for the list of possible encodings.
+# This tag specifies the encoding used for all characters in the configuration
+# file that follow. The default is UTF-8 which is also the encoding used for all
+# text before the first occurrence of this tag. Doxygen uses libiconv (or the
+# iconv built into libc) for the transcoding. See
+# https://www.gnu.org/software/libiconv/ for the list of possible encodings.
 # The default value is: UTF-8.
 
 DOXYFILE_ENCODING      = UTF-8
@@ -93,6 +93,14 @@ ALLOW_UNICODE_NAMES    = NO
 
 OUTPUT_LANGUAGE        = English
 
+# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all generated output in the proper direction.
+# Possible values are: None, LTR, RTL and Context.
+# The default value is: None.
+
+OUTPUT_TEXT_DIRECTION  = None
+
 # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
 # descriptions after the members that are listed in the file and class
 # documentation (similar to Javadoc). Set to NO to disable this.
@@ -189,6 +197,16 @@ SHORT_NAMES            = NO
 
 JAVADOC_AUTOBRIEF      = NO
 
+# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line
+# such as
+# /***************
+# as being the beginning of a Javadoc-style comment "banner". If set to NO, the
+# Javadoc-style will behave just like regular comments and it will not be
+# interpreted by doxygen.
+# The default value is: NO.
+
+JAVADOC_BANNER         = NO
+
 # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
 # line (until the first dot) of a Qt-style comment as the brief description. If
 # set to NO, the Qt-style will behave just like regular Qt-style comments (thus
@@ -236,7 +254,12 @@ TAB_SIZE               = 8
 # will allow you to put the command \sideeffect (or @sideeffect) in the
 # documentation, which will result in a user-defined paragraph with heading
 # "Side Effects:". You can put \n's in the value part of an alias to insert
-# newlines.
+# newlines (in the resulting output). You can put ^^ in the value part of an
+# alias to insert a newline as if a physical newline was in the original file.
+# When you need a literal { or } or , in the value part of an alias you have to
+# escape them by means of a backslash (\), this can lead to conflicts with the
+# commands \{ and \} for these it is advised to use the version @{ and @} or use
+# a double escape (\\{ and \\})
 
 ALIASES                =
 
@@ -274,17 +297,26 @@ OPTIMIZE_FOR_FORTRAN   = NO
 
 OPTIMIZE_OUTPUT_VHDL   = NO
 
+# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice
+# sources only. Doxygen will then generate output that is more tailored for that
+# language. For instance, namespaces will be presented as modules, types will be
+# separated into more groups, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_SLICE  = NO
+
 # Doxygen selects the parser to use depending on the extension of the files it
 # parses. With this tag you can assign which parser to use for a given
 # extension. Doxygen has a built-in mapping, but you can override or extend it
 # using this tag. The format is ext=language, where ext is a file extension, and
-# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
-# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:
-# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:
-# Fortran. In the later case the parser tries to guess whether the code is fixed
-# or free formatted code, this is the default for Fortran type files), VHDL. For
-# instance to make doxygen treat .inc files as Fortran files (default is PHP),
-# and .f files as C (default is Fortran), use: inc=Fortran f=C.
+# language is one of the parsers supported by doxygen: IDL, Java, JavaScript,
+# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice,
+# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran:
+# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser
+# tries to guess whether the code is fixed or free formatted code, this is the
+# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat
+# .inc files as Fortran files (default is PHP), and .f files as C (default is
+# Fortran), use: inc=Fortran f=C.
 #
 # Note: For files without extension you can use no_extension as a placeholder.
 #
@@ -296,7 +328,7 @@ EXTENSION_MAPPING      = cuh=C++ \
 
 # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
 # according to the Markdown format, which allows for more readable
-# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# documentation. See https://daringfireball.net/projects/markdown/ for details.
 # The output of markdown processing is further processed by doxygen, so you can
 # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
 # case of backward compatibilities issues.
@@ -308,7 +340,7 @@ MARKDOWN_SUPPORT       = YES
 # to that level are automatically included in the table of contents, even if
 # they do not have an id attribute.
 # Note: This feature currently applies only to Markdown headings.
-# Minimum value: 0, maximum value: 99, default value: 0.
+# Minimum value: 0, maximum value: 99, default value: 5.
 # This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
 
 TOC_INCLUDE_HEADINGS   = 0
@@ -338,7 +370,7 @@ BUILTIN_STL_SUPPORT    = NO
 CPP_CLI_SUPPORT        = NO
 
 # Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
-# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen
 # will parse them like normal C++ but will assume all classes use public instead
 # of private inheritance when no explicit protection keyword is present.
 # The default value is: NO.
@@ -444,6 +476,12 @@ EXTRACT_ALL            = YES
 
 EXTRACT_PRIVATE        = NO
 
+# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual
+# methods of a class will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIV_VIRTUAL   = NO
+
 # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
 # scope will be included in the documentation.
 # The default value is: NO.
@@ -498,8 +536,8 @@ HIDE_UNDOC_MEMBERS     = NO
 HIDE_UNDOC_CLASSES     = NO
 
 # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
-# (class|struct|union) declarations. If set to NO, these declarations will be
-# included in the documentation.
+# declarations. If set to NO, these declarations will be included in the
+# documentation.
 # The default value is: NO.
 
 HIDE_FRIEND_COMPOUNDS  = NO
@@ -522,7 +560,7 @@ INTERNAL_DOCS          = NO
 # names in lower-case letters. If set to YES, upper-case letters are also
 # allowed. This is useful if you have classes or files whose names only differ
 # in case and if your file system supports case sensitive file names. Windows
-# and Mac users are advised to set this option to NO.
+# (including Cygwin) ands Mac users are advised to set this option to NO.
 # The default value is: system dependent.
 
 CASE_SENSE_NAMES       = NO
@@ -709,7 +747,7 @@ LAYOUT_FILE            =
 # The CITE_BIB_FILES tag can be used to specify one or more bib files containing
 # the reference definitions. This must be a list of .bib files. The .bib
 # extension is automatically appended if omitted. This requires the bibtex tool
-# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info.
 # For LaTeX the style of the bibliography can be controlled using
 # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
 # search path. See also \cite for info how to create references.
@@ -754,7 +792,8 @@ WARN_IF_DOC_ERROR      = YES
 # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
 # are documented, but have no documentation for their parameters or return
 # value. If set to NO, doxygen will only warn about wrong or incomplete
-# parameter documentation, but not about the absence of documentation.
+# parameter documentation, but not about the absence of documentation. If
+# EXTRACT_ALL is set to YES then this flag will automatically be disabled.
 # The default value is: NO.
 
 WARN_NO_PARAMDOC       = NO
@@ -791,14 +830,14 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = ./backend  \
+INPUT                  = ./backend \
                          ./topology \
                          .
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
 # libiconv (or the iconv built into libc) for the transcoding. See the libiconv
-# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
 # possible encodings.
 # The default value is: UTF-8.
 
@@ -815,8 +854,10 @@ INPUT_ENCODING         = UTF-8
 # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
 # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
 # *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
-# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08,
-# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf.
+# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment),
+# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen
+# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f, *.for, *.tcl, *.vhd,
+# *.vhdl, *.ucf, *.qsf and *.ice.
 
 FILE_PATTERNS          = *.h \
                          *.cuh \
@@ -997,7 +1038,7 @@ INLINE_SOURCES         = NO
 STRIP_CODE_COMMENTS    = YES
 
 # If the REFERENCED_BY_RELATION tag is set to YES then for each documented
-# function all documented functions referencing it will be listed.
+# entity all documented functions referencing it will be listed.
 # The default value is: NO.
 
 REFERENCED_BY_RELATION = NO
@@ -1029,12 +1070,12 @@ SOURCE_TOOLTIPS        = YES
 # If the USE_HTAGS tag is set to YES then the references to source code will
 # point to the HTML generated by the htags(1) tool instead of doxygen built-in
 # source browser. The htags tool is part of GNU's global source tagging system
-# (see http://www.gnu.org/software/global/global.html). You will need version
+# (see https://www.gnu.org/software/global/global.html). You will need version
 # 4.8.6 or higher.
 #
 # To use it do the following:
 # - Install the latest version of global
-# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file
 # - Make sure the INPUT points to the root of the source tree
 # - Run doxygen as normal
 #
@@ -1062,7 +1103,7 @@ VERBATIM_HEADERS       = YES
 # rich C++ code for which doxygen's built-in parser lacks the necessary type
 # information.
 # Note: The availability of this option depends on whether or not doxygen was
-# generated with the -Duse-libclang=ON option for CMake.
+# generated with the -Duse_libclang=ON option for CMake.
 # The default value is: NO.
 
 CLANG_ASSISTED_PARSING = NO
@@ -1075,6 +1116,16 @@ CLANG_ASSISTED_PARSING = NO
 
 CLANG_OPTIONS          =
 
+# If clang assisted parsing is enabled you can provide the clang parser with the
+# path to the compilation database (see:
+# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) used when the files
+# were built. This is equivalent to specifying the "-p" option to a clang tool,
+# such as clang-check. These options will then be passed to the parser.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse_libclang=ON option for CMake.
+
+CLANG_DATABASE_PATH    =
+
 #---------------------------------------------------------------------------
 # Configuration options related to the alphabetical class index
 #---------------------------------------------------------------------------
@@ -1193,7 +1244,7 @@ HTML_EXTRA_FILES       =
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
 # this color. Hue is specified as an angle on a colorwheel, see
-# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# https://en.wikipedia.org/wiki/Hue for more information. For instance the value
 # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
 # purple, and 360 is red again.
 # Minimum value: 0, maximum value: 359, default value: 220.
@@ -1229,6 +1280,17 @@ HTML_COLORSTYLE_GAMMA  = 80
 
 HTML_TIMESTAMP         = YES
 
+# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
+# documentation will contain a main index with vertical navigation menus that
+# are dynamically created via JavaScript. If disabled, the navigation index will
+# consists of multiple levels of tabs that are statically embedded in every HTML
+# page. Disable this option to support browsers that do not have JavaScript,
+# like the Qt help browser.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_MENUS     = YES
+
 # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
 # documentation will contain sections that can be hidden and shown after the
 # page has loaded.
@@ -1252,13 +1314,13 @@ HTML_INDEX_NUM_ENTRIES = 100
 
 # If the GENERATE_DOCSET tag is set to YES, additional index files will be
 # generated that can be used as input for Apple's Xcode 3 integrated development
-# environment (see: http://developer.apple.com/tools/xcode/), introduced with
-# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# environment (see: https://developer.apple.com/xcode/), introduced with OSX
+# 10.5 (Leopard). To create a documentation set, doxygen will generate a
 # Makefile in the HTML output directory. Running make will produce the docset in
 # that directory and running make install will install the docset in
 # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
-# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
-# for more information.
+# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy
+# genXcode/_index.html for more information.
 # The default value is: NO.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
@@ -1297,7 +1359,7 @@ DOCSET_PUBLISHER_NAME  = Publisher
 # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
 # additional HTML index files: index.hhp, index.hhc, and index.hhk. The
 # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
-# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on
 # Windows.
 #
 # The HTML Help Workshop contains a compiler that can convert all HTML output
@@ -1373,7 +1435,7 @@ QCH_FILE               =
 
 # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
 # Project output. For more information please see Qt Help Project / Namespace
-# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace).
 # The default value is: org.doxygen.Project.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1381,7 +1443,7 @@ QHP_NAMESPACE          = org.doxygen.Project
 
 # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
 # Help Project output. For more information please see Qt Help Project / Virtual
-# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-
 # folders).
 # The default value is: doc.
 # This tag requires that the tag GENERATE_QHP is set to YES.
@@ -1390,7 +1452,7 @@ QHP_VIRTUAL_FOLDER     = doc
 
 # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
 # filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1398,7 +1460,7 @@ QHP_CUST_FILTER_NAME   =
 
 # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
 # custom filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1406,7 +1468,7 @@ QHP_CUST_FILTER_ATTRS  =
 
 # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
 # project's filter section matches. Qt Help Project / Filter Attributes (see:
-# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHP_SECT_FILTER_ATTRS  =
@@ -1499,7 +1561,7 @@ EXT_LINKS_IN_WINDOW    = NO
 
 FORMULA_FONTSIZE       = 10
 
-# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
 # generated for formulas are transparent PNGs. Transparent PNGs are not
 # supported properly for IE 6.0, but are supported on all modern browsers.
 #
@@ -1510,8 +1572,14 @@ FORMULA_FONTSIZE       = 10
 
 FORMULA_TRANSPARENT    = YES
 
+# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands
+# to create new LaTeX commands to be used in formulas as building blocks. See
+# the section "Including formulas" for details.
+
+FORMULA_MACROFILE      =
+
 # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
-# http://www.mathjax.org) which uses client side Javascript for the rendering
+# https://www.mathjax.org) which uses client side JavaScript for the rendering
 # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
 # installed or if you want to formulas look prettier in the HTML output. When
 # enabled you may also need to install MathJax separately and configure the path
@@ -1538,8 +1606,8 @@ MATHJAX_FORMAT         = HTML-CSS
 # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
 # Content Delivery Network so you can quickly see the result without installing
 # MathJax. However, it is strongly recommended to install a local copy of
-# MathJax from http://www.mathjax.org before deployment.
-# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# MathJax from https://www.mathjax.org before deployment.
+# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
 MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?...
@@ -1581,7 +1649,7 @@ MATHJAX_CODEFILE       =
 SEARCHENGINE           = YES
 
 # When the SERVER_BASED_SEARCH tag is enabled the search engine will be
-# implemented using a web server instead of a web client using Javascript. There
+# implemented using a web server instead of a web client using JavaScript. There
 # are two flavors of web server based searching depending on the EXTERNAL_SEARCH
 # setting. When disabled, doxygen will generate a PHP script for searching and
 # an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
@@ -1600,7 +1668,7 @@ SERVER_BASED_SEARCH    = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: http://xapian.org/).
+# Xapian (see: https://xapian.org/).
 #
 # See the section "External Indexing and Searching" for details.
 # The default value is: NO.
@@ -1613,7 +1681,7 @@ EXTERNAL_SEARCH        = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Xapian (see: https://xapian.org/). See the section "External Indexing and
 # Searching" for details.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
@@ -1665,21 +1733,35 @@ LATEX_OUTPUT           = latex
 # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
 # invoked.
 #
-# Note that when enabling USE_PDFLATEX this option is only used for generating
-# bitmaps for formulas in the HTML output, but not in the Makefile that is
-# written to the output directory.
-# The default file is: latex.
+# Note that when not enabling USE_PDFLATEX the default is latex when enabling
+# USE_PDFLATEX the default is pdflatex and when in the later case latex is
+# chosen this is overwritten by pdflatex. For specific output languages the
+# default can have been set differently, this depends on the implementation of
+# the output language.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_CMD_NAME         = latex
 
 # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
 # index for LaTeX.
+# Note: This tag is used in the Makefile / make.bat.
+# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file
+# (.tex).
 # The default file is: makeindex.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 MAKEINDEX_CMD_NAME     = makeindex
 
+# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to
+# generate index for LaTeX. In case there is no backslash (\) as first character
+# it will be automatically added in the LaTeX code.
+# Note: This tag is used in the generated output file (.tex).
+# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat.
+# The default value is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_MAKEINDEX_CMD    = makeindex
+
 # If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
 # documents. This may be useful for small projects and may help to save some
 # trees in general.
@@ -1800,7 +1882,7 @@ LATEX_SOURCE_CODE      = NO
 
 # The LATEX_BIB_STYLE tag can be used to specify the style to use for the
 # bibliography, e.g. plainnat, or ieeetr. See
-# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# https://en.wikipedia.org/wiki/BibTeX and \cite for more info.
 # The default value is: plain.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
@@ -1814,6 +1896,14 @@ LATEX_BIB_STYLE        = plain
 
 LATEX_TIMESTAMP        = NO
 
+# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute)
+# path from which the emoji images will be read. If a relative path is entered,
+# it will be relative to the LATEX_OUTPUT directory. If left blank the
+# LATEX_OUTPUT directory will be used.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EMOJI_DIRECTORY  =
+
 #---------------------------------------------------------------------------
 # Configuration options related to the RTF output
 #---------------------------------------------------------------------------
@@ -1853,9 +1943,9 @@ COMPACT_RTF            = NO
 
 RTF_HYPERLINKS         = NO
 
-# Load stylesheet definitions from file. Syntax is similar to doxygen's config
-# file, i.e. a series of assignments. You only have to provide replacements,
-# missing definitions are set to their default value.
+# Load stylesheet definitions from file. Syntax is similar to doxygen's
+# configuration file, i.e. a series of assignments. You only have to provide
+# replacements, missing definitions are set to their default value.
 #
 # See also section "Doxygen usage" for information on how to generate the
 # default style sheet that doxygen normally uses.
@@ -1864,8 +1954,8 @@ RTF_HYPERLINKS         = NO
 RTF_STYLESHEET_FILE    =
 
 # Set optional variables used in the generation of an RTF document. Syntax is
-# similar to doxygen's config file. A template extensions file can be generated
-# using doxygen -e rtf extensionFile.
+# similar to doxygen's configuration file. A template extensions file can be
+# generated using doxygen -e rtf extensionFile.
 # This tag requires that the tag GENERATE_RTF is set to YES.
 
 RTF_EXTENSIONS_FILE    =
@@ -1951,6 +2041,13 @@ XML_OUTPUT             = xml
 
 XML_PROGRAMLISTING     = YES
 
+# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include
+# namespace members in file scope as well, matching the HTML output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_NS_MEMB_FILE_SCOPE = NO
+
 #---------------------------------------------------------------------------
 # Configuration options related to the DOCBOOK output
 #---------------------------------------------------------------------------
@@ -1983,9 +2080,9 @@ DOCBOOK_PROGRAMLISTING = NO
 #---------------------------------------------------------------------------
 
 # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
-# AutoGen Definitions (see http://autogen.sf.net) file that captures the
-# structure of the code including all documentation. Note that this feature is
-# still experimental and incomplete at the moment.
+# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
+# the structure of the code including all documentation. Note that this feature
+# is still experimental and incomplete at the moment.
 # The default value is: NO.
 
 GENERATE_AUTOGEN_DEF   = NO
@@ -2152,12 +2249,6 @@ EXTERNAL_GROUPS        = YES
 
 EXTERNAL_PAGES         = YES
 
-# The PERL_PATH should be the absolute path and name of the perl script
-# interpreter (i.e. the result of 'which perl').
-# The default file (with absolute path) is: /usr/bin/perl.
-
-PERL_PATH              = /usr/bin/perl
-
 #---------------------------------------------------------------------------
 # Configuration options related to the dot tool
 #---------------------------------------------------------------------------
@@ -2171,15 +2262,6 @@ PERL_PATH              = /usr/bin/perl
 
 CLASS_DIAGRAMS         = NO
 
-# You can define message sequence charts within doxygen comments using the \msc
-# command. Doxygen will then run the mscgen tool (see:
-# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
-# documentation. The MSCGEN_PATH tag allows you to specify the directory where
-# the mscgen tool resides. If left empty the tool is assumed to be found in the
-# default search path.
-
-MSCGEN_PATH            =
-
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The
 # DIA_PATH tag allows you to specify the directory where the dia binary resides.
diff --git a/inc/dg/adaptive_t.cu b/inc/dg/adaptive_t.cu
index c4bbe0e49..a44054448 100644
--- a/inc/dg/adaptive_t.cu
+++ b/inc/dg/adaptive_t.cu
@@ -37,7 +37,7 @@ int main()
 {
     std::cout << "Program to test correct implementation of adaptive methods in adaptive.h at the example of the damped driven harmonic oscillator. Errors should be small! \n";
     std::cout << std::scientific;
-    //![doxygen.]
+    //![doxygen]
     //... in main
     //set start and end time
     const double t_start = 0., t_end = 1.;
@@ -53,7 +53,7 @@ int main()
     //now compute error
     dg::blas1::axpby( 1., solution(t_end, damping, omega_0, omega_drive), -1., u_end);
     std::cout << "With "<<counter<<"\t Dormand Prince steps norm of error is "<<sqrt(dg::blas1::dot( u_end, u_end))<<"\n";
-    //![doxygen.]
+    //![doxygen]
     std::cout << "Explicit Methods \n";
     std::vector<std::string> names{
         "Heun-Euler-2-1-2",
diff --git a/inc/dg/backend/exblas/Doxyfile b/inc/dg/backend/exblas/Doxyfile
index a83827126..6919fc84e 100644
--- a/inc/dg/backend/exblas/Doxyfile
+++ b/inc/dg/backend/exblas/Doxyfile
@@ -1,4 +1,4 @@
-# Doxyfile 1.8.11
+# Doxyfile 1.8.17
 
 #---------------------------------------------------------------------------
 # Project related configuration options
@@ -12,6 +12,7 @@ OUTPUT_DIRECTORY       = ./doc
 CREATE_SUBDIRS         = NO
 ALLOW_UNICODE_NAMES    = NO
 OUTPUT_LANGUAGE        = English
+OUTPUT_TEXT_DIRECTION  = None
 BRIEF_MEMBER_DESC      = YES
 REPEAT_BRIEF           = YES
 ABBREVIATE_BRIEF       = "The $name class" \
@@ -32,6 +33,7 @@ STRIP_FROM_PATH        =
 STRIP_FROM_INC_PATH    =
 SHORT_NAMES            = NO
 JAVADOC_AUTOBRIEF      = NO
+JAVADOC_BANNER         = NO
 QT_AUTOBRIEF           = NO
 MULTILINE_CPP_IS_BRIEF = NO
 INHERIT_DOCS           = YES
@@ -43,8 +45,10 @@ OPTIMIZE_OUTPUT_FOR_C  = NO
 OPTIMIZE_OUTPUT_JAVA   = NO
 OPTIMIZE_FOR_FORTRAN   = NO
 OPTIMIZE_OUTPUT_VHDL   = NO
+OPTIMIZE_OUTPUT_SLICE  = NO
 EXTENSION_MAPPING      = cuh=C++
 MARKDOWN_SUPPORT       = YES
+TOC_INCLUDE_HEADINGS   = 5
 AUTOLINK_SUPPORT       = YES
 BUILTIN_STL_SUPPORT    = NO
 CPP_CLI_SUPPORT        = NO
@@ -62,6 +66,7 @@ LOOKUP_CACHE_SIZE      = 0
 #---------------------------------------------------------------------------
 EXTRACT_ALL            = NO
 EXTRACT_PRIVATE        = NO
+EXTRACT_PRIV_VIRTUAL   = NO
 EXTRACT_PACKAGE        = NO
 EXTRACT_STATIC         = YES
 EXTRACT_LOCAL_CLASSES  = YES
@@ -147,6 +152,7 @@ USE_HTAGS              = NO
 VERBATIM_HEADERS       = YES
 CLANG_ASSISTED_PARSING = NO
 CLANG_OPTIONS          =
+CLANG_DATABASE_PATH    =
 #---------------------------------------------------------------------------
 # Configuration options related to the alphabetical class index
 #---------------------------------------------------------------------------
@@ -168,6 +174,7 @@ HTML_COLORSTYLE_HUE    = 220
 HTML_COLORSTYLE_SAT    = 100
 HTML_COLORSTYLE_GAMMA  = 80
 HTML_TIMESTAMP         = NO
+HTML_DYNAMIC_MENUS     = YES
 HTML_DYNAMIC_SECTIONS  = NO
 HTML_INDEX_NUM_ENTRIES = 100
 GENERATE_DOCSET        = NO
@@ -199,6 +206,7 @@ TREEVIEW_WIDTH         = 250
 EXT_LINKS_IN_WINDOW    = NO
 FORMULA_FONTSIZE       = 10
 FORMULA_TRANSPARENT    = YES
+FORMULA_MACROFILE      =
 USE_MATHJAX            = NO
 MATHJAX_FORMAT         = HTML-CSS
 MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
@@ -218,6 +226,7 @@ GENERATE_LATEX         = NO
 LATEX_OUTPUT           = latex
 LATEX_CMD_NAME         = latex
 MAKEINDEX_CMD_NAME     = makeindex
+LATEX_MAKEINDEX_CMD    = makeindex
 COMPACT_LATEX          = NO
 PAPER_TYPE             = a4
 EXTRA_PACKAGES         =
@@ -232,6 +241,7 @@ LATEX_HIDE_INDICES     = NO
 LATEX_SOURCE_CODE      = NO
 LATEX_BIB_STYLE        = plain
 LATEX_TIMESTAMP        = NO
+LATEX_EMOJI_DIRECTORY  =
 #---------------------------------------------------------------------------
 # Configuration options related to the RTF output
 #---------------------------------------------------------------------------
@@ -256,6 +266,7 @@ MAN_LINKS              = NO
 GENERATE_XML           = NO
 XML_OUTPUT             = xml
 XML_PROGRAMLISTING     = YES
+XML_NS_MEMB_FILE_SCOPE = NO
 #---------------------------------------------------------------------------
 # Configuration options related to the DOCBOOK output
 #---------------------------------------------------------------------------
@@ -293,12 +304,10 @@ GENERATE_TAGFILE       =
 ALLEXTERNALS           = NO
 EXTERNAL_GROUPS        = YES
 EXTERNAL_PAGES         = YES
-PERL_PATH              = /usr/bin/perl
 #---------------------------------------------------------------------------
 # Configuration options related to the dot tool
 #---------------------------------------------------------------------------
 CLASS_DIAGRAMS         = NO
-MSCGEN_PATH            =
 DIA_PATH               =
 HIDE_UNDOC_RELATIONS   = YES
 HAVE_DOT               = YES
@@ -325,6 +334,7 @@ DOTFILE_DIRS           =
 MSCFILE_DIRS           =
 DIAFILE_DIRS           =
 PLANTUML_JAR_PATH      =
+PLANTUML_CFG_FILE      =
 PLANTUML_INCLUDE_PATH  =
 DOT_GRAPH_MAX_NODES    = 50
 MAX_DOT_GRAPH_DEPTH    = 0
diff --git a/inc/dg/backend/exblas/mpi_accumulate.h b/inc/dg/backend/exblas/mpi_accumulate.h
index 03eac6593..83635b17e 100644
--- a/inc/dg/backend/exblas/mpi_accumulate.h
+++ b/inc/dg/backend/exblas/mpi_accumulate.h
@@ -14,10 +14,12 @@
 
 namespace exblas {
 
+///@cond
 namespace detail{
 //we keep track of communicators that were created in the past
 static std::map<MPI_Comm, std::array<MPI_Comm, 2>> comm_mods;
 }
+///@endcond
 /**
  * @brief This function can be used to partition communicators for the \c exblas::reduce_mpi_cpu function
  *
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 238d903aa..0a5ea48ec 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -160,10 +160,11 @@ struct MultigridCG2d
     /**
      * @brief Nested iterations (USE THIS ONE!)
      *
-     * - Compute residual with given initial guess.
-     * - Project residual down to the coarsest grid.
-     * - Solve equation on the coarse grid
-     * - interpolate solution up to next finer grid and repeat until the original grid is reached.
+     * Equivalent to the following
+     * -# Compute residual with given initial guess.
+     * -# Project residual down to the coarsest grid.
+     * -# Solve equation on the coarse grid.
+     * -# interpolate solution up to next finer grid and repeat 3 and 4 until the original grid is reached.
      * @note The preconditioner for the CG solver is taken from the \c precond() method in the \c SymmetricOp class
      * @copydoc hide_symmetric_op
      * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index 4105af40a..24653713f 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -25,6 +25,8 @@ class NoRoot1d: public std::exception
      *
      * \param x_min left boundary
      * \param x_max right boundary
+     * @param wert_min value at left boundary
+     * @param wert_max value at right boundary
      */
     NoRoot1d(double x_min, double x_max, double wert_min, double wert_max): x1(x_min), x2(x_max), wert_min(wert_min), wert_max(wert_max){}
     /*! @brief display left and right boundary on std::cerr
diff --git a/inc/file/Doxyfile b/inc/file/Doxyfile
index d7047966e..7cc781f81 100644
--- a/inc/file/Doxyfile
+++ b/inc/file/Doxyfile
@@ -1,4 +1,4 @@
-# Doxyfile 1.8.11
+# Doxyfile 1.8.17
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project.
@@ -17,11 +17,11 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 
-# This tag specifies the encoding used for all characters in the config file
-# that follow. The default is UTF-8 which is also the encoding used for all text
-# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
-# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
-# for the list of possible encodings.
+# This tag specifies the encoding used for all characters in the configuration
+# file that follow. The default is UTF-8 which is also the encoding used for all
+# text before the first occurrence of this tag. Doxygen uses libiconv (or the
+# iconv built into libc) for the transcoding. See
+# https://www.gnu.org/software/libiconv/ for the list of possible encodings.
 # The default value is: UTF-8.
 
 DOXYFILE_ENCODING      = UTF-8
@@ -93,6 +93,14 @@ ALLOW_UNICODE_NAMES    = NO
 
 OUTPUT_LANGUAGE        = English
 
+# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all generated output in the proper direction.
+# Possible values are: None, LTR, RTL and Context.
+# The default value is: None.
+
+OUTPUT_TEXT_DIRECTION  = None
+
 # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
 # descriptions after the members that are listed in the file and class
 # documentation (similar to Javadoc). Set to NO to disable this.
@@ -189,6 +197,16 @@ SHORT_NAMES            = NO
 
 JAVADOC_AUTOBRIEF      = NO
 
+# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line
+# such as
+# /***************
+# as being the beginning of a Javadoc-style comment "banner". If set to NO, the
+# Javadoc-style will behave just like regular comments and it will not be
+# interpreted by doxygen.
+# The default value is: NO.
+
+JAVADOC_BANNER         = NO
+
 # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
 # line (until the first dot) of a Qt-style comment as the brief description. If
 # set to NO, the Qt-style will behave just like regular Qt-style comments (thus
@@ -236,7 +254,12 @@ TAB_SIZE               = 4
 # will allow you to put the command \sideeffect (or @sideeffect) in the
 # documentation, which will result in a user-defined paragraph with heading
 # "Side Effects:". You can put \n's in the value part of an alias to insert
-# newlines.
+# newlines (in the resulting output). You can put ^^ in the value part of an
+# alias to insert a newline as if a physical newline was in the original file.
+# When you need a literal { or } or , in the value part of an alias you have to
+# escape them by means of a backslash (\), this can lead to conflicts with the
+# commands \{ and \} for these it is advised to use the version @{ and @} or use
+# a double escape (\\{ and \\})
 
 ALIASES                =
 
@@ -274,17 +297,26 @@ OPTIMIZE_FOR_FORTRAN   = NO
 
 OPTIMIZE_OUTPUT_VHDL   = NO
 
+# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice
+# sources only. Doxygen will then generate output that is more tailored for that
+# language. For instance, namespaces will be presented as modules, types will be
+# separated into more groups, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_SLICE  = NO
+
 # Doxygen selects the parser to use depending on the extension of the files it
 # parses. With this tag you can assign which parser to use for a given
 # extension. Doxygen has a built-in mapping, but you can override or extend it
 # using this tag. The format is ext=language, where ext is a file extension, and
-# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
-# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:
-# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:
-# Fortran. In the later case the parser tries to guess whether the code is fixed
-# or free formatted code, this is the default for Fortran type files), VHDL. For
-# instance to make doxygen treat .inc files as Fortran files (default is PHP),
-# and .f files as C (default is Fortran), use: inc=Fortran f=C.
+# language is one of the parsers supported by doxygen: IDL, Java, JavaScript,
+# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice,
+# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran:
+# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser
+# tries to guess whether the code is fixed or free formatted code, this is the
+# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat
+# .inc files as Fortran files (default is PHP), and .f files as C (default is
+# Fortran), use: inc=Fortran f=C.
 #
 # Note: For files without extension you can use no_extension as a placeholder.
 #
@@ -295,7 +327,7 @@ EXTENSION_MAPPING      =
 
 # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
 # according to the Markdown format, which allows for more readable
-# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# documentation. See https://daringfireball.net/projects/markdown/ for details.
 # The output of markdown processing is further processed by doxygen, so you can
 # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
 # case of backward compatibilities issues.
@@ -303,6 +335,15 @@ EXTENSION_MAPPING      =
 
 MARKDOWN_SUPPORT       = YES
 
+# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up
+# to that level are automatically included in the table of contents, even if
+# they do not have an id attribute.
+# Note: This feature currently applies only to Markdown headings.
+# Minimum value: 0, maximum value: 99, default value: 5.
+# This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
+
+TOC_INCLUDE_HEADINGS   = 5
+
 # When enabled doxygen tries to link words that correspond to documented
 # classes, or namespaces to their corresponding documentation. Such a link can
 # be prevented in individual cases by putting a % sign in front of the word or
@@ -328,7 +369,7 @@ BUILTIN_STL_SUPPORT    = NO
 CPP_CLI_SUPPORT        = NO
 
 # Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
-# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen
 # will parse them like normal C++ but will assume all classes use public instead
 # of private inheritance when no explicit protection keyword is present.
 # The default value is: NO.
@@ -434,6 +475,12 @@ EXTRACT_ALL            = NO
 
 EXTRACT_PRIVATE        = NO
 
+# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual
+# methods of a class will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIV_VIRTUAL   = NO
+
 # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
 # scope will be included in the documentation.
 # The default value is: NO.
@@ -488,8 +535,8 @@ HIDE_UNDOC_MEMBERS     = NO
 HIDE_UNDOC_CLASSES     = NO
 
 # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
-# (class|struct|union) declarations. If set to NO, these declarations will be
-# included in the documentation.
+# declarations. If set to NO, these declarations will be included in the
+# documentation.
 # The default value is: NO.
 
 HIDE_FRIEND_COMPOUNDS  = NO
@@ -512,7 +559,7 @@ INTERNAL_DOCS          = NO
 # names in lower-case letters. If set to YES, upper-case letters are also
 # allowed. This is useful if you have classes or files whose names only differ
 # in case and if your file system supports case sensitive file names. Windows
-# and Mac users are advised to set this option to NO.
+# (including Cygwin) ands Mac users are advised to set this option to NO.
 # The default value is: system dependent.
 
 CASE_SENSE_NAMES       = NO
@@ -699,7 +746,7 @@ LAYOUT_FILE            =
 # The CITE_BIB_FILES tag can be used to specify one or more bib files containing
 # the reference definitions. This must be a list of .bib files. The .bib
 # extension is automatically appended if omitted. This requires the bibtex tool
-# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info.
 # For LaTeX the style of the bibliography can be controlled using
 # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
 # search path. See also \cite for info how to create references.
@@ -744,7 +791,8 @@ WARN_IF_DOC_ERROR      = YES
 # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
 # are documented, but have no documentation for their parameters or return
 # value. If set to NO, doxygen will only warn about wrong or incomplete
-# parameter documentation, but not about the absence of documentation.
+# parameter documentation, but not about the absence of documentation. If
+# EXTRACT_ALL is set to YES then this flag will automatically be disabled.
 # The default value is: NO.
 
 WARN_NO_PARAMDOC       = NO
@@ -786,7 +834,7 @@ INPUT                  = .
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
 # libiconv (or the iconv built into libc) for the transcoding. See the libiconv
-# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
 # possible encodings.
 # The default value is: UTF-8.
 
@@ -803,8 +851,10 @@ INPUT_ENCODING         = UTF-8
 # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
 # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
 # *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
-# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl,
-# *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js.
+# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment),
+# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen
+# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f, *.for, *.tcl, *.vhd,
+# *.vhdl, *.ucf, *.qsf and *.ice.
 
 FILE_PATTERNS          = *.h
 
@@ -959,7 +1009,7 @@ INLINE_SOURCES         = NO
 STRIP_CODE_COMMENTS    = YES
 
 # If the REFERENCED_BY_RELATION tag is set to YES then for each documented
-# function all documented functions referencing it will be listed.
+# entity all documented functions referencing it will be listed.
 # The default value is: NO.
 
 REFERENCED_BY_RELATION = NO
@@ -991,12 +1041,12 @@ SOURCE_TOOLTIPS        = YES
 # If the USE_HTAGS tag is set to YES then the references to source code will
 # point to the HTML generated by the htags(1) tool instead of doxygen built-in
 # source browser. The htags tool is part of GNU's global source tagging system
-# (see http://www.gnu.org/software/global/global.html). You will need version
+# (see https://www.gnu.org/software/global/global.html). You will need version
 # 4.8.6 or higher.
 #
 # To use it do the following:
 # - Install the latest version of global
-# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file
 # - Make sure the INPUT points to the root of the source tree
 # - Run doxygen as normal
 #
@@ -1024,7 +1074,7 @@ VERBATIM_HEADERS       = YES
 # rich C++ code for which doxygen's built-in parser lacks the necessary type
 # information.
 # Note: The availability of this option depends on whether or not doxygen was
-# generated with the -Duse-libclang=ON option for CMake.
+# generated with the -Duse_libclang=ON option for CMake.
 # The default value is: NO.
 
 CLANG_ASSISTED_PARSING = NO
@@ -1037,6 +1087,16 @@ CLANG_ASSISTED_PARSING = NO
 
 CLANG_OPTIONS          =
 
+# If clang assisted parsing is enabled you can provide the clang parser with the
+# path to the compilation database (see:
+# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) used when the files
+# were built. This is equivalent to specifying the "-p" option to a clang tool,
+# such as clang-check. These options will then be passed to the parser.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse_libclang=ON option for CMake.
+
+CLANG_DATABASE_PATH    =
+
 #---------------------------------------------------------------------------
 # Configuration options related to the alphabetical class index
 #---------------------------------------------------------------------------
@@ -1155,7 +1215,7 @@ HTML_EXTRA_FILES       =
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
 # this color. Hue is specified as an angle on a colorwheel, see
-# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# https://en.wikipedia.org/wiki/Hue for more information. For instance the value
 # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
 # purple, and 360 is red again.
 # Minimum value: 0, maximum value: 359, default value: 220.
@@ -1191,6 +1251,17 @@ HTML_COLORSTYLE_GAMMA  = 80
 
 HTML_TIMESTAMP         = YES
 
+# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
+# documentation will contain a main index with vertical navigation menus that
+# are dynamically created via JavaScript. If disabled, the navigation index will
+# consists of multiple levels of tabs that are statically embedded in every HTML
+# page. Disable this option to support browsers that do not have JavaScript,
+# like the Qt help browser.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_MENUS     = YES
+
 # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
 # documentation will contain sections that can be hidden and shown after the
 # page has loaded.
@@ -1214,13 +1285,13 @@ HTML_INDEX_NUM_ENTRIES = 100
 
 # If the GENERATE_DOCSET tag is set to YES, additional index files will be
 # generated that can be used as input for Apple's Xcode 3 integrated development
-# environment (see: http://developer.apple.com/tools/xcode/), introduced with
-# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# environment (see: https://developer.apple.com/xcode/), introduced with OSX
+# 10.5 (Leopard). To create a documentation set, doxygen will generate a
 # Makefile in the HTML output directory. Running make will produce the docset in
 # that directory and running make install will install the docset in
 # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
-# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
-# for more information.
+# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy
+# genXcode/_index.html for more information.
 # The default value is: NO.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
@@ -1259,7 +1330,7 @@ DOCSET_PUBLISHER_NAME  = Publisher
 # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
 # additional HTML index files: index.hhp, index.hhc, and index.hhk. The
 # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
-# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on
 # Windows.
 #
 # The HTML Help Workshop contains a compiler that can convert all HTML output
@@ -1335,7 +1406,7 @@ QCH_FILE               =
 
 # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
 # Project output. For more information please see Qt Help Project / Namespace
-# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace).
 # The default value is: org.doxygen.Project.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1343,7 +1414,7 @@ QHP_NAMESPACE          = org.doxygen.Project
 
 # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
 # Help Project output. For more information please see Qt Help Project / Virtual
-# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-
 # folders).
 # The default value is: doc.
 # This tag requires that the tag GENERATE_QHP is set to YES.
@@ -1352,7 +1423,7 @@ QHP_VIRTUAL_FOLDER     = doc
 
 # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
 # filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1360,7 +1431,7 @@ QHP_CUST_FILTER_NAME   =
 
 # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
 # custom filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1368,7 +1439,7 @@ QHP_CUST_FILTER_ATTRS  =
 
 # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
 # project's filter section matches. Qt Help Project / Filter Attributes (see:
-# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHP_SECT_FILTER_ATTRS  =
@@ -1461,7 +1532,7 @@ EXT_LINKS_IN_WINDOW    = NO
 
 FORMULA_FONTSIZE       = 10
 
-# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
 # generated for formulas are transparent PNGs. Transparent PNGs are not
 # supported properly for IE 6.0, but are supported on all modern browsers.
 #
@@ -1472,8 +1543,14 @@ FORMULA_FONTSIZE       = 10
 
 FORMULA_TRANSPARENT    = YES
 
+# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands
+# to create new LaTeX commands to be used in formulas as building blocks. See
+# the section "Including formulas" for details.
+
+FORMULA_MACROFILE      =
+
 # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
-# http://www.mathjax.org) which uses client side Javascript for the rendering
+# https://www.mathjax.org) which uses client side JavaScript for the rendering
 # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
 # installed or if you want to formulas look prettier in the HTML output. When
 # enabled you may also need to install MathJax separately and configure the path
@@ -1500,8 +1577,8 @@ MATHJAX_FORMAT         = HTML-CSS
 # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
 # Content Delivery Network so you can quickly see the result without installing
 # MathJax. However, it is strongly recommended to install a local copy of
-# MathJax from http://www.mathjax.org before deployment.
-# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# MathJax from https://www.mathjax.org before deployment.
+# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
 MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?...
@@ -1543,7 +1620,7 @@ MATHJAX_CODEFILE       =
 SEARCHENGINE           = YES
 
 # When the SERVER_BASED_SEARCH tag is enabled the search engine will be
-# implemented using a web server instead of a web client using Javascript. There
+# implemented using a web server instead of a web client using JavaScript. There
 # are two flavors of web server based searching depending on the EXTERNAL_SEARCH
 # setting. When disabled, doxygen will generate a PHP script for searching and
 # an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
@@ -1562,7 +1639,7 @@ SERVER_BASED_SEARCH    = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: http://xapian.org/).
+# Xapian (see: https://xapian.org/).
 #
 # See the section "External Indexing and Searching" for details.
 # The default value is: NO.
@@ -1575,7 +1652,7 @@ EXTERNAL_SEARCH        = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Xapian (see: https://xapian.org/). See the section "External Indexing and
 # Searching" for details.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
@@ -1627,21 +1704,35 @@ LATEX_OUTPUT           = latex
 # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
 # invoked.
 #
-# Note that when enabling USE_PDFLATEX this option is only used for generating
-# bitmaps for formulas in the HTML output, but not in the Makefile that is
-# written to the output directory.
-# The default file is: latex.
+# Note that when not enabling USE_PDFLATEX the default is latex when enabling
+# USE_PDFLATEX the default is pdflatex and when in the later case latex is
+# chosen this is overwritten by pdflatex. For specific output languages the
+# default can have been set differently, this depends on the implementation of
+# the output language.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_CMD_NAME         = latex
 
 # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
 # index for LaTeX.
+# Note: This tag is used in the Makefile / make.bat.
+# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file
+# (.tex).
 # The default file is: makeindex.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 MAKEINDEX_CMD_NAME     = makeindex
 
+# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to
+# generate index for LaTeX. In case there is no backslash (\) as first character
+# it will be automatically added in the LaTeX code.
+# Note: This tag is used in the generated output file (.tex).
+# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat.
+# The default value is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_MAKEINDEX_CMD    = makeindex
+
 # If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
 # documents. This may be useful for small projects and may help to save some
 # trees in general.
@@ -1762,7 +1853,7 @@ LATEX_SOURCE_CODE      = NO
 
 # The LATEX_BIB_STYLE tag can be used to specify the style to use for the
 # bibliography, e.g. plainnat, or ieeetr. See
-# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# https://en.wikipedia.org/wiki/BibTeX and \cite for more info.
 # The default value is: plain.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
@@ -1776,6 +1867,14 @@ LATEX_BIB_STYLE        = plain
 
 LATEX_TIMESTAMP        = NO
 
+# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute)
+# path from which the emoji images will be read. If a relative path is entered,
+# it will be relative to the LATEX_OUTPUT directory. If left blank the
+# LATEX_OUTPUT directory will be used.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EMOJI_DIRECTORY  =
+
 #---------------------------------------------------------------------------
 # Configuration options related to the RTF output
 #---------------------------------------------------------------------------
@@ -1815,9 +1914,9 @@ COMPACT_RTF            = NO
 
 RTF_HYPERLINKS         = NO
 
-# Load stylesheet definitions from file. Syntax is similar to doxygen's config
-# file, i.e. a series of assignments. You only have to provide replacements,
-# missing definitions are set to their default value.
+# Load stylesheet definitions from file. Syntax is similar to doxygen's
+# configuration file, i.e. a series of assignments. You only have to provide
+# replacements, missing definitions are set to their default value.
 #
 # See also section "Doxygen usage" for information on how to generate the
 # default style sheet that doxygen normally uses.
@@ -1826,8 +1925,8 @@ RTF_HYPERLINKS         = NO
 RTF_STYLESHEET_FILE    =
 
 # Set optional variables used in the generation of an RTF document. Syntax is
-# similar to doxygen's config file. A template extensions file can be generated
-# using doxygen -e rtf extensionFile.
+# similar to doxygen's configuration file. A template extensions file can be
+# generated using doxygen -e rtf extensionFile.
 # This tag requires that the tag GENERATE_RTF is set to YES.
 
 RTF_EXTENSIONS_FILE    =
@@ -1913,6 +2012,13 @@ XML_OUTPUT             = xml
 
 XML_PROGRAMLISTING     = YES
 
+# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include
+# namespace members in file scope as well, matching the HTML output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_NS_MEMB_FILE_SCOPE = NO
+
 #---------------------------------------------------------------------------
 # Configuration options related to the DOCBOOK output
 #---------------------------------------------------------------------------
@@ -1945,9 +2051,9 @@ DOCBOOK_PROGRAMLISTING = NO
 #---------------------------------------------------------------------------
 
 # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
-# AutoGen Definitions (see http://autogen.sf.net) file that captures the
-# structure of the code including all documentation. Note that this feature is
-# still experimental and incomplete at the moment.
+# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
+# the structure of the code including all documentation. Note that this feature
+# is still experimental and incomplete at the moment.
 # The default value is: NO.
 
 GENERATE_AUTOGEN_DEF   = NO
@@ -2114,12 +2220,6 @@ EXTERNAL_GROUPS        = YES
 
 EXTERNAL_PAGES         = YES
 
-# The PERL_PATH should be the absolute path and name of the perl script
-# interpreter (i.e. the result of 'which perl').
-# The default file (with absolute path) is: /usr/bin/perl.
-
-PERL_PATH              = /usr/bin/perl
-
 #---------------------------------------------------------------------------
 # Configuration options related to the dot tool
 #---------------------------------------------------------------------------
@@ -2133,15 +2233,6 @@ PERL_PATH              = /usr/bin/perl
 
 CLASS_DIAGRAMS         = NO
 
-# You can define message sequence charts within doxygen comments using the \msc
-# command. Doxygen will then run the mscgen tool (see:
-# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
-# documentation. The MSCGEN_PATH tag allows you to specify the directory where
-# the mscgen tool resides. If left empty the tool is assumed to be found in the
-# default search path.
-
-MSCGEN_PATH            =
-
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The
 # DIA_PATH tag allows you to specify the directory where the dia binary resides.
@@ -2371,6 +2462,11 @@ DIAFILE_DIRS           =
 
 PLANTUML_JAR_PATH      =
 
+# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a
+# configuration file for plantuml.
+
+PLANTUML_CFG_FILE      =
+
 # When using plantuml, the specified paths are searched for files specified by
 # the !include statement in a plantuml block.
 
diff --git a/inc/geometries/Doxyfile b/inc/geometries/Doxyfile
index dbe9526e4..4fb23e550 100644
--- a/inc/geometries/Doxyfile
+++ b/inc/geometries/Doxyfile
@@ -1,4 +1,4 @@
-# Doxyfile 1.8.11
+# Doxyfile 1.8.17
 
 # This file describes the settings to be used by the documentation system
 # doxygen (www.doxygen.org) for a project.
@@ -17,11 +17,11 @@
 # Project related configuration options
 #---------------------------------------------------------------------------
 
-# This tag specifies the encoding used for all characters in the config file
-# that follow. The default is UTF-8 which is also the encoding used for all text
-# before the first occurrence of this tag. Doxygen uses libiconv (or the iconv
-# built into libc) for the transcoding. See http://www.gnu.org/software/libiconv
-# for the list of possible encodings.
+# This tag specifies the encoding used for all characters in the configuration
+# file that follow. The default is UTF-8 which is also the encoding used for all
+# text before the first occurrence of this tag. Doxygen uses libiconv (or the
+# iconv built into libc) for the transcoding. See
+# https://www.gnu.org/software/libiconv/ for the list of possible encodings.
 # The default value is: UTF-8.
 
 DOXYFILE_ENCODING      = UTF-8
@@ -93,6 +93,14 @@ ALLOW_UNICODE_NAMES    = NO
 
 OUTPUT_LANGUAGE        = English
 
+# The OUTPUT_TEXT_DIRECTION tag is used to specify the direction in which all
+# documentation generated by doxygen is written. Doxygen will use this
+# information to generate all generated output in the proper direction.
+# Possible values are: None, LTR, RTL and Context.
+# The default value is: None.
+
+OUTPUT_TEXT_DIRECTION  = None
+
 # If the BRIEF_MEMBER_DESC tag is set to YES, doxygen will include brief member
 # descriptions after the members that are listed in the file and class
 # documentation (similar to Javadoc). Set to NO to disable this.
@@ -189,6 +197,16 @@ SHORT_NAMES            = NO
 
 JAVADOC_AUTOBRIEF      = NO
 
+# If the JAVADOC_BANNER tag is set to YES then doxygen will interpret a line
+# such as
+# /***************
+# as being the beginning of a Javadoc-style comment "banner". If set to NO, the
+# Javadoc-style will behave just like regular comments and it will not be
+# interpreted by doxygen.
+# The default value is: NO.
+
+JAVADOC_BANNER         = NO
+
 # If the QT_AUTOBRIEF tag is set to YES then doxygen will interpret the first
 # line (until the first dot) of a Qt-style comment as the brief description. If
 # set to NO, the Qt-style will behave just like regular Qt-style comments (thus
@@ -236,7 +254,12 @@ TAB_SIZE               = 8
 # will allow you to put the command \sideeffect (or @sideeffect) in the
 # documentation, which will result in a user-defined paragraph with heading
 # "Side Effects:". You can put \n's in the value part of an alias to insert
-# newlines.
+# newlines (in the resulting output). You can put ^^ in the value part of an
+# alias to insert a newline as if a physical newline was in the original file.
+# When you need a literal { or } or , in the value part of an alias you have to
+# escape them by means of a backslash (\), this can lead to conflicts with the
+# commands \{ and \} for these it is advised to use the version @{ and @} or use
+# a double escape (\\{ and \\})
 
 ALIASES                =
 
@@ -274,17 +297,26 @@ OPTIMIZE_FOR_FORTRAN   = NO
 
 OPTIMIZE_OUTPUT_VHDL   = NO
 
+# Set the OPTIMIZE_OUTPUT_SLICE tag to YES if your project consists of Slice
+# sources only. Doxygen will then generate output that is more tailored for that
+# language. For instance, namespaces will be presented as modules, types will be
+# separated into more groups, etc.
+# The default value is: NO.
+
+OPTIMIZE_OUTPUT_SLICE  = NO
+
 # Doxygen selects the parser to use depending on the extension of the files it
 # parses. With this tag you can assign which parser to use for a given
 # extension. Doxygen has a built-in mapping, but you can override or extend it
 # using this tag. The format is ext=language, where ext is a file extension, and
-# language is one of the parsers supported by doxygen: IDL, Java, Javascript,
-# C#, C, C++, D, PHP, Objective-C, Python, Fortran (fixed format Fortran:
-# FortranFixed, free formatted Fortran: FortranFree, unknown formatted Fortran:
-# Fortran. In the later case the parser tries to guess whether the code is fixed
-# or free formatted code, this is the default for Fortran type files), VHDL. For
-# instance to make doxygen treat .inc files as Fortran files (default is PHP),
-# and .f files as C (default is Fortran), use: inc=Fortran f=C.
+# language is one of the parsers supported by doxygen: IDL, Java, JavaScript,
+# Csharp (C#), C, C++, D, PHP, md (Markdown), Objective-C, Python, Slice,
+# Fortran (fixed format Fortran: FortranFixed, free formatted Fortran:
+# FortranFree, unknown formatted Fortran: Fortran. In the later case the parser
+# tries to guess whether the code is fixed or free formatted code, this is the
+# default for Fortran type files), VHDL, tcl. For instance to make doxygen treat
+# .inc files as Fortran files (default is PHP), and .f files as C (default is
+# Fortran), use: inc=Fortran f=C.
 #
 # Note: For files without extension you can use no_extension as a placeholder.
 #
@@ -296,7 +328,7 @@ EXTENSION_MAPPING      = cuh=C++ \
 
 # If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
 # according to the Markdown format, which allows for more readable
-# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# documentation. See https://daringfireball.net/projects/markdown/ for details.
 # The output of markdown processing is further processed by doxygen, so you can
 # mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
 # case of backward compatibilities issues.
@@ -304,6 +336,15 @@ EXTENSION_MAPPING      = cuh=C++ \
 
 MARKDOWN_SUPPORT       = YES
 
+# When the TOC_INCLUDE_HEADINGS tag is set to a non-zero value, all headings up
+# to that level are automatically included in the table of contents, even if
+# they do not have an id attribute.
+# Note: This feature currently applies only to Markdown headings.
+# Minimum value: 0, maximum value: 99, default value: 5.
+# This tag requires that the tag MARKDOWN_SUPPORT is set to YES.
+
+TOC_INCLUDE_HEADINGS   = 5
+
 # When enabled doxygen tries to link words that correspond to documented
 # classes, or namespaces to their corresponding documentation. Such a link can
 # be prevented in individual cases by putting a % sign in front of the word or
@@ -329,7 +370,7 @@ BUILTIN_STL_SUPPORT    = NO
 CPP_CLI_SUPPORT        = NO
 
 # Set the SIP_SUPPORT tag to YES if your project consists of sip (see:
-# http://www.riverbankcomputing.co.uk/software/sip/intro) sources only. Doxygen
+# https://www.riverbankcomputing.com/software/sip/intro) sources only. Doxygen
 # will parse them like normal C++ but will assume all classes use public instead
 # of private inheritance when no explicit protection keyword is present.
 # The default value is: NO.
@@ -435,6 +476,12 @@ EXTRACT_ALL            = YES
 
 EXTRACT_PRIVATE        = NO
 
+# If the EXTRACT_PRIV_VIRTUAL tag is set to YES, documented private virtual
+# methods of a class will be included in the documentation.
+# The default value is: NO.
+
+EXTRACT_PRIV_VIRTUAL   = NO
+
 # If the EXTRACT_PACKAGE tag is set to YES, all members with package or internal
 # scope will be included in the documentation.
 # The default value is: NO.
@@ -489,8 +536,8 @@ HIDE_UNDOC_MEMBERS     = NO
 HIDE_UNDOC_CLASSES     = NO
 
 # If the HIDE_FRIEND_COMPOUNDS tag is set to YES, doxygen will hide all friend
-# (class|struct|union) declarations. If set to NO, these declarations will be
-# included in the documentation.
+# declarations. If set to NO, these declarations will be included in the
+# documentation.
 # The default value is: NO.
 
 HIDE_FRIEND_COMPOUNDS  = NO
@@ -513,7 +560,7 @@ INTERNAL_DOCS          = NO
 # names in lower-case letters. If set to YES, upper-case letters are also
 # allowed. This is useful if you have classes or files whose names only differ
 # in case and if your file system supports case sensitive file names. Windows
-# and Mac users are advised to set this option to NO.
+# (including Cygwin) ands Mac users are advised to set this option to NO.
 # The default value is: system dependent.
 
 CASE_SENSE_NAMES       = NO
@@ -700,7 +747,7 @@ LAYOUT_FILE            =
 # The CITE_BIB_FILES tag can be used to specify one or more bib files containing
 # the reference definitions. This must be a list of .bib files. The .bib
 # extension is automatically appended if omitted. This requires the bibtex tool
-# to be installed. See also http://en.wikipedia.org/wiki/BibTeX for more info.
+# to be installed. See also https://en.wikipedia.org/wiki/BibTeX for more info.
 # For LaTeX the style of the bibliography can be controlled using
 # LATEX_BIB_STYLE. To use this feature you need bibtex and perl available in the
 # search path. See also \cite for info how to create references.
@@ -745,7 +792,8 @@ WARN_IF_DOC_ERROR      = YES
 # This WARN_NO_PARAMDOC option can be enabled to get warnings for functions that
 # are documented, but have no documentation for their parameters or return
 # value. If set to NO, doxygen will only warn about wrong or incomplete
-# parameter documentation, but not about the absence of documentation.
+# parameter documentation, but not about the absence of documentation. If
+# EXTRACT_ALL is set to YES then this flag will automatically be disabled.
 # The default value is: NO.
 
 WARN_NO_PARAMDOC       = NO
@@ -787,7 +835,7 @@ INPUT                  = .
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
 # libiconv (or the iconv built into libc) for the transcoding. See the libiconv
-# documentation (see: http://www.gnu.org/software/libiconv) for the list of
+# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
 # possible encodings.
 # The default value is: UTF-8.
 
@@ -804,8 +852,10 @@ INPUT_ENCODING         = UTF-8
 # If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
 # *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
 # *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
-# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f, *.for, *.tcl,
-# *.vhd, *.vhdl, *.ucf, *.qsf, *.as and *.js.
+# *.m, *.markdown, *.md, *.mm, *.dox (to be provided as doxygen C comment),
+# *.doc (to be provided as doxygen C comment), *.txt (to be provided as doxygen
+# C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, *.f, *.for, *.tcl, *.vhd,
+# *.vhdl, *.ucf, *.qsf and *.ice.
 
 FILE_PATTERNS          = *.h \
                          *.cuh
@@ -856,7 +906,9 @@ EXCLUDE_SYMBOLS        = hide_*
 # that contain example code fragments that are included (see the \include
 # command).
 
-EXAMPLE_PATH           = ds_t.cu hector_t.cu flux_t.cu
+EXAMPLE_PATH           = ds_t.cu \
+                         hector_t.cu \
+                         flux_t.cu
 
 # If the value of the EXAMPLE_PATH tag contains directories, you can use the
 # EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp and
@@ -961,7 +1013,7 @@ INLINE_SOURCES         = NO
 STRIP_CODE_COMMENTS    = YES
 
 # If the REFERENCED_BY_RELATION tag is set to YES then for each documented
-# function all documented functions referencing it will be listed.
+# entity all documented functions referencing it will be listed.
 # The default value is: NO.
 
 REFERENCED_BY_RELATION = NO
@@ -993,12 +1045,12 @@ SOURCE_TOOLTIPS        = YES
 # If the USE_HTAGS tag is set to YES then the references to source code will
 # point to the HTML generated by the htags(1) tool instead of doxygen built-in
 # source browser. The htags tool is part of GNU's global source tagging system
-# (see http://www.gnu.org/software/global/global.html). You will need version
+# (see https://www.gnu.org/software/global/global.html). You will need version
 # 4.8.6 or higher.
 #
 # To use it do the following:
 # - Install the latest version of global
-# - Enable SOURCE_BROWSER and USE_HTAGS in the config file
+# - Enable SOURCE_BROWSER and USE_HTAGS in the configuration file
 # - Make sure the INPUT points to the root of the source tree
 # - Run doxygen as normal
 #
@@ -1026,7 +1078,7 @@ VERBATIM_HEADERS       = YES
 # rich C++ code for which doxygen's built-in parser lacks the necessary type
 # information.
 # Note: The availability of this option depends on whether or not doxygen was
-# generated with the -Duse-libclang=ON option for CMake.
+# generated with the -Duse_libclang=ON option for CMake.
 # The default value is: NO.
 
 CLANG_ASSISTED_PARSING = NO
@@ -1039,6 +1091,16 @@ CLANG_ASSISTED_PARSING = NO
 
 CLANG_OPTIONS          =
 
+# If clang assisted parsing is enabled you can provide the clang parser with the
+# path to the compilation database (see:
+# http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html) used when the files
+# were built. This is equivalent to specifying the "-p" option to a clang tool,
+# such as clang-check. These options will then be passed to the parser.
+# Note: The availability of this option depends on whether or not doxygen was
+# generated with the -Duse_libclang=ON option for CMake.
+
+CLANG_DATABASE_PATH    =
+
 #---------------------------------------------------------------------------
 # Configuration options related to the alphabetical class index
 #---------------------------------------------------------------------------
@@ -1157,7 +1219,7 @@ HTML_EXTRA_FILES       =
 # The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. Doxygen
 # will adjust the colors in the style sheet and background images according to
 # this color. Hue is specified as an angle on a colorwheel, see
-# http://en.wikipedia.org/wiki/Hue for more information. For instance the value
+# https://en.wikipedia.org/wiki/Hue for more information. For instance the value
 # 0 represents red, 60 is yellow, 120 is green, 180 is cyan, 240 is blue, 300
 # purple, and 360 is red again.
 # Minimum value: 0, maximum value: 359, default value: 220.
@@ -1193,6 +1255,17 @@ HTML_COLORSTYLE_GAMMA  = 80
 
 HTML_TIMESTAMP         = YES
 
+# If the HTML_DYNAMIC_MENUS tag is set to YES then the generated HTML
+# documentation will contain a main index with vertical navigation menus that
+# are dynamically created via JavaScript. If disabled, the navigation index will
+# consists of multiple levels of tabs that are statically embedded in every HTML
+# page. Disable this option to support browsers that do not have JavaScript,
+# like the Qt help browser.
+# The default value is: YES.
+# This tag requires that the tag GENERATE_HTML is set to YES.
+
+HTML_DYNAMIC_MENUS     = YES
+
 # If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML
 # documentation will contain sections that can be hidden and shown after the
 # page has loaded.
@@ -1216,13 +1289,13 @@ HTML_INDEX_NUM_ENTRIES = 100
 
 # If the GENERATE_DOCSET tag is set to YES, additional index files will be
 # generated that can be used as input for Apple's Xcode 3 integrated development
-# environment (see: http://developer.apple.com/tools/xcode/), introduced with
-# OSX 10.5 (Leopard). To create a documentation set, doxygen will generate a
+# environment (see: https://developer.apple.com/xcode/), introduced with OSX
+# 10.5 (Leopard). To create a documentation set, doxygen will generate a
 # Makefile in the HTML output directory. Running make will produce the docset in
 # that directory and running make install will install the docset in
 # ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find it at
-# startup. See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html
-# for more information.
+# startup. See https://developer.apple.com/library/archive/featuredarticles/Doxy
+# genXcode/_index.html for more information.
 # The default value is: NO.
 # This tag requires that the tag GENERATE_HTML is set to YES.
 
@@ -1261,7 +1334,7 @@ DOCSET_PUBLISHER_NAME  = Publisher
 # If the GENERATE_HTMLHELP tag is set to YES then doxygen generates three
 # additional HTML index files: index.hhp, index.hhc, and index.hhk. The
 # index.hhp is a project file that can be read by Microsoft's HTML Help Workshop
-# (see: http://www.microsoft.com/en-us/download/details.aspx?id=21138) on
+# (see: https://www.microsoft.com/en-us/download/details.aspx?id=21138) on
 # Windows.
 #
 # The HTML Help Workshop contains a compiler that can convert all HTML output
@@ -1337,7 +1410,7 @@ QCH_FILE               =
 
 # The QHP_NAMESPACE tag specifies the namespace to use when generating Qt Help
 # Project output. For more information please see Qt Help Project / Namespace
-# (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#namespace).
+# (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#namespace).
 # The default value is: org.doxygen.Project.
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1345,7 +1418,7 @@ QHP_NAMESPACE          = org.doxygen.Project
 
 # The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating Qt
 # Help Project output. For more information please see Qt Help Project / Virtual
-# Folders (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#virtual-
+# Folders (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#virtual-
 # folders).
 # The default value is: doc.
 # This tag requires that the tag GENERATE_QHP is set to YES.
@@ -1354,7 +1427,7 @@ QHP_VIRTUAL_FOLDER     = doc
 
 # If the QHP_CUST_FILTER_NAME tag is set, it specifies the name of a custom
 # filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1362,7 +1435,7 @@ QHP_CUST_FILTER_NAME   =
 
 # The QHP_CUST_FILTER_ATTRS tag specifies the list of the attributes of the
 # custom filter to add. For more information please see Qt Help Project / Custom
-# Filters (see: http://qt-project.org/doc/qt-4.8/qthelpproject.html#custom-
+# Filters (see: https://doc.qt.io/archives/qt-4.8/qthelpproject.html#custom-
 # filters).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
@@ -1370,7 +1443,7 @@ QHP_CUST_FILTER_ATTRS  =
 
 # The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this
 # project's filter section matches. Qt Help Project / Filter Attributes (see:
-# http://qt-project.org/doc/qt-4.8/qthelpproject.html#filter-attributes).
+# https://doc.qt.io/archives/qt-4.8/qthelpproject.html#filter-attributes).
 # This tag requires that the tag GENERATE_QHP is set to YES.
 
 QHP_SECT_FILTER_ATTRS  =
@@ -1463,7 +1536,7 @@ EXT_LINKS_IN_WINDOW    = NO
 
 FORMULA_FONTSIZE       = 10
 
-# Use the FORMULA_TRANPARENT tag to determine whether or not the images
+# Use the FORMULA_TRANSPARENT tag to determine whether or not the images
 # generated for formulas are transparent PNGs. Transparent PNGs are not
 # supported properly for IE 6.0, but are supported on all modern browsers.
 #
@@ -1474,8 +1547,14 @@ FORMULA_FONTSIZE       = 10
 
 FORMULA_TRANSPARENT    = YES
 
+# The FORMULA_MACROFILE can contain LaTeX \newcommand and \renewcommand commands
+# to create new LaTeX commands to be used in formulas as building blocks. See
+# the section "Including formulas" for details.
+
+FORMULA_MACROFILE      =
+
 # Enable the USE_MATHJAX option to render LaTeX formulas using MathJax (see
-# http://www.mathjax.org) which uses client side Javascript for the rendering
+# https://www.mathjax.org) which uses client side JavaScript for the rendering
 # instead of using pre-rendered bitmaps. Use this if you do not have LaTeX
 # installed or if you want to formulas look prettier in the HTML output. When
 # enabled you may also need to install MathJax separately and configure the path
@@ -1502,8 +1581,8 @@ MATHJAX_FORMAT         = HTML-CSS
 # MATHJAX_RELPATH should be ../mathjax. The default value points to the MathJax
 # Content Delivery Network so you can quickly see the result without installing
 # MathJax. However, it is strongly recommended to install a local copy of
-# MathJax from http://www.mathjax.org before deployment.
-# The default value is: http://cdn.mathjax.org/mathjax/latest.
+# MathJax from https://www.mathjax.org before deployment.
+# The default value is: https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.5/.
 # This tag requires that the tag USE_MATHJAX is set to YES.
 
 MATHJAX_RELPATH        = https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?...
@@ -1545,7 +1624,7 @@ MATHJAX_CODEFILE       =
 SEARCHENGINE           = YES
 
 # When the SERVER_BASED_SEARCH tag is enabled the search engine will be
-# implemented using a web server instead of a web client using Javascript. There
+# implemented using a web server instead of a web client using JavaScript. There
 # are two flavors of web server based searching depending on the EXTERNAL_SEARCH
 # setting. When disabled, doxygen will generate a PHP script for searching and
 # an index file used by the script. When EXTERNAL_SEARCH is enabled the indexing
@@ -1564,7 +1643,7 @@ SERVER_BASED_SEARCH    = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: http://xapian.org/).
+# Xapian (see: https://xapian.org/).
 #
 # See the section "External Indexing and Searching" for details.
 # The default value is: NO.
@@ -1577,7 +1656,7 @@ EXTERNAL_SEARCH        = NO
 #
 # Doxygen ships with an example indexer (doxyindexer) and search engine
 # (doxysearch.cgi) which are based on the open source search engine library
-# Xapian (see: http://xapian.org/). See the section "External Indexing and
+# Xapian (see: https://xapian.org/). See the section "External Indexing and
 # Searching" for details.
 # This tag requires that the tag SEARCHENGINE is set to YES.
 
@@ -1629,21 +1708,35 @@ LATEX_OUTPUT           = latex
 # The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be
 # invoked.
 #
-# Note that when enabling USE_PDFLATEX this option is only used for generating
-# bitmaps for formulas in the HTML output, but not in the Makefile that is
-# written to the output directory.
-# The default file is: latex.
+# Note that when not enabling USE_PDFLATEX the default is latex when enabling
+# USE_PDFLATEX the default is pdflatex and when in the later case latex is
+# chosen this is overwritten by pdflatex. For specific output languages the
+# default can have been set differently, this depends on the implementation of
+# the output language.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 LATEX_CMD_NAME         = latex
 
 # The MAKEINDEX_CMD_NAME tag can be used to specify the command name to generate
 # index for LaTeX.
+# Note: This tag is used in the Makefile / make.bat.
+# See also: LATEX_MAKEINDEX_CMD for the part in the generated output file
+# (.tex).
 # The default file is: makeindex.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
 MAKEINDEX_CMD_NAME     = makeindex
 
+# The LATEX_MAKEINDEX_CMD tag can be used to specify the command name to
+# generate index for LaTeX. In case there is no backslash (\) as first character
+# it will be automatically added in the LaTeX code.
+# Note: This tag is used in the generated output file (.tex).
+# See also: MAKEINDEX_CMD_NAME for the part in the Makefile / make.bat.
+# The default value is: makeindex.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_MAKEINDEX_CMD    = makeindex
+
 # If the COMPACT_LATEX tag is set to YES, doxygen generates more compact LaTeX
 # documents. This may be useful for small projects and may help to save some
 # trees in general.
@@ -1764,7 +1857,7 @@ LATEX_SOURCE_CODE      = NO
 
 # The LATEX_BIB_STYLE tag can be used to specify the style to use for the
 # bibliography, e.g. plainnat, or ieeetr. See
-# http://en.wikipedia.org/wiki/BibTeX and \cite for more info.
+# https://en.wikipedia.org/wiki/BibTeX and \cite for more info.
 # The default value is: plain.
 # This tag requires that the tag GENERATE_LATEX is set to YES.
 
@@ -1778,6 +1871,14 @@ LATEX_BIB_STYLE        = plain
 
 LATEX_TIMESTAMP        = NO
 
+# The LATEX_EMOJI_DIRECTORY tag is used to specify the (relative or absolute)
+# path from which the emoji images will be read. If a relative path is entered,
+# it will be relative to the LATEX_OUTPUT directory. If left blank the
+# LATEX_OUTPUT directory will be used.
+# This tag requires that the tag GENERATE_LATEX is set to YES.
+
+LATEX_EMOJI_DIRECTORY  =
+
 #---------------------------------------------------------------------------
 # Configuration options related to the RTF output
 #---------------------------------------------------------------------------
@@ -1817,9 +1918,9 @@ COMPACT_RTF            = NO
 
 RTF_HYPERLINKS         = NO
 
-# Load stylesheet definitions from file. Syntax is similar to doxygen's config
-# file, i.e. a series of assignments. You only have to provide replacements,
-# missing definitions are set to their default value.
+# Load stylesheet definitions from file. Syntax is similar to doxygen's
+# configuration file, i.e. a series of assignments. You only have to provide
+# replacements, missing definitions are set to their default value.
 #
 # See also section "Doxygen usage" for information on how to generate the
 # default style sheet that doxygen normally uses.
@@ -1828,8 +1929,8 @@ RTF_HYPERLINKS         = NO
 RTF_STYLESHEET_FILE    =
 
 # Set optional variables used in the generation of an RTF document. Syntax is
-# similar to doxygen's config file. A template extensions file can be generated
-# using doxygen -e rtf extensionFile.
+# similar to doxygen's configuration file. A template extensions file can be
+# generated using doxygen -e rtf extensionFile.
 # This tag requires that the tag GENERATE_RTF is set to YES.
 
 RTF_EXTENSIONS_FILE    =
@@ -1915,6 +2016,13 @@ XML_OUTPUT             = xml
 
 XML_PROGRAMLISTING     = YES
 
+# If the XML_NS_MEMB_FILE_SCOPE tag is set to YES, doxygen will include
+# namespace members in file scope as well, matching the HTML output.
+# The default value is: NO.
+# This tag requires that the tag GENERATE_XML is set to YES.
+
+XML_NS_MEMB_FILE_SCOPE = NO
+
 #---------------------------------------------------------------------------
 # Configuration options related to the DOCBOOK output
 #---------------------------------------------------------------------------
@@ -1947,9 +2055,9 @@ DOCBOOK_PROGRAMLISTING = NO
 #---------------------------------------------------------------------------
 
 # If the GENERATE_AUTOGEN_DEF tag is set to YES, doxygen will generate an
-# AutoGen Definitions (see http://autogen.sf.net) file that captures the
-# structure of the code including all documentation. Note that this feature is
-# still experimental and incomplete at the moment.
+# AutoGen Definitions (see http://autogen.sourceforge.net/) file that captures
+# the structure of the code including all documentation. Note that this feature
+# is still experimental and incomplete at the moment.
 # The default value is: NO.
 
 GENERATE_AUTOGEN_DEF   = NO
@@ -2116,12 +2224,6 @@ EXTERNAL_GROUPS        = NO
 
 EXTERNAL_PAGES         = NO
 
-# The PERL_PATH should be the absolute path and name of the perl script
-# interpreter (i.e. the result of 'which perl').
-# The default file (with absolute path) is: /usr/bin/perl.
-
-PERL_PATH              = /usr/bin/perl
-
 #---------------------------------------------------------------------------
 # Configuration options related to the dot tool
 #---------------------------------------------------------------------------
@@ -2135,15 +2237,6 @@ PERL_PATH              = /usr/bin/perl
 
 CLASS_DIAGRAMS         = NO
 
-# You can define message sequence charts within doxygen comments using the \msc
-# command. Doxygen will then run the mscgen tool (see:
-# http://www.mcternan.me.uk/mscgen/)) to produce the chart and insert it in the
-# documentation. The MSCGEN_PATH tag allows you to specify the directory where
-# the mscgen tool resides. If left empty the tool is assumed to be found in the
-# default search path.
-
-MSCGEN_PATH            =
-
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The
 # DIA_PATH tag allows you to specify the directory where the dia binary resides.
@@ -2373,6 +2466,11 @@ DIAFILE_DIRS           =
 
 PLANTUML_JAR_PATH      =
 
+# When using plantuml, the PLANTUML_CFG_FILE tag can be used to specify a
+# configuration file for plantuml.
+
+PLANTUML_CFG_FILE      =
+
 # When using plantuml, the specified paths are searched for files specified by
 # the !include statement in a plantuml block.
 
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 3cd5c0bf0..45d94f5f5 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -145,7 +145,7 @@ struct RealCurvilinearProductGrid3d : public dg::aRealProductGeometry3d<real_typ
     }
 
 
-    ///@copydoc CurvilinearGrid2d::generator()const
+    ///@copydoc RealCurvilinearGrid2d::generator()const
     const aRealGenerator2d<real_type> & generator() const{return *handle_;}
     virtual RealCurvilinearProductGrid3d* clone()const override final{return new RealCurvilinearProductGrid3d(*this);}
     private:
diff --git a/inc/geometries/ribeiro.h b/inc/geometries/ribeiro.h
index e4f5d9e84..2c0d7af31 100644
--- a/inc/geometries/ribeiro.h
+++ b/inc/geometries/ribeiro.h
@@ -194,7 +194,6 @@ struct Ribeiro : public aGenerator2d
     /**
      * @brief Construct a near-conformal grid generator
      *
-     * @param psi psi is the flux function in Cartesian coordinates (x,y), psiX is its derivative in x, psiY the derivative in y, psiXX the second derivative in x, etc.
      * @param psi \f$ \psi(x,y)\f$ the flux function and its derivatives in Cartesian coordinates (x,y)
      * @param psi_0 first boundary
      * @param psi_1 second boundary
-- 
GitLab


From a0b2285b585c50e6de76a26ab6ff8c74bb51c70a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 5 Aug 2020 15:59:50 +0200
Subject: [PATCH 285/540] BugFix geometry_diag.cu

---
 inc/geometries/geometry_diag.cu | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 509f85479..0d739bd06 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -32,8 +32,8 @@ struct Parameters
     double profile_alpha;
     Parameters( const Json::Value& js){
         n = js.get("n",3).asUInt();
-        Nx = js.get("Nx",100).asUInt();
-        Ny = js.get("Ny",100).asUInt();
+        Nx = js.get("Nx",100).asUInt()/js["compression"].get(0u,1).asUInt();
+        Ny = js.get("Ny",100).asUInt()/js["compression"].get(1u,1).asUInt();
         Nz = js.get("Nz", 1).asUInt();
         Npsi = js.get("Npsi", 32).asUInt();
         boxscaleRm = js["box"]["scaleR"].get(0u, 1.1).asDouble();
@@ -241,6 +241,7 @@ int main( int argc, char* argv[])
         std::cout << "Generate X-point flux-aligned grid ... \n";
         double RX = gp.R_0-1.1*gp.triangularity*gp.a;
         double ZX = -1.1*gp.elongation*gp.a;
+        dg::geo::findXpoint( mag.get_psip(), RX, ZX);
         double psipX = mag.psip()(RX, ZX);
         std::cout << "Found X-point at "<<RX<<" "<<ZX<<" with Psip = "<<psipX<<std::endl;
         if( fabs(psipX ) > 1e-10)
@@ -248,7 +249,6 @@ int main( int argc, char* argv[])
             std::cerr << " Psip at X-point is not zero. Unable to construct grid\n";
             return -1;
         }
-        dg::geo::findXpoint( mag.get_psip(), RX, ZX);
         dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), RX, ZX) ;
         dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipO, RX, ZX, mag.R0(), 0, 0, false);
         double fx_0 = 1./8.;
-- 
GitLab


From 909dd2dc59c012df7db6d921608c760478d09e8e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 10 Aug 2020 23:46:55 +0200
Subject: [PATCH 286/540] Fix unstable iteration in finite beta simulations

by avoiding apar updates during inversion
---
 src/feltor/implicit.h | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index c75eb611c..64b996404 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -128,16 +128,23 @@ struct ImplicitVelocity
                 m_multi_induction[u].elliptic().set_compute_in_2d(true);
         }
         m_multi_chi = m_multigrid.project( m_temp);
-        m_old_apar = dg::Extrapolation<Container>( 1, dg::evaluate( dg::zero, g));
+        m_old_apar = dg::Extrapolation<Container>( 2, dg::evaluate( dg::zero, g));
     }
     void set_density( const std::array<Container, 2>& dens){
-        dg::blas1::transform( dens, m_fields[0], dg::PLUS<double>(+1));
-        dg::blas1::axpby(  m_p.beta/m_p.mu[1], m_fields[0][1],
-                          -m_p.beta/m_p.mu[0], m_fields[0][0], m_temp);
-        //m_induction.set_chi( m_temp);
-        m_multigrid.project( m_temp, m_multi_chi);
-        for( unsigned u=0; u<m_p.stages; u++)
-            m_multi_induction[u].set_chi( m_multi_chi[u]);
+        if( m_p.beta != 0)
+        {
+            dg::blas1::transform( dens, m_fields[0], dg::PLUS<double>(+1));
+            dg::blas1::axpby(  m_p.beta/m_p.mu[1], m_fields[0][1],
+                              -m_p.beta/m_p.mu[0], m_fields[0][0], m_temp);
+            //m_induction.set_chi( m_temp);
+            m_multigrid.project( m_temp, m_multi_chi);
+            for( unsigned u=0; u<m_p.stages; u++)
+                m_multi_induction[u].set_chi( m_multi_chi[u]);
+        }
+    }
+    void update(){
+        if( m_p.beta != 0)
+            m_old_apar.update( m_apar);
     }
 
     void operator()( double t, const std::array<Container,2>& w,
@@ -158,9 +165,10 @@ struct ImplicitVelocity
             //m_invert( m_induction, m_apar, m_temp, weights(),
             //    inv_weights(), precond());
             m_old_apar.extrapolate( m_apar);
+            //dg::blas1::scal( m_apar, 0.);
             std::vector<unsigned> number = m_multigrid.direct_solve(
                 m_multi_induction, m_apar, m_temp, {m_p.eps_pol,m_p.eps_pol,m_p.eps_pol});
-            m_old_apar.update( m_apar);
+            //m_old_apar.update( m_apar); //don't update here: makes the solver potentially unstable
             if(  number[0] == m_multigrid.max_iter())
                 throw dg::Fail( m_p.eps_pol);
 
@@ -275,6 +283,7 @@ struct FeltorSpecialSolver
         m_solver.solve( alpha, m_imdens, t, y[0], rhs[0]);
         m_imvelo.set_density( y[0]);
         m_solver.solve( alpha, m_imvelo, t, y[1], rhs[1]);
+        m_imvelo.update();
     }
     private:
     dg::DefaultSolver<std::array<Container,2>> m_solver;
-- 
GitLab


From b777e8d9216aa8494b1a1875ff4fb935801063cf Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 17 Aug 2020 14:19:47 +0200
Subject: [PATCH 287/540] FIX: static 2d variables not properly written

This should not affect current diagnostics since they were
meant as mere consistency tests anyway
---
 src/feltor/feltor_hpc.cu | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index f7b8340e2..917fd5e1b 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -353,9 +353,9 @@ int main( int argc, char* argv[])
             "long_name", record.long_name.size(), record.long_name.data());
         MPI_OUT err = nc_enddef( ncid);
         MPI_OUT std::cout << "Computing2d "<<record.name<<"\n";
-        record.function( transferH, var, g3d_out);
-        //record.function( resultH, var, grid);
-        //dg::blas2::symv( projectH, resultH, transferH);
+        //record.function( transferH, var, g3d_out); //ATTENTION: This does not work because feltor internal varialbes return full grid functions
+        record.function( resultH, var, grid);
+        dg::blas2::symv( projectH, resultH, transferH);
         if(write2d)file::put_var_double( ncid, vecID, *g2d_out_ptr, transferH);
         MPI_OUT err = nc_redef(ncid);
     }
-- 
GitLab


From 50e87cd4d6c87aaeaf7f1d2e75ab8a18c5dfe36a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 18 Aug 2020 00:10:54 +0200
Subject: [PATCH 288/540] Improve diagnostics around separatrix

but general problem still not solved
---
 inc/dg/enums.h                  |  6 +++---
 inc/dg/topology/interpolation.h |  1 +
 src/feltor/feltordiag.cu        | 14 ++++++++------
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index 7371ac869..e54c4bc29 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -101,9 +101,9 @@ enum norm{
 //
 ///@brief Direction of a discrete derivative
 enum direction{
-    forward, //!< forward derivative
-    backward, //!< backward derivative
-    centered //!< centered derivative
+    forward, //!< forward derivative (cell to the right and current cell)
+    backward, //!< backward derivative (cell to the left and current cell)
+    centered //!< centered derivative (cell to the left and right and current cell)
 };
 
 
diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index 2b0c31dd0..b154a0486 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -77,6 +77,7 @@ std::vector<real_type> coefficients( real_type xn, unsigned n)
  * point will be mirrored at the boundary: \c dg::NEU will then simply interpolate at the resulting point, \c dg::DIR will take the negative of the interpolation.
  (\c dg::DIR_NEU and \c dg::NEU_DIR apply \c dg::NEU / \c dg::DIR to the respective left or right boundary )
  * This means the result of the interpolation is as if the interpolated function were Fourier transformed with the correct boundary condition and thus extended beyond the grid boundaries.
+ * Note that if a point lies directly on the boundary between two grid cells, the value of the polynomial to the right is taken.
 */
 
 /**
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 86436496c..185caa008 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -129,7 +129,8 @@ int main( int argc, char* argv[])
     std::cout << "psi max in gridX2d is "<<gridX2d.x1()<<"\n";
     std::cout << "DONE!\n";
     //Create 1d grid
-    dg::Grid1d g1d_out(psipO, psipmax, 3, Npsi, dg::DIR_NEU); //inner value is always 0
+    dg::Grid1d g1d_out(psipO, psipmax, npsi, Npsi, dg::DIR_NEU); //inner value is always 0
+    std::cout << "Cell separatrix boundary is "<<Npsi*(1.-fx_0)*g1d_out.h()+g1d_out.x0()<<"\n";
     const double f0 = ( gridX2d.x1() - gridX2d.x0() ) / ( psipmax - psipO );
     dg::HVec t1d = dg::evaluate( dg::zero, g1d_out), fsa1d( t1d);
     dg::HVec transfer1d = dg::evaluate(dg::zero,g1d_out);
@@ -198,7 +199,8 @@ int main( int argc, char* argv[])
 
     dg::HVec dvdpsip2d = dg::evaluate( dg::zero, g2d_out);
     dg::blas2::symv( fsa2rzmatrix, dvdpsip, dvdpsip2d);
-    dg::HMatrix dpsi = dg::create::dx( g1d_out, dg::DIR_NEU);
+    dg::HMatrix dpsi = dg::create::dx( g1d_out, dg::DIR_NEU, dg::backward); //we need to avoid involving cells outside LCFS in computation (also avoids right boundary)
+    //although the first point outside LCFS is still wrong
 
     // define 2d and 1d and 0d dimensions and variables
     int dim_ids[3], tvarID;
@@ -390,14 +392,14 @@ int main( int argc, char* argv[])
                         dg::blas2::symv( dpsi, fsa1d, t1d);
                         dg::blas1::pointwiseDivide( t1d, dvdpsip, transfer1d);
 
-                        result = dg::interpolate( dg::xspace, fsa1d, 0., g1d_out);
+                        result = dg::interpolate( dg::xspace, fsa1d, -1e-12, g1d_out);
                     }
                     else
                     {
                         dg::blas1::pointwiseDot( fsa1d, dvdpsip, t1d);
                         transfer1d = dg::integrate( t1d, g1d_out);
 
-                        result = dg::interpolate( dg::xspace, transfer1d, 0., g1d_out);
+                        result = dg::interpolate( dg::xspace, transfer1d, -1e-12, g1d_out); //make sure to take inner cell for interpolation
                     }
                     err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_ifs"),
                         start1d_out, count1d, transfer1d.data());
@@ -412,7 +414,7 @@ int main( int argc, char* argv[])
                         dg::blas1::pointwiseDot( t1d, t1d, t1d);//dvjv2
                         dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);//dvjv2
                         transfer1d = dg::integrate( t1d, g1d_out);
-                        result = dg::interpolate( dg::xspace, transfer1d, 0., g1d_out);
+                        result = dg::interpolate( dg::xspace, transfer1d, -1e-12, g1d_out);
                         result = sqrt(result);
                     }
                     else
@@ -421,7 +423,7 @@ int main( int argc, char* argv[])
                         dg::blas1::pointwiseDot( t1d, dvdpsip, t1d);
                         transfer1d = dg::integrate( t1d, g1d_out);
 
-                        result = dg::interpolate( dg::xspace, transfer1d, 0., g1d_out);
+                        result = dg::interpolate( dg::xspace, transfer1d, -1e-12, g1d_out);
                         result = sqrt(result);
                     }
                     err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
-- 
GitLab


From d29cb921201c24bb1b4e769d5c9f5a5a8ae93156 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Tue, 18 Aug 2020 20:33:11 +0200
Subject: [PATCH 289/540] Fix jsoncpp library in m100 config file

This bug will probably show on other platforms. We should find a long-term solution for that
---
 config/m100.mk | 1 +
 1 file changed, 1 insertion(+)

diff --git a/config/m100.mk b/config/m100.mk
index 6669a212b..8dc38fc00 100644
--- a/config/m100.mk
+++ b/config/m100.mk
@@ -14,6 +14,7 @@ NVCCARCH=-arch sm_70 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppre
 NVCCFLAGS= -std=c++14 -Xcompiler "-mcpu=power9 -Wall"# -mavx -mfma" #flags for NVCC
 
 INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
+JSONLIB=-L$(HOME)/include/json/../../lib -ljsoncpp_static # json library for input parameters
 LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
 LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
 #is the novel jsoncpp lib folder changed?
-- 
GitLab


From e1ad9bdc37bca367493dd49f12b6dcfe58a54e8a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 19 Aug 2020 10:35:06 +0200
Subject: [PATCH 290/540] HotFix: forgot to update density in implicit.h

---
 src/feltor/implicit.h         |  3 +--
 src/feltor/init.h             | 17 ++++++++++++++++-
 src/feltor/input/default.json | 10 +++++-----
 3 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 64b996404..a61b37136 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -131,12 +131,11 @@ struct ImplicitVelocity
         m_old_apar = dg::Extrapolation<Container>( 2, dg::evaluate( dg::zero, g));
     }
     void set_density( const std::array<Container, 2>& dens){
+        dg::blas1::transform( dens, m_fields[0], dg::PLUS<double>(+1));
         if( m_p.beta != 0)
         {
-            dg::blas1::transform( dens, m_fields[0], dg::PLUS<double>(+1));
             dg::blas1::axpby(  m_p.beta/m_p.mu[1], m_fields[0][1],
                               -m_p.beta/m_p.mu[0], m_fields[0][0], m_temp);
-            //m_induction.set_chi( m_temp);
             m_multigrid.project( m_temp, m_multi_chi);
             for( unsigned u=0; u<m_p.stages; u++)
                 m_multi_induction[u].set_chi( m_multi_chi[u]);
diff --git a/src/feltor/init.h b/src/feltor/init.h
index a4ab868c4..7b229fb4b 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -305,7 +305,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
 
 std::map<std::string, std::function< HVec(
     bool& fixed_profile, //indicate whether a profile should be forced (yes or no)
-    HVec& ne_profile,    // if fixed_profile is yes you need to construct something here, if no then you can ignore the parameter; if you construct something it will show in the output file in any case
+    HVec& ne_profile,    // if fixed_profile is yes you need to construct something here, if no then you can ignore the parameter; if you construct something it will show in the output file
     Geometry& grid, const feltor::Parameters& p,
     const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
 > > source_profiles =
@@ -350,6 +350,21 @@ std::map<std::string, std::function< HVec(
             return source_profile;
         }
     },
+    //{"tcv",
+    //    []( bool& fixed_profile, HVec& ne_profile,
+    //    Geometry& grid, const feltor::Parameters& p,
+    //    const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+    //    {
+    //        const double psi0 = 0.4, R_0 = 1.075/1e-3, Z_0 = -0.01/1e-3, sigma = 9.3e-3;
+
+    //        fixed_profile = false;
+    //        //ignore ne_profile
+    //        HVec source_profile = dg::pullback(
+    //            dg::compose( dg::Gaussian( R_0));
+
+
+    //    }
+    //},
     {"gaussian",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index 69ed1e765..6fe58fff3 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -2,7 +2,7 @@
     "n"  : 3,
     "Nx" : 48,
     "Ny" : 96,
-    "Nz" : 32,
+    "Nz" : 8,
     "dt" : 1e-2,
     "compression" : [2,2],
     "FCI":
@@ -42,7 +42,7 @@
     },
     "initne"     : "turbulence",
     "initphi"    : "zero",
-    "amplitude" : 0.2,
+    "amplitude" : 0.002,
     "sigma"     : 2.0,
     "posX"      : 0.6,
     "posY"      : 0,
@@ -55,15 +55,15 @@
     },
     "source" :
     {
-        "rate": 1e-2,
+        "rate": 1e-3,
         "type": "influx",
-        "boundary": 0.5,
+        "boundary": 0.55,
         "alpha": 0.2
     },
     "damping":
     {
         "rate": 1e-2,
         "boundary": 1.1,
-        "alpha": 0.1
+        "alpha": 0.2
     }
 }
-- 
GitLab


From f5c7bb37a09691825aa2e8eea6d669a74576baae Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 19 Aug 2020 16:06:18 +0200
Subject: [PATCH 291/540] Correct npsi in feltordiag.cu

---
 src/feltor/feltordiag.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 185caa008..4dd43f6a8 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -181,7 +181,7 @@ int main( int argc, char* argv[])
     map1d.emplace_back("psit1d", psit,
         "Toroidal flux label psi_t integrated using q-profile");
     //we need to avoid integrating >=0 for total psi_t
-    dg::Grid1d g1d_fine(psipO<0. ? psipO : 0., psipO<0. ? 0. : psipO, 3 ,Npsi,dg::DIR_NEU);
+    dg::Grid1d g1d_fine(psipO<0. ? psipO : 0., psipO<0. ? 0. : psipO, npsi ,Npsi,dg::DIR_NEU);
     qprofile = dg::evaluate( qprof, g1d_fine);
     dg::HVec w1d = dg::create::weights( g1d_fine);
     double psit_tot = dg::blas1::dot( w1d, qprofile);
-- 
GitLab


From 7909de153effd2485d8078be1ced459276370614 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 19 Aug 2020 21:41:30 +0200
Subject: [PATCH 292/540] Add enums and unified MagneticParameters

a first step towards a unified magnetic field creation and
description
---
 inc/geometries/magnetic_field.h | 419 +++++++++++++++++++-------------
 src/feltor/feltor.tex           |  14 +-
 2 files changed, 255 insertions(+), 178 deletions(-)

diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 3d36215a6..f00189c43 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -20,6 +20,73 @@ namespace geo
 ///@addtogroup magnetic
 ///@{
 
+///@brief How flux-function is computed
+enum class equilibrium
+{
+    solovev, //!< dg::geo::solovev::Psip
+    taylor, //!< dg::geo::taylor::Psip
+    //polynomial, ///!< dg::geo::polynomial::Psip
+    guenther, //!< dg::geo::guenther::Psip
+    toroidal, //!< dg::geo::toroidal::Psip
+    circular //!< dg::geo::circular::Psip
+};
+///@brief How flux-function is modified
+enum class modifier
+{
+    none, //!< no modification
+    heaviside //!< Psip is dampened to a constant outside a critical value
+};
+///@brief How flux-function looks like
+enum class form
+{
+    standardO, //!< closed flux surfaces centered around an O-point located near (R_0, 0); flux-aligned grids can be constructed
+    standardX, //!< closed flux surfaces centered around an O-point located near (R_0, 0) and bordered by a separatrix with a single X-point; flux-aligned X-grids can be constructed
+    square, //!< closed flux surfaces centered around an O-point and bordered by a square  with four X-points in the corners (mainly the Guenther field)
+    centeredX //!< one X-point in the middle, no O-point, only open flux surfaces, X-grids cannot be constructed
+};
+///@cond
+static const std::map<std::string, equilibrium> str2equilibrium{
+    {"solovev", equilibrium::solovev},
+    {"taylor", equilibrium::taylor},
+    //{"polynomial": equilibrium::polynomial},
+    {"guenther", equilibrium::guenther},
+    {"toroidal", equilibrium::toroidal},
+    {"circular", equilibrium::circular}
+};
+static const std::map<std::string, modifier> str2modifier{
+    {"none", modifier::none},
+    {"heaviside", modifier::heaviside}
+};
+static const std::map<std::string, form> str2form{
+    {"standardO", form::standardO},
+    {"standardX", form::standardX},
+    {"square", form::square},
+    {"centeredX", form::centeredX}
+};
+///@endcond
+
+//Meta-data about magnetic fields
+//
+/**
+ * @brief Meta-data about the magnetic field in particular the flux function
+ *
+ * The purpose of this is to give a unified set of parameters for
+ * all equilibria that can be used to stear program execution.
+ *
+ * For example it is very hard to automatically detect if the construction
+ * of a flux aligned X-grid is possible, but for a human it is very easy.
+ * Here we give the \c form specifier that can be used in an if-else statement.
+ */
+struct MagneticFieldParameters
+{
+    double a, //!< The minor radius; the purpose of this parameter is not to be exact but to serve as a refernce of how to setup the size of a simulation box
+           elongation, //!< (maximum Z - minimum Z of lcfs)/2a; 1 for a circle; the purpose of this parameter is not to be exact but more to be a reference of how to setup the aspect ratio of a simulation box
+           triangularity; //!< (R_0 - R_X) /a;  The purpose of this parameter is to find the approximate location of R_X (if an X-point is present, Z_X is given by elongation) the exact location can be computed by the \c findXpoint function
+    equilibrium equ; //!< the way the flux function is computed
+    modifier mod; //!<  the way the flux function is modified
+    form frm; //!< human readable descriptor of how the flux function looks
+};
+
 /**
 * @brief A tokamak field as given by R0, Psi and Ipol
 
@@ -36,42 +103,49 @@ struct TokamakMagneticField
     ///as long as the field stays empty the access functions are undefined
     TokamakMagneticField(){}
     TokamakMagneticField( double R0, const CylindricalFunctorsLvl2& psip, const
-            CylindricalFunctorsLvl1& ipol): R0_(R0), psip_(psip), ipol_(ipol){}
+            CylindricalFunctorsLvl1& ipol
+            //, MagneticFieldParameters gp
+            ): m_R0(R0), m_psip(psip), m_ipol(ipol) {}//, m_params(gp){}
     void set( double R0, const CylindricalFunctorsLvl2& psip, const
-            CylindricalFunctorsLvl1& ipol)
+            CylindricalFunctorsLvl1& ipol
+            //, MagneticFieldParameters gp
+            )
     {
-        R0_=R0;
-        psip_=psip;
-        ipol_=ipol;
+        m_R0=R0;
+        m_psip=psip;
+        m_ipol=ipol;
+        //m_params = gp;
     }
     /// \f$ R_0 \f$
-    double R0()const {return R0_;}
+    double R0()const {return m_R0;}
     /// \f$ \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const CylindricalFunctor& psip()const{return psip_.f();}
+    const CylindricalFunctor& psip()const{return m_psip.f();}
     /// \f$ \partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const CylindricalFunctor& psipR()const{return psip_.dfx();}
+    const CylindricalFunctor& psipR()const{return m_psip.dfx();}
     /// \f$ \partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const CylindricalFunctor& psipZ()const{return psip_.dfy();}
+    const CylindricalFunctor& psipZ()const{return m_psip.dfy();}
     /// \f$ \partial_R\partial_R \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const CylindricalFunctor& psipRR()const{return psip_.dfxx();}
+    const CylindricalFunctor& psipRR()const{return m_psip.dfxx();}
     /// \f$ \partial_R\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const CylindricalFunctor& psipRZ()const{return psip_.dfxy();}
+    const CylindricalFunctor& psipRZ()const{return m_psip.dfxy();}
     /// \f$ \partial_Z\partial_Z \psi_p(R,Z)\f$, where R, Z are cylindrical coordinates
-    const CylindricalFunctor& psipZZ()const{return psip_.dfyy();}
+    const CylindricalFunctor& psipZZ()const{return m_psip.dfyy();}
     /// \f$ I(\psi_p) \f$ the current
-    const CylindricalFunctor& ipol()const{return ipol_.f();}
+    const CylindricalFunctor& ipol()const{return m_ipol.f();}
     /// \f$ \partial_R I(\psi_p) \f$
-    const CylindricalFunctor& ipolR()const{return ipol_.dfx();}
+    const CylindricalFunctor& ipolR()const{return m_ipol.dfx();}
     /// \f$ \partial_Z I(\psi_p) \f$
-    const CylindricalFunctor& ipolZ()const{return ipol_.dfy();}
+    const CylindricalFunctor& ipolZ()const{return m_ipol.dfy();}
 
-    const CylindricalFunctorsLvl2& get_psip() const{return psip_;}
-    const CylindricalFunctorsLvl1& get_ipol() const{return ipol_;}
+    const CylindricalFunctorsLvl2& get_psip() const{return m_psip;}
+    const CylindricalFunctorsLvl1& get_ipol() const{return m_ipol;}
+    //const MagneticFieldParameters& params() const{return m_params;}
 
     private:
-    double R0_;
-    CylindricalFunctorsLvl2 psip_;
-    CylindricalFunctorsLvl1 ipol_;
+    double m_R0;
+    CylindricalFunctorsLvl2 m_psip;
+    CylindricalFunctorsLvl1 m_ipol;
+    //MagneticFieldParamters m_params;
 };
 
 ///@cond
@@ -110,7 +184,6 @@ CylindricalFunctorsLvl2 periodify( const CylindricalFunctorsLvl2& in, double R0,
  */
 TokamakMagneticField periodify( const TokamakMagneticField& mag, double R0, double R1, double Z0, double Z1, dg::bc bcx, dg::bc bcy)
 {
-    //what if Dirichlet BC in the current? Won't that generate a NaN?
     return TokamakMagneticField( mag.R0(),
             periodify( mag.get_psip(), R0, R1, Z0, Z1, bcx, bcy),
             //what if Dirichlet BC in the current? Won't that generate a NaN?
@@ -120,14 +193,14 @@ TokamakMagneticField periodify( const TokamakMagneticField& mag, double R0, doub
 ///@brief \f$   |B| = R_0\sqrt{I^2+(\nabla\psi)^2}/R   \f$
 struct Bmodule : public aCylindricalFunctor<Bmodule>
 {
-    Bmodule( const TokamakMagneticField& mag): mag_(mag)  { }
+    Bmodule( const TokamakMagneticField& mag): m_mag(mag)  { }
     double do_compute(double R, double Z) const
     {
-        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
-        return mag_.R0()/R*sqrt(ipol*ipol+psipR*psipR +psipZ*psipZ);
+        double psipR = m_mag.psipR()(R,Z), psipZ = m_mag.psipZ()(R,Z), ipol = m_mag.ipol()(R,Z);
+        return m_mag.R0()/R*sqrt(ipol*ipol+psipR*psipR +psipZ*psipZ);
     }
   private:
-    TokamakMagneticField mag_;
+    TokamakMagneticField m_mag;
 };
 
 /**
@@ -139,14 +212,14 @@ struct Bmodule : public aCylindricalFunctor<Bmodule>
  */
 struct InvB : public aCylindricalFunctor<InvB>
 {
-    InvB(  const TokamakMagneticField& mag): mag_(mag){ }
+    InvB(  const TokamakMagneticField& mag): m_mag(mag){ }
     double do_compute(double R, double Z) const
     {
-        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
-        return R/(mag_.R0()*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
+        double psipR = m_mag.psipR()(R,Z), psipZ = m_mag.psipZ()(R,Z), ipol = m_mag.ipol()(R,Z);
+        return R/(m_mag.R0()*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
     }
   private:
-    TokamakMagneticField mag_;
+    TokamakMagneticField m_mag;
 };
 
 /**
@@ -158,14 +231,14 @@ struct InvB : public aCylindricalFunctor<InvB>
  */
 struct LnB : public aCylindricalFunctor<LnB>
 {
-    LnB(const TokamakMagneticField& mag): mag_(mag) { }
+    LnB(const TokamakMagneticField& mag): m_mag(mag) { }
     double do_compute(double R, double Z) const
     {
-        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
-        return log(mag_.R0()/R*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
+        double psipR = m_mag.psipR()(R,Z), psipZ = m_mag.psipZ()(R,Z), ipol = m_mag.ipol()(R,Z);
+        return log(m_mag.R0()/R*sqrt(ipol*ipol + psipR*psipR +psipZ*psipZ)) ;
     }
   private:
-    TokamakMagneticField mag_;
+    TokamakMagneticField m_mag;
 };
 
 /**
@@ -180,16 +253,16 @@ struct LnB : public aCylindricalFunctor<LnB>
  */
 struct BR: public aCylindricalFunctor<BR>
 {
-    BR(const TokamakMagneticField& mag): invB_(mag), mag_(mag) { }
+    BR(const TokamakMagneticField& mag): m_invB(mag), m_mag(mag) { }
     double do_compute(double R, double Z) const
     {
         double Rn;
-        Rn = R/mag_.R0();
-        return -1./R/invB_(R,Z) + invB_(R,Z)/Rn/Rn*(mag_.ipol()(R,Z)*mag_.ipolR()(R,Z) + mag_.psipR()(R,Z)*mag_.psipRR()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipRZ()(R,Z));
+        Rn = R/m_mag.R0();
+        return -1./R/m_invB(R,Z) + m_invB(R,Z)/Rn/Rn*(m_mag.ipol()(R,Z)*m_mag.ipolR()(R,Z) + m_mag.psipR()(R,Z)*m_mag.psipRR()(R,Z) + m_mag.psipZ()(R,Z)*m_mag.psipRZ()(R,Z));
     }
   private:
-    InvB invB_;
-    TokamakMagneticField mag_;
+    InvB m_invB;
+    TokamakMagneticField m_mag;
 };
 
 /**
@@ -202,16 +275,16 @@ struct BR: public aCylindricalFunctor<BR>
  */
 struct BZ: public aCylindricalFunctor<BZ>
 {
-    BZ(const TokamakMagneticField& mag ): mag_(mag), invB_(mag) { }
+    BZ(const TokamakMagneticField& mag ): m_mag(mag), m_invB(mag) { }
     double do_compute(double R, double Z) const
     {
         double Rn;
-        Rn = R/mag_.R0();
-        return (invB_(R,Z)/Rn/Rn)*(mag_.ipol()(R,Z)*mag_.ipolZ()(R,Z) + mag_.psipR()(R,Z)*mag_.psipRZ()(R,Z) + mag_.psipZ()(R,Z)*mag_.psipZZ()(R,Z));
+        Rn = R/m_mag.R0();
+        return (m_invB(R,Z)/Rn/Rn)*(m_mag.ipol()(R,Z)*m_mag.ipolZ()(R,Z) + m_mag.psipR()(R,Z)*m_mag.psipRZ()(R,Z) + m_mag.psipZ()(R,Z)*m_mag.psipZZ()(R,Z));
     }
   private:
-    TokamakMagneticField mag_;
-    InvB invB_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
 };
 
 ///@brief Approximate \f$ \mathcal{K}^{R}_{\nabla B} \f$
@@ -220,7 +293,7 @@ struct BZ: public aCylindricalFunctor<BZ>
 ///@copydoc hide_toroidal_approximation_note
 struct CurvatureNablaBR: public aCylindricalFunctor<CurvatureNablaBR>
 {
-    CurvatureNablaBR(const TokamakMagneticField& mag, int sign): invB_(mag), bZ_(mag) {
+    CurvatureNablaBR(const TokamakMagneticField& mag, int sign): m_invB(mag), m_bZ(mag) {
         if( sign >0)
             m_sign = +1.;
         else
@@ -228,12 +301,12 @@ struct CurvatureNablaBR: public aCylindricalFunctor<CurvatureNablaBR>
     }
     double do_compute( double R, double Z) const
     {
-        return -m_sign*invB_(R,Z)*invB_(R,Z)*bZ_(R,Z);
+        return -m_sign*m_invB(R,Z)*m_invB(R,Z)*m_bZ(R,Z);
     }
     private:
     double m_sign;
-    InvB invB_;
-    BZ bZ_;
+    InvB m_invB;
+    BZ m_bZ;
 };
 
 ///@brief Approximate \f$  \mathcal{K}^{Z}_{\nabla B}  \f$
@@ -242,7 +315,7 @@ struct CurvatureNablaBR: public aCylindricalFunctor<CurvatureNablaBR>
 ///@copydoc hide_toroidal_approximation_note
 struct CurvatureNablaBZ: public aCylindricalFunctor<CurvatureNablaBZ>
 {
-    CurvatureNablaBZ( const TokamakMagneticField& mag, int sign): invB_(mag), bR_(mag) {
+    CurvatureNablaBZ( const TokamakMagneticField& mag, int sign): m_invB(mag), m_bR(mag) {
         if( sign >0)
             m_sign = +1.;
         else
@@ -250,12 +323,12 @@ struct CurvatureNablaBZ: public aCylindricalFunctor<CurvatureNablaBZ>
     }
     double do_compute( double R, double Z) const
     {
-        return m_sign*invB_(R,Z)*invB_(R,Z)*bR_(R,Z);
+        return m_sign*m_invB(R,Z)*m_invB(R,Z)*m_bR(R,Z);
     }
     private:
     double m_sign;
-    InvB invB_;
-    BR bR_;
+    InvB m_invB;
+    BR m_bR;
 };
 
 ///@brief Approximate \f$ \mathcal{K}^{R}_{\vec{\kappa}}=0 \f$
@@ -279,7 +352,7 @@ struct CurvatureKappaR: public aCylindricalFunctor<CurvatureKappaR>
 ///@copydoc hide_toroidal_approximation_note
 struct CurvatureKappaZ: public aCylindricalFunctor<CurvatureKappaZ>
 {
-    CurvatureKappaZ( const TokamakMagneticField& mag, int sign): invB_(mag) {
+    CurvatureKappaZ( const TokamakMagneticField& mag, int sign): m_invB(mag) {
         if( sign >0)
             m_sign = +1.;
         else
@@ -287,11 +360,11 @@ struct CurvatureKappaZ: public aCylindricalFunctor<CurvatureKappaZ>
     }
     double do_compute( double R, double Z) const
     {
-        return -m_sign*invB_(R,Z)/R;
+        return -m_sign*m_invB(R,Z)/R;
     }
     private:
     double m_sign;
-    InvB invB_;
+    InvB m_invB;
 };
 
 ///@brief Approximate \f$  \vec{\nabla}\cdot \mathcal{K}_{\vec{\kappa}}  \f$
@@ -300,7 +373,7 @@ struct CurvatureKappaZ: public aCylindricalFunctor<CurvatureKappaZ>
 ///@copydoc hide_toroidal_approximation_note
 struct DivCurvatureKappa: public aCylindricalFunctor<DivCurvatureKappa>
 {
-    DivCurvatureKappa( const TokamakMagneticField& mag, int sign): invB_(mag), bZ_(mag){
+    DivCurvatureKappa( const TokamakMagneticField& mag, int sign): m_invB(mag), m_bZ(mag){
         if( sign >0)
             m_sign = +1.;
         else
@@ -308,12 +381,12 @@ struct DivCurvatureKappa: public aCylindricalFunctor<DivCurvatureKappa>
     }
     double do_compute( double R, double Z) const
     {
-        return m_sign*bZ_(R,Z)*invB_(R,Z)*invB_(R,Z)/R;
+        return m_sign*m_bZ(R,Z)*m_invB(R,Z)*m_invB(R,Z)/R;
     }
     private:
     double m_sign;
-    InvB invB_;
-    BZ bZ_;
+    InvB m_invB;
+    BZ m_bZ;
 };
 
 ///@brief Approximate \f$  \vec{\nabla}\cdot \mathcal{K}_{\nabla B}  \f$
@@ -322,30 +395,30 @@ struct DivCurvatureKappa: public aCylindricalFunctor<DivCurvatureKappa>
 ///@copydoc hide_toroidal_approximation_note
 struct DivCurvatureNablaB: public aCylindricalFunctor<DivCurvatureNablaB>
 {
-    DivCurvatureNablaB( const TokamakMagneticField& mag, int sign): div_(mag, sign){ }
+    DivCurvatureNablaB( const TokamakMagneticField& mag, int sign): m_div(mag, sign){ }
     double do_compute( double R, double Z) const
     {
-        return -div_(R,Z);
+        return -m_div(R,Z);
     }
     private:
-    DivCurvatureKappa div_;
+    DivCurvatureKappa m_div;
 };
 ///@brief True \f$ \mathcal{K}^{R}_{\nabla B} \f$
 ///
 /// \f$ \mathcal{K}^R_{\nabla B} =-\frac{R_0I}{ B^3R}  \frac{\partial B}{\partial Z}  \f$
 struct TrueCurvatureNablaBR: public aCylindricalFunctor<TrueCurvatureNablaBR>
 {
-    TrueCurvatureNablaBR(const TokamakMagneticField& mag): R0_(mag.R0()), c_(mag), invB_(mag), bZ_(mag) { }
+    TrueCurvatureNablaBR(const TokamakMagneticField& mag): m_R0(mag.R0()), m_mag(mag), m_invB(mag), m_bZ(mag) { }
     double do_compute( double R, double Z) const
     {
-        double invB = invB_(R,Z), ipol = c_.ipol()(R,Z);
-        return -invB*invB*invB*ipol*R0_/R*bZ_(R,Z);
+        double invB = m_invB(R,Z), ipol = m_mag.ipol()(R,Z);
+        return -invB*invB*invB*ipol*m_R0/R*m_bZ(R,Z);
     }
     private:
-    double R0_;
-    TokamakMagneticField c_;
-    InvB invB_;
-    BZ bZ_;
+    double m_R0;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
+    BZ m_bZ;
 };
 
 ///@brief True \f$ \mathcal{K}^{Z}_{\nabla B} \f$
@@ -353,17 +426,17 @@ struct TrueCurvatureNablaBR: public aCylindricalFunctor<TrueCurvatureNablaBR>
 /// \f$ \mathcal{K}^Z_{\nabla B} =\frac{R_0I}{ B^3R}  \frac{\partial B}{\partial R}  \f$
 struct TrueCurvatureNablaBZ: public aCylindricalFunctor<TrueCurvatureNablaBZ>
 {
-    TrueCurvatureNablaBZ(const TokamakMagneticField& mag): R0_(mag.R0()), c_(mag), invB_(mag), bR_(mag) { }
+    TrueCurvatureNablaBZ(const TokamakMagneticField& mag): m_R0(mag.R0()), m_mag(mag), m_invB(mag), m_bR(mag) { }
     double do_compute( double R, double Z) const
     {
-        double invB = invB_(R,Z), ipol = c_.ipol()(R,Z);
-        return invB*invB*invB*ipol*R0_/R*bR_(R,Z);
+        double invB = m_invB(R,Z), ipol = m_mag.ipol()(R,Z);
+        return invB*invB*invB*ipol*m_R0/R*m_bR(R,Z);
     }
     private:
-    double R0_;
-    TokamakMagneticField c_;
-    InvB invB_;
-    BR bR_;
+    double m_R0;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
+    BR m_bR;
 };
 
 ///@brief True \f$ \mathcal{K}^{\varphi}_{\nabla B} \f$
@@ -371,91 +444,91 @@ struct TrueCurvatureNablaBZ: public aCylindricalFunctor<TrueCurvatureNablaBZ>
 /// \f$ \mathcal{K}^\varphi_{\nabla B} =\frac{1}{ B^3R^2}\left( \frac{\partial\psi}{\partial Z} \frac{\partial B}{\partial Z} + \frac{\partial \psi}{\partial R}\frac{\partial B}{\partial R} \right) \f$
 struct TrueCurvatureNablaBP: public aCylindricalFunctor<TrueCurvatureNablaBP>
 {
-    TrueCurvatureNablaBP(const TokamakMagneticField& mag): c_(mag), invB_(mag),bR_(mag), bZ_(mag) { }
+    TrueCurvatureNablaBP(const TokamakMagneticField& mag): m_mag(mag), m_invB(mag),m_bR(mag), m_bZ(mag) { }
     double do_compute( double R, double Z) const
     {
-        double invB = invB_(R,Z);
-        return c_.R0()*invB*invB*invB/R/R*(c_.psipZ()(R,Z)*bZ_(R,Z) + c_.psipR()(R,Z)*bR_(R,Z));
+        double invB = m_invB(R,Z);
+        return m_mag.R0()*invB*invB*invB/R/R*(m_mag.psipZ()(R,Z)*m_bZ(R,Z) + m_mag.psipR()(R,Z)*m_bR(R,Z));
     }
     private:
-    TokamakMagneticField c_;
-    InvB invB_;
-    BR bR_;
-    BZ bZ_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
+    BR m_bR;
+    BZ m_bZ;
 };
 
 ///@brief True \f$ \mathcal{K}^R_{\vec{\kappa}} \f$
 struct TrueCurvatureKappaR: public aCylindricalFunctor<TrueCurvatureKappaR>
 {
-    TrueCurvatureKappaR( const TokamakMagneticField& mag):c_(mag), invB_(mag), bZ_(mag){ }
+    TrueCurvatureKappaR( const TokamakMagneticField& mag):m_mag(mag), m_invB(mag), m_bZ(mag){ }
     double do_compute( double R, double Z) const
     {
-        double invB = invB_(R,Z);
-        return c_.R0()*invB*invB/R*(c_.ipolZ()(R,Z) - c_.ipol()(R,Z)*invB*bZ_(R,Z));
+        double invB = m_invB(R,Z);
+        return m_mag.R0()*invB*invB/R*(m_mag.ipolZ()(R,Z) - m_mag.ipol()(R,Z)*invB*m_bZ(R,Z));
     }
     private:
-    TokamakMagneticField c_;
-    InvB invB_;
-    BZ bZ_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
+    BZ m_bZ;
 };
 
 ///@brief True \f$ \mathcal{K}^Z_{\vec{\kappa}} \f$
 struct TrueCurvatureKappaZ: public aCylindricalFunctor<TrueCurvatureKappaZ>
 {
-    TrueCurvatureKappaZ( const TokamakMagneticField& mag):c_(mag), invB_(mag), bR_(mag){ }
+    TrueCurvatureKappaZ( const TokamakMagneticField& mag):m_mag(mag), m_invB(mag), m_bR(mag){ }
     double do_compute( double R, double Z) const
     {
-        double invB = invB_(R,Z);
-        return c_.R0()*invB*invB/R*( - c_.ipolR()(R,Z) + c_.ipol()(R,Z)*invB*bR_(R,Z));
+        double invB = m_invB(R,Z);
+        return m_mag.R0()*invB*invB/R*( - m_mag.ipolR()(R,Z) + m_mag.ipol()(R,Z)*invB*m_bR(R,Z));
     }
     private:
-    TokamakMagneticField c_;
-    InvB invB_;
-    BR bR_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
+    BR m_bR;
 };
 ///@brief True \f$ \mathcal{K}^\varphi_{\vec{\kappa}} \f$
 struct TrueCurvatureKappaP: public aCylindricalFunctor<TrueCurvatureKappaP>
 {
-    TrueCurvatureKappaP( const TokamakMagneticField& mag):c_(mag), invB_(mag), bR_(mag), bZ_(mag){ }
+    TrueCurvatureKappaP( const TokamakMagneticField& mag):m_mag(mag), m_invB(mag), m_bR(mag), m_bZ(mag){ }
     double do_compute( double R, double Z) const
     {
-        double invB = invB_(R,Z);
-        return c_.R0()*invB*invB/R/R*(
-            + invB*c_.psipZ()(R,Z)*bZ_(R,Z) + invB *c_.psipR()(R,Z)*bR_(R,Z)
-            + c_.psipR()(R,Z)/R - c_.psipRR()(R,Z) - c_.psipZZ()(R,Z));
+        double invB = m_invB(R,Z);
+        return m_mag.R0()*invB*invB/R/R*(
+            + invB*m_mag.psipZ()(R,Z)*m_bZ(R,Z) + invB *m_mag.psipR()(R,Z)*m_bR(R,Z)
+            + m_mag.psipR()(R,Z)/R - m_mag.psipRR()(R,Z) - m_mag.psipZZ()(R,Z));
     }
     private:
-    TokamakMagneticField c_;
-    InvB invB_;
-    BR bR_;
-    BZ bZ_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
+    BR m_bR;
+    BZ m_bZ;
 };
 
 ///@brief True \f$  \vec{\nabla}\cdot \mathcal{K}_{\vec{\kappa}}  \f$
 struct TrueDivCurvatureKappa: public aCylindricalFunctor<TrueDivCurvatureKappa>
 {
-    TrueDivCurvatureKappa( const TokamakMagneticField& mag): c_(mag), invB_(mag), bR_(mag), bZ_(mag){}
+    TrueDivCurvatureKappa( const TokamakMagneticField& mag): m_mag(mag), m_invB(mag), m_bR(mag), m_bZ(mag){}
     double do_compute( double R, double Z) const
     {
-        double invB = invB_(R,Z);
-        return c_.R0()*invB*invB*invB/R*( c_.ipolR()(R,Z)*bZ_(R,Z) - c_.ipolZ()(R,Z)*bR_(R,Z) );
+        double invB = m_invB(R,Z);
+        return m_mag.R0()*invB*invB*invB/R*( m_mag.ipolR()(R,Z)*m_bZ(R,Z) - m_mag.ipolZ()(R,Z)*m_bR(R,Z) );
     }
     private:
-    TokamakMagneticField c_;
-    InvB invB_;
-    BR bR_;
-    BZ bZ_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
+    BR m_bR;
+    BZ m_bZ;
 };
 
 ///@brief True \f$  \vec{\nabla}\cdot \mathcal{K}_{\nabla B}  \f$
 struct TrueDivCurvatureNablaB: public aCylindricalFunctor<TrueDivCurvatureNablaB>
 {
-    TrueDivCurvatureNablaB( const TokamakMagneticField& mag): div_(mag){}
+    TrueDivCurvatureNablaB( const TokamakMagneticField& mag): m_div(mag){}
     double do_compute( double R, double Z) const {
-        return - div_(R,Z);
+        return - m_div(R,Z);
     }
     private:
-    TrueDivCurvatureKappa div_;
+    TrueDivCurvatureKappa m_div;
 };
 
 /**
@@ -465,17 +538,17 @@ struct TrueDivCurvatureNablaB: public aCylindricalFunctor<TrueDivCurvatureNablaB
  */
 struct GradLnB: public aCylindricalFunctor<GradLnB>
 {
-    GradLnB( const TokamakMagneticField& mag): mag_(mag), invB_(mag), bR_(mag), bZ_(mag) { }
+    GradLnB( const TokamakMagneticField& mag): m_mag(mag), m_invB(mag), m_bR(mag), m_bZ(mag) { }
     double do_compute( double R, double Z) const
     {
-        double invB = invB_(R,Z);
-        return mag_.R0()*invB*invB*(bR_(R,Z)*mag_.psipZ()(R,Z)-bZ_(R,Z)*mag_.psipR()(R,Z))/R ;
+        double invB = m_invB(R,Z);
+        return m_mag.R0()*invB*invB*(m_bR(R,Z)*m_mag.psipZ()(R,Z)-m_bZ(R,Z)*m_mag.psipR()(R,Z))/R ;
     }
     private:
-    TokamakMagneticField mag_;
-    InvB invB_;
-    BR bR_;
-    BZ bZ_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
+    BR m_bR;
+    BZ m_bZ;
 };
 /**
  * @brief \f$  \nabla \cdot \vec b \f$
@@ -497,94 +570,94 @@ struct Divb: public aCylindricalFunctor<Divb>
 ///@brief \f$ B^\varphi = R_0I/R^2\f$
 struct BFieldP: public aCylindricalFunctor<BFieldP>
 {
-    BFieldP( const TokamakMagneticField& mag): mag_(mag){}
+    BFieldP( const TokamakMagneticField& mag): m_mag(mag){}
     double do_compute( double R, double Z) const
     {
-        return mag_.R0()*mag_.ipol()(R,Z)/R/R;
+        return m_mag.R0()*m_mag.ipol()(R,Z)/R/R;
     }
     private:
 
-    TokamakMagneticField mag_;
+    TokamakMagneticField m_mag;
 };
 
 ///@brief \f$ B^R = R_0\psi_Z /R\f$
 struct BFieldR: public aCylindricalFunctor<BFieldR>
 {
-    BFieldR( const TokamakMagneticField& mag): mag_(mag){}
+    BFieldR( const TokamakMagneticField& mag): m_mag(mag){}
     double do_compute( double R, double Z) const
     {
-        return  mag_.R0()/R*mag_.psipZ()(R,Z);
+        return  m_mag.R0()/R*m_mag.psipZ()(R,Z);
     }
     private:
-    TokamakMagneticField mag_;
+    TokamakMagneticField m_mag;
 
 };
 
 ///@brief \f$ B^Z = -R_0\psi_R /R\f$
 struct BFieldZ: public aCylindricalFunctor<BFieldZ>
 {
-    BFieldZ( const TokamakMagneticField& mag): mag_(mag){}
+    BFieldZ( const TokamakMagneticField& mag): m_mag(mag){}
     double do_compute( double R, double Z) const
     {
-        return -mag_.R0()/R*mag_.psipR()(R,Z);
+        return -m_mag.R0()/R*m_mag.psipR()(R,Z);
     }
     private:
-    TokamakMagneticField mag_;
+    TokamakMagneticField m_mag;
 };
 
 ///@brief \f$  B^{\theta} = B^R\partial_R\theta + B^Z\partial_Z\theta\f$
 struct BFieldT: public aCylindricalFunctor<BFieldT>
 {
-    BFieldT( const TokamakMagneticField& mag):  R_0_(mag.R0()), fieldR_(mag), fieldZ_(mag){}
+    BFieldT( const TokamakMagneticField& mag):  m_R0(mag.R0()), m_fieldR(mag), m_fieldZ(mag){}
     double do_compute(double R, double Z) const
     {
-        double r2 = (R-R_0_)*(R-R_0_) + Z*Z;
-        return fieldR_(R,Z)*(-Z/r2) + fieldZ_(R,Z)*(R-R_0_)/r2;
+        double r2 = (R-m_R0)*(R-m_R0) + Z*Z;
+        return m_fieldR(R,Z)*(-Z/r2) + m_fieldZ(R,Z)*(R-m_R0)/r2;
     }
     private:
-    double R_0_;
-    BFieldR fieldR_;
-    BFieldZ fieldZ_;
+    double m_R0;
+    BFieldR m_fieldR;
+    BFieldZ m_fieldZ;
 };
 
 ///@brief \f$ b^R = B^R/|B|\f$
 struct BHatR: public aCylindricalFunctor<BHatR>
 {
-    BHatR( const TokamakMagneticField& mag): mag_(mag), invB_(mag){ }
+    BHatR( const TokamakMagneticField& mag): m_mag(mag), m_invB(mag){ }
     double do_compute( double R, double Z) const
     {
-        return  invB_(R,Z)*mag_.R0()/R*mag_.psipZ()(R,Z);
+        return  m_invB(R,Z)*m_mag.R0()/R*m_mag.psipZ()(R,Z);
     }
     private:
-    TokamakMagneticField mag_;
-    InvB invB_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
 
 };
 
 ///@brief \f$ b^Z = B^Z/|B|\f$
 struct BHatZ: public aCylindricalFunctor<BHatZ>
 {
-    BHatZ( const TokamakMagneticField& mag): mag_(mag), invB_(mag){ }
+    BHatZ( const TokamakMagneticField& mag): m_mag(mag), m_invB(mag){ }
     double do_compute( double R, double Z) const
     {
-        return  -invB_(R,Z)*mag_.R0()/R*mag_.psipR()(R,Z);
+        return  -m_invB(R,Z)*m_mag.R0()/R*m_mag.psipR()(R,Z);
     }
     private:
-    TokamakMagneticField mag_;
-    InvB invB_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
 };
 
 ///@brief \f$ b^\varphi = B^\varphi/|B|\f$
 struct BHatP: public aCylindricalFunctor<BHatP>
 {
-    BHatP( const TokamakMagneticField& mag): mag_(mag), invB_(mag){ }
+    BHatP( const TokamakMagneticField& mag): m_mag(mag), m_invB(mag){ }
     double do_compute( double R, double Z) const
     {
-        return invB_(R,Z)*mag_.R0()*mag_.ipol()(R,Z)/R/R;
+        return m_invB(R,Z)*m_mag.R0()*m_mag.ipol()(R,Z)/R/R;
     }
     private:
-    TokamakMagneticField mag_;
-    InvB invB_;
+    TokamakMagneticField m_mag;
+    InvB m_invB;
 };
 
 /**
@@ -666,57 +739,57 @@ inline CylindricalVectorLvl0 createGradPsip( const TokamakMagneticField& mag){
 ///@brief \f$ \nabla_\parallel b^R \f$
 struct GradBHatR: public aCylindricalFunctor<GradBHatR>
 {
-    GradBHatR( const TokamakMagneticField& mag): bhatR_(mag), divb_(mag), mag_(mag){}
+    GradBHatR( const TokamakMagneticField& mag): m_bhatR(mag), m_divb(mag), m_mag(mag){}
     double do_compute( double R, double Z) const
     {
-        double ipol = mag_.ipol()(R,Z);
-        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z);
-        double psipZZ = mag_.psipZZ()(R,Z), psipRZ = mag_.psipRZ()(R,Z);
-        return  divb_(R,Z)*bhatR_(R,Z) +
+        double ipol = m_mag.ipol()(R,Z);
+        double psipR = m_mag.psipR()(R,Z), psipZ = m_mag.psipZ()(R,Z);
+        double psipZZ = m_mag.psipZZ()(R,Z), psipRZ = m_mag.psipRZ()(R,Z);
+        return  m_divb(R,Z)*m_bhatR(R,Z) +
                 ( psipZ*(psipRZ-psipZ/R) - psipZZ*psipR  )/
                     (ipol*ipol + psipR*psipR + psipZ*psipZ);
     }
     private:
-    BHatR bhatR_;
-    Divb divb_;
-    TokamakMagneticField mag_;
+    BHatR m_bhatR;
+    Divb m_divb;
+    TokamakMagneticField m_mag;
 };
 ///@brief \f$ \nabla_\parallel b^Z \f$
 struct GradBHatZ: public aCylindricalFunctor<GradBHatZ>
 {
-    GradBHatZ( const TokamakMagneticField& mag): bhatZ_(mag), divb_(mag), mag_(mag){}
+    GradBHatZ( const TokamakMagneticField& mag): m_bhatZ(mag), m_divb(mag), m_mag(mag){}
     double do_compute( double R, double Z) const
     {
-        double ipol = mag_.ipol()(R,Z);
-        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z);
-        double psipRR = mag_.psipRR()(R,Z), psipRZ = mag_.psipRZ()(R,Z);
+        double ipol = m_mag.ipol()(R,Z);
+        double psipR = m_mag.psipR()(R,Z), psipZ = m_mag.psipZ()(R,Z);
+        double psipRR = m_mag.psipRR()(R,Z), psipRZ = m_mag.psipRZ()(R,Z);
 
-        return  divb_(R,Z)*bhatZ_(R,Z) +
+        return  m_divb(R,Z)*m_bhatZ(R,Z) +
                 (psipR*(psipRZ+psipZ/R) - psipRR*psipZ)/
                     (ipol*ipol + psipR*psipR + psipZ*psipZ);
     }
     private:
-    BHatZ bhatZ_;
-    Divb divb_;
-    TokamakMagneticField mag_;
+    BHatZ m_bhatZ;
+    Divb m_divb;
+    TokamakMagneticField m_mag;
 };
 ///@brief \f$ \nabla_\parallel b^\varphi \f$
 struct GradBHatP: public aCylindricalFunctor<GradBHatP>
 {
-    GradBHatP( const TokamakMagneticField& mag): bhatP_(mag), divb_(mag), mag_(mag){}
+    GradBHatP( const TokamakMagneticField& mag): m_bhatP(mag), m_divb(mag), m_mag(mag){}
     double do_compute( double R, double Z) const
     {
-        double ipol = mag_.ipol()(R,Z), ipolR = mag_.ipolR()(R,Z), ipolZ  = mag_.ipolZ()(R,Z);
-        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z);
+        double ipol = m_mag.ipol()(R,Z), ipolR = m_mag.ipolR()(R,Z), ipolZ  = m_mag.ipolZ()(R,Z);
+        double psipR = m_mag.psipR()(R,Z), psipZ = m_mag.psipZ()(R,Z);
 
-        return  divb_(R,Z)*bhatP_(R,Z) +
+        return  m_divb(R,Z)*m_bhatP(R,Z) +
              (psipZ*(ipolR/R - 2.*ipol/R/R) - ipolZ/R*psipR)/
                     (ipol*ipol + psipR*psipR + psipZ*psipZ);
     }
     private:
-    BHatP bhatP_;
-    Divb divb_;
-    TokamakMagneticField mag_;
+    BHatP m_bhatP;
+    Divb m_divb;
+    TokamakMagneticField m_mag;
 };
 
 ///@brief \f$ \sqrt{\psi_p/ \psi_{p,\min}} \f$
@@ -740,17 +813,17 @@ struct RhoP: public aCylindricalFunctor<RhoP>
 ///@brief Inertia factor \f$ \mathcal I_0 \f$
 struct Hoo : public dg::geo::aCylindricalFunctor<Hoo>
 {
-    Hoo( dg::geo::TokamakMagneticField mag): mag_(mag){}
+    Hoo( dg::geo::TokamakMagneticField mag): m_mag(mag){}
     double do_compute( double R, double Z) const
     {
-        double psipR = mag_.psipR()(R,Z), psipZ = mag_.psipZ()(R,Z), ipol = mag_.ipol()(R,Z);
+        double psipR = m_mag.psipR()(R,Z), psipZ = m_mag.psipZ()(R,Z), ipol = m_mag.ipol()(R,Z);
         double psip2 = psipR*psipR+psipZ*psipZ;
         if( psip2 == 0)
             psip2 = 1e-16;
         return (ipol*ipol + psip2)/R/R/psip2;
     }
     private:
-    dg::geo::TokamakMagneticField mag_;
+    dg::geo::TokamakMagneticField m_mag;
 };
 ///@}
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index a15599d13..9665ba88c 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -854,6 +854,10 @@ We can choose the constant influx
 \begin{align} \label{eq:electron_source_influx}
     S_{prof}(R,Z) &= \Theta_{\alpha_p/2}\left( \rho_{p,s} - \frac{\alpha_p}{2} - \rho_p(R,Z) \right) H(Z-Z_X)
 \end{align}
+%or a ringed Gaussian source of the form
+%\begin{align}
+%    S_{prof}(R,Z) &= \exp\left( -\frac{(\psi_p-\psi_{p,0})^2}{\sigma^2}\right)H(Z-Z_X)
+%\end{align}
 or a Torpex inspired source profile
 \begin{align} \label{eq:electron_source_torpex}
   S_{prof}(R,Z) &=
@@ -884,7 +888,7 @@ Now, our idea is to dampen the density and velocity in the region defined by the
 magnetic field straightening.
 The idea for the terms $S_U$ is mainly to provide more numerical stability
 in the corner regions of the domain, where the parallel derivative may lead
-to unfavourable numerical instabilities.
+to unwelcome numerical instabilities.
 For both electrons and ions we choose
 \begin{subequations} \label{eq:velocity_source}
 \begin{align}
@@ -1090,8 +1094,8 @@ The relevant terms in the output file are
 \bottomrule
 \end{longtable}
 
-\subsubsection{Vorticity and toroidal ExB angular momentum equation} \label{sec:vorticity_eq}
-We write the vorticity equation (in the LWL)
+\subsubsection{Toroidal ExB angular momentum equation} \label{sec:vorticity_eq}
+We integrate the polarisation equation over volume and derive by time (in the LWL)
 \begin{align}
     &\partial_t \RA{\Omega} + \frac{\partial}{\partial v}\frac{\d v}{\d\psi_p}\RA{\vec j_\Omega\cn\psi_p} = -\RA{F_{L,\varphi}} + \RA{\mathcal S_\Omega} \label{eq:vorticity_average} \\
 \Omega &:= \mu_i N_i \left(\frac{\vn\psi_p\cn\phi}{B^2} + \tau_i \vn\ln N_i\cn\psi_p\right) \equiv \mu_i N_i(u_{E,\varphi} + u_{D,\varphi}) \\
@@ -1100,7 +1104,7 @@ We write the vorticity equation (in the LWL)
     F_{L,\varphi} &:=  -(z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) - (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p) \\
     \mathcal S_\Omega &:= \mu_i S_{n_e} \frac{\vn\psi_p\cn \phi}{B^2} + \mu_i\tau_i\vn\psi_p\cn S_{n_e} \label{eq:em_source}
 \end{align}
-Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB velocity
+Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB angular momentum
 \begin{align}
 &\partial_t \RA{\Omega_E} + \frac{\partial}{\partial v} \frac{\d v}{\d \psi_p}\RA{ \vec j_{\Omega_E}\cn\psi_p} = -\RA{F_{L,\varphi}}+ \RA{\mathcal S_{\Omega_E}} + \RA{\Lambda_{\Omega_E}} \label{eq:exb_average} \\
 \Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \equiv \mu_i N_i u_{E,\varphi} \\
@@ -1438,7 +1442,7 @@ source & dict & & & Density source, cf. the output \texttt{sne\_tt\_ifs} in \tex
 \\
 \qquad boundary & float & 0.2  & 0.5 & Source region boundary $\rho_{p,b}$: yields in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
 \qquad alpha  & float & 0.2 & 0.2 & Transition width $\alpha_p$ in the Heaviside
-in the density Eq.~\eqref{eq:density_profile} (with $rho_{p,b}=0$ and source profiles Eq.~\eqref{eq:electron_source} (should be
+in the density Eq.~\eqref{eq:density_profile} (with $\rho_{p,b}=0$ and source profiles Eq.~\eqref{eq:electron_source} (should be
 small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes, must not be zero)
 \\
 damping & dict & & & magnetic and density damping region \\
-- 
GitLab


From 80239955e7bfe95b066408a54f34f9b19c67f935 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 19 Aug 2020 22:14:59 +0200
Subject: [PATCH 293/540] Update guenther params

and remove equilibrium from solovev::Parameters; the purpose
is to read the equilibrium before constructing the Parameters
---
 inc/geometries/README                |  1 -
 inc/geometries/average_t.cu          |  2 +-
 inc/geometries/geometry_diag.cu      |  2 +-
 inc/geometries/guenther.h            |  1 -
 inc/geometries/guenther_parameters.h | 74 ----------------------------
 inc/geometries/guenther_params.js    | 22 ---------
 inc/geometries/guenther_params.json  | 10 ++++
 inc/geometries/magnetic_field.h      |  1 +
 inc/geometries/solovev_parameters.h  |  8 ++-
 9 files changed, 16 insertions(+), 105 deletions(-)
 delete mode 100644 inc/geometries/guenther_parameters.h
 delete mode 100644 inc/geometries/guenther_params.js
 create mode 100644 inc/geometries/guenther_params.json

diff --git a/inc/geometries/README b/inc/geometries/README
index bab0c1ba6..33c41eb83 100644
--- a/inc/geometries/README
+++ b/inc/geometries/README
@@ -26,7 +26,6 @@
     solovev_parameters.h
     taylor.h
     guenther.h
-    guenther_parameters.h
     toroidal.h
 
     #utilities
diff --git a/inc/geometries/average_t.cu b/inc/geometries/average_t.cu
index 20b65e968..d258683ca 100644
--- a/inc/geometries/average_t.cu
+++ b/inc/geometries/average_t.cu
@@ -68,7 +68,7 @@ int main( int argc, char* argv[])
         map1d.emplace_back("psi_fsa",   dg::evaluate( fsa,      grid1d),
             "Flux surface average of psi with delta function");
         dg::HVec qprofile;
-        if( gp.equilibrium == "solovev")
+        //if( gp.equilibrium == "solovev")
         {
             dg::geo::SafetyFactor qprof( mag);
             qprofile = dg::evaluate( qprof, grid1d);
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 0d739bd06..34a4b370f 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -313,7 +313,7 @@ int main( int argc, char* argv[])
         dg::blas1::transform( rho, rho, dg::SQRT<double>());
         map1d.emplace_back("rho_p", rho,
             "Alternative flux label rho_p = Sqrt[-psi/psimin + 1]");
-        if( gp.equilibrium == "solovev")
+        //if( gp.equilibrium == "solovev")
         {
             dg::geo::SafetyFactor qprof( mag);
             dg::HVec qprofile = dg::evaluate( qprof, grid1d);
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 95ec59003..0d571feb5 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -6,7 +6,6 @@
 
 #include "dg/blas.h"
 
-#include "guenther_parameters.h"
 #include "magnetic_field.h"
 
 //TODO somebody document the functions as in solovev/geometry.h
diff --git a/inc/geometries/guenther_parameters.h b/inc/geometries/guenther_parameters.h
deleted file mode 100644
index c0502bb61..000000000
--- a/inc/geometries/guenther_parameters.h
+++ /dev/null
@@ -1,74 +0,0 @@
-#pragma once
-#include <vector>
-/*!@file
- *
- * Geometry parameters for guenther field
- */
-namespace dg
-{
-namespace geo
-{
-namespace guenther
-{
-/**
- * @brief Constructs and display geometric parameters for the guenther field
- *
- * @ingroup geom
- * @note include \c json/json.h before \c geometries.h in order to activate json functionality
- */
-struct Parameters
-{
-    double I_0, //!< the current
-           R_0, //!< central tokamak radius
-           a,  //!<  little tokamak radius
-           elongation, //!< elongation of the magnetic surfaces
-           triangularity, //!< triangularity of the magnetic surfaces
-           alpha, //!< damping width
-           rk4eps,  //!< accuracy for the field line mapping
-           psipmin, //!< for source
-           psipmax, //!< for profile
-           psipmaxcut, //!< for cutting
-           psipmaxlim; //!< for limiter
-    std::vector<double> c;  //!< coefficients for the solovev equilibrium
-#ifdef JSONCPP_VERSION_STRING
-    /**
-     * @brief Construct from Json dataset
-     * @param js Must contain the variables "I_0", "R_0", "inverseaspectratio", "elongation", "triangularity", "alpha", "rk4eps" (1e-5), "psip_min" (0), "psip_max" (0), "psip_max_cut" (0), "psip_max_lim" (1e10)
-     * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
-     */
-    Parameters( const Json::Value& js) {
-        I_0  = js["I_0"].asDouble();
-        R_0  = js["R_0"].asDouble();
-        a  = R_0*js["inverseaspectratio"].asDouble();
-        elongation=js["elongation"].asDouble();
-        triangularity=js["triangularity"].asDouble();
-        alpha=js["alpha"].asDouble();
-        rk4eps=js.get("rk4eps",1e-5).asDouble();
-        psipmin= js.get("psip_min",0).asDouble();
-        psipmax= js.get("psip_max",0).asDouble();
-        psipmaxcut= js.get("psip_max_cut",0).asDouble();
-        psipmaxlim= js.get("psip_max_lim",1e10).asDouble();
-    }
-#endif // JSONCPP_VERSION_STRING
-    ///Write variables as a formatted string
-    void display( std::ostream& os = std::cout ) const
-    {
-        os << "Geometrical parameters are: \n"
-            <<" I0            = "<<I_0<<"\n"
-            <<" R0            = "<<R_0<<"\n"
-            <<" epsilon_a     = "<<a/R_0<<"\n"
-            <<" elongation    = "<<elongation<<"\n"
-            <<" triangularity = "<<triangularity<<"\n"
-            <<" alpha         = "<<alpha<<"\n"
-            <<" rk4 epsilon   = "<<rk4eps<<"\n"
-            <<" psipmin       = "<<psipmin<<"\n"
-            <<" psipmax       = "<<psipmax<<"\n"
-            <<" psipmaxcut    = "<<psipmaxcut<<"\n"
-            <<" psipmaxlim    = "<<psipmaxlim<<"\n";
-        os << std::flush;
-
-    }
-};
-} //namespace guenther
-} //namespace geo
-} //namespace dg
diff --git a/inc/geometries/guenther_params.js b/inc/geometries/guenther_params.js
deleted file mode 100644
index 12738c44c..000000000
--- a/inc/geometries/guenther_params.js
+++ /dev/null
@@ -1,22 +0,0 @@
-
-                            //Guenther equi
-          //-----------------------------------------------------------
-{
-    //----------------------geometric coefficients----------------------
-    "I_0"                    :  20.,
-    "R_0"                    :  10.,     //(in units of rho_s)
-    "inverseaspectratio"     :  0.1, //(a/R0)
-    "elongation"             :  1.0,
-    "triangularity"          :  0.0,
-    //----------------------Miscellaneous----------------------
-    "alpha"                 :  0.1, //(Damping thickness)
-    "rk4eps"                :  1e-4,
-    "psip_min"              : -1e10,
-    "psip_max"              :  1e10, //(> -7.4)
-    "psip_max_cut"          :  1e10,
-    "psip_max_lim"          :  1e10
-    //@ ------------------------------------------------------------------------
-}
-
-
-
diff --git a/inc/geometries/guenther_params.json b/inc/geometries/guenther_params.json
new file mode 100644
index 000000000..c1ba5b170
--- /dev/null
+++ b/inc/geometries/guenther_params.json
@@ -0,0 +1,10 @@
+{
+    "equilibrium"       : "guenther",
+    "I_0"               :  20.,
+    "R_0"               :  10.,
+    "a"                 :  1,
+    "form"              : "square"
+}
+
+
+
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index f00189c43..662aa0fa3 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <map>
 #include "fluxfunctions.h"
 
 /*!@file
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index 551748652..11d0cfa96 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -29,13 +29,12 @@ struct Parameters
            elongation, //!< elongation of the magnetic surfaces
            triangularity; //!< triangularity of the magnetic surfaces
     std::vector<double> c;  //!< 12 coefficients for the solovev equilibrium;
-    std::string equilibrium;
 #ifdef JSONCPP_VERSION_STRING
     /**
      * @brief Construct from Json dataset
      * @param js Can contain the variables "A" (0), "c" (0), "PP" (1.), "PI"
      * (1.), "R_0" , "inverseaspectratio" , "elongation" (1), "triangularity"
-     * (0), "equilibrium" ("solovev")
+     * (0)
      * @param mode determine what happens when a key is missing
      * @note the default values in brackets are taken if the variables are not found in the input file
      * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
@@ -52,7 +51,6 @@ struct Parameters
         a  = R_0*file::get( mode, js, "inverseaspectratio", 0.).asDouble();
         elongation=file::get( mode, js, "elongation", 1.).asDouble();
         triangularity=file::get( mode, js, "triangularity", 0.).asDouble();
-        equilibrium = file::get( mode, js, "equilibrium", "solovev").asString();
     }
     /**
      * @brief Put values into a json string
@@ -71,7 +69,7 @@ struct Parameters
         js["inverseaspectratio"] = a/R_0;
         js["elongation"] = elongation;
         js["triangularity"] = triangularity;
-        js[ "equilibrium"] = equilibrium;
+        js[ "equilibrium"] = "solovev";
         return js;
     }
 #endif // JSONCPP_VERSION_STRING
@@ -104,7 +102,7 @@ struct Parameters
     ///Write variables as a formatted string
     void display( std::ostream& os = std::cout ) const
     {
-        os << "Geometrical parameters are: \n"
+        os << "Solovev Geometrical parameters are: \n"
             <<" A               = "<<A<<"\n"
             <<" Prefactor Psi   = "<<pp<<"\n"
             <<" Prefactor I     = "<<pi<<"\n";
-- 
GitLab


From 6c394d220be25dde08b51d069c337422eb38f83c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 19 Aug 2020 22:29:50 +0200
Subject: [PATCH 294/540] Rename geometry files to json

---
 .../COMPASS/BLOB/geometry_params.js           | 34 -------------
 inc/geometries/COMPASS/BLOB/input.txt         | 48 -------------------
 ...eometry_params.js => geometry_params.json} |  0
 ..._Xpoint.js => geometry_params_Xpoint.json} |  0
 ...s => geometry_params_Xpoint_harmonic.json} |  0
 ....js => geometry_params_Xpoint_taylor.json} |  0
 ... geometry_params_circ_Iconst_largeR0.json} |  0
 src/heat/geometry/geometry_params_g.js        | 22 ---------
 src/heat/geometry/guenther_params.json        | 10 ++++
 9 files changed, 10 insertions(+), 104 deletions(-)
 delete mode 100644 inc/geometries/COMPASS/BLOB/geometry_params.js
 delete mode 100644 inc/geometries/COMPASS/BLOB/input.txt
 rename inc/geometries/{geometry_params.js => geometry_params.json} (100%)
 rename inc/geometries/{geometry_params_Xpoint.js => geometry_params_Xpoint.json} (100%)
 rename inc/geometries/{geometry_params_Xpoint_harmonic.js => geometry_params_Xpoint_harmonic.json} (100%)
 rename inc/geometries/{geometry_params_Xpoint_taylor.js => geometry_params_Xpoint_taylor.json} (100%)
 rename inc/geometries/{geometry_params_circ_Iconst_largeR0.js => geometry_params_circ_Iconst_largeR0.json} (100%)
 delete mode 100644 src/heat/geometry/geometry_params_g.js
 create mode 100644 src/heat/geometry/guenther_params.json

diff --git a/inc/geometries/COMPASS/BLOB/geometry_params.js b/inc/geometries/COMPASS/BLOB/geometry_params.js
deleted file mode 100644
index ea33df2fd..000000000
--- a/inc/geometries/COMPASS/BLOB/geometry_params.js
+++ /dev/null
@@ -1,34 +0,0 @@
-//          * Input-File for COMPASS axisymmetric solovev equilibrium *
-//    I const without X-point, Force free, circular Cross section, Te_50eV,B0_1T,deuterium
-//          -----------------------------------------------------------
-//
-{
-	"A" : 1,
-	"R_0" : 547.89200000000005,
-	"alpha" : 0.050000000000000003,
-	"c" :
-	[
-		0.1042258469898235,
-		-0.10978486643179905,
-		0.17290644239764172,
-		-0.032573200490467011,
-		0.010084171188467612,
-		-0.0033426939773477794,
-		-0.00010801904592074435,
-		0,
-		0,
-		0,
-		0,
-		0
-	],
-	"elongation" : 1,
-	"equilibrium" : "solovev",
-	"inverseaspectratio" : 0.41071400000000002,
-	"psip_max" : 0,
-	"psip_max_cut" : 10000000000,
-	"psip_max_lim" : 10000000000,
-	"psip_min" : -6,
-	"qampl" : 1,
-	"rk4eps" : 1.0000000000000001e-05,
-	"triangularity" : 0
-}
\ No newline at end of file
diff --git a/inc/geometries/COMPASS/BLOB/input.txt b/inc/geometries/COMPASS/BLOB/input.txt
deleted file mode 100644
index d5036b408..000000000
--- a/inc/geometries/COMPASS/BLOB/input.txt
+++ /dev/null
@@ -1,48 +0,0 @@
-                * Input-File for "FELTOR" *
-                ---------------------------
-
-
-@-----------------------------Space and Time discretization------------
-1)  n  (# of x,y-polynomials)            =   3      (3)
-2)  nx (grid points in x)                =   160 (192)
-3)  ny (grid points in y)                =   160 (192)
-4)  nz (grid points in z)                =   15(>16)
-5)  dt (time step in units c_s/rho_s)    =   5e-3 (0.01)
-
-----------------------------------Output parameters--------------------
-6)  n_out (# of x-y polynomials in output)  =  3
-7)  nx_out (# grid points in output field)  =  160
-8)  ny_out (# grid points in output field)  =  160
-9)  nz_out (# grid points in output field)  =  15
-10) itstp  (steps between outputs)          =  1000 (1000)
-11) total # of outputs (excluding first)    =  300 (300)
-
--------------------------Algorithmic parameters------------------------
-12)  eps_pol (stop for polarisation)        =   1e-4 (1e-6)
-13)  eps_max (stop for induction)           =   0.0 (1e-6)
-14)  eps_gamma (stop for Gamma CG)          =   1e-5 (1e-8)
-15)  eps_time ( stop for time inversion )   =   1e-12
--------------------------Physical parameters----------------------------
-16) mu_e (-m_e/m_i)                         = -0.000272121, (-0.000544617,-0.000272121,-0.000181372 )
-17) tau (Ti/Te)                             =  1.0    (0.0)
-18) beta_e0                                 =  0.0  (1e-4)
-19) nu_perp                                 =  1e-4
-20) nu_parallel  (viscosity)                =  1    (0.001)
-21) para resistivity (c)                    =  1e-5   (3e-5 gives a drift wave)
-
-------------------------Initial perturbation parameters---------------------
-22) amp (blob amplitude)                    =  1.0    (1.0)
-23) sigma (blob variance in units of rho_s) =  7.0     (10)
-24) x-position ( in units of a)             =  0.3   (0.4)
-25) y-position ( in units of a)             =  0.00   (0.5)
-26) sigma_z (variance in units of R_0)      =  2.0       (12.5 for oo)
-27) k_psi (zonal modes)                     =  1.0
-28) Profile amplitude                       =  0.0  (peak amplitude)
-29) Background Prof amplitude               =  1.   (density on the boundary)
-----------------------------------Miscellaneous----------------------------
-30) particle source amplitude               =  0.0
-31) boxscale R+                             =  0.6 (a little larger than 1)
-32) boxscale R-                             =  0.6 (a little larger than 1)
-33) boxscale Z+                             =  0.6 (a little larger than 1)
-34) boxscale Z-                             =  0.6 (a little larger than 1)
-@ ------------------------------------------------------------
diff --git a/inc/geometries/geometry_params.js b/inc/geometries/geometry_params.json
similarity index 100%
rename from inc/geometries/geometry_params.js
rename to inc/geometries/geometry_params.json
diff --git a/inc/geometries/geometry_params_Xpoint.js b/inc/geometries/geometry_params_Xpoint.json
similarity index 100%
rename from inc/geometries/geometry_params_Xpoint.js
rename to inc/geometries/geometry_params_Xpoint.json
diff --git a/inc/geometries/geometry_params_Xpoint_harmonic.js b/inc/geometries/geometry_params_Xpoint_harmonic.json
similarity index 100%
rename from inc/geometries/geometry_params_Xpoint_harmonic.js
rename to inc/geometries/geometry_params_Xpoint_harmonic.json
diff --git a/inc/geometries/geometry_params_Xpoint_taylor.js b/inc/geometries/geometry_params_Xpoint_taylor.json
similarity index 100%
rename from inc/geometries/geometry_params_Xpoint_taylor.js
rename to inc/geometries/geometry_params_Xpoint_taylor.json
diff --git a/inc/geometries/geometry_params_circ_Iconst_largeR0.js b/inc/geometries/geometry_params_circ_Iconst_largeR0.json
similarity index 100%
rename from inc/geometries/geometry_params_circ_Iconst_largeR0.js
rename to inc/geometries/geometry_params_circ_Iconst_largeR0.json
diff --git a/src/heat/geometry/geometry_params_g.js b/src/heat/geometry/geometry_params_g.js
deleted file mode 100644
index 6dc662e31..000000000
--- a/src/heat/geometry/geometry_params_g.js
+++ /dev/null
@@ -1,22 +0,0 @@
-
-                            //Guenther equi
-          //-----------------------------------------------------------
-{
-    //----------------------geometric coefficients----------------------
-    "I_0"                    :  20.,
-    "R_0"                    :  10.,     //(in units of rho_s)
-    "inverseaspectratio"     :  0.1, //(a/R0)
-    "elongation"             :  1.0,
-    "triangularity"          :  0.0,
-    //----------------------Miscellaneous----------------------
-    "alpha"                 :  0.1, //(Damping thickness)
-    "rk4eps"                :  1e-4,
-    "psip_min"              : -1e10.,
-    "psip_max"              :  1e10, //(> -7.4)
-    "psip_max_cut"          :  1e10,
-    "psip_max_lim"          :  1e10
-    //@ ------------------------------------------------------------------------
-}
-
-
-
diff --git a/src/heat/geometry/guenther_params.json b/src/heat/geometry/guenther_params.json
new file mode 100644
index 000000000..c1ba5b170
--- /dev/null
+++ b/src/heat/geometry/guenther_params.json
@@ -0,0 +1,10 @@
+{
+    "equilibrium"       : "guenther",
+    "I_0"               :  20.,
+    "R_0"               :  10.,
+    "a"                 :  1,
+    "form"              : "square"
+}
+
+
+
-- 
GitLab


From fc3797cc94a9548b4ba3aa46cb5926859c37dee5 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 20 Aug 2020 19:54:34 +0200
Subject: [PATCH 295/540] Add form geometry parameter to json files

This is an experimental addition to see if we can use it to
decide on Xgrid construction
---
 inc/geometries/geometry_params.json                 | 11 ++---------
 inc/geometries/geometry_params_Xpoint.json          |  9 +--------
 inc/geometries/geometry_params_Xpoint_harmonic.json | 10 ++--------
 inc/geometries/geometry_params_Xpoint_taylor.json   | 11 ++---------
 .../geometry_params_circ_Iconst_largeR0.json        | 13 ++++---------
 src/feltor/geometry/compass.json                    |  1 +
 src/feltor/geometry/geometry_params.json            |  1 +
 src/feltor/geometry/geometry_paramsXpoint.json      |  1 +
 src/feltor/geometry/torpex.json                     |  2 ++
 src/heat/geometry/geometry_params.json              |  3 ++-
 10 files changed, 18 insertions(+), 44 deletions(-)

diff --git a/inc/geometries/geometry_params.json b/inc/geometries/geometry_params.json
index 0f641c01e..856130505 100644
--- a/inc/geometries/geometry_params.json
+++ b/inc/geometries/geometry_params.json
@@ -22,16 +22,9 @@
     "inverseaspectratio" : 0.41071428571428575, //a/R_0
     "elongation"         : 1.00,
     "triangularity"      : 0.0,
-    //----------------------Miscellaneous----------------------
-    "alpha"         :  0.05, //damping thickness
-    "rk4eps"        :  1e-5,
-    "psip_min"      : -6.,
-    "psip_max"      :  0.0, // (> -7.4)
-    "psip_max_cut"  :  1e10,
-    "psip_max_lim"  :  1e10,
-    "qampl"         :  1
+    "equilibrium"        : "solovev",
+    "form"               : "standardO"
 }
-
 //@ ----------------------------------------------------------------
 
 
diff --git a/inc/geometries/geometry_params_Xpoint.json b/inc/geometries/geometry_params_Xpoint.json
index c99bc4393..c1a2c2a9e 100644
--- a/inc/geometries/geometry_params_Xpoint.json
+++ b/inc/geometries/geometry_params_Xpoint.json
@@ -23,14 +23,7 @@
     "elongation"         : 1.75,
     "triangularity"      : 0.47,
     "equilibrium"  : "solovev",
-    //----------------------Miscellaneous----------------------
-    "alpha"         :  0.05, //damping thickness
-    "rk4eps"        :  1e-6,
-    "psip_min"      : -6.,
-    "psip_max"      :  0.0, // (> -7.4)
-    "psip_max_cut"  :  1e10,
-    "psip_max_lim"  :  1e10,
-    "qampl"         :  1
+    "form" : "standardX"
 }
 //@ ----------------------------------------------------------
 
diff --git a/inc/geometries/geometry_params_Xpoint_harmonic.json b/inc/geometries/geometry_params_Xpoint_harmonic.json
index 5795da0bc..217e4af0b 100644
--- a/inc/geometries/geometry_params_Xpoint_harmonic.json
+++ b/inc/geometries/geometry_params_Xpoint_harmonic.json
@@ -23,14 +23,8 @@
     "inverseaspectratio" : 0.41071428571428575, //a/R_0
     "elongation"         : 1.75,
     "triangularity"      : 0.47,
-    //----------------------Miscellaneous----------------------
-    "alpha"         :  0.05, //damping thickness
-    "rk4eps"        :  1e-4,
-    "psip_min"      : -6.,
-    "psip_max"      :  0.0, // (> -7.4)
-    "psip_max_cut"  :  1e10,
-    "psip_max_lim"  :  1e10,
-    "qampl"         :  1
+    "equilibrium" : "solovev",
+    "form" : "standardX"
 }
 //@ ----------------------------------------------------------
 
diff --git a/inc/geometries/geometry_params_Xpoint_taylor.json b/inc/geometries/geometry_params_Xpoint_taylor.json
index acf56fc6b..66c407bba 100644
--- a/inc/geometries/geometry_params_Xpoint_taylor.json
+++ b/inc/geometries/geometry_params_Xpoint_taylor.json
@@ -5,6 +5,7 @@
  *          ----------------------------------------------------------- */
 {
     "equilibrium" : "taylor", //taylor state
+    "form"               : "standardX"
     //----------------------Taylor coefficients---------------
     "c" :[-0.730004529864617913443309878090712685057289011596696572774289301646,
     -0.013752279867251353918502943858642437963550235876487492144645446786828705,
@@ -22,15 +23,7 @@
     "R_0"                : 547.891714877869, //(in units of rhos)
     "inverseaspectratio" : 0.41071428571428575, //a/R_0
     "elongation"         : 1.75,
-    "triangularity"      : 0.25,
-    //----------------------Miscellaneous----------------------
-    "alpha"         :  0.05, //damping thickness
-    "rk4eps"        :  1e-4,
-    "psip_min"      : -100,
-    "psip_max"      :  0.0, // (> -7.4)
-    "psip_max_cut"  :  1e10,
-    "psip_max_lim"  :  1e10,
-    "qampl"         :  1
+    "triangularity"      : 0.25
 }
 //@ ----------------------------------------------------------
 
diff --git a/inc/geometries/geometry_params_circ_Iconst_largeR0.json b/inc/geometries/geometry_params_circ_Iconst_largeR0.json
index ad6678f33..cb16bfea2 100644
--- a/inc/geometries/geometry_params_circ_Iconst_largeR0.json
+++ b/inc/geometries/geometry_params_circ_Iconst_largeR0.json
@@ -22,13 +22,8 @@
 		0
 	],
 	"elongation" : 1,
+	"triangularity" : 0,
 	"equilibrium" : "solovev",
-	"inverseaspectratio" : 0.016666666666666666,
-	"psip_max" : 0,
-	"psip_max_cut" : 0,
-	"psip_max_lim" : 0.20000000000000001,
-	"psip_min" : -0.69999999999999996,
-	"qampl" : 1,
-	"rk4eps" : 1.0000000000000001e-05,
-	"triangularity" : 0
-}
\ No newline at end of file
+    "form": "standardO",
+	"inverseaspectratio" : 0.016666666666666666
+}
diff --git a/src/feltor/geometry/compass.json b/src/feltor/geometry/compass.json
index 9a6328b9a..4f13af462 100644
--- a/src/feltor/geometry/compass.json
+++ b/src/feltor/geometry/compass.json
@@ -20,6 +20,7 @@
    	],
    	"elongation": 1.6594,
    	"equilibrium": "solovev",
+    "form" : "standardX",
    	"inverseaspectratio": 0.2857142857142857,
    	"triangularity": 0.4
 }
diff --git a/src/feltor/geometry/geometry_params.json b/src/feltor/geometry/geometry_params.json
index 23f05ee17..2ffc090ac 100644
--- a/src/feltor/geometry/geometry_params.json
+++ b/src/feltor/geometry/geometry_params.json
@@ -24,6 +24,7 @@
 	],
 	"elongation" : 1,
 	"equilibrium" : "solovev",
+    "form" : "standardO",
 	"inverseaspectratio" : 0.1666666666666667,
 	"triangularity" : 0
 }
diff --git a/src/feltor/geometry/geometry_paramsXpoint.json b/src/feltor/geometry/geometry_paramsXpoint.json
index 336ca0790..febe3e4e3 100644
--- a/src/feltor/geometry/geometry_paramsXpoint.json
+++ b/src/feltor/geometry/geometry_paramsXpoint.json
@@ -24,6 +24,7 @@
 	],
 	"elongation" : 1.7,
 	"equilibrium" : "solovev",
+	"form" : "standardX",
 	"inverseaspectratio" : 0.16666666666666669,
 	"triangularity" : 0.33000000000000002
 }
diff --git a/src/feltor/geometry/torpex.json b/src/feltor/geometry/torpex.json
index 8c2338da4..f655d9cfe 100644
--- a/src/feltor/geometry/torpex.json
+++ b/src/feltor/geometry/torpex.json
@@ -12,6 +12,8 @@
     "PP": -4.752951718389757e-06,
     "PI": -1,
     "R_0": 500,
+    "equilibrium" : "solovev", 
+    "form" : "centeredX",
     "elongation": 1,
     "triangularity": 0,
     "inverseaspectratio": 0.2
diff --git a/src/heat/geometry/geometry_params.json b/src/heat/geometry/geometry_params.json
index 2a9eb5766..cf604c462 100644
--- a/src/heat/geometry/geometry_params.json
+++ b/src/heat/geometry/geometry_params.json
@@ -22,7 +22,8 @@
     "inverseaspectratio" : 0.41071428571428575, //a/R_0
     "elongation"         : 1.75,
     "triangularity"      : 0.47,
-    "qampl"         :  1
+    "equilibrium" : "solovev",
+    "form" : "standardX"
 }
 //@ ----------------------------------------------------------
 
-- 
GitLab


From 4338139e387ea8e98f45a2b06429b4d67e5b434d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 20 Aug 2020 20:58:09 +0200
Subject: [PATCH 296/540] Add Meta-data to TokamakMagneticField

now we can get rid of GeometryParams in principle
---
 inc/geometries/geometry_elliptic_mpib.cu |  2 +-
 inc/geometries/guenther.h                |  8 +++-----
 inc/geometries/magnetic_field.h          | 25 ++++++++++++------------
 inc/geometries/solovev.h                 | 12 ++++++------
 inc/geometries/solovev_parameters.h      | 14 +++++++++++++
 inc/geometries/taylor.h                  |  8 +++-----
 inc/geometries/toroidal.h                | 18 ++++++-----------
 7 files changed, 45 insertions(+), 42 deletions(-)

diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 1921d4c32..5632568ef 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -35,7 +35,7 @@ int main(int argc, char**argv)
         is >> js;
     }
     dg::geo::solovev::Parameters gp(js);
-    dg::geo::TokamakMagneticField c = dg::geo::solovev::createMagField(gp);
+    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
     if(rank==0)std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
     if(rank==0)std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 0d571feb5..3a8db6e12 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -145,10 +145,6 @@ static inline CylindricalFunctorsLvl1 createIpol( double I_0)
 {
     return CylindricalFunctorsLvl1( Ipol(I_0), IpolR(), IpolZ());
 }
-static inline TokamakMagneticField createMagField( double R_0, double I_0)
-{
-    return TokamakMagneticField( R_0, createPsip(R_0), createIpol(I_0));
-}
 ///@}
 } //namespace guenther
 
@@ -163,7 +159,9 @@ static inline TokamakMagneticField createMagField( double R_0, double I_0)
  */
 static inline dg::geo::TokamakMagneticField createGuentherField( double R_0, double I_0)
 {
-    return TokamakMagneticField( R_0, guenther::createPsip(R_0), guenther::createIpol(I_0));
+    MagneticFieldParameters params = { 1., 1., 0.,
+            equilibrium::guenther, modifier::none, form::square};
+    return TokamakMagneticField( R_0, guenther::createPsip(R_0), guenther::createIpol(I_0), params);
 }
 } //namespace geo
 }//namespace dg
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 662aa0fa3..406612ce5 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -21,7 +21,7 @@ namespace geo
 ///@addtogroup magnetic
 ///@{
 
-///@brief How flux-function is computed
+///@brief How flux-function is computed. Decides how to construct magnetic field.
 enum class equilibrium
 {
     solovev, //!< dg::geo::solovev::Psip
@@ -37,11 +37,12 @@ enum class modifier
     none, //!< no modification
     heaviside //!< Psip is dampened to a constant outside a critical value
 };
-///@brief How flux-function looks like
+///@brief How flux function looks like. Decider on whether and what flux aligned grid to construct
 enum class form
 {
     standardO, //!< closed flux surfaces centered around an O-point located near (R_0, 0); flux-aligned grids can be constructed
     standardX, //!< closed flux surfaces centered around an O-point located near (R_0, 0) and bordered by a separatrix with a single X-point; flux-aligned X-grids can be constructed
+    none, //!< no shaping: Purely toroidal magnetic field
     square, //!< closed flux surfaces centered around an O-point and bordered by a square  with four X-points in the corners (mainly the Guenther field)
     centeredX //!< one X-point in the middle, no O-point, only open flux surfaces, X-grids cannot be constructed
 };
@@ -62,6 +63,7 @@ static const std::map<std::string, form> str2form{
     {"standardO", form::standardO},
     {"standardX", form::standardX},
     {"square", form::square},
+    {"none", form::none},
     {"centeredX", form::centeredX}
 };
 ///@endcond
@@ -89,7 +91,7 @@ struct MagneticFieldParameters
 };
 
 /**
-* @brief A tokamak field as given by R0, Psi and Ipol
+* @brief A tokamak field as given by R0, Psi and Ipol plus Meta-data like shape and equilibrium
 
  This is the representation of toroidally axisymmetric magnetic fields that can be modeled in the form
  \f$
@@ -104,18 +106,15 @@ struct TokamakMagneticField
     ///as long as the field stays empty the access functions are undefined
     TokamakMagneticField(){}
     TokamakMagneticField( double R0, const CylindricalFunctorsLvl2& psip, const
-            CylindricalFunctorsLvl1& ipol
-            //, MagneticFieldParameters gp
-            ): m_R0(R0), m_psip(psip), m_ipol(ipol) {}//, m_params(gp){}
+            CylindricalFunctorsLvl1& ipol , MagneticFieldParameters gp
+            ): m_R0(R0), m_psip(psip), m_ipol(ipol), m_params(gp){}
     void set( double R0, const CylindricalFunctorsLvl2& psip, const
-            CylindricalFunctorsLvl1& ipol
-            //, MagneticFieldParameters gp
-            )
+            CylindricalFunctorsLvl1& ipol , MagneticFieldParameters gp)
     {
         m_R0=R0;
         m_psip=psip;
         m_ipol=ipol;
-        //m_params = gp;
+        m_params = gp;
     }
     /// \f$ R_0 \f$
     double R0()const {return m_R0;}
@@ -140,13 +139,13 @@ struct TokamakMagneticField
 
     const CylindricalFunctorsLvl2& get_psip() const{return m_psip;}
     const CylindricalFunctorsLvl1& get_ipol() const{return m_ipol;}
-    //const MagneticFieldParameters& params() const{return m_params;}
+    const MagneticFieldParameters& params() const{return m_params;}
 
     private:
     double m_R0;
     CylindricalFunctorsLvl2 m_psip;
     CylindricalFunctorsLvl1 m_ipol;
-    //MagneticFieldParamters m_params;
+    MagneticFieldParameters m_params;
 };
 
 ///@cond
@@ -188,7 +187,7 @@ TokamakMagneticField periodify( const TokamakMagneticField& mag, double R0, doub
     return TokamakMagneticField( mag.R0(),
             periodify( mag.get_psip(), R0, R1, Z0, Z1, bcx, bcy),
             //what if Dirichlet BC in the current? Won't that generate a NaN?
-            periodify( mag.get_ipol(), R0, R1, Z0, Z1, bcx, bcy));
+            periodify( mag.get_ipol(), R0, R1, Z0, Z1, bcx, bcy), mag.params());
 }
 
 ///@brief \f$   |B| = R_0\sqrt{I^2+(\nabla\psi)^2}/R   \f$
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index f70cbbe77..d268cf23c 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -345,10 +345,6 @@ static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp)
     return CylindricalFunctorsLvl1( Ipol(gp), IpolR(gp), IpolZ(gp));
 }
 
-static inline dg::geo::TokamakMagneticField createMagField( Parameters gp)
-{
-    return TokamakMagneticField( gp.R_0, createPsip(gp), createIpol(gp));
-}
 ///@}
 
 ///@cond
@@ -545,8 +541,10 @@ static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp,
 static inline dg::geo::TokamakMagneticField createSolovevField(
     dg::geo::solovev::Parameters gp)
 {
+    MagneticFieldParameters params = { gp.a, gp.elongation, gp.triangularity,
+            equilibrium::solovev, modifier::none, str2form.at( gp.form)};
     return TokamakMagneticField( gp.R_0, solovev::createPsip(gp),
-        solovev::createIpol(gp));
+        solovev::createIpol(gp), params);
 }
 /**
  * @brief Create a modified Solovev Magnetic field
@@ -568,8 +566,10 @@ static inline dg::geo::TokamakMagneticField createSolovevField(
 static inline dg::geo::TokamakMagneticField createModifiedSolovevField(
     dg::geo::solovev::Parameters gp, double psi0, double alpha, double sign = -1)
 {
+    MagneticFieldParameters params = { gp.a, gp.elongation, gp.triangularity,
+            equilibrium::solovev, modifier::heaviside, str2form.at( gp.form)};
     return TokamakMagneticField( gp.R_0, solovev::mod::createPsip(gp, psi0,
-    alpha, sign), solovev::mod::createIpol(gp, psi0, alpha, sign));
+    alpha, sign), solovev::mod::createIpol(gp, psi0, alpha, sign), params);
 }
 
 } //namespace geo
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index 11d0cfa96..e84833070 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -29,6 +29,7 @@ struct Parameters
            elongation, //!< elongation of the magnetic surfaces
            triangularity; //!< triangularity of the magnetic surfaces
     std::vector<double> c;  //!< 12 coefficients for the solovev equilibrium;
+    std::string form;
 #ifdef JSONCPP_VERSION_STRING
     /**
      * @brief Construct from Json dataset
@@ -51,6 +52,17 @@ struct Parameters
         a  = R_0*file::get( mode, js, "inverseaspectratio", 0.).asDouble();
         elongation=file::get( mode, js, "elongation", 1.).asDouble();
         triangularity=file::get( mode, js, "triangularity", 0.).asDouble();
+        try{
+            form = file::get( file::error::is_throw, js, "form", "standardX").asString();
+        } catch ( std::exception& err)
+        {
+            if( isToroidal())
+                form = "none";
+            else if( !hasXpoint())
+                form = "standardO";
+            else
+                form = "standardX";
+        }
     }
     /**
      * @brief Put values into a json string
@@ -70,6 +82,7 @@ struct Parameters
         js["elongation"] = elongation;
         js["triangularity"] = triangularity;
         js[ "equilibrium"] = "solovev";
+        js[ "form"] = form;
         return js;
     }
 #endif // JSONCPP_VERSION_STRING
@@ -112,6 +125,7 @@ struct Parameters
         os  <<" R0            = "<<R_0<<"\n"
             <<" a             = "<<a<<"\n"
             <<" epsilon_a     = "<<a/R_0<<"\n"
+            <<" form          = "<<form<<"\n"
             <<" elongation    = "<<elongation<<"\n"
             <<" triangularity = "<<triangularity<<"\n";
         os << std::flush;
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index eb864b610..f40839f10 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -301,10 +301,6 @@ static inline CylindricalFunctorsLvl1 createIpol( solovev::Parameters gp)
 {
     return CylindricalFunctorsLvl1( Ipol(gp), IpolR(gp), IpolZ(gp));
 }
-static inline dg::geo::TokamakMagneticField createMagField( solovev::Parameters gp)
-{
-    return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp));
-}
 
 ///@}
 
@@ -320,7 +316,9 @@ static inline dg::geo::TokamakMagneticField createMagField( solovev::Parameters
  */
 static inline dg::geo::TokamakMagneticField createTaylorField( dg::geo::solovev::Parameters gp)
 {
-    return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp));
+    MagneticFieldParameters params = { gp.a, gp.elongation, gp.triangularity,
+            equilibrium::solovev, modifier::none, str2form.at( gp.form)};
+    return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp), params);
 }
 } //namespace geo
 
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 87b8d8cb7..38e2a7023 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -27,16 +27,6 @@ static inline CylindricalFunctorsLvl1 createIpol( )
     return ipol;
 }
 
-/**
- * @brief Models a slab toroidal field
- *
- * \f$ B=\frac{R_0}{R}\f$, \f$ \psi_p = 1\f$ and \f$ I = 1\f$.
- @note The solovev field can also be made to model a todoidal slab field
- */
-static inline TokamakMagneticField createMagField( double R0)
-{
-    return TokamakMagneticField( R0, createPsip(), createIpol());
-}
 ///@}
 
 }//namespace toroidal
@@ -117,7 +107,9 @@ static inline CylindricalFunctorsLvl1 createIpol( double I0 )
  */
 static inline dg::geo::TokamakMagneticField createToroidalField( double R0)
 {
-    return TokamakMagneticField( R0, toroidal::createPsip(), toroidal::createIpol());
+    MagneticFieldParameters params = { 1., 1., 0.,
+            equilibrium::circular, modifier::none, form::none};
+    return TokamakMagneticField( R0, toroidal::createPsip(), toroidal::createIpol(), params);
 }
 /**
  * @brief Create a Magnetic field with circular flux surfaces and constant current
@@ -130,7 +122,9 @@ static inline dg::geo::TokamakMagneticField createToroidalField( double R0)
  */
 static inline dg::geo::TokamakMagneticField createCircularField( double R0, double I0)
 {
-    return TokamakMagneticField( R0, circular::createPsip(R0), circular::createIpol(I0));
+    MagneticFieldParameters params = { 1., 1., 0.,
+            equilibrium::circular, modifier::none, form::standardO};
+    return TokamakMagneticField( R0, circular::createPsip(R0), circular::createIpol(I0), params);
 }
 
 }//namespace geo
-- 
GitLab


From c3a315f7d598f1170005db20629ab8b111e93799 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 20 Aug 2020 21:37:57 +0200
Subject: [PATCH 297/540] Add polynomial magnetic field equilibrium

and redesign MagneticFieldParameters because the access looks ugly
---
 inc/geometries/geometries.h            |   1 +
 inc/geometries/magnetic_field.h        |  54 +++-
 inc/geometries/polynomial.h            | 363 +++++++++++++++++++++++++
 inc/geometries/polynomial_parameters.h | 115 ++++++++
 4 files changed, 525 insertions(+), 8 deletions(-)
 create mode 100644 inc/geometries/polynomial.h
 create mode 100644 inc/geometries/polynomial_parameters.h

diff --git a/inc/geometries/geometries.h b/inc/geometries/geometries.h
index 9e44986fc..b1159ccea 100644
--- a/inc/geometries/geometries.h
+++ b/inc/geometries/geometries.h
@@ -23,6 +23,7 @@
 #include "solovev.h"
 #include "guenther.h"
 #include "toroidal.h"
+#include "polynomial.h"
 
 #include "fluxfunctions.h"
 #include "magnetic_field.h"
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 406612ce5..87d88238f 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -26,7 +26,7 @@ enum class equilibrium
 {
     solovev, //!< dg::geo::solovev::Psip
     taylor, //!< dg::geo::taylor::Psip
-    //polynomial, ///!< dg::geo::polynomial::Psip
+    polynomial, ///!< dg::geo::polynomial::Psip
     guenther, //!< dg::geo::guenther::Psip
     toroidal, //!< dg::geo::toroidal::Psip
     circular //!< dg::geo::circular::Psip
@@ -50,7 +50,7 @@ enum class form
 static const std::map<std::string, equilibrium> str2equilibrium{
     {"solovev", equilibrium::solovev},
     {"taylor", equilibrium::taylor},
-    //{"polynomial": equilibrium::polynomial},
+    {"polynomial", equilibrium::polynomial},
     {"guenther", equilibrium::guenther},
     {"toroidal", equilibrium::toroidal},
     {"circular", equilibrium::circular}
@@ -82,12 +82,50 @@ static const std::map<std::string, form> str2form{
  */
 struct MagneticFieldParameters
 {
-    double a, //!< The minor radius; the purpose of this parameter is not to be exact but to serve as a refernce of how to setup the size of a simulation box
-           elongation, //!< (maximum Z - minimum Z of lcfs)/2a; 1 for a circle; the purpose of this parameter is not to be exact but more to be a reference of how to setup the aspect ratio of a simulation box
-           triangularity; //!< (R_0 - R_X) /a;  The purpose of this parameter is to find the approximate location of R_X (if an X-point is present, Z_X is given by elongation) the exact location can be computed by the \c findXpoint function
-    equilibrium equ; //!< the way the flux function is computed
-    modifier mod; //!<  the way the flux function is modified
-    form frm; //!< human readable descriptor of how the flux function looks
+    /**
+     * @brief Default values are for a Toroidal field
+     */
+    MagneticFieldParameters( ){
+        m_a = 1, m_elongation = 1, m_triangularity = 0;
+        m_equilibrium = equilibrium::toroidal;
+        m_modifier = modifier::none;
+        m_form = form::none;
+    }
+    /**
+     * @brief Constructor
+     *
+     * @param a The minor radius; the purpose of this parameter is not to be exact but to serve as a refernce of how to setup the size of a simulation box
+     * @param elongation (maximum Z - minimum Z of lcfs)/2a; 1 for a circle; the purpose of this parameter is not to be exact but more to be a reference of how to setup the aspect ratio of a simulation box
+     * @param triangularity (R_0 - R_X) /a;  The purpose of this parameter is to find the approximate location of R_X (if an X-point is present, Z_X is given by elongation) the exact location can be computed by the \c findXpoint function
+     * @param equ the way the flux function is computed
+     * @param mod the way the flux function is modified
+     * @param frm human readable descriptor of how the flux function looks
+     */
+    MagneticFieldParameters( double a, double elongation, double triangularity,
+            equilibrium equ, modifier mod, form frm): m_a(a),
+        m_elongation(elongation),
+        m_triangularity( triangularity),
+        m_equilibrium( equ),
+        m_modifier(mod), m_form( frm){}
+ //!< The minor radius; the purpose of this parameter is not to be exact but to serve as a refernce of how to setup the size of a simulation box
+    double a() const{return m_a;}
+ //!< (maximum Z - minimum Z of lcfs)/2a; 1 for a circle; the purpose of this parameter is not to be exact but more to be a reference of how to setup the aspect ratio of a simulation box
+    double elongation() const{return m_elongation;}
+ //!< (R_0 - R_X) /a;  The purpose of this parameter is to find the approximate location of R_X (if an X-point is present, Z_X is given by elongation) the exact location can be computed by the \c findXpoint function
+    double triangularity() const{return m_triangularity;}
+ //!< the way the flux function is computed
+    equilibrium getEquilibrium() const{return m_equilibrium;}
+ //!<  the way the flux function is modified
+    modifier getModifier() const{return m_modifier;}
+ //!< human readable descriptor of how the flux function looks
+    form getForm() const{return m_form;}
+    private:
+    double m_a,
+           m_elongation,
+           m_triangularity;
+    equilibrium m_equilibrium;
+    modifier m_modifier;
+    form m_form;
 };
 
 /**
diff --git a/inc/geometries/polynomial.h b/inc/geometries/polynomial.h
new file mode 100644
index 000000000..ff5ec0fb8
--- /dev/null
+++ b/inc/geometries/polynomial.h
@@ -0,0 +1,363 @@
+#pragma once
+
+#include <iostream>
+#include <cmath>
+#include <vector>
+
+#include "dg/blas.h"
+
+#include "dg/topology/functions.h"
+#include "dg/functors.h"
+#include "polynomial_parameters.h"
+#include "magnetic_field.h"
+
+
+/*!@file
+ *
+ * MagneticField objects
+ */
+namespace dg
+{
+namespace geo
+{
+
+/**
+ * @brief \f[ \sum_{i=0}^{M-1} \sum_{j=0}^{N-1} c_{i*N+j} x^i y^j  \f]
+ */
+struct Horner2d
+{
+    Horner2d(): m_c( 1, 1), m_M(1), m_N(1){}
+    Horner2d( std::vector<double> c, unsigned M, unsigned N): m_c(c), m_M(M), m_N(N){}
+    double operator()( double x, double y) const
+    {
+        std::vector<double> cx( m_M);
+        for( unsigned i=0; i<m_M; i++)
+            cx[i] = horner( &m_c[i*m_N], m_N, y);
+        return horner( &cx[0], m_M, x);
+    }
+    private:
+    double horner( const double * c, unsigned M, double x) const
+    {
+        double b = c[M-1];
+        for( unsigned i=0; i<M-1; i++)
+            b = c[M-2-i] + b*x;
+        return b;
+    }
+    std::vector<double> m_c;
+    unsigned m_M, m_N;
+};
+
+/**
+ * @brief Contains a polynomial approximation type flux function
+ */
+namespace polynomial
+{
+///@addtogroup polynomial
+///@{
+
+/**
+ * @brief \f[ \hat{\psi}_p  \f]
+ *
+ * \f[ \hat{\psi}_p(R,Z) =
+      \hat{R}_0P_{\psi}\Bigg\{ \sum_i \sum_j c_{i*N+j} x^i y^j \Bigg\}
+      \Bigg\} \f]
+      with \f$ \bar R := \frac{ R}{R_0} \f$ and \f$\bar Z := \frac{Z}{R_0}\f$
+ *
+ */
+struct Psip: public aCylindricalFunctor<Psip>
+{
+    /**
+     * @brief Construct from given geometric parameters
+     *
+     * @param gp geometric parameters
+     */
+    Psip( Parameters gp ): m_R0(gp.R_0),  m_pp(gp.pp), m_horner(gp.c, gp.M, gp.N) {}
+    double do_compute(double R, double Z) const
+    {
+        return m_R0*m_pp*m_horner( R/m_R0,Z/m_R0);
+    }
+  private:
+    double m_R0, m_pp;
+    Horner2d m_horner;
+};
+
+struct PsipR: public aCylindricalFunctor<PsipR>
+{
+    ///@copydoc Psip::Psip()
+    PsipR( Parameters gp ): m_R0(gp.R_0),  m_pp(gp.pp){
+        std::vector<double>  beta ( (gp.M-1)*gp.N);
+        for( unsigned i=0; i<gp.M-1; i++)
+            for( unsigned j=0; j<gp.N; j++)
+                beta[i*gp.N+j] = (double)(i+1)*gp.c[ ( i+1)*gp.N +j];
+        m_horner = Horner2d( beta, gp.M-1, gp.N);
+    }
+    double do_compute(double R, double Z) const
+    {
+        return m_pp*m_horner( R/m_R0,Z/m_R0);
+    }
+  private:
+    double m_R0, m_pp;
+    Horner2d m_horner;
+};
+struct PsipRR: public aCylindricalFunctor<PsipRR>
+{
+    ///@copydoc Psip::Psip()
+    PsipRR( Parameters gp ): m_R0(gp.R_0),  m_pp(gp.pp){
+        std::vector<double>  beta ( (gp.M-2)*gp.N);
+        for( unsigned i=0; i<gp.M-2; i++)
+            for( unsigned j=0; j<gp.N; j++)
+                beta[i*gp.N+j] = (double)((i+2)*(i+1))*gp.c[ (i+2)*gp.N +j];
+        m_horner = Horner2d( beta, gp.M-2, gp.N);
+    }
+    double do_compute(double R, double Z) const
+    {
+        return m_pp/m_R0*m_horner( R/m_R0,Z/m_R0);
+    }
+  private:
+    double m_R0, m_pp;
+    Horner2d m_horner;
+};
+struct PsipZ: public aCylindricalFunctor<PsipZ>
+{
+    ///@copydoc Psip::Psip()
+    PsipZ( Parameters gp ): m_R0(gp.R_0),  m_pp(gp.pp){
+        std::vector<double>  beta ( gp.M*(gp.N-1));
+        for( unsigned i=0; i<gp.M; i++)
+            for( unsigned j=0; j<gp.N-1; j++)
+                beta[i*(gp.N-1)+j] = (double)(j+1)*gp.c[ i*gp.N +j+1];
+        m_horner = Horner2d( beta, gp.M, gp.N-1);
+    }
+    double do_compute(double R, double Z) const
+    {
+        return m_pp*m_horner( R/m_R0,Z/m_R0);
+    }
+  private:
+    double m_R0, m_pp;
+    Horner2d m_horner;
+};
+struct PsipZZ: public aCylindricalFunctor<PsipZZ>
+{
+    ///@copydoc Psip::Psip()
+    PsipZZ( Parameters gp ): m_R0(gp.R_0),  m_pp(gp.pp){
+        std::vector<double>  beta ( gp.M*(gp.N-2));
+        for( unsigned i=0; i<gp.M; i++)
+            for( unsigned j=0; j<gp.N-2; j++)
+                beta[i*(gp.N-2)+j] = (double)((j+2)*(j+1))*gp.c[ i*gp.N +j+2];
+        m_horner = Horner2d( beta, gp.M, gp.N-2);
+    }
+    double do_compute(double R, double Z) const
+    {
+        return m_pp/m_R0*m_horner(R/m_R0,Z/m_R0);
+    }
+  private:
+    double m_R0, m_pp;
+    Horner2d m_horner;
+};
+struct PsipRZ: public aCylindricalFunctor<PsipRZ>
+{
+    ///@copydoc Psip::Psip()
+    PsipRZ( Parameters gp ): m_R0(gp.R_0),  m_pp(gp.pp){
+        std::vector<double>  beta ( (gp.M-1)*(gp.N-1));
+        for( unsigned i=0; i<gp.M-1; i++)
+            for( unsigned j=0; j<gp.N-1; j++)
+                beta[i*(gp.N-1)+j] = (double)((j+1)*(i+1))*gp.c[ (i+1)*gp.N +j+1];
+        m_horner = Horner2d( beta, gp.M-1, gp.N-1);
+    }
+    double do_compute(double R, double Z) const
+    {
+        return m_pp/m_R0*m_horner(R/m_R0,Z/m_R0);
+    }
+  private:
+    double m_R0, m_pp;
+    Horner2d m_horner;
+};
+
+static inline dg::geo::CylindricalFunctorsLvl2 createPsip( Parameters gp)
+{
+    return CylindricalFunctorsLvl2( Psip(gp), PsipR(gp), PsipZ(gp),
+        PsipRR(gp), PsipRZ(gp), PsipZZ(gp));
+}
+static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp)
+{
+    return CylindricalFunctorsLvl1( Constant( gp.pi), Constant(0), Constant(0));
+}
+
+///@}
+
+///@cond
+namespace mod
+{
+
+struct Psip: public aCylindricalFunctor<Psip>
+{
+    Psip( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_ipoly( psi0, alpha, sign), m_psip(gp)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        return m_ipoly( psip);
+    }
+    private:
+    dg::IPolynomialHeaviside m_ipoly;
+    polynomial::Psip m_psip;
+};
+struct PsipR: public aCylindricalFunctor<PsipR>
+{
+    PsipR( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_psip(gp), m_psipR(gp)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        double psipR = m_psipR(R,Z);
+        return psipR*m_poly( psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    polynomial::Psip m_psip;
+    polynomial::PsipR m_psipR;
+};
+struct PsipZ: public aCylindricalFunctor<PsipZ>
+{
+    PsipZ( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_psip(gp), m_psipZ(gp)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        double psipZ = m_psipZ(R,Z);
+        return psipZ*m_poly( psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    polynomial::Psip m_psip;
+    polynomial::PsipZ m_psipZ;
+};
+
+struct PsipZZ: public aCylindricalFunctor<PsipZZ>
+{
+    PsipZZ( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipZ(gp), m_psipZZ(gp)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        double psipZ = m_psipZ(R,Z);
+        double psipZZ = m_psipZZ(R,Z);
+        return psipZZ*m_poly( psip) + psipZ*psipZ*m_dpoly(psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    dg::DPolynomialHeaviside m_dpoly;
+    polynomial::Psip m_psip;
+    polynomial::PsipZ m_psipZ;
+    polynomial::PsipZZ m_psipZZ;
+};
+struct PsipRR: public aCylindricalFunctor<PsipRR>
+{
+    PsipRR( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipR(gp), m_psipRR(gp)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        double psipR = m_psipR(R,Z);
+        double psipRR = m_psipRR(R,Z);
+        return psipRR*m_poly( psip) + psipR*psipR*m_dpoly(psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    dg::DPolynomialHeaviside m_dpoly;
+    polynomial::Psip m_psip;
+    polynomial::PsipR m_psipR;
+    polynomial::PsipRR m_psipRR;
+};
+struct PsipRZ: public aCylindricalFunctor<PsipRZ>
+{
+    PsipRZ( Parameters gp, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipR(gp), m_psipZ(gp), m_psipRZ(gp)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        double psipR = m_psipR(R,Z);
+        double psipZ = m_psipZ(R,Z);
+        double psipRZ = m_psipRZ(R,Z);
+        return psipRZ*m_poly( psip) + psipR*psipZ*m_dpoly(psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    dg::DPolynomialHeaviside m_dpoly;
+    polynomial::Psip m_psip;
+    polynomial::PsipR m_psipR;
+    polynomial::PsipZ m_psipZ;
+    polynomial::PsipRZ m_psipRZ;
+};
+
+static inline dg::geo::CylindricalFunctorsLvl2 createPsip( Parameters gp,
+    double psi0, double alpha, double sign = -1)
+{
+    return CylindricalFunctorsLvl2( Psip(gp, psi0, alpha, sign), PsipR(gp,
+    psi0, alpha, sign), PsipZ(gp, psi0, alpha, sign), PsipRR(gp, psi0, alpha,
+    sign), PsipRZ(gp, psi0, alpha, sign), PsipZZ(gp, psi0, alpha, sign));
+}
+static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp,
+    double psi0, double alpha, double sign = -1)
+{
+    return CylindricalFunctorsLvl1( Constant( gp.pi), Constant( 0), Constant( 0));
+}
+
+} //namespace mod
+///@endcond
+
+///////////////////////////////////////introduce fields into polynomial namespace
+
+
+} //namespace polynomial
+
+/**
+ * @brief Create a Polynomial Magnetic field
+ *
+ * Based on \c dg::geo::polynomial::Psip(gp) and \c dg::geo::polynomial::Ipol(gp)
+ * @param gp Polynomial parameters
+ * @return A magnetic field object
+ * @ingroup geom
+ */
+static inline dg::geo::TokamakMagneticField createPolynomialField(
+    dg::geo::polynomial::Parameters gp)
+{
+    MagneticFieldParameters params( gp.a, gp.elongation, gp.triangularity,
+            equilibrium::polynomial, modifier::none, str2form.at( gp.form));
+    return TokamakMagneticField( gp.R_0, polynomial::createPsip(gp),
+        polynomial::createIpol(gp), params);
+}
+/**
+ * @brief Create a modified Polynomial Magnetic field
+ *
+ * Based on \c dg::geo::polynomial::mod::Psip(gp) and
+ * \c dg::geo::polynomial::mod::Ipol(gp)
+ * We modify psi above a certain value to a constant using the
+ * \c dg::IPolynomialHeaviside function (an approximation to the integrated Heaviside
+ * function with width alpha), i.e. we replace psi with IPolynomialHeaviside(psi).
+ * This subsequently modifies all derivatives of psi and the poloidal
+ * current.
+ * @param gp Polynomial parameters
+ * @param psi0 boundary value where psi is modified to a constant psi0
+ * @param alpha radius of the transition region where the modification acts (smaller is quicker)
+ * @param sign determines which side of Psi to dampen (negative or positive, forwarded to \c dg::IPolynomialHeaviside)
+ * @return A magnetic field object
+ * @ingroup geom
+ */
+static inline dg::geo::TokamakMagneticField createModifiedPolynomialField(
+    dg::geo::polynomial::Parameters gp, double psi0, double alpha, double sign = -1)
+{
+    MagneticFieldParameters params( gp.a, gp.elongation, gp.triangularity,
+            equilibrium::polynomial, modifier::heaviside, str2form.at( gp.form));
+    return TokamakMagneticField( gp.R_0, polynomial::mod::createPsip(gp, psi0,
+    alpha, sign), polynomial::mod::createIpol(gp, psi0, alpha, sign), params);
+}
+
+} //namespace geo
+} //namespace dg
+
diff --git a/inc/geometries/polynomial_parameters.h b/inc/geometries/polynomial_parameters.h
new file mode 100644
index 000000000..c0644dd99
--- /dev/null
+++ b/inc/geometries/polynomial_parameters.h
@@ -0,0 +1,115 @@
+#pragma once
+#include <string>
+#include <vector>
+#ifdef JSONCPP_VERSION_STRING
+#include <dg/file/json_utilities.h>
+#endif
+/*!@file
+ *
+ * Geometry parameters
+ */
+namespace dg
+{
+namespace geo
+{
+namespace polynomial
+{
+/**
+ * @brief Constructs and display geometric parameters for the polynomial fields
+ * @ingroup geom
+ * @note include \c json/json.h before \c geometries.h in order to activate json functionality
+ */
+struct Parameters
+{
+    double R_0, //!< major tokamak radius
+           pp, //!< prefactor for Psi_p
+           pi, //!< prefactor for current I
+           a,  //!<  little tokamak radius
+           elongation, //!< elongation of the magnetic surfaces
+           triangularity; //!< triangularity of the magnetic surfaces
+    unsigned M, //!< number of coefficients in R
+             N; //!< number of coefficients in Z
+    std::vector<double> c;  //!< M*N coefficients for the polynomial equilibrium, \c c[i*N+j] corresponds to R^i Z^j;
+    std::string form;
+#ifdef JSONCPP_VERSION_STRING
+    /**
+     * @brief Construct from Json dataset
+     * @param js Can contain the variables "M" (1), "N" (1), "c" (0), "PP" (1.), "PI"
+     * (1.), "R_0" , "inverseaspectratio" , "elongation" (1), "triangularity"
+     * (0)
+     * @param mode determine what happens when a key is missing
+     * @note the default values in brackets are taken if the variables are not found in the input file
+     * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
+     */
+    Parameters( const Json::Value& js, file::error mode = file::error::is_silent) {
+        pp  = file::get( mode, js, "PP", 1).asDouble();
+        pi  = file::get( mode, js, "PI", 1).asDouble();
+        M = file::get( mode, js, "M", 1).asUInt();
+        M = file::get( mode, js, "N", 1).asUInt();
+        c.resize(M*N);
+        for (unsigned i=0;i<M*N;i++)
+            c[i] = file::get_idx( mode, js, "c", i, 0.).asDouble();
+
+        R_0  = file::get( mode, js, "R_0", 0.).asDouble();
+        a  = R_0*file::get( mode, js, "inverseaspectratio", 0.).asDouble();
+        elongation=file::get( mode, js, "elongation", 1.).asDouble();
+        triangularity=file::get( mode, js, "triangularity", 0.).asDouble();
+        form = file::get( mode, js, "form", "standardX").asString();
+    }
+    /**
+     * @brief Put values into a json string
+     *
+     * @return Json value
+     * @attention This member is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
+     */
+    Json::Value dump( ) const
+    {
+        Json::Value js;
+        js["M"] = M;
+        js["N"] = N;
+        js["PP"] = pp;
+        js["PI"] = pi;
+        for (unsigned i=0;i<N*N;i++) js["c"][i] = c[i];
+        js["R_0"] = R_0;
+        js["inverseaspectratio"] = a/R_0;
+        js["elongation"] = elongation;
+        js["triangularity"] = triangularity;
+        js[ "equilibrium"] = "polynomial";
+        js[ "form"] = form;
+        return js;
+    }
+#endif // JSONCPP_VERSION_STRING
+    /**
+    * @brief True if \c pp==0
+    *
+    * @return \c true if the flux function is a constant
+    */
+    bool isToroidal() const{
+        if( pp == 0)
+            return true;
+        return false;
+    }
+    ///Write variables as a formatted string
+    void display( std::ostream& os = std::cout ) const
+    {
+        os << "Polynomial Geometrical parameters are: \n"
+            <<" Prefactor Psi   = "<<pp<<"\n"
+            <<" Prefactor I     = "<<pi<<"\n"
+            <<" number in R     = "<<M<<"\n"
+            <<" number in Z     = "<<N<<"\n";
+        for( unsigned i=0; i<M*N; i++)
+            os<<" c"<<i+1<<"\t\t = "<<c[i]<<"\n";
+
+        os  <<" R0            = "<<R_0<<"\n"
+            <<" a             = "<<a<<"\n"
+            <<" epsilon_a     = "<<a/R_0<<"\n"
+            <<" form          = "<<form<<"\n"
+            <<" elongation    = "<<elongation<<"\n"
+            <<" triangularity = "<<triangularity<<"\n";
+        os << std::flush;
+
+    }
+};
+} //namespace polynomial
+} //namespace geo
+} //namespace dg
-- 
GitLab


From 5e258e6628bbde53030a853576273a0c72e08659 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 20 Aug 2020 21:51:26 +0200
Subject: [PATCH 298/540] Make geometry_diag use novel form field

first step towards general magnetic field parameters
---
 inc/geometries/geometry_diag.cu | 35 +++++++++++++++++----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 34a4b370f..92ffd6d6b 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -123,22 +123,23 @@ int main( int argc, char* argv[])
     const dg::geo::solovev::Parameters gp(geom_js);
     p.display( std::cout);
     gp.display( std::cout);
+    //Test coefficients
+    dg::geo::TokamakMagneticField mag_origin = dg::geo::createSolovevField(gp);
+    dg::geo::TokamakMagneticField mag = mag_origin;
     std::string input = input_js.toStyledString();
     std::string geom = geom_js.toStyledString();
     unsigned n, Nx, Ny, Nz;
     n = p.n, Nx = p.Nx, Ny = p.Ny, Nz = p.Nz;
-    double Rmin=gp.R_0-p.boxscaleRm*gp.a;
-    double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
-    double Rmax=gp.R_0+p.boxscaleRp*gp.a;
-    double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    double Rmin=mag.R0()-p.boxscaleRm*mag.params().a();
+    double Zmin=-p.boxscaleZm*mag.params().a()*mag.params().elongation();
+    double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
+    double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
 
-    //Test coefficients
-    dg::geo::TokamakMagneticField mag_origin = dg::geo::createSolovevField(gp);
-    dg::geo::TokamakMagneticField mag = mag_origin;
     //Find O-point
-    double RO = gp.R_0, ZO = 0.;
+    double RO = mag.R0(), ZO = 0.;
     int point = 1;
-    if( !gp.isToroidal() )
+    dg::geo::form mag_form = mag.params().getForm();
+    if( mag_form == dg::geo::form::standardX || mag_form == dg::geo::form::standardO )
         point = dg::geo::findOpoint( mag.get_psip(), RO, ZO);
     const double psipO = mag.psip()( RO, ZO);
     std::cout << "O-point found at "<<RO<<" "<<ZO<<" with Psip "<<psipO<<std::endl;
@@ -146,7 +147,7 @@ int main( int argc, char* argv[])
         std::cout << " (minimum)"<<std::endl;
     if( point == 2 )
         std::cout << " (maximum)"<<std::endl;
-    const double psip0 = mag.psip()(gp.R_0, 0);
+    const double psip0 = mag.psip()(mag.R0(), 0);
     std::cout << "psip( R_0, 0) = "<<psip0<<"\n";
     if( p.damping_alpha > 0.)
     {
@@ -228,7 +229,7 @@ int main( int argc, char* argv[])
         {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::compose( dg::TanhProfX( -3*p.source_alpha, p.source_alpha, -1), mag.psip())},
         ////
         {"BathRZ", "A randomized field", dg::BathRZ( 16, 16, Rmin,Zmin, 30.,2, p.amp)},
-        {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(gp.R_0+p.posX*gp.a, p.posY*gp.a,
+        {"Gaussian3d", "A Gaussian field", dg::Gaussian3d(mag.R0()+p.posX*mag.params().a(), p.posY*mag.params().a(),
             M_PI, p.sigma, p.sigma, p.sigma, p.amp)},
         { "Hoo", "The novel h02 factor", dg::geo::Hoo( mag) }
     };
@@ -236,11 +237,11 @@ int main( int argc, char* argv[])
     /// -------  Elements for fsa on X-point grid ----------------
     double psipmax = dg::blas1::reduce( psipog2d, 0., thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     std::unique_ptr<dg::geo::CurvilinearGridX2d> gX2d;
-    if( gp.hasXpoint())
+    if( mag.params().getForm() == dg::geo::form::standardX)
     {
         std::cout << "Generate X-point flux-aligned grid ... \n";
-        double RX = gp.R_0-1.1*gp.triangularity*gp.a;
-        double ZX = -1.1*gp.elongation*gp.a;
+        double RX = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
+        double ZX = -1.1*mag.params().elongation()*mag.params().a();
         dg::geo::findXpoint( mag.get_psip(), RX, ZX);
         double psipX = mag.psip()(RX, ZX);
         std::cout << "Found X-point at "<<RX<<" "<<ZX<<" with Psip = "<<psipX<<std::endl;
@@ -304,7 +305,7 @@ int main( int argc, char* argv[])
     /// --------- More flux labels --------------------------------
     dg::Grid1d grid1d(psipO<psipmax ? psipO : psipmax,
             psipO<psipmax ? psipmax : psipO, npsi ,Npsi,dg::DIR_NEU); //inner value is always zero
-    if( !gp.isToroidal())
+    if( mag_form != dg::geo::form::none && mag_form != dg::geo::form::centeredX)
     {
         dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
         dg::blas1::axpby( -1./psipO, rho, +1., 1., rho); //transform psi to rho
@@ -313,7 +314,7 @@ int main( int argc, char* argv[])
         dg::blas1::transform( rho, rho, dg::SQRT<double>());
         map1d.emplace_back("rho_p", rho,
             "Alternative flux label rho_p = Sqrt[-psi/psimin + 1]");
-        //if( gp.equilibrium == "solovev")
+        if( mag_form == dg::geo::form::standardX || mag_form == dg::geo::form::standardO)
         {
             dg::geo::SafetyFactor qprof( mag);
             dg::HVec qprofile = dg::evaluate( qprof, grid1d);
@@ -364,7 +365,7 @@ int main( int argc, char* argv[])
             pair.first.data(), pair.second.size(), pair.second.data());
 
     int dim1d_ids[1], dim2d_ids[2], dim3d_ids[3] ;
-    if( gp.hasXpoint())
+    if( mag_form == dg::geo::form::standardX)
     {
         int dim_idsX[2] = {0,0};
         err = file::define_dimensions( ncid, dim_idsX, gX2d->grid(), {"eta", "zeta"} );
-- 
GitLab


From e575ea2584ad7521b9ccc42bff88d63a28ec144b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 21 Aug 2020 15:31:06 +0200
Subject: [PATCH 299/540] Minor Fix definition of magnetization stress

---
 src/feltor/feltor.tex    | 2 +-
 src/feltor/feltordiag.cu | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 9665ba88c..7e3aff433 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1109,7 +1109,7 @@ Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continui
 &\partial_t \RA{\Omega_E} + \frac{\partial}{\partial v} \frac{\d v}{\d \psi_p}\RA{ \vec j_{\Omega_E}\cn\psi_p} = -\RA{F_{L,\varphi}}+ \RA{\mathcal S_{\Omega_E}} + \RA{\Lambda_{\Omega_E}} \label{eq:exb_average} \\
 \Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \equiv \mu_i N_i u_{E,\varphi} \\
 \vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
-    - \vn A_\parallel\cn\psi_p \left(\frac{1}{\beta} \frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \tau_i N_iU_i\right) \\
+    - \vn A_\parallel\cn\psi_p \left(\frac{1}{\beta} \frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \mu_i \tau_i N_iU_i\right) \\
     \mathcal S_{\Omega_E} &:= \mu_i S_{n_e} \frac{\vn\psi_p\cn\phi}{B^2} \quad
     \Lambda_{\Omega_E} := \mu_i \Lambda_{n_e}\frac{\vn\psi_p\cn\phi}{B^2}
 \end{align}
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 4dd43f6a8..c6a6735c5 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -55,6 +55,7 @@ int main( int argc, char* argv[])
 
     //-----------------Create Netcdf output file with attributes----------//
     int ncid_out;
+    //maybe we should issue a warning if file exists?
     err = nc_create(argv[argc-1],NC_NETCDF4|NC_CLOBBER, &ncid_out);
 
     /// Set global attributes
-- 
GitLab


From fa72993ef5bb4d8d36403a4b6a2cb95c737c6825 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 21 Aug 2020 21:13:03 +0200
Subject: [PATCH 300/540] Unifiy magnetic field generation

in a single function
geometry Parameter classes should now not be used any more in
application programs
---
 inc/geometries/geometry_diag.cu     | 48 +++++++++-------
 inc/geometries/guenther_params.json |  4 +-
 inc/geometries/make_field.h         | 89 +++++++++++++++++++++++++++++
 3 files changed, 117 insertions(+), 24 deletions(-)
 create mode 100644 inc/geometries/make_field.h

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 92ffd6d6b..12fd9b2a5 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -10,8 +10,7 @@
 #include "dg/algorithm.h"
 #include "dg/file/file.h"
 
-#include "solovev.h"
-//#include "taylor.h"
+#include "make_field.h"
 #include "magnetic_field.h"
 #include "testfunctors.h"
 #include "curvilinearX.h"
@@ -124,7 +123,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     gp.display( std::cout);
     //Test coefficients
-    dg::geo::TokamakMagneticField mag_origin = dg::geo::createSolovevField(gp);
+    dg::geo::TokamakMagneticField mag_origin = dg::geo::createMagneticField(geom_js, file::error::is_warning);
     dg::geo::TokamakMagneticField mag = mag_origin;
     std::string input = input_js.toStyledString();
     std::string geom = geom_js.toStyledString();
@@ -135,26 +134,33 @@ int main( int argc, char* argv[])
     double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
     double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
 
-    //Find O-point
-    double RO = mag.R0(), ZO = 0.;
-    int point = 1;
     dg::geo::form mag_form = mag.params().getForm();
+    double psipO = 1.;
     if( mag_form == dg::geo::form::standardX || mag_form == dg::geo::form::standardO )
-        point = dg::geo::findOpoint( mag.get_psip(), RO, ZO);
-    const double psipO = mag.psip()( RO, ZO);
-    std::cout << "O-point found at "<<RO<<" "<<ZO<<" with Psip "<<psipO<<std::endl;
-    if( point == 1 )
-        std::cout << " (minimum)"<<std::endl;
-    if( point == 2 )
-        std::cout << " (maximum)"<<std::endl;
-    const double psip0 = mag.psip()(mag.R0(), 0);
-    std::cout << "psip( R_0, 0) = "<<psip0<<"\n";
-    if( p.damping_alpha > 0.)
     {
-        double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
-        double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
-        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2., fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
+        //Find O-point
+        double RO = mag.R0(), ZO = 0.;
+        int point = dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+        const double psipO = mag.psip()( RO, ZO);
+        std::cout << "O-point found at "<<RO<<" "<<ZO<<" with Psip "<<psipO<<std::endl;
+        if( point == 1 )
+            std::cout << " (minimum)"<<std::endl;
+        if( point == 2 )
+            std::cout << " (maximum)"<<std::endl;
+        double psip0 = mag.psip()(mag.R0(), 0);
+        std::cout << "psip( R_0, 0) = "<<psip0<<"\n";
+        if( p.damping_alpha > 0.)
+        {
+            double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
+            double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
+            std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
+            Json::Value jsmod;
+            jsmod["modifier"] = "heaviside";
+            jsmod["psi0"] = damping_psi0p + damping_alphap/2.;
+            jsmod["alpha"] = fabs( damping_alphap/2.);
+            jsmod["sign"] = ((psipO>0)-(psipO<0));
+            mag = dg::geo::createModifiedField(geom_js, jsmod, file::error::is_warning);
+        }
     }
 
 
@@ -237,7 +243,7 @@ int main( int argc, char* argv[])
     /// -------  Elements for fsa on X-point grid ----------------
     double psipmax = dg::blas1::reduce( psipog2d, 0., thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     std::unique_ptr<dg::geo::CurvilinearGridX2d> gX2d;
-    if( mag.params().getForm() == dg::geo::form::standardX)
+    if( mag_form == dg::geo::form::standardX)
     {
         std::cout << "Generate X-point flux-aligned grid ... \n";
         double RX = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
diff --git a/inc/geometries/guenther_params.json b/inc/geometries/guenther_params.json
index c1ba5b170..acacca4c1 100644
--- a/inc/geometries/guenther_params.json
+++ b/inc/geometries/guenther_params.json
@@ -1,9 +1,7 @@
 {
     "equilibrium"       : "guenther",
     "I_0"               :  20.,
-    "R_0"               :  10.,
-    "a"                 :  1,
-    "form"              : "square"
+    "R_0"               :  10.
 }
 
 
diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
new file mode 100644
index 000000000..60b40c00a
--- /dev/null
+++ b/inc/geometries/make_field.h
@@ -0,0 +1,89 @@
+#include "magnetic_field.h"
+#include "solovev.h"
+#include "guenther.h"
+#include "polynomial.h"
+#include "toroidal.h"
+//#include "taylor.h" //only if boost is included
+#include <dg/file/json_utilities.h>
+
+
+namespace dg{
+namespace geo{
+
+TokamakMagneticField createMagneticField( Json::Value js, file::error mode)
+{
+    std::string e = file::get( mode, js, "equilibrium", "solovev" ).asString();
+    equilibrium equi = str2equilibrium.at( e);
+    switch( equi){
+        case equilibrium::polynomial:
+        {
+            polynomial::Parameters gp( js, mode);
+            return createPolynomialField( gp);
+        }
+        case equilibrium::toroidal:
+        {
+            double R0 = file::get( mode, js, "R_0", 10).asDouble();
+            return createToroidalField( R0);
+        }
+        case equilibrium::guenther:
+        {
+            double I0 = file::get( mode, js, "I_0", 20).asDouble();
+            double R0 = file::get( mode, js, "R_0", 10).asDouble();
+            return createGuentherField( R0, I0);
+        }
+        case equilibrium::circular:
+        {
+            double I0 = file::get( mode, js, "I_0", 20).asDouble();
+            double R0 = file::get( mode, js, "R_0", 10).asDouble();
+            return createCircularField( R0, I0);
+        }
+        default:
+        {
+            solovev::Parameters gp( js, mode);
+            return createSolovevField( gp);
+        }
+    }
+}
+TokamakMagneticField createModifiedField( Json::Value js, Json::Value jsmod, file::error mode)
+{
+    std::string e = file::get( mode, js, "equilibrium", "solovev" ).asString();
+    equilibrium equi = str2equilibrium.at( e);
+    std::string m = file::get( mode, jsmod, "modifier", "heaviside" ).asString();
+    modifier mod = str2modifier.at( m);
+    switch (mod) {
+        default:
+        {
+            return createMagneticField( js, mode);
+        }
+        case modifier::heaviside:
+        {
+            double psi0 = file::get( mode, jsmod, "psi0", 0. ).asDouble();
+            double alpha = file::get( mode, jsmod, "alpha", 0. ).asDouble();
+            double sign = file::get( mode, jsmod, "sign", -1. ).asDouble();
+            switch( equi){
+                case equilibrium::polynomial:
+                {
+                    polynomial::Parameters gp( js, mode);
+                    return createModifiedPolynomialField( gp, psi0, alpha, sign );
+                }
+                case equilibrium::solovev:
+                {
+                    solovev::Parameters gp( js, mode);
+                    return createModifiedSolovevField( gp, psi0, alpha, sign );
+                }
+                default:
+                    std::stringstream message;
+                    message << "*** "<<e<<" has no modification implemented!";
+                    if( file::error::is_throw == mode)
+                        throw std::runtime_error( message.str());
+                    else if (file::error::is_warning == mode)
+                        std::cerr <<"WARNING "<< message.str()<<"\n";
+                    else
+                        ;
+                    return createMagneticField( js, mode);
+            }
+        }
+    }
+}
+} //namespace geo
+}//namespace dg
-- 
GitLab


From 468f4f4b71ee3f1e76c284214651fa87db401de7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 21 Aug 2020 21:17:24 +0200
Subject: [PATCH 301/540] Add tcv geometry json file

---
 src/feltor/geometry/tcv.json | 50 ++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 src/feltor/geometry/tcv.json

diff --git a/src/feltor/geometry/tcv.json b/src/feltor/geometry/tcv.json
new file mode 100644
index 000000000..57bc286ed
--- /dev/null
+++ b/src/feltor/geometry/tcv.json
@@ -0,0 +1,50 @@
+{
+    "M": 6,
+    "N": 6,
+    "c": [
+        -1.0484078057117057,
+        -1.0842606353509106,
+        27.536407585400283,
+        20.08700802501453,
+        -116.02565935574977,
+        -120.66010077293852,
+        7.303125619957273,
+        5.818210372553227,
+        -154.9885146764427,
+        -102.29730950402626,
+        622.85031890258,
+        604.7877151679331,
+        -19.03494023088786,
+        -11.402531138983106,
+        348.4024179884296,
+        198.3558695933294,
+        -1325.376409836957,
+        -1179.8289027290296,
+        24.223202203614758,
+        10.268162419571157,
+        -388.13523888449845,
+        -183.31357499053175,
+        1393.7746630606455,
+        1117.3199053627043,
+        -14.796032650579809,
+        -4.217844314114564,
+        212.35696838126282,
+        80.01889289206099,
+        -720.5651955679641,
+        -509.77624096800616,
+        3.45586986984333,
+        0.6153789172952369,
+        -45.45086372840671,
+        -12.955518132950145,
+        146.06735359643812,
+        88.72649045842803
+    ],
+    "PP": -1,
+    "PI": -1,
+    "R_0": 906.38,
+    "elongation": 1.5,
+    "triangularity": 0.4,
+    "inverseaspectratio": 0.2759381898454746,
+    "equilibrium": "polynomial",
+    "form": "standardX"
+}
-- 
GitLab


From 9997bed20bfd0c8386f3ee6736de5fcdd3371aed Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 21 Aug 2020 23:04:21 +0200
Subject: [PATCH 302/540] Update solvev normalize to general fields

---
 inc/geometries/geometry_diag.cu        | 12 ++--
 inc/geometries/normalize_solovev_t.cu  | 50 ++++++++++---
 inc/geometries/polynomial_parameters.h |  2 +-
 src/feltor/geometry/tcv.json           | 98 +++++++++++++-------------
 src/feltor/input/compass.json          | 69 ++++++++++++++++++
 5 files changed, 167 insertions(+), 64 deletions(-)
 create mode 100644 src/feltor/input/compass.json

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 12fd9b2a5..798e4eb43 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -119,11 +119,9 @@ int main( int argc, char* argv[])
     }
     std::cout << input_js<<std::endl;
     const Parameters p(input_js);
-    const dg::geo::solovev::Parameters gp(geom_js);
     p.display( std::cout);
-    gp.display( std::cout);
     //Test coefficients
-    dg::geo::TokamakMagneticField mag_origin = dg::geo::createMagneticField(geom_js, file::error::is_warning);
+    dg::geo::TokamakMagneticField mag_origin = dg::geo::createMagneticField(geom_js, file::error::is_throw);
     dg::geo::TokamakMagneticField mag = mag_origin;
     std::string input = input_js.toStyledString();
     std::string geom = geom_js.toStyledString();
@@ -135,13 +133,13 @@ int main( int argc, char* argv[])
     double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
 
     dg::geo::form mag_form = mag.params().getForm();
-    double psipO = 1.;
+    double psipO = -1.;
     if( mag_form == dg::geo::form::standardX || mag_form == dg::geo::form::standardO )
     {
         //Find O-point
         double RO = mag.R0(), ZO = 0.;
         int point = dg::geo::findOpoint( mag.get_psip(), RO, ZO);
-        const double psipO = mag.psip()( RO, ZO);
+        psipO = mag.psip()( RO, ZO);
         std::cout << "O-point found at "<<RO<<" "<<ZO<<" with Psip "<<psipO<<std::endl;
         if( point == 1 )
             std::cout << " (minimum)"<<std::endl;
@@ -159,7 +157,7 @@ int main( int argc, char* argv[])
             jsmod["psi0"] = damping_psi0p + damping_alphap/2.;
             jsmod["alpha"] = fabs( damping_alphap/2.);
             jsmod["sign"] = ((psipO>0)-(psipO<0));
-            mag = dg::geo::createModifiedField(geom_js, jsmod, file::error::is_warning);
+            mag = dg::geo::createModifiedField(geom_js, jsmod, file::error::is_throw);
         }
     }
 
@@ -260,7 +258,7 @@ int main( int argc, char* argv[])
         dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipO, RX, ZX, mag.R0(), 0, 0, false);
         double fx_0 = 1./8.;
         psipmax = -fx_0/(1.-fx_0)*psipO;
-        //std::cout << "psi 1 is          "<<psipmax<<"\n";
+        std::cout << "psi 1 is          "<<psipmax<<"\n";
         gX2d = std::make_unique<dg::geo::CurvilinearGridX2d>(generator, fx_0, 0., npsi, Npsi, 640, dg::DIR, dg::NEU);
         std::cout << "DONE! \n";
         dg::Average<dg::HVec > avg_eta( gX2d->grid(), dg::coo2d::y);
diff --git a/inc/geometries/normalize_solovev_t.cu b/inc/geometries/normalize_solovev_t.cu
index faa232795..23634ad0a 100644
--- a/inc/geometries/normalize_solovev_t.cu
+++ b/inc/geometries/normalize_solovev_t.cu
@@ -2,8 +2,7 @@
 #include <fstream>
 #include "dg/algorithm.h"
 #include "dg/file/file.h"
-#include "solovev.h"
-#include "fluxfunctions.h"
+#include "make_field.h"
 
 int main( int argc, char* argv[])
 {
@@ -19,11 +18,34 @@ int main( int argc, char* argv[])
         std::cerr << " Usage: "<< argv[0]<<" [input.json] [normalized.json]\n";
         return -1;
     }
-    dg::geo::solovev::Parameters gp(geom_js);
+
     std::cout << "Input file: \n"<< geom_js.toStyledString();
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    double RX = gp.R_0-1.1*gp.triangularity*gp.a;
-    double ZX = -1.1*gp.elongation*gp.a;
+    dg::geo::TokamakMagneticField mag;
+    std::string e = file::get( file::error::is_throw, geom_js, "equilibrium", "solovev" ).asString();
+    dg::geo::equilibrium equi = dg::geo::str2equilibrium.at( e);
+    switch( equi){
+        case dg::geo::equilibrium::polynomial:
+        {
+            std::cout << "Creating polynomial Field!\n";
+            dg::geo::polynomial::Parameters gp( geom_js, file::error::is_warning);
+            mag = dg::geo::createPolynomialField( gp);
+            break;
+        }
+        case dg::geo::equilibrium::solovev:
+        {
+            std::cout << "Creating Solovev Field!\n";
+            dg::geo::solovev::Parameters gp( geom_js, file::error::is_warning);
+            mag = dg::geo::createSolovevField( gp);
+            break;
+        }
+        default:
+        {
+            std::cerr << "Equilibrium "<<e<<" cannot be normalized\n";
+            return -1;
+        }
+    }
+    double RX = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
+    double ZX = -1.1*mag.params().elongation()*mag.params().a();
     try{
         dg::geo::findXpoint( mag.get_psip(), RX, ZX);
     }catch ( std::exception& e)
@@ -33,8 +55,20 @@ int main( int argc, char* argv[])
     }
     const double psipX = mag.psip()( RX, ZX);
     std::cout << "X-point found at "<<RX<<" "<<ZX<<" with Psip = "<<psipX<<std::endl;
-    gp.c[0] = gp.c[0] - psipX/gp.pp/gp.R_0;
-    Json::Value output = gp.dump();
+
+    Json::Value output;
+    if( equi == dg::geo::equilibrium::solovev)
+    {
+        dg::geo::solovev::Parameters gp(geom_js);
+        gp.c[0] = gp.c[0] - psipX/gp.pp/gp.R_0;
+        output = gp.dump();
+    }
+    else if( equi == dg::geo::equilibrium::polynomial)
+    {
+        dg::geo::polynomial::Parameters gp(geom_js);
+        gp.c[0] = gp.c[0] - psipX/gp.pp/gp.R_0;
+        output = gp.dump();
+    }
     std::cout << "Output file "<<argv[2]<<": \n"<< output.toStyledString();
     std::fstream file( argv[2], std::fstream::out | std::fstream::trunc);
     file << output.toStyledString();
diff --git a/inc/geometries/polynomial_parameters.h b/inc/geometries/polynomial_parameters.h
index c0644dd99..ab5606555 100644
--- a/inc/geometries/polynomial_parameters.h
+++ b/inc/geometries/polynomial_parameters.h
@@ -45,7 +45,7 @@ struct Parameters
         pp  = file::get( mode, js, "PP", 1).asDouble();
         pi  = file::get( mode, js, "PI", 1).asDouble();
         M = file::get( mode, js, "M", 1).asUInt();
-        M = file::get( mode, js, "N", 1).asUInt();
+        N = file::get( mode, js, "N", 1).asUInt();
         c.resize(M*N);
         for (unsigned i=0;i<M*N;i++)
             c[i] = file::get_idx( mode, js, "c", i, 0.).asDouble();
diff --git a/src/feltor/geometry/tcv.json b/src/feltor/geometry/tcv.json
index 57bc286ed..7da3be901 100644
--- a/src/feltor/geometry/tcv.json
+++ b/src/feltor/geometry/tcv.json
@@ -1,50 +1,52 @@
 {
-    "M": 6,
-    "N": 6,
-    "c": [
-        -1.0484078057117057,
-        -1.0842606353509106,
-        27.536407585400283,
-        20.08700802501453,
-        -116.02565935574977,
-        -120.66010077293852,
-        7.303125619957273,
-        5.818210372553227,
-        -154.9885146764427,
-        -102.29730950402626,
-        622.85031890258,
-        604.7877151679331,
-        -19.03494023088786,
-        -11.402531138983106,
-        348.4024179884296,
-        198.3558695933294,
-        -1325.376409836957,
-        -1179.8289027290296,
-        24.223202203614758,
-        10.268162419571157,
-        -388.13523888449845,
-        -183.31357499053175,
-        1393.7746630606455,
-        1117.3199053627043,
-        -14.796032650579809,
-        -4.217844314114564,
-        212.35696838126282,
-        80.01889289206099,
-        -720.5651955679641,
-        -509.77624096800616,
-        3.45586986984333,
-        0.6153789172952369,
-        -45.45086372840671,
-        -12.955518132950145,
-        146.06735359643812,
-        88.72649045842803
-    ],
-    "PP": -1,
-    "PI": -1,
-    "R_0": 906.38,
-    "elongation": 1.5,
-    "triangularity": 0.4,
-    "inverseaspectratio": 0.2759381898454746,
-    "equilibrium": "polynomial",
-    "form": "standardX"
+	"M" : 6,
+	"N" : 6,
+	"PI" : -1.0,
+	"PP" : -1.0,
+	"R_0" : 906.38,
+	"c" : 
+	[
+		-1.1279927930530271,
+		-1.0842606353509106,
+		27.536407585400283,
+		20.08700802501453,
+		-116.02565935574977,
+		-120.66010077293852,
+		7.3031256199572727,
+		5.818210372553227,
+		-154.98851467644269,
+		-102.29730950402626,
+		622.85031890258006,
+		604.78771516793313,
+		-19.034940230887859,
+		-11.402531138983106,
+		348.40241798842959,
+		198.3558695933294,
+		-1325.376409836957,
+		-1179.8289027290296,
+		24.223202203614758,
+		10.268162419571157,
+		-388.13523888449845,
+		-183.31357499053175,
+		1393.7746630606455,
+		1117.3199053627043,
+		-14.796032650579809,
+		-4.2178443141145641,
+		212.35696838126282,
+		80.01889289206099,
+		-720.56519556796411,
+		-509.77624096800616,
+		3.4558698698433301,
+		0.61537891729523686,
+		-45.450863728406709,
+		-12.955518132950145,
+		146.06735359643812,
+		88.726490458428032
+	],
+	"elongation" : 1.5,
+	"equilibrium" : "polynomial",
+	"form" : "standardX",
+	"inverseaspectratio" : 0.27593818984547458,
+	"triangularity" : 0.40000000000000002
 }
+
diff --git a/src/feltor/input/compass.json b/src/feltor/input/compass.json
new file mode 100644
index 000000000..f450330a9
--- /dev/null
+++ b/src/feltor/input/compass.json
@@ -0,0 +1,69 @@
+{
+    "n"  : 3,
+    "Nx" : 224,
+    "Ny" : 384,
+    "Nz" : 32,
+    "dt" : 2e-3,
+    "compression" : [2,2],
+    "FCI":
+    {
+        "refine": [1,1],
+        "rk4eps": 1e-6,
+        "periodify": true
+    },
+    "inner_loop": 5, 
+    "itstp": 500, 
+    "maxout": 50, 
+    "eps_pol"    : 1e-6,
+    "jumpfactor" : 1,
+    "eps_gamma"  : 1e-6,
+    "stages"     : 3,
+    "eps_time"   : 1e-10,
+    "rtol"       : 1e-5,
+    "mu"          : -0.000272121,
+    "tau"         : 1.0,
+    "beta"        : 0e-4,
+    "nu_perp"     : 2e-3,
+    "perp_diff"   : "hyperviscous",
+    "nu_parallel" : 4e2,
+    "resistivity" : 1e-4,
+    "curvmode"   : "toroidal",
+    "symmetric"  : false,
+    "bc" :
+    {
+        "density" : ["DIR", "DIR"],
+        "velocity": ["NEU", "NEU"],
+        "potential":["DIR", "DIR"]
+    },
+    "box" :
+    {
+        "scaleR" :  [1.3,1.25],
+        "scaleZ" :  [1.4,1.3]
+    },
+    "initne"     : "turbulence",
+    "initphi"    : "zero",
+    "amplitude" : 0.001,
+    "sigma"     : 2.0,
+    "posX"      : 0.6,
+    "posY"      : 0,
+    "sigma_z"   : 0.25,
+    "k_psi"     : 0,
+    "profile":
+    {
+        "amp": 0,
+        "alpha": 0.2
+    },
+    "source" : 
+    {
+        "rate": 2e-3,
+        "type": "influx",
+        "boundary": 0.55,
+        "alpha" : 0.2
+    },
+    "damping":
+    {
+        "rate" : 1e-2,
+        "boundary": 1.1,
+        "alpha": 0.25
+    }
+}
-- 
GitLab


From e8d9233181b13d4b50f2bdac4bb6d8fb03c4aa15 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 21 Aug 2020 23:05:03 +0200
Subject: [PATCH 303/540] Rename normalize program

---
 inc/geometries/{normalize_solovev_t.cu => normalize_params_t.cu} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename inc/geometries/{normalize_solovev_t.cu => normalize_params_t.cu} (100%)

diff --git a/inc/geometries/normalize_solovev_t.cu b/inc/geometries/normalize_params_t.cu
similarity index 100%
rename from inc/geometries/normalize_solovev_t.cu
rename to inc/geometries/normalize_params_t.cu
-- 
GitLab


From 2c1366b3c64dd82db6be1a99c2768ec77debd0b4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 21 Aug 2020 23:19:59 +0200
Subject: [PATCH 304/540] Add TCV source to feltor

---
 src/feltor/feltor.tex |  9 +++++----
 src/feltor/init.h     | 31 +++++++++++++++++--------------
 2 files changed, 22 insertions(+), 18 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 7e3aff433..ffa377b39 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -854,10 +854,11 @@ We can choose the constant influx
 \begin{align} \label{eq:electron_source_influx}
     S_{prof}(R,Z) &= \Theta_{\alpha_p/2}\left( \rho_{p,s} - \frac{\alpha_p}{2} - \rho_p(R,Z) \right) H(Z-Z_X)
 \end{align}
-%or a ringed Gaussian source of the form
-%\begin{align}
-%    S_{prof}(R,Z) &= \exp\left( -\frac{(\psi_p-\psi_{p,0})^2}{\sigma^2}\right)H(Z-Z_X)
-%\end{align}
+or a ringed Gaussian TCV source of the form
+\begin{align}
+    S_{prof}(R,Z) &= \exp\left( -\frac{(\psi_p-\psi_{p,0})^2}{\sigma^2}\right)H(Z-Z_X)
+\end{align}
+with $\psi_{p,0} = \psi_p(1075, -10)$ and $\sigma = 0.0093\psi_{p,0}/0.4$,
 or a Torpex inspired source profile
 \begin{align} \label{eq:electron_source_torpex}
   S_{prof}(R,Z) &=
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 7b229fb4b..9dcafdaf6 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -350,21 +350,24 @@ std::map<std::string, std::function< HVec(
             return source_profile;
         }
     },
-    //{"tcv",
-    //    []( bool& fixed_profile, HVec& ne_profile,
-    //    Geometry& grid, const feltor::Parameters& p,
-    //    const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
-    //    {
-    //        const double psi0 = 0.4, R_0 = 1.075/1e-3, Z_0 = -0.01/1e-3, sigma = 9.3e-3;
-
-    //        fixed_profile = false;
-    //        //ignore ne_profile
-    //        HVec source_profile = dg::pullback(
-    //            dg::compose( dg::Gaussian( R_0));
-
+    {"tcv",
+        []( bool& fixed_profile, HVec& ne_profile,
+        Geometry& grid, const feltor::Parameters& p,
+        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        {
+            const double R_0 = 1075., Z_0 = -10.;
+            const double psip0 = mag.psip()( R_0, Z_0);
+            const double sigma = 9.3e-3*psip0/0.4;
 
-    //    }
-    //},
+            fixed_profile = false;
+            ne_profile = dg::construct<HVec>( detail::profile(grid, p,gp,mag));
+            dg::HVec source_profile = dg::pullback(
+                dg::compose( dg::GaussianX( psip0, sigma, 1.),  mag.psip() ), grid);
+            dg::blas1::pointwiseDot( detail::xpoint_damping(grid,p,gp,mag),
+                   source_profile, source_profile);
+            return source_profile;
+        }
+    },
     {"gaussian",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
-- 
GitLab


From 702fb8298ffbc1f6dee4e29e7525948b678cf9c7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 21 Aug 2020 23:26:31 +0200
Subject: [PATCH 305/540] Replace gp calls in init.h

Now we can remove Parameters
---
 inc/geometries/geometries.h |  1 +
 src/feltor/init.h           | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/inc/geometries/geometries.h b/inc/geometries/geometries.h
index b1159ccea..e2c4e8d3e 100644
--- a/inc/geometries/geometries.h
+++ b/inc/geometries/geometries.h
@@ -24,6 +24,7 @@
 #include "guenther.h"
 #include "toroidal.h"
 #include "polynomial.h"
+#include "make_field.h"
 
 #include "fluxfunctions.h"
 #include "magnetic_field.h"
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 9dcafdaf6..e28a8b497 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -43,7 +43,7 @@ HVec circular_damping( const Geometry& grid,
     if( p.profile_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
     HVec circular = dg::pullback( dg::compose(
-                dg::PolynomialHeaviside( gp.a, gp.a*p.profile_alpha/2., -1),
+                dg::PolynomialHeaviside( mag.params().a(), mag.params().a()*p.profile_alpha/2., -1),
                 Radius( mag.R0(), 0.)), grid);
     return circular;
 }
@@ -54,10 +54,10 @@ HVec xpoint_damping(const Geometry& grid,
     const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
 {
     HVec xpoint_damping = dg::evaluate( dg::one, grid);
-    if( gp.hasXpoint() )
+    if( mag.params().getForm() == dg::geo::form::standardX)
     {
-        double RX = gp.R_0 - 1.1*gp.triangularity*gp.a;
-        double ZX = -1.1*gp.elongation*gp.a;
+        double RX = mag.R0() - 1.1*mag.params().triangularity()*mag.params().a();
+        double ZX = -1.1*mag.params().elongation()*mag.params().a();
         dg::geo::findXpoint( mag.get_psip(), RX, ZX);
         xpoint_damping = dg::pullback(
             dg::geo::ZCutter(ZX), grid);
@@ -162,7 +162,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
             if( p.sigma == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 in straight blob initial condition\n");
-            dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
+            dg::Gaussian init0( mag.R0()+p.posX*mag.params().a(), p.posY*mag.params().a(), p.sigma, p.sigma, p.amp);
             if( p.symmetric)
                 ntilde = dg::pullback( init0, grid);
             else
@@ -194,7 +194,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
             if( p.sigma == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 in straight blob initial condition\n");
-            dg::Gaussian init0( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma, p.sigma, p.amp);
+            dg::Gaussian init0( mag.R0()+p.posX*mag.params().a(), p.posY*mag.params().a(), p.sigma, p.sigma, p.amp);
             if( p.symmetric)
                 ntilde = dg::pullback( init0, grid);
             else
@@ -270,7 +270,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
         {
             if( p.sigma == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 in turbulence on gaussian\n");
-            dg::Gaussian prof( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
+            dg::Gaussian prof( mag.R0()+p.posX*mag.params().a(), p.posY*mag.params().a(), p.sigma,
                 p.sigma, p.nprofamp);
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(
@@ -339,11 +339,11 @@ std::map<std::string, std::function< HVec(
         {
             if( p.sigma == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 for torpex source profile\n");
-            dg::Gaussian prof( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
+            dg::Gaussian prof( mag.R0()+p.posX*mag.params().a(), p.posY*mag.params().a(), p.sigma,
                 p.sigma, p.nprofamp);
             ne_profile = dg::pullback( prof, grid);
             fixed_profile = false;
-            double rhosinm = 0.98 / gp.R_0;
+            double rhosinm = 0.98 / mag.R0();
             double rhosinm2 = rhosinm*rhosinm;
             HVec source_profile = dg::construct<HVec> ( dg::pullback(
                 detail::TorpexSource(0.98/rhosinm, -0.02/rhosinm, 0.0335/rhosinm, 0.05/rhosinm, 565*rhosinm2 ), grid) );
@@ -376,7 +376,7 @@ std::map<std::string, std::function< HVec(
             fixed_profile = false;
             if( p.sigma == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 for gaussian source profile\n");
-            dg::Gaussian prof( gp.R_0+p.posX*gp.a, p.posY*gp.a, p.sigma,
+            dg::Gaussian prof( mag.R0()+p.posX*mag.params().a(), p.posY*mag.params().a(), p.sigma,
                 p.sigma, 1.);
             return dg::pullback( prof, grid);
         }
-- 
GitLab


From 3b34a5cea968b164a8bf5b639fce6231d86173f9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 21 Aug 2020 23:33:41 +0200
Subject: [PATCH 306/540] Remove gp from feltor function calls

---
 src/feltor/feltor.cu     |  8 ++--
 src/feltor/feltor_hpc.cu |  8 ++--
 src/feltor/feltordiag.h  |  5 +--
 src/feltor/init.h        | 82 ++++++++++++++++++++--------------------
 4 files changed, 51 insertions(+), 52 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 6b1b76638..66b33162f 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -74,7 +74,7 @@ int main( int argc, char* argv[])
     HVec damping_profile = dg::evaluate( dg::zero, grid);
     if( p.damping_alpha > 0.)
     {
-        damping_profile = feltor::wall_damping( grid, p, gp, mag);
+        damping_profile = feltor::wall_damping( grid, p, mag);
         double RO=mag.R0(), ZO=0.;
         dg::geo::findOpoint( mag.get_psip(), RO, ZO);
         double psipO = mag.psip()( RO, ZO);
@@ -103,7 +103,7 @@ int main( int argc, char* argv[])
     gradPsip[2] =  result; //zero
     DVec hoo = dg::pullback( dg::geo::Hoo( mag), grid);
     feltor::Variables var = {
-        feltor, p,gp,mag, gradPsip, gradPsip, hoo
+        feltor, p,mag, gradPsip, gradPsip, hoo
     };
 
 
@@ -112,7 +112,7 @@ int main( int argc, char* argv[])
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
     try{
-        y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+        y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,mag );
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
         return -1;
@@ -123,7 +123,7 @@ int main( int argc, char* argv[])
     HVec source_profile;
     try{
         source_profile = feltor::source_profiles.at(p.source_type)(
-            fixed_profile, profile, grid, p, gp, mag);
+            fixed_profile, profile, grid, p,  mag);
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
         return -1;
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 917fd5e1b..c007bea3f 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -186,7 +186,7 @@ int main( int argc, char* argv[])
     HVec damping_profile = dg::evaluate( dg::zero, grid);
     if( p.damping_alpha > 0.)
     {
-        damping_profile = feltor::wall_damping( grid, p, gp, mag);
+        damping_profile = feltor::wall_damping( grid, p, mag);
         double RO=mag.R0(), ZO=0.;
         dg::geo::findOpoint( mag.get_psip(), RO, ZO);
         double psipO = mag.psip()( RO, ZO);
@@ -225,7 +225,7 @@ int main( int argc, char* argv[])
     gradPsip[2] =  resultD; //zero
     DVec hoo = dg::pullback( dg::geo::Hoo( mag), grid);
     feltor::Variables var = {
-        feltor, p, gp, mag, gradPsip, gradPsip, hoo
+        feltor, p, mag, gradPsip, gradPsip, hoo
     };
     // the vector ids
     std::map<std::string, int> id3d, id4d, restart_ids;
@@ -239,7 +239,7 @@ int main( int argc, char* argv[])
     if( argc == 4)
     {
         try{
-            y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,gp,mag );
+            y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,mag );
         }catch ( std::out_of_range& error){
             MPI_OUT std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)" << std::endl;
 #ifdef FELTOR_MPI
@@ -266,7 +266,7 @@ int main( int argc, char* argv[])
         bool fixed_profile;
         HVec profile, source_profile;
         source_profile = feltor::source_profiles.at(p.source_type)(
-            fixed_profile, profile, grid, p, gp, mag);
+            fixed_profile, profile, grid, p, mag);
         feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
             p.source_rate, dg::construct<DVec>(source_profile),
             p.damping_rate, dg::construct<DVec>(damping_profile)
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 733efa052..75adf9e82 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -150,7 +150,6 @@ void jacobian(
 struct Variables{
     feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec>& f;
     feltor::Parameters p;
-    dg::geo::solovev::Parameters gp;
     dg::geo::TokamakMagneticField mag;
     std::array<DVec, 3> gradPsip;
     std::array<DVec, 3> tmp;
@@ -205,7 +204,7 @@ std::vector<Record_static> diagnostics3d_static_list = {
             result = dg::evaluate( dg::zero, grid);
             bool fixed_profile;
             HVec source = feltor::source_profiles.at(v.p.source_type)(
-                fixed_profile, result, grid, v.p, v.gp, v.mag);
+                fixed_profile, result, grid, v.p, v.mag);
         }
     },
     { "Source", "Source region",
@@ -213,7 +212,7 @@ std::vector<Record_static> diagnostics3d_static_list = {
             bool fixed_profile;
             HVec profile;
             result = feltor::source_profiles.at(v.p.source_type)(
-                fixed_profile, profile, grid, v.p, v.gp, v.mag);
+                fixed_profile, profile, grid, v.p, v.mag);
         }
     },
     { "xc", "x-coordinate in Cartesian coordinate system",
diff --git a/src/feltor/init.h b/src/feltor/init.h
index e28a8b497..2988e622a 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -38,7 +38,7 @@ struct Radius : public dg::geo::aCylindricalFunctor<Radius>
 };
 HVec circular_damping( const Geometry& grid,
     const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+    const dg::geo::TokamakMagneticField& mag )
 {
     if( p.profile_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
@@ -51,7 +51,7 @@ HVec circular_damping( const Geometry& grid,
 
 HVec xpoint_damping(const Geometry& grid,
     const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+    const dg::geo::TokamakMagneticField& mag )
 {
     HVec xpoint_damping = dg::evaluate( dg::one, grid);
     if( mag.params().getForm() == dg::geo::form::standardX)
@@ -66,31 +66,31 @@ HVec xpoint_damping(const Geometry& grid,
 }
 HVec profile_damping(const Geometry& grid,
     const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+    const dg::geo::TokamakMagneticField& mag )
 {
     if( p.profile_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
     HVec profile_damping = dg::pullback( dg::compose(dg::PolynomialHeaviside(
         1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)), grid);
-    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag),
+    dg::blas1::pointwiseDot( xpoint_damping(grid,p,mag),
         profile_damping, profile_damping);
     return profile_damping;
 }
 HVec profile(const Geometry& grid,
     const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag ){
+    const dg::geo::TokamakMagneticField& mag ){
     double RO=mag.R0(), ZO=0.;
     dg::geo::findOpoint( mag.get_psip(), RO, ZO);
     double psipO = mag.psip()( RO, ZO);
     //First the profile and the source (on the host since we want to output those)
     HVec profile = dg::pullback( dg::compose(dg::LinearX(
         p.nprofamp/psipO, 0.), mag.psip()), grid);
-    dg::blas1::pointwiseDot( profile_damping(grid,p,gp,mag), profile, profile);
+    dg::blas1::pointwiseDot( profile_damping(grid,p,mag), profile, profile);
     return profile;
 }
 HVec source_damping(const Geometry& grid,
     const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+    const dg::geo::TokamakMagneticField& mag )
 {
     if( p.source_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: source alpha must not be 0\n");
@@ -98,7 +98,7 @@ HVec source_damping(const Geometry& grid,
         dg::compose(dg::PolynomialHeaviside(
             p.source_boundary-p.source_alpha/2.,
         p.source_alpha/2., -1 ), dg::geo::RhoP(mag)), grid);
-    dg::blas1::pointwiseDot( xpoint_damping(grid,p,gp,mag),
+    dg::blas1::pointwiseDot( xpoint_damping(grid,p,mag),
            source_damping, source_damping);
     return source_damping;
 }
@@ -108,7 +108,7 @@ void init_ni(
     std::array<std::array<DVec,2>,2>& y0,
     Explicit<Geometry, IDMatrix, DMatrix, DVec>& feltor,
     const Geometry& grid, const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+    dg::geo::TokamakMagneticField& mag )
 {
 #ifdef FELTOR_MPI
     int rank;
@@ -129,7 +129,7 @@ void init_ni(
 //for wall shadow
 HVec wall_damping(const Geometry& grid,
     const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, const dg::geo::TokamakMagneticField& mag )
+    const dg::geo::TokamakMagneticField& mag )
 {
     if( p.source_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: damping alpha must not be 0\n");
@@ -146,16 +146,16 @@ HVec wall_damping(const Geometry& grid,
 std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
     const Geometry& grid, const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+    dg::geo::TokamakMagneticField& mag )
 > > initial_conditions =
 {
     { "blob",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
             const Geometry& grid, const feltor::Parameters& p,
-            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+            dg::geo::TokamakMagneticField& mag )
         {
             std::array<std::array<DVec,2>,2> y0;
-            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
             if( p.sigma_z == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in straight blob initial condition\n");
@@ -174,7 +174,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 3);
             }
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-            detail::init_ni( y0, f,grid,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -184,10 +184,10 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     { "straight blob",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
             const Geometry& grid, const feltor::Parameters& p,
-            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+            dg::geo::TokamakMagneticField& mag )
         {
             std::array<std::array<DVec,2>,2> y0;
-            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
             if( p.sigma_z == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in straight blob initial condition\n");
@@ -206,7 +206,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-            detail::init_ni( y0, f,grid,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -216,10 +216,10 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     { "turbulence",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
             const Geometry& grid, const feltor::Parameters& p,
-            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+            dg::geo::TokamakMagneticField& mag )
         {
             std::array<std::array<DVec,2>,2> y0;
-            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
             if( p.sigma_z == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in turbulence initial condition\n");
@@ -235,9 +235,9 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 //For turbulence the exact evaluate is maybe not so important (thus takes less memory)
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
-            dg::blas1::pointwiseDot( detail::profile_damping(grid,p,gp,mag), ntilde, ntilde);
+            dg::blas1::pointwiseDot( detail::profile_damping(grid,p,mag), ntilde, ntilde);
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-            detail::init_ni( y0, f,grid,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -247,16 +247,16 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     { "zonal",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
             const Geometry& grid, const feltor::Parameters& p,
-            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+            dg::geo::TokamakMagneticField& mag )
         {
             std::array<std::array<DVec,2>,2> y0;
-            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,gp,mag));
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,mag));
             HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::SinX sinX( p.amp, 0., p.k_psi);
             ntilde = dg::pullback( dg::compose( sinX, mag.psip()), grid);
-            dg::blas1::pointwiseDot( detail::profile_damping(grid,p,gp,mag), ntilde, ntilde);
+            dg::blas1::pointwiseDot( detail::profile_damping(grid,p,mag), ntilde, ntilde);
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
-            detail::init_ni( y0, f,grid,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -266,7 +266,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     { "turbulence on gaussian",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
             const Geometry& grid, const feltor::Parameters& p,
-            const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+            dg::geo::TokamakMagneticField& mag )
         {
             if( p.sigma == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 in turbulence on gaussian\n");
@@ -292,9 +292,9 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
                 ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
             }
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0] );
-            dg::blas1::pointwiseDot( dg::construct<DVec>(detail::circular_damping(grid,p,gp,mag)),
+            dg::blas1::pointwiseDot( dg::construct<DVec>(detail::circular_damping(grid,p,mag)),
                 y0[0][0], y0[0][0] );
-            detail::init_ni( y0, f,grid,p,gp,mag);
+            detail::init_ni( y0, f,grid,p,mag);
 
             dg::blas1::copy( 0., y0[1][0]); //set we = 0
             dg::blas1::copy( 0., y0[1][1]); //set Wi = 0
@@ -307,35 +307,35 @@ std::map<std::string, std::function< HVec(
     bool& fixed_profile, //indicate whether a profile should be forced (yes or no)
     HVec& ne_profile,    // if fixed_profile is yes you need to construct something here, if no then you can ignore the parameter; if you construct something it will show in the output file
     Geometry& grid, const feltor::Parameters& p,
-    const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+    dg::geo::TokamakMagneticField& mag )
 > > source_profiles =
 {
     {"profile",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
-        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = true;
-            ne_profile = dg::construct<HVec>( detail::profile(grid, p,gp,mag));
-            HVec source_profile = dg::construct<HVec> ( detail::source_damping( grid, p,gp,mag));
+            ne_profile = dg::construct<HVec>( detail::profile(grid, p,mag));
+            HVec source_profile = dg::construct<HVec> ( detail::source_damping( grid, p,mag));
             return source_profile;
         }
     },
     {"influx",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
-        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = false;
-            ne_profile = dg::construct<HVec>( detail::profile(grid, p,gp,mag));
-            HVec source_profile = dg::construct<HVec> ( detail::source_damping( grid, p,gp,mag));
+            ne_profile = dg::construct<HVec>( detail::profile(grid, p,mag));
+            HVec source_profile = dg::construct<HVec> ( detail::source_damping( grid, p,mag));
             return source_profile;
         }
     },
     {"torpex",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
-        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        dg::geo::TokamakMagneticField& mag )
         {
             if( p.sigma == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma must not be 0 for torpex source profile\n");
@@ -353,17 +353,17 @@ std::map<std::string, std::function< HVec(
     {"tcv",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
-        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        dg::geo::TokamakMagneticField& mag )
         {
             const double R_0 = 1075., Z_0 = -10.;
             const double psip0 = mag.psip()( R_0, Z_0);
             const double sigma = 9.3e-3*psip0/0.4;
 
             fixed_profile = false;
-            ne_profile = dg::construct<HVec>( detail::profile(grid, p,gp,mag));
-            dg::HVec source_profile = dg::pullback(
+            ne_profile = dg::construct<HVec>( detail::profile(grid, p,mag));
+            HVec source_profile = dg::pullback(
                 dg::compose( dg::GaussianX( psip0, sigma, 1.),  mag.psip() ), grid);
-            dg::blas1::pointwiseDot( detail::xpoint_damping(grid,p,gp,mag),
+            dg::blas1::pointwiseDot( detail::xpoint_damping(grid,p,mag),
                    source_profile, source_profile);
             return source_profile;
         }
@@ -371,7 +371,7 @@ std::map<std::string, std::function< HVec(
     {"gaussian",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
-        const dg::geo::solovev::Parameters& gp, dg::geo::TokamakMagneticField& mag )
+        dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = false;
             if( p.sigma == 0)
-- 
GitLab


From dfe154e5f90c709d3f7bff1df145e828b50c3ea0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 21 Aug 2020 23:40:56 +0200
Subject: [PATCH 307/540] Removed gp from feltor and feltor_hpc

---
 src/feltor/feltor.cu     | 22 +++++++++++++---------
 src/feltor/feltor_hpc.cu | 22 +++++++++++++---------
 2 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 66b33162f..d499c7e44 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -58,18 +58,17 @@ int main( int argc, char* argv[])
     }
 
     const feltor::Parameters p(js);
-    const dg::geo::solovev::Parameters gp(gs);
     p.display( std::cout);
-    gp.display( std::cout);
+    std::cout << gs.toStyledString() << std::endl;
+    dg::geo::TokamakMagneticField mag = dg::geo::createMagneticField(gs, file::error::is_warning);
     /////////////////////////////////////////////////////////////////////////
-    double Rmin=gp.R_0-p.boxscaleRm*gp.a;
-    double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
-    double Rmax=gp.R_0+p.boxscaleRp*gp.a;
-    double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    double Rmin=mag.R0()-p.boxscaleRm*mag.params().a();
+    double Zmin=-p.boxscaleZm*mag.params().a()*mag.params().elongation();
+    double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
+    double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
     //Make grid
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     //Wall damping has to be constructed before modification (!)
     HVec damping_profile = dg::evaluate( dg::zero, grid);
     if( p.damping_alpha > 0.)
@@ -80,8 +79,13 @@ int main( int argc, char* argv[])
         double psipO = mag.psip()( RO, ZO);
         double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
         double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2.,
-                fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
+        std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
+        Json::Value jsmod;
+        jsmod["modifier"] = "heaviside";
+        jsmod["psi0"] = damping_psi0p + damping_alphap/2.;
+        jsmod["alpha"] = fabs( damping_alphap/2.);
+        jsmod["sign"] = ((psipO>0)-(psipO<0));
+        mag = dg::geo::createModifiedField(gs, jsmod, file::error::is_warning);
     }
     if( p.periodify)
         mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index c007bea3f..d937372c4 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -144,10 +144,10 @@ int main( int argc, char* argv[])
         }
     }
     const feltor::Parameters p( js);
-    const dg::geo::solovev::Parameters gp(gs);
     MPI_OUT p.display( std::cout);
-    MPI_OUT gp.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
+    MPI_OUT std::cout << geom << std::endl;
+    dg::geo::TokamakMagneticField mag = dg::geo::createMagneticField(gs, file::error::is_warning);
 #ifdef FELTOR_MPI
     if( np[2] >= (int)p.Nz)
     {
@@ -157,10 +157,10 @@ int main( int argc, char* argv[])
     }
 #endif //FELTOR_MPI
     ////////////////////////////////set up computations///////////////////////////
-    double Rmin=gp.R_0-p.boxscaleRm*gp.a;
-    double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
-    double Rmax=gp.R_0+p.boxscaleRp*gp.a;
-    double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    double Rmin=mag.R0()-p.boxscaleRm*mag.params().a();
+    double Zmin=-p.boxscaleZm*mag.params().a()*mag.params().elongation();
+    double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
+    double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
     //Make grids
     Geometry grid( Rmin, Rmax, Zmin, Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER
@@ -181,7 +181,6 @@ int main( int argc, char* argv[])
     unsigned local_size2d = g2d_out_ptr->size();
 #endif
 
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     //Wall damping has to be constructed before modification (!)
     HVec damping_profile = dg::evaluate( dg::zero, grid);
     if( p.damping_alpha > 0.)
@@ -192,8 +191,13 @@ int main( int argc, char* argv[])
         double psipO = mag.psip()( RO, ZO);
         double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
         double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2.,
-                fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
+        std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
+        Json::Value jsmod;
+        jsmod["modifier"] = "heaviside";
+        jsmod["psi0"] = damping_psi0p + damping_alphap/2.;
+        jsmod["alpha"] = fabs( damping_alphap/2.);
+        jsmod["sign"] = ((psipO>0)-(psipO<0));
+        mag = dg::geo::createModifiedField(gs, jsmod, file::error::is_warning);
     }
     if( p.periodify)
         mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
-- 
GitLab


From 95290d484865224ab5b754f2d96a3f4261dbce10 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 26 Aug 2020 16:46:56 +0200
Subject: [PATCH 308/540] Add integrate between coarse grid function

does not change fsa but makes ta nicer
---
 inc/geometries/fieldaligned.h | 34 +++++++++++++++++++++++++++++++-
 src/feltor/feltordiag.cu      | 37 +++++++++++++++++++++++++++++++++--
 2 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 435d73d35..1dae3afae 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -360,9 +360,17 @@ struct Fieldaligned
     * @param coarse the coarse input vector
     *
     * @return the input interpolated onto the grid given in the constructor
-    * @note the interpolation weights are taken in the phi distance not the s-distancewhich makes the interpolation linear in phi
+    * @note the interpolation weights are taken in the phi distance not the s-distance, which makes the interpolation linear in phi
     */
     container interpolate_from_coarse_grid( const ProductGeometry& grid_coarse, const container& coarse);
+    /**
+    * @brief Integrate a 2d function on the fine grid \f[ \frac{1}{\Delta\varphi} \int_{-\Delta\varphi}^{\Delta\varphi}d \varphi w(\varphi) f(R(\varphi), Z(\varphi) \f]
+    *
+    * @param grid_coarse The coarse grid (\c coarse_grid.Nz() must integer divide \c Nz from input grid) The x and y dimensions must be equal
+    * @param coarse the 2d input vector
+    * @param out the integral (2d vector)
+    */
+    void integrate_between_coarse_grid( const ProductGeometry& grid_coarse, const container& coarse, container& out );
     private:
     void ePlus( enum whichMatrix which, const container& in, container& out);
     void eMinus(enum whichMatrix which, const container& in, container& out);
@@ -589,6 +597,30 @@ container Fieldaligned<G, I,container>::interpolate_from_coarse_grid( const G& g
         }
     return out;
 }
+template<class G, class I, class container>
+void Fieldaligned<G, I,container>::integrate_between_coarse_grid( const G& grid, const container& in, container& out)
+{
+    assert( m_g->Nz() % grid.Nz() == 0);
+    unsigned Nz_coarse = grid.Nz(), Nz = m_g->Nz();
+    unsigned cphi = Nz / Nz_coarse;
+
+    out = in;
+    container helperP( in), helperM(in), tempP(in), tempM(in);
+
+    //1. Apply plus and minus T and sum up
+    for( int j=1; j<(int)cphi; j++)
+    {
+        //!!! The value of f at the plus plane is I^- of the current plane
+        dg::blas2::symv( m_minus, helperP, tempP);
+        dg::blas1::axpby( (double)(cphi-j)/(double)cphi, tempP, 1., out  );
+        helperP.swap(tempP);
+        //!!! The value of f at the minus plane is I^+ of the current plane
+        dg::blas2::symv( m_plus, helperM, tempM);
+        dg::blas1::axpby( (double)(cphi-j)/(double)cphi, tempM, 1., out  );
+        helperM.swap(tempM);
+    }
+    dg::blas1::scal( out, 1./(double)cphi);
+}
 
 template<class G, class I, class container>
 void Fieldaligned<G, I, container >::operator()(enum whichMatrix which, const container& f, container& fe)
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index c6a6735c5..af39c62b3 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -85,9 +85,15 @@ int main( int argc, char* argv[])
     const double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
     const double Rmax=gp.R_0+p.boxscaleRp*gp.a;
     const double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    const unsigned FACTOR=10;
 
-    dg::Grid2d   g2d_out( Rmin,Rmax, Zmin,Zmax,
+    dg::Grid2d g2d_out( Rmin,Rmax, Zmin,Zmax,
         p.n_out, p.Nx_out, p.Ny_out, p.bcxN, p.bcyN);
+    /////////////////////////////////////////////////////////////////////////
+    Geometry g3d( Rmin, Rmax, Zmin, Zmax, 0., 2.*M_PI,
+        p.n_out, p.Nx_out, p.Ny_out, p.Nz, p.bcxN, p.bcyN, dg::PER);
+    Geometry g3d_fine( Rmin, Rmax, Zmin, Zmax, 0., 2.*M_PI,
+        p.n_out, p.Nx_out, p.Ny_out, FACTOR*p.Nz, p.bcxN, p.bcyN, dg::PER);
 
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     double RO=mag.R0(), ZO=0.;
@@ -105,12 +111,19 @@ int main( int argc, char* argv[])
 
     dg::HVec transferH2d = dg::evaluate(dg::zero,g2d_out);
     dg::HVec t2d_mp = dg::evaluate(dg::zero,g2d_out);
+    std::cout << "Construct Fieldaligned derivative ... \n";
+
+    auto bhat = dg::geo::createBHat( mag);
+    dg::geo::Fieldaligned<Geometry, IDMatrix, DVec> fieldaligned(
+        bhat, g3d_fine, dg::NEU, dg::NEU, dg::geo::NoLimiter(), //let's take NEU bc because N is not homogeneous
+        p.rk4eps, 5, 5);
 
 
     ///--------------- Construct X-point grid ---------------------//
 
 
     //std::cout << "Type X-point grid resolution (n(3), Npsi(32), Neta(640)) Must be divisible by 8\n";
+    //we use so many Neta so that we get close to the X-point
     std::cout << "Using default X-point grid resolution (n(3), Npsi(64), Neta(640))\n";
     unsigned npsi = 3, Npsi = 64, Neta = 640;//set number of psivalues (NPsi % 8 == 0)
     //std::cin >> npsi >> Npsi >> Neta;
@@ -243,6 +256,15 @@ int main( int argc, char* argv[])
         err = nc_put_att_text( ncid_out, id2d[name], "long_name", long_name.size(),
             long_name.data());
 
+        name = record_name + "_cta2d";
+        if( name[0] == 'j')
+            name[1] = 's';
+        long_name = record.long_name + " (Convoluted toroidal average on 2d plane.)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
+            &id2d[name]);
+        err = nc_put_att_text( ncid_out, id2d[name], "long_name", long_name.size(),
+            long_name.data());
+
         name = record_name + "_fsa2d";
         long_name = record.long_name + " (Flux surface average interpolated to 2d plane.)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
@@ -304,7 +326,7 @@ int main( int argc, char* argv[])
         }
         err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
         err = nc_inq_dimlen( ncid, timeID, &steps);
-        //steps = 3;
+        steps = 3;
         for( unsigned i=0; i<steps; i++)//timestepping
         {
             if( j > 1 && i == 0)
@@ -342,6 +364,10 @@ int main( int argc, char* argv[])
                 {
                     err = nc_get_vara_double( ncid, dataID,
                         start2d, count2d, transferH2d.data());
+                    DVec transferD2d = transferH2d;
+                    fieldaligned.integrate_between_coarse_grid( g3d, transferD2d, transferD2d);
+                    transferH2d = transferD2d;
+                    t2d_mp = transferH2d; //save toroidal average
                     //2. Compute fsa and output fsa
                     dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
                     dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
@@ -357,11 +383,18 @@ int main( int argc, char* argv[])
                 {
                     dg::blas1::scal( fsa1d, 0.);
                     dg::blas1::scal( transferH2d, 0.);
+                    dg::blas1::scal( t2d_mp, 0.);
                 }
                 err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_fsa"),
                     start1d_out, count1d, fsa1d.data());
                 err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fsa2d"),
                     start2d_out, count2d, transferH2d.data() );
+                if( record_name[0] == 'j')
+                    record_name[1] = 's';
+                err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_cta2d"),
+                    start2d_out, count2d, t2d_mp.data() ); //this is still js...
+                if( record_name[0] == 'j')
+                    record_name[1] = 'v';
                 //4. Read 2d variable and compute fluctuations
                 available = true;
                 try{
-- 
GitLab


From a8e06e2ab3bc8c6c9925ce13db0b8b3b1684b159 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 26 Aug 2020 18:53:39 +0200
Subject: [PATCH 309/540] Add standard deviation to feltordiag

---
 src/feltor/feltordiag.cu | 34 +++++++++++++++++++++++++---------
 1 file changed, 25 insertions(+), 9 deletions(-)

diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index af39c62b3..e64d511aa 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -257,8 +257,6 @@ int main( int argc, char* argv[])
             long_name.data());
 
         name = record_name + "_cta2d";
-        if( name[0] == 'j')
-            name[1] = 's';
         long_name = record.long_name + " (Convoluted toroidal average on 2d plane.)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 3, dim_ids,
             &id2d[name]);
@@ -274,6 +272,12 @@ int main( int argc, char* argv[])
 
         name = record_name + "_fsa";
         long_name = record.long_name + " (Flux surface average.)";
+        err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
+            &id1d[name]);
+        err = nc_put_att_text( ncid_out, id1d[name], "long_name", long_name.size(),
+            long_name.data());
+        name = record_name + "_std_fsa";
+        long_name = record.long_name + " (Flux surface average standard deviation on outboard midplane.)";
         err = nc_def_var( ncid_out, name.data(), NC_DOUBLE, 2, dim_ids1d,
             &id1d[name]);
         err = nc_put_att_text( ncid_out, id1d[name], "long_name", long_name.size(),
@@ -373,9 +377,11 @@ int main( int argc, char* argv[])
                     dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
                     poloidal_average( transferH2dX, t1d, false); //average over eta
                     dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
-                    dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
-                    if( record_name[0] == 'j')
-                        dg::blas1::pointwiseDot( fsa1d, dvdpsip, fsa1d );
+                    dg::blas1::copy( 0., fsa1d); //get rid of previous nan in fsa1d (nasty bug)
+                    if( record_name[0] != 'j')
+                        dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
+                    else
+                        dg::blas1::copy( t1d, fsa1d);
                     //3. Interpolate fsa on 2d plane : <f>
                     dg::blas2::gemv(fsa2rzmatrix, fsa1d, transferH2d); //fsa on RZ grid
                 }
@@ -390,11 +396,9 @@ int main( int argc, char* argv[])
                 err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_fsa2d"),
                     start2d_out, count2d, transferH2d.data() );
                 if( record_name[0] == 'j')
-                    record_name[1] = 's';
+                    dg::blas1::pointwiseDot( t2d_mp, dvdpsip2d, t2d_mp );//make it jv
                 err = nc_put_vara_double( ncid_out, id2d.at(record_name+"_cta2d"),
-                    start2d_out, count2d, t2d_mp.data() ); //this is still js...
-                if( record_name[0] == 'j')
-                    record_name[1] = 'v';
+                    start2d_out, count2d, t2d_mp.data() );
                 //4. Read 2d variable and compute fluctuations
                 available = true;
                 try{
@@ -462,6 +466,16 @@ int main( int argc, char* argv[])
                     }
                     err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
                         start2d_out, count2d, &result );
+                    //7. Compute midplane fluctuation amplitudes
+                    dg::blas1::pointwiseDot( transferH2d, transferH2d, transferH2d);
+                    dg::blas2::symv( grid2gridX2d, transferH2d, transferH2dX); //interpolate onto X-point grid
+                    dg::blas1::pointwiseDot( transferH2dX, volX2d, transferH2dX); //multiply by sqrt(g)
+                    poloidal_average( transferH2dX, t1d, false); //average over eta
+                    dg::blas1::scal( t1d, 4*M_PI*M_PI*f0); //
+                    dg::blas1::pointwiseDivide( t1d, dvdpsip, fsa1d );
+                    dg::blas1::transform ( fsa1d, fsa1d, dg::SQRT<double>() );
+                    err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_std_fsa"),
+                        start1d_out, count1d, fsa1d.data());
                 }
                 else
                 {
@@ -476,6 +490,8 @@ int main( int argc, char* argv[])
                         start2d_out, count2d, &result );
                     err = nc_put_vara_double( ncid_out, id0d.at(record_name+"_ifs_norm"),
                         start2d_out, count2d, &result );
+                    err = nc_put_vara_double( ncid_out, id1d.at(record_name+"_std_fsa"),
+                        start1d_out, count1d, transfer1d.data());
                 }
 
             }
-- 
GitLab


From 95e2cf070ad450e6caf4b9ad908cfedf2049eba8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 27 Aug 2020 09:51:41 +0200
Subject: [PATCH 310/540] Fix: remove stepsize limitation in feltordiag

---
 src/feltor/feltordiag.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index e64d511aa..62b010ccf 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -330,7 +330,7 @@ int main( int argc, char* argv[])
         }
         err = nc_inq_unlimdim( ncid, &timeID); //Attention: Finds first unlimited dim, which hopefully is time and not energy_time
         err = nc_inq_dimlen( ncid, timeID, &steps);
-        steps = 3;
+        //steps = 3;
         for( unsigned i=0; i<steps; i++)//timestepping
         {
             if( j > 1 && i == 0)
-- 
GitLab


From 97613243257bdab7834fc2022d826d523b4621ea Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 27 Aug 2020 11:48:53 +0200
Subject: [PATCH 311/540] Document additional diagnostics cta2d and std_fsa

---
 doc/related_pages/newcommands.tex |   3 +
 src/feltor/feltor.tex             | 112 ++++++++++++++++++++++++++++++
 2 files changed, 115 insertions(+)

diff --git a/doc/related_pages/newcommands.tex b/doc/related_pages/newcommands.tex
index 1cc99ef18..e5fa5ee27 100644
--- a/doc/related_pages/newcommands.tex
+++ b/doc/related_pages/newcommands.tex
@@ -60,6 +60,9 @@
 \newcommand{\Tp}{\mathcal T^+_{\Delta\varphi}}
 \newcommand{\Tm}{\mathcal T^-_{\Delta\varphi}}
 \newcommand{\Tpm}{\mathcal T^\pm_{\Delta\varphi}}
+\newcommand{\Tdp}{\mathcal T^+_{\delta\varphi}}
+\newcommand{\Tdm}{\mathcal T^-_{\delta\varphi}}
+\newcommand{\Tdpm}{\mathcal T^\pm_{\delta\varphi}}
 %%%%%%%%Some useful abbreviations %%%%%%%%%%%%%%%%
 \def\feltor{{\textsc{Feltor }}}
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index ffa377b39..70e8746c0 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -392,6 +392,7 @@ K_{\vn\times\bhat}^\varphi &= \frac{R_0}{R^2B^2}\left(
 \label{}
 \end{align}
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsection{Flux surface averaging and safety factor}
 \subsubsection{Preliminary}
 Recall that the {\bf Dirac delta-function} has the property (in any dimension):
@@ -448,6 +449,7 @@ Notice that numerically we can integrate in flux-aligned coordinates by generati
 grid and pulling back (interpolating) the relevant fields to this grid. This is the second method
 to numerically compute area integrals.
 
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Flux surface average}
 
 
@@ -549,6 +551,114 @@ and refine the stepsize until machine-precision is reached.
 Notice that the safety factor diverges on the last closed flux
 surface whereas Eq.~\eqref{eq:total_flux}
 remains finite due to the $\vn\psi_p$ factor.
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\subsubsection{Toroidal averages}
+Here, we comment on the $\varphi$ average that is part of the flux-surface average Eq.~\eqref{eq:fsa_vol}.
+One simple approach is
+quadrature of the form
+\begin{align}\label{eq:toroidal_summation}
+    \bar f = \frac{1}{N} \sum_{i=0}^{N-1} f_i (R,Z)
+\end{align}
+where $N=32$ in most of our simulations and $f_i$ is the $i$-th toroidal plane.
+Since the boundary conditions in $\varphi$ are periodic this amounts to the trapezoidal rule.
+A low number of toroidal planes is sufficient in simulations when we use the toroidal field
+approximation in combination with the
+flux-coordinate independent (FCI) approach for the parallel derivatives.
+However, since the actual $\varphi$ direction is
+under-resolved the
+integration gives a wrong answer to the actual $\varphi$ average (seen in 2d plots as little humps).
+This is because the resulting structures are predominantly field aligned and not toroidally symmetric.
+
+In order to improve the toroidal average we now have the following idea:
+if we, before we do the $\varphi$ integration,
+interpolate the function to integrate onto a large number of toroidal
+planes then the result should
+be more accurate than before.
+In other words we interpolate the function given on the coarse $\varphi$ simulation grid
+onto a hypothetic fine $\varphi$ grid along the magnetic field lines
+and only then compute the $\varphi$ average.
+
+Let us divide the $\varphi$ direction between two original planes into $N_\varphi+1$ (a large number) equidistant planes
+of distance $\delta \varphi$ and integrate the magnetic field $\vec B$ in between.
+\begin{subequations}
+\begin{align}
+    \frac{\d R}{\d\varphi}&= \frac{B^R}{B^\varphi},\\ %\frac{R}{I}\frac{\partial\psi}{\partial Z},\\
+    \frac{\d Z}{\d\varphi}&=\frac{B^Z}{B^\varphi},\\%-\frac{R}{I}\frac{\partial\psi}{\partial R}.
+\end{align}
+\label{eq:fieldline}
+\end{subequations}
+We integrate Eqs.~\eqref{eq:fieldline} from $\varphi=0$ to $\varphi=\pm \Delta \varphi$
+with initial condition
+\begin{align}
+    (R(0), Z(0) ) = (R, Z).
+    \label{}
+\end{align}
+Let us characterize the solution $(R(\pm \delta \varphi), Z(\pm \delta \varphi))$ to Eqs.~\eqref{eq:fieldline} as the flow generated by $\vec B/B^\varphi$
+\begin{align}
+    \Tdpm\vec z \equiv \Tdpm[R, Z, \varphi]:= ( R(\pm \delta\varphi), Z( \pm \delta\varphi), \varphi\pm\delta \varphi),
+    \label{}
+\end{align}
+Obviously we have $\Tdm\circ\Tdp = 1$, but $\Tdpm$ is not necessarily unitary since $\vec B/B^\varphi$ is in general
+not divergence free.
+We are now able to extend the function $f$ given on the coarse $\varphi$ grid unto the fine $\varphi$ grid via
+\begin{align}
+    f(R,Z,\varphi_0+j\delta\varphi) = {\Tdm}^j f(R,Z,\varphi_0)\\
+    f(R,Z,\varphi_0-j\delta \varphi) = {\Tdp}^j f(R,Z,\varphi_0)
+\end{align}
+This gives simple 0-th order extrapolation of our function.
+Let us call $f_i := f(R,Z,\varphi_i)$ the $i$-th toroidal plane and $N_\varphi$ even. Then
+we have the following integration, where we consider the original toroidal planes as cell-centered
+\begin{align}
+    \RA{f}_\varphi &= \frac{1}{(N_\varphi+1) N} \left[\left(
+    {\Tdp}^{N_\varphi/2} f_0 + ... + \Tdp f_0 + f_0 + \Tdm f_0 ... + {\Tdm}^{N_\varphi/2} f_0\right)\right. \nonumber\\
+    &\left. +\left( {\Tdp}^{N_\varphi/2} f_1 + ... + \Tdp f_1 + f_1 + \Tdm f_1 ... + {\Tdm}^{N_\varphi/2} f_1\right)  + ... \right] \nonumber\\
+    &= \frac{1}{N (N_\varphi+1)} \sum_{i=0}^{N-1} \left[f_i + \sum_{j=1}^{N_\varphi/2} \left( {\Tdm}^j f_i + {\Tdp}^jf_i\right)\right] \nonumber\\
+    &=
+    \frac{1}{N_\varphi+1} \left[ \sum_{j=0}^{N_\varphi/2}  {\Tdm}^j \left(\frac{1}{N}\sum_{i=0}^{N} f_i\right)
+    +
+    \sum_{j=1}^{N_\varphi/2}  {\Tdp}^j \left(\frac{1}{N}\sum_{i=0}^{N} f_i\right)\right]
+    \nonumber\\
+    &=
+    \frac{1}{N_\varphi+1} \left[ \sum_{j=0}^{N_\varphi/2}  {\Tdm}^j \bar f(R,Z)
+    +
+    \sum_{j=1}^{N_\varphi/2}  {\Tdp}^j \bar f(R,Z)\right]
+\end{align}
+Here, we used that the push-forward operator $\Tdm$ is linear that is $\Tdm f_0 + \Tdm f_1 = \Tdm (f_0+f_1)$
+and recover the simple toroidal summation $\bar f$ Eq.~\eqref{eq:toroidal_summation}.
+Now, we can see that in the limit $N_\varphi \rightarrow\infty$ the discrete sum represents the integral
+of the form
+\begin{align}
+    \RA{f}_\varphi(R,Z) = \frac{1}{\Delta\varphi}\int_{-\Delta\varphi/2}^{\Delta\varphi/2}\d\varphi \bar f(R(\varphi),Z(\varphi))
+\end{align}
+A consistency test of this approach is to simply use $\vec B = e_\varphi$. Then
+$\Tdpm = 1$ and we recover the original integration $\RA{f}_\varphi= \bar f$.
+Now, instead of doing a 0-th order interpolation let us try a linear interpolation along field-lines in between planes that is ( assuming $N_\varphi$ toroidal planes)
+\begin{align}
+    f(R,Z,\varphi_i + j\delta\varphi) = \left(1-\frac{j}{N_\varphi}\right){\Tdm}^j f_i + \frac{j}{N_\varphi} {\Tdp}^{N_\varphi -j}f_{i+1}
+\end{align}
+\begin{align}
+    \RA{ f}_\varphi &= \frac{1}{N_\varphi N} (( f_0 + (1-\alpha_1)\Tdm f_0 + \alpha_1 (\Tdp)^{N_\varphi -1} f_1 + (1-\alpha_2)(\Tdm)^2 f_0 + \alpha_2 (\Tdp)^{N_\varphi-2} f_1 ...  )+ (f_1 + ...) + ...)\nonumber\\
+    &= \frac{1}{ N_\varphi} \sum_{j=0}^{N_\varphi-1}  (1-\alpha_j)(\Tdm)^j \bar f+\alpha_j (\Tdp)^{N_\varphi -j} \bar f
+    = \frac{1}{\Delta\varphi}\int_{-\Delta\varphi}^{\Delta\varphi}\d\varphi w(\varphi) \bar f (R(\varphi),Z(\varphi))
+    \label{eq:cta}
+\end{align}
+with $\alpha_j = \frac{j}{N_\varphi}$ and $w(\varphi)$ a linear weight function (pyramid shape) with $\int_{-\Delta\varphi}^{\Delta\varphi} w(\varphi) = \Delta\varphi$. Taking $\Tdpm=1$ again leads to the old result.
+
+
+Now, an interesting question is, what happens if we are trying to apply the above
+results to a function that is not field-aligned like $B(R,Z)$ of $\vec K\cdot \nabla\psi_p$ for instance? For those functions $\bar f_\mathrm{old}$ actually yields the exact
+result, while the convolution is an approximation.
+Here, we have to test.
+Typically, the functions that we use are slowly varying in $R$ and $Z$ and
+so the convolution should not change the result too much.
+A good test candidate is still $\langle \mathcal K(\psi_p)\rangle_{\psi_p}=0$.
+
+In all practical tests so far the flux-suface average is not or only very slightly changed by this procedure.
+This means that it is not
+necessary to follow the smoothing procedure if one is only interested in the flux-surface average.
+This makes sense because the toroidal and poloidal averages commute.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsection{Alternative flux labels}
 We find the toroidal flux $\psi_t$ by integrating the q-profile $\psi_t = \int^{\psi_p} \d\psi_p q(\psi_p)$. Since $q$ diverges, $\psi_t$, in contrast to $\psi_p$,
 is not defined outside the last closed flux-surface (but has a finite value on the last closed flux surface). We now define the normalized poloidal and toroidal flux labels $\rho_p$ and $\rho_t$
@@ -1590,7 +1700,9 @@ rho\_p           & Dataset & 1 (psi) & poloidal flux label $\rho_p:= \sqrt{1 - \
 rho\_t           & Dataset & 1 (psi) & Toroidal flux label $\rho_t := \sqrt{\psi_t/\psi_{t,\mathrm{sep}}}$ (is similar to $\rho$ in the edge but $\rho_t$ is nicer in the core domain, because equidistant $\rho_t$ make more equidistant flux-surfaces)\\
 Z\_fluc2d        & Dataset & 3 (time,y,x) & Fluctuation level on selected plane ($\varphi= 0$) $\delta Z := Z(R,Z,0) - \RA{ Z}(R,Z)$ \\
 Z\_fsa2d         & Dataset & 3 (time, y,x) & Flux surface average $\RA{ Z}$ interpolated onto 2d plane Eq.~\eqref{eq:fsa_vol} \\
+Z\_cta2d         & Dataset & 3 (time, y,x) & Convoluted toroidal average Eq.~\eqref{eq:cta} \\
 Z\_fsa           & Dataset & 2 (time, psi) & Flux surface average $\RA{ Z}$ Eq.~\eqref{eq:fsa_vol} \\
+Z\_std\_fsa      & Dataset & 2 (time, psi) & Standard deviation of flux surface average on outboard midplane $\sqrt{\RA{(\delta Z)^2}}$ \\
 Z\_ifs           & Dataset & 2 (time, psi) & Volume integrated flux surface average $\int\d v\RA{ Z}$ unless Z is a current, then it is the volume derived flux-surface average $\partial_v \RA{ Z}$ \\
 Z\_ifs\_lcfs     & Dataset & 1 (time) & Volume integrated flux surface average evaluated on last closed flux surface $\int_0^{v(0)}\d v\RA{ Z}$ unless Z is a current, then it is the fsa evaluated $\RA{ j_v}(0)$ \\
 Z\_ifs\_norm     & Dataset & 1 (time) & Volume integrated square flux surface average $\sqrt{\int \d v \RA{Z}^2}$, unless Z is a current, then it is the square derivative of the flux surface average $\sqrt{\int\d v (\partial_v \RA{j^v})^2}$\\
-- 
GitLab


From ef77ec24459e4f19da6a860349728f9c3dd6994e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 9 Sep 2020 23:39:46 +0200
Subject: [PATCH 312/540] Add ion density terms in feltordiag

the purpose is to start finding the reason for terms not adding up
in the fsa diagnostics
---
 src/feltor/feltor.tex   | 54 ++++++++++++++---------
 src/feltor/feltordiag.h | 95 +++++++++++++++++++++++++++++++----------
 2 files changed, 106 insertions(+), 43 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 70e8746c0..2a6e444ea 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1073,29 +1073,31 @@ In the output file we have
 \end{longtable}
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
-The density equation directly yields the particle conservation
+The density equations directly yield the particle conservation
 \begin{align} \label{eq:mass_theorem}
-  \frac{\partial}{\partial t} n_e
-  + \nc\vec{ j_{n_e}}
-  =  \Lambda_{n_e}+S_{n_e}
+  \frac{\partial}{\partial t} N
+  + \nc\vec{ j_{N}}
+  =  \Lambda_{N}+S_{N}
 \end{align}
 The terms of the particle conservation thus read
 \begin{align}
-  n_e= & n_e,\\
-  \vec j_{n_e} =& n_e\left(
-  \vec u_E + \vec u_C + \vec u_{K} +u_e\left(\bhat+{\vec b}_\perp\right)  \right) \nonumber\\
-  =& n_e \left(\frac{\bhat\times \vn\phi}{B} 
-  + \tau_e \frac{\bhat\times\vn n_e}{n_eB} 
-  + \mu_e u_e^2\vec K_{\vn\times\bhat} 
-  + u_e(\bhat + {\vec b}_\perp) \right), \label{eq:particle_flux} \\
-  \Lambda_{n_e} =&
-  \nu_\perp\Delta_\perp n_e + \nu_\parallel\Delta_\parallel n_e
+  N= & N,\\
+  \vec j_{N} =& N\left(
+  \vec u_\psi + \vec u_C + \vec u_{K} +U\left(\bhat+{\vec b}_\perp\right)  \right)
+\label{eq:particle_flux}\\
+  %\nonumber\\
+  %=& N \left(\frac{\bhat\times \vn\phi}{B}
+  %+ \tau_e \frac{\bhat\times\vn n_e}{n_eB}
+  %+ \mu_e u_e^2\vec K_{\vn\times\bhat}
+  %+ u_e(\bhat + {\vec b}_\perp) \right), \\
+  \Lambda_{N} =&
+  \nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N
 \\
-  S_{n_e} =&  S_{n_e}
+  S_{N} =&  S_{N}
 \end{align}
 Notice that
 \begin{align}
-n_e \vec K = n_e\vn\times\frac{\bhat}{B} = \vn\times n_e\frac{\bhat}{B} + \frac{\bhat\times\vn n_e}{B}
+\tau N \vec K = \tau N\vn\times\frac{\bhat}{B} = \tau \vn\times N\frac{\bhat}{B} + \tau \frac{\bhat\times\vn N}{B}
 \label{}
 \end{align}
 such that we can define the diamagnetic flux in the particle flux since
@@ -1118,12 +1120,21 @@ The relevant terms in the output file are
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} & \textbf{Name} &  \textbf{Equation}\\
 \midrule
     electrons & $n_e$ &
-    jsne\_tt &$ n_e (\vec u_E + \vec u_K + \vec u_C )\cn \psi_p$ \\
+    jsneC\_tt &$ n_e ( \vec u_K + \vec u_C )\cn \psi_p$ \\
     jsneA\_tt &$ n_e u_e \vec{ b}_\perp  \cn \psi_p$ &
     jsneE\_tt & $ n_e \vec u_E\cn\psi_p$ \\
     lneperp\_tt &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ &
     lneparallel\_tt &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
-    sne\_tt & $S_{n_e}$ & \\
+    sne\_tt & $S_{n_e}$ &
+    jsdiae\_tt & $\tau_e \bhat \times \vn n_e \cn \psi_p /B$\\
+    ions & $N_i$ &
+    jsniC\_tt &$ N_i ( \vec u_K + \vec u_C )\cn \psi_p$ \\
+    jsniA\_tt &$ N_i U_i \vec{ b}_\perp  \cn \psi_p$ &
+    jsniE\_tt & $ N_i \vec u^i_E\cn\psi_p$ \\
+    lniperp\_tt &$ \Lambda_{\perp,N_i} = \nu_\perp \Delta_\perp N_i$ or $-\nu_\perp \Delta^2_\perp N_i$ &
+    lniparallel\_tt &$ \Lambda_{\parallel,N_i} = \nu_\parallel \Delta_\parallel N_i$ \\
+    sni\_tt & $S_{N_i}$ &
+    jsdiai\_tt & $\tau_i \bhat \times \vn N_i \cn \psi_p /B$\\
 \bottomrule
 \end{longtable}
 
@@ -1206,7 +1217,7 @@ The relevant terms in the output file are
 \end{longtable}
 
 \subsubsection{Toroidal ExB angular momentum equation} \label{sec:vorticity_eq}
-We integrate the polarisation equation over volume and derive by time (in the LWL)
+We integrate the polarisation equation over volume and derive by time. In the drift-ordering up to order $\mathcal O(\delta^3)$ we get
 \begin{align}
     &\partial_t \RA{\Omega} + \frac{\partial}{\partial v}\frac{\d v}{\d\psi_p}\RA{\vec j_\Omega\cn\psi_p} = -\RA{F_{L,\varphi}} + \RA{\mathcal S_\Omega} \label{eq:vorticity_average} \\
 \Omega &:= \mu_i N_i \left(\frac{\vn\psi_p\cn\phi}{B^2} + \tau_i \vn\ln N_i\cn\psi_p\right) \equiv \mu_i N_i(u_{E,\varphi} + u_{D,\varphi}) \\
@@ -1215,7 +1226,8 @@ We integrate the polarisation equation over volume and derive by time (in the LW
     F_{L,\varphi} &:=  -(z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) - (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p) \\
     \mathcal S_\Omega &:= \mu_i S_{n_e} \frac{\vn\psi_p\cn \phi}{B^2} + \mu_i\tau_i\vn\psi_p\cn S_{n_e} \label{eq:em_source}
 \end{align}
-Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB angular momentum
+Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB angular momentum. Again up to order $\mathcal O(\delta^3)$ in the drift ordering we obtain
+(the diffusive term is for testing purposes)
 \begin{align}
 &\partial_t \RA{\Omega_E} + \frac{\partial}{\partial v} \frac{\d v}{\d \psi_p}\RA{ \vec j_{\Omega_E}\cn\psi_p} = -\RA{F_{L,\varphi}}+ \RA{\mathcal S_{\Omega_E}} + \RA{\Lambda_{\Omega_E}} \label{eq:exb_average} \\
 \Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \equiv \mu_i N_i u_{E,\varphi} \\
@@ -1256,7 +1268,7 @@ In the output file we have
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsubsection{Parallel momentum balance}
-The flux surface average over the parallel momentum equation under species summation and the LWL yields
+The flux surface average over the parallel momentum equation under species summation  yields up to order $\mathcal O(\delta^3)$ in the drift-ordering
 \begin{align}
   \frac{\partial}{\partial t}\RA{\mu_iN_iU_{i} }
     % \nonumber\\
@@ -1265,7 +1277,7 @@ The flux surface average over the parallel momentum equation under species summa
    = \sum_s\RA{-z_s\tau_s N_s\npar \ln B} + \mu_i \RA{ S_{N_i} U_i}
    \label{eq:parallel_momentum}
 \end{align}
-while the toroidal parallel angular momentum contribution reads
+while the toroidal parallel angular momentum contribution reads up to order $\mathcal O(\delta^3)$
 \begin{align}\label{eq:parallel_momentum_direction}
     \frac{\partial}{\partial t}  \RA{\mu_iN_iU_i b_\varphi}
     + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i b_\varphi\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_s N_s + z_s\mu_sN_sU_s^2) b_\varphi b_{\perp}^{\;v} }
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 75adf9e82..cceca4786 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -24,24 +24,20 @@ struct RadialParticleFlux{
     RadialParticleFlux( double tau, double mu):
         m_tau(tau), m_mu(mu){
     }
+    //jsNC
     DG_DEVICE double operator()( double ne, double ue,
-        double d0P, double d1P, double d2P, //Phi
         double d0S, double d1S, double d2S, //Psip
-        double b_0,         double b_1,         double b_2,
         double curv0,       double curv1,       double curv2,
         double curvKappa0,  double curvKappa1,  double curvKappa2
         ){
         double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
         double curvS = curv0*d0S+curv1*d1S+curv2*d2S;
-        double PS = b_0*( d1P*d2S-d2P*d1S)+
-                    b_1*( d2P*d0S-d0P*d2S)+
-                    b_2*( d0P*d1S-d1P*d0S);
         double JPsi =
-            + ne * PS
             + ne * m_mu*ue*ue*curvKappaS
             + ne * m_tau*curvS;
         return JPsi;
     }
+    //jsNA
     DG_DEVICE double operator()( double ne, double ue, double A,
         double d0A, double d1A, double d2A,
         double d0S, double d1S, double d2S, //Psip
@@ -447,11 +443,6 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
-    {"curvne_tt", "Curvature operator applied to electron density", true,
-        []( DVec& result, Variables& v ) {
-            routines::dot( v.f.curv(), v.f.gradN(0), result);
-        }
-    },
     /// ------------------ Correlation terms --------------------//
     {"ne2", "Square of electron density", false,
         []( DVec& result, Variables& v ) {
@@ -472,19 +463,31 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------ Density terms ------------------------//
-    {"jsne_tt", "Radial electron particle flux without induction contribution (Time average)", true,
+    {"jsneE_tt", "Radial electron particle flux: ExB contribution (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
+            dg::blas1::pointwiseDot( result, v.f.density(0), result);
+        }
+    },
+    {"jsneC_tt", "Radial electron particle flux: curvature contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
                 v.f.density(0), v.f.velocity(0),
-                v.f.gradP(0)[0], v.f.gradP(0)[1], v.f.gradP(0)[2],
                 v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
-                v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
         }
     },
+    {"jsdiae_tt", "Radial electron particle flux: diamagnetic contribution (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            // u_D Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradN(0), v.gradPsip, result);
+            dg::blas1::scal( result, v.p.tau[0]);
+        }
+    },
     {"jsneA_tt", "Radial electron particle flux: induction contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
@@ -497,13 +500,6 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
-    {"jsneE_tt", "Radial electron particle flux: ExB contribution (Time average)", true,
-        []( DVec& result, Variables& v ) {
-            // ExB Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
-            dg::blas1::pointwiseDot( result, v.f.density(0), result);
-        }
-    },
     {"lneperp_tt", "Perpendicular electron diffusion (Time average)", true,
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(0), v.tmp[0], result);
@@ -517,11 +513,66 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., result);
         }
     },
-    {"sne_tt", "Source term for electron density", true,
+    {"sne_tt", "Source term for electron density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::copy( v.f.density_source(0), result);
         }
     },
+    {"jsniE_tt", "Radial ion particle flux: ExB contribution (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(1), v.gradPsip, result);
+            dg::blas1::pointwiseDot( result, v.f.density(1), result);
+        }
+    },
+    {"jsniC_tt", "Radial ion particle flux: curvature contribution (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                routines::RadialParticleFlux( v.p.tau[1], v.p.mu[1]),
+                v.f.density(1), v.f.velocity(1),
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
+                v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
+                v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
+            );
+        }
+    },
+    {"jsdiai_tt", "Radial ion particle flux: diamagnetic contribution (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            // u_D Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradN(1), v.gradPsip, result);
+            dg::blas1::scal( result, v.p.tau[1]);
+        }
+    },
+    {"jsniA_tt", "Radial ion particle flux: induction contribution (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::evaluate( result, dg::equals(),
+                routines::RadialParticleFlux( v.p.tau[1], v.p.mu[1]),
+                v.f.density(1), v.f.velocity(1), v.f.induction(),
+                v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
+                v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
+                v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
+            );
+        }
+    },
+    {"lniperp_tt", "Perpendicular ion diffusion (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            v.f.compute_diffusive_lapMperpN( v.f.density(1), v.tmp[0], result);
+            dg::blas1::scal( result, -v.p.nu_perp);
+        }
+    },
+    {"lniparallel_tt", "Parallel ion diffusion (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(1),
+                                     0., result);
+            dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(1), 1., result);
+        }
+    },
+    {"sni_tt", "Source term for ion density (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::copy( v.f.density_source(1), result);
+        }
+    },
     /// ------------------- Energy terms ------------------------//
     {"nelnne", "Entropy electrons", false,
         []( DVec& result, Variables& v ) {
-- 
GitLab


From c9202425a090e6a790a0e94129529ce84cf42d40 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Fri, 11 Sep 2020 13:57:16 +0200
Subject: [PATCH 313/540] Fix M100 makefile with jsonlib line

---
 config/m100.mk | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/config/m100.mk b/config/m100.mk
index 8dc38fc00..81e734d1b 100644
--- a/config/m100.mk
+++ b/config/m100.mk
@@ -13,9 +13,9 @@ NVCC=nvcc #CUDA compiler
 NVCCARCH=-arch sm_70 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
 NVCCFLAGS= -std=c++14 -Xcompiler "-mcpu=power9 -Wall"# -mavx -mfma" #flags for NVCC
 
-INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
-JSONLIB=-L$(HOME)/include/json/../../lib -ljsoncpp_static # json library for input parameters
-LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
+INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC) -I$(JSONCPP_INC)
+JSONLIB=-L$(JSONCPP_LIB) -ljsoncpp
+#JSONLIB=-L$(HOME)/include/json/../../lib -ljsoncpp_static # json library for input parameters
+LIBS    =-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
 LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
-#is the novel jsoncpp lib folder changed?
 endif
-- 
GitLab


From f549b9f9d0e16980616a151dd7bbff2416258f1c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 11 Sep 2020 15:43:01 +0200
Subject: [PATCH 314/540] Fix geometry read in feltor_hpc

---
 src/feltor/feltor.cu       | 15 +++++++++++----
 src/feltor/feltor_hpc.cu   | 18 ++++++++++++++----
 src/feltor/manufactured.cu | 18 +++---------------
 3 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index d499c7e44..ce517ccf7 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -37,7 +37,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     try{
-        file::file2Json( inputfile, js, file::comments::are_forbidden);
+        file::file2Json( inputfile, js, file::comments::are_forbidden, file::error::is_throw);
         feltor::Parameters(js, file::error::is_throw);
     }catch(std::runtime_error& e)
     {
@@ -47,8 +47,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     try{
-        file::file2Json( geomfile, gs, file::comments::are_discarded);
-        dg::geo::solovev::Parameters(gs, file::error::is_throw);
+        file::file2Json( geomfile, gs, file::comments::are_discarded, file::error::is_throw);
     }catch(std::runtime_error& e)
     {
 
@@ -60,7 +59,15 @@ int main( int argc, char* argv[])
     const feltor::Parameters p(js);
     p.display( std::cout);
     std::cout << gs.toStyledString() << std::endl;
-    dg::geo::TokamakMagneticField mag = dg::geo::createMagneticField(gs, file::error::is_warning);
+    dg::geo::TokamakMagneticField mag;
+    try{
+        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
+    }catch(std::runtime_error& e)
+    {
+        std::cerr << "ERROR in geometry file "<<geomfile<<std::endl;
+        std::cerr <<e.what()<<std::endl;
+        return -1;
+    }
     /////////////////////////////////////////////////////////////////////////
     double Rmin=mag.R0()-p.boxscaleRm*mag.params().a();
     double Zmin=-p.boxscaleZm*mag.params().a()*mag.params().elongation();
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index d937372c4..733da6be7 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -121,7 +121,7 @@ int main( int argc, char* argv[])
     else
     {
         try{
-            file::file2Json( argv[1], js, file::comments::are_discarded);
+            file::file2Json( argv[1], js, file::comments::are_discarded, file::error::is_throw);
             feltor::Parameters( js, file::error::is_throw);
         } catch( std::exception& e) {
             MPI_OUT std::cerr << "ERROR in input parameter file "<<argv[1]<<std::endl;
@@ -132,8 +132,7 @@ int main( int argc, char* argv[])
             return -1;
         }
         try{
-            file::file2Json( argv[2], gs, file::comments::are_discarded);
-            dg::geo::solovev::Parameters( gs, file::error::is_throw);
+            file::file2Json( argv[2], gs, file::comments::are_discarded, file::error::is_throw);
         } catch( std::exception& e) {
             MPI_OUT std::cerr << "ERROR in geometry file "<<argv[2]<<std::endl;
             MPI_OUT std::cerr << e.what()<<std::endl;
@@ -147,7 +146,18 @@ int main( int argc, char* argv[])
     MPI_OUT p.display( std::cout);
     std::string input = js.toStyledString(), geom = gs.toStyledString();
     MPI_OUT std::cout << geom << std::endl;
-    dg::geo::TokamakMagneticField mag = dg::geo::createMagneticField(gs, file::error::is_warning);
+    dg::geo::TokamakMagneticField mag;
+    try{
+        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
+    }catch(std::runtime_error& e)
+    {
+        std::cerr << "ERROR in geometry file "<<argv[2]<<std::endl;
+        std::cerr <<e.what()<<std::endl;
+#ifdef FELTOR_MPI
+            MPI_Abort(MPI_COMM_WORLD, -1);
+#endif //FELTOR_MPI
+        return -1;
+    }
 #ifdef FELTOR_MPI
     if( np[2] >= (int)p.Nz)
     {
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 819aa31f4..75ecf4fff 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -29,7 +29,8 @@ int main( int argc, char* argv[])
         return -1;
     }
     const feltor::Parameters p( js, file::error::is_throw);// p.display( std::cout);
-    std::cout << "# "<<p.n<<" x "<<p.Nx<<" x "<<p.Ny<<" x "<<p.Nz<<"\n";
+    p.display( std::cout);
+//std::cout << "# "<<p.n<<" x "<<p.Nx<<" x "<<p.Ny<<" x "<<p.Nz<<"\n";
     const double R_0 = 10;
     const double I_0 = 20; //q factor at r=1 is I_0/R_0
     const double a  = 1; //small radius
@@ -83,22 +84,12 @@ int main( int argc, char* argv[])
         feltor::FeltorSpecialSolver<
             dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
         > karniadakis( grid, p, mag);
-    double time = 0, dt_new = p.dt, TMAX = 1e-3;
+    double time = 0, TMAX = 0.1;
     karniadakis.init( feltor, im, time, y0, p.dt);
     while( time < TMAX)
     {
-        if( time + dt_new > TMAX)
-            dt_new = TMAX - time;
-
         try{
             karniadakis.step( feltor, im, time, y0);
-            //do
-            //{
-            //    adaptive.step( feltor, im, time, y0, time, y0, dt_new,
-            //        dg::pid_control, dg::l2norm, p.rtol, 1e-10);
-            //    if( adaptive.failed())
-            //        std::cout << "FAILED STEP! REPEAT!\n";
-            //}while ( adaptive.failed());
         }
         catch( dg::Fail& fail) {
             std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
@@ -144,9 +135,6 @@ int main( int argc, char* argv[])
               <<"    phie: "<<sqrt(dg::blas2::dot( w3d,sol_phi[0]))/normphie<<"\t"<<normphie<<"\n"
               <<"    phii: "<<sqrt(dg::blas2::dot( w3d,sol_phi[1]))/normphii<<"\t"<<normphii<<"\n"
               <<"    apar: "<<sqrt(dg::blas2::dot( w3d,sol_apar))/normapar<<"\t"<<normapar<<"\n";
-    //feltor.update_quantities();
-    //feltor.quantities().display();
-
 
     return 0;
 
-- 
GitLab


From 7abf34e19da2ef0cf465686bc137dabd0c73a50c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 16 Sep 2020 11:53:43 +0200
Subject: [PATCH 315/540] Rewrite U as U_parallel in feltor.tex

---
 src/feltor/feltor.tex | 244 +++++++++++++++++++++---------------------
 1 file changed, 122 insertions(+), 122 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 2a6e444ea..e821bbbe8 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -743,23 +743,23 @@ where $a\in\{e,i\}$ is the species label and $z$ is the charge number.
 Omitting the species label we arrive at (dividing the density equation by $\Omega_0n_0$ and the velocity equation by $\Omega_0 c_s$)
 \begin{align}
 \frac{\partial}{\partial t} N &+ \vec\nc\left( N \left(
-    \vec u_E + \vec u_K + \vec u_{C} + U\left(\bhat + {\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
-\mu N \frac{\partial}{\partial t} U &+ \mu N \left(
-    \vec u_E + \vec u_K + \vec u_{C} + U\left(\bhat + {\vec b}_\perp\right)
-    \right)\cn U  \nonumber \\
-    &+ 2\mu \nc ( NU \vec u_{\vn\times\bhat})
-    -\mu NU\nc \vec u_{\vn\times\bhat}
-    + \mu NU\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
+    \vec u_E + \vec u_K + \vec u_{C} + U_\parallel\left(\bhat + {\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
+\mu N \frac{\partial}{\partial t} U_\parallel &+ \mu N \left(
+    \vec u_E + \vec u_K + \vec u_{C} + U_\parallel\left(\bhat + {\vec b}_\perp\right)
+    \right)\cn U_\parallel  \nonumber \\
+    &+ 2\mu \nc ( NU_\parallel \vec u_{\vn\times\bhat})
+    -\mu NU_\parallel\nc \vec u_{\vn\times\bhat}
+    + \mu NU_\parallel\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
     &= -\tau \left(\bhat + {\vec b}_\perp\right)\cn N
     -N \left( \left(\bhat+{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right)
-    - \eta n_e^2(U_i-u_e) + \mu N\Lambda_U + \mu N S_U
+    - \eta n_e^2(U_{\parallel,i}-u_{\parallel,e}) + \mu N\Lambda_U + \mu N S_U
 \label{}
 \end{align}
 with
 \begin{align}
 \vec u_E := \frac{\bhat\times\vn\psi}{B},\quad
 \vec u_{K} := \tau \left(\vec{\mathcal K_{\vn B}} + \vec{\mathcal K_{\vn\times\bhat}}\right)=\tau\vec{\mathcal K}  ,\nonumber\\
-\vec u_C := \mu U^2\vec{\mathcal K_{\vn\times\bhat}},\quad
+\vec u_C := \mu U_\parallel^2\vec{\mathcal K_{\vn\times\bhat}},\quad
 \vec u_{\vn\times\bhat} := \tau\vec{\mathcal K_{\vn\times\bhat}},\quad
 {\vec b}_\perp = \frac{\vn\times A_\parallel \bhat}{B}.
 \label{}
@@ -769,7 +769,7 @@ The electric potential \(\phi\) and parallel magnetic vector potential \(A_\para
 computed by the polarisation and induction equations (with $q_e=-e$ and $q_i=+e$)
 \begin{align}
  -\nc\left(\frac{\mu_iN_i}{B^2} \np \phi\right) &=  \Gamma_{1,i} N_i -n_e, \quad \Gamma_{1,i}^{-1} := 1-\frac{1}{2}\mu_i\tau_i\Delta_\perp , \\
-  -\frac{1}{\beta} \Delta_\perp A_\parallel &= \left(N_i U_i-n_e u_e \right)
+  -\frac{1}{\beta} \Delta_\perp A_\parallel &= \left(N_i U_{\parallel,i}-n_e u_{\parallel,e} \right)
   \label{eq:polarisation_dimensional}
 \end{align}
 Given $\phi$ we define the generalised electric potential
@@ -782,20 +782,20 @@ on in the electric potential $\phi$ and 0th order FLR effects in the parallel ma
 potential $A_\parallel$.
 We have the continuity equation for the electron density \(n_e\) and the ion gyro-centre
 density \(N_i\) and the momentum conservation equation for
-the parallel electron velocity \(u_e\) and the parallel ion gyro-centre velocity \(U_i\)~\cite{WiesenbergerPhD, HeldPhD}.
+the parallel electron velocity \(u_{\parallel,e}\) and the parallel ion gyro-centre velocity \(U_{\parallel,i}\)~\cite{WiesenbergerPhD, HeldPhD}.
 \subsection{ Scale invariance}
 \subsubsection{Sign reversals of the magnetic field}\label{sec:field_reversal}
 If we change the direction of the magnetic field vector $\bhat$, we immediately see that all perpendicular
-drifts and $U\bhat$ change directions. On the other side, the diffusive and resistive terms remain unchanged.
+drifts and $U_\parallel\bhat$ change directions. On the other side, the diffusive and resistive terms remain unchanged.
 Without resistivity and diffusion a change in direction of the magnetic field thus corresponds to
 a time reversal $t\rightarrow t'=-t$.
 In the code $\bhat$ changes sign by using both $-\mathcal P_\psi$ and $-\mathcal P_I$.
 
 Also note that changing the sign of the magnetic field only in the parallel derivatives $\npar \rightarrow -\npar$ does not
-have any effect. This can be seen by simply renormalizing $U'=-U$. This reverts the equations back to the original equations.
+have any effect. This can be seen by simply renormalizing $U_\parallel'=-U_\parallel$. This reverts the equations back to the original equations.
 \subsubsection{Scaling of density}
-If $N, U, \phi, A_\parallel$ are a solution to the model equations
-then so are $N'=\alpha N$, $U'=U$, $\phi'=\phi$ and $A_\parallel'=A_\parallel$ with the changed parameters $S_N' = \alpha S_N$, $\eta' = \eta/\alpha$ and $ \beta' = \beta/\alpha$. If $N$
+If $N, U_\parallel, \phi, A_\parallel$ are a solution to the model equations
+then so are $N'=\alpha N$, $U_\parallel'=U_\parallel$, $\phi'=\phi$ and $A_\parallel'=A_\parallel$ with the changed parameters $S_N' = \alpha S_N$, $\eta' = \eta/\alpha$ and $ \beta' = \beta/\alpha$. If $N$
 has a Dirichlet boundary condition, then $N'$ satisfies a correspondingly scaled boundary condition.
 
 
@@ -807,7 +807,7 @@ $\eta_\parallel := \frac{0.51 m_e \nu_{ei}}{n_e e^2}$ and $\nu_{ei} = \sqrt{2} z
     \label{eq:resistivity}
 \end{align}
 with $\ln \lambda \approx 10$.
- The approximate Spitzer current \(J_{\parallel,s}:= n_e \left(U_i - u_e\right)\)
+ The approximate Spitzer current \(J_{\parallel,s}:= n_e \left(U_{\parallel,i} - u_{\parallel,e}\right)\)
  determines the parallel resistive terms to $R_\parallel:= n_e\eta J_{\parallel,s}$.
 
 The dissipative terms can be decomposed into perpendicular and parallel components
@@ -821,15 +821,15 @@ For numerical stabilisation we choose:
 \begin{align}
 \Lambda_{n_e,\parallel} &= \nu_\parallel \Delta_\parallel n_e &
 \Lambda_{N_i,\parallel} &= \nu_\parallel \Delta_\parallel N_i \\
-\Lambda_{u_e,\parallel} &= \nu_\parallel \Delta_\parallel u_e &
-\Lambda_{U_i,\parallel} &= \nu_\parallel \Delta_\parallel U_i
+\Lambda_{u_e,\parallel} &= \nu_\parallel \Delta_\parallel u_{\parallel,e} &
+\Lambda_{U_i,\parallel} &= \nu_\parallel \Delta_\parallel U_{\parallel,i}
 \end{align}
 Similarly, for the perpendicular dissipation we apply viscous or hyperviscous terms.
 \begin{align}\label{eq:perpdiffNT}
  \Lambda_{n_e,\perp} &=  \nu_\perp \Delta_\perp n_e \text{ or } -\nu_\perp \Delta_\perp^2 n_e&
  \Lambda_{N_i,\perp} &=  \nu_\perp \Delta_\perp N_i \text{ or } -\nu_\perp \Delta_\perp^2 N_i & \\
- \Lambda_{u_e,\perp} &=  \nu_\perp \Delta_\perp u_e \text{ or } -\nu_\perp \Delta_\perp^2 u_e &
- \Lambda_{U_i,\perp} &=  \nu_\perp \Delta_\perp U_i \text{ or } -\nu_\perp \Delta_\perp^2 U_i
+ \Lambda_{u_e,\perp} &=  \nu_\perp \Delta_\perp u_{\parallel,e} \text{ or } -\nu_\perp \Delta_\perp^2 u_{\parallel,e} &
+ \Lambda_{U_i,\perp} &=  \nu_\perp \Delta_\perp U_{\parallel,i} \text{ or } -\nu_\perp \Delta_\perp^2 U_{\parallel,i}
 \end{align}
 Here the mass diffusion coefficient coincides with the viscous coefficient, hence we fixed the Schmidt number \(\mathit{Sc}_\parallel:= \frac{\nu_U}{\nu_N}\) to unity.
 The drift-fluid corresponding diffusion gives an order-of-magnitude estimate for $\nu_\perp$.
@@ -860,21 +860,21 @@ where $a$ is the minor radius, $e$ is the elongation of the flux surfaces and
 the $\varepsilon$ are free parameters to be specified by the user.
 
 We choose boundary conditions separately on input for the variables
-$n_e$, $u_e$ and $\phi$. The boundary condition for $N_i$, $U_i$ and
-$\psi$ are equal to $n_e$, $u_e$ and $\phi$ respectively.
-We choose $A_\parallel$ to have equal boundary conditions as $u_e$ and $U_i$.
-This will later enable us to treat the sum of $U$ and $A_\parallel$
-in the same way as $U$.
+$n_e$, $u_{\parallel,e}$ and $\phi$. The boundary condition for $N_i$, $U_{\parallel,i}$ and
+$\psi$ are equal to $n_e$, $u_{\parallel,e}$ and $\phi$ respectively.
+We choose $A_\parallel$ to have equal boundary conditions as $u_{\parallel,e}$ and $U_{\parallel,i}$.
+This will later enable us to treat the sum of $U_\parallel$ and $A_\parallel$
+in the same way as $U_\parallel$.
 Typically,
 \begin{align}
-n_e = n_0, \quad u_e = \phi = 0
-\text{ or } \hat n \cn n_e = \hat n \cn u_e = 0
+n_e = n_0, \quad u_{\parallel,e} = \phi = 0
+\text{ or } \hat n \cn n_e = \hat n \cn u_{\parallel,e} = 0
 \end{align}
 where $\hat n$ is the normal vector to the boundary.
 
 We initialize the parallel velocity to zero
 \begin{align}
-  u_e(R,Z,\varphi,0) = U_i(R,Z,\varphi,0) = 0
+  u_{\parallel,e}(R,Z,\varphi,0) = U_{\parallel,i}(R,Z,\varphi,0) = 0
   \label{}
 \end{align}
 which in turn initializes $A_\parallel = 0$
@@ -1005,7 +1005,7 @@ For both electrons and ions we choose
 \begin{align}
     S^d_{n_e}(R,Z,\varphi,t) &:= -\omega_d (n_e-1)\Theta_{\alpha_p/2}\left(\rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2}\right)\\
     S^d_{N_i}(R,Z,\varphi,t) &= \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S^d_{n_e} -\nc\left( \frac{\mu_i S^d_{n_e}}{B^2}\np \phi\right)\\
-S^d_U(R,Z,\varphi, t)& := -\omega_d U \Theta_{\alpha_p/2}\left(  \rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2} \right)
+S^d_U(R,Z,\varphi, t)& := -\omega_d U_\parallel \Theta_{\alpha_p/2}\left(  \rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2} \right)
 \end{align}
 \end{subequations}
 
@@ -1016,29 +1016,29 @@ two functions for which we have no boundary conditions
     \begin{align}
     \frac{\partial}{\partial t} N =&
         - \frac{1}{B}[\psi, N]_{\perp}%\nonumber\\
-        - \bar \npar \left( NU\right)
-        - NU\left(\vec \nc\bhat+\vec \nc{\vec b}_\perp\right)
+        - \bar \npar \left( NU_\parallel\right)
+        - NU_\parallel\left(\vec \nc\bhat+\vec \nc{\vec b}_\perp\right)
         - \tau \mathcal K(N) \nonumber \\&
         - N \mathcal K(\psi)
-        -\mu \mathcal K_{\vn\times\bhat}(NU^2)
-        -\mu NU^2\nc \vec{ \mathcal K_{\vn\times\bhat}}
+        -\mu \mathcal K_{\vn\times\bhat}(NU_\parallel^2)
+        -\mu NU_\parallel^2\nc \vec{ \mathcal K_{\vn\times\bhat}}
         + \nu_\perp\Delta_\perp N + \nu_\parallel \Delta_\parallel N + S_N, \\
     \frac{\partial}{\partial t} W =&
-        - \frac{1}{B}\left[\psi, U\right]_{\perp}%& \nonumber\\
+        - \frac{1}{B}\left[\psi, U_\parallel\right]_{\perp}%& \nonumber\\
         - \frac{1}{\mu} \bar \npar \psi% \nonumber\\
-        - \frac{1}{2}\bar \npar U^2
+        - \frac{1}{2}\bar \npar U_\parallel^2
         -\frac{\tau}{\mu} \bar \npar \ln N
-        - U\mathcal K_{\vn\times\bhat}(\psi)
-        - \tau \mathcal K(U)
-        -\tau U\nc\vec{ \mathcal K_{\vn\times\bhat}}\nonumber\\&
-        - \left(2\tau + {\mu}U^2\right) \mathcal K_{\vn\times\bhat} (U)
-        -2\tau U\mathcal K_{\vn\times\bhat}(\ln N)
-        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_i - u_e) \nonumber\\&
-        + \nu_\perp\Delta_\perp U
-        + \nu_\parallel \Delta_\parallel U
+        - U_\parallel\mathcal K_{\vn\times\bhat}(\psi)
+        - \tau \mathcal K(U_\parallel)
+        -\tau U_\parallel\nc\vec{ \mathcal K_{\vn\times\bhat}}\nonumber\\&
+        - \left(2\tau + {\mu}U_\parallel^2\right) \mathcal K_{\vn\times\bhat} (U_\parallel)
+        -2\tau U_\parallel\mathcal K_{\vn\times\bhat}(\ln N)
+        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_{\parallel,i} - u_{\parallel,e}) \nonumber\\&
+        + \nu_\perp\Delta_\perp U_\parallel
+        + \nu_\parallel \Delta_\parallel U_\parallel
         + S_U,
         \label{eq:EgyrofluidU} \\
-        W&:= \left( U + \frac{A_\parallel}{\mu}\right)
+        W_\parallel&:= \left( U_\parallel + \frac{A_\parallel}{\mu}\right)
     \end{align}
     \label{eq:Egyrofluid}
 \end{subequations}
@@ -1051,7 +1051,7 @@ and
     -\nc\left( \frac{N_i}{B^2}\np \phi \right) &= \Gamma_{1,i} N_i - n_e, \quad\quad
     \Gamma_{1,i}^{-1} = 1-\frac{1}{2}\tau_i\mu_i \Delta_\perp \\
     \psi_e = \phi, \quad \psi_i &= \Gamma_{1,i}\phi -\frac{\mu_i}{2}\frac{(\np\phi)^2}{B^2} \\
-    \left(\frac{\beta}{\mu_i}N_i - \frac{\beta}{\mu_e}n_e-\Delta_\perp\right)
+    \left(\frac{\beta}{\mu_i}N_i - \frac{\beta}{\mu_{\parallel,e}}n_e-\Delta_\perp\right)
     A_\parallel &= \beta\left(N_iW_i-n_e w_e\right)
   \end{align}
 \end{subequations}
@@ -1064,8 +1064,8 @@ In the output file we have
 \midrule
     electrons &$n_e$ &
     ions &$N_i$ \\
-    Ue &$u_e$ &
-    Ui &$U_i$ \\
+    Ue &$u_{\parallel,e}$ &
+    Ui &$U_{\parallel,i}$ \\
     potential &$\phi$ &
     psi &$\psi$ \\
     induction &$A_\parallel$ & \\
@@ -1083,13 +1083,13 @@ The terms of the particle conservation thus read
 \begin{align}
   N= & N,\\
   \vec j_{N} =& N\left(
-  \vec u_\psi + \vec u_C + \vec u_{K} +U\left(\bhat+{\vec b}_\perp\right)  \right)
+  \vec u_\psi + \vec u_C + \vec u_{K} +U_\parallel\left(\bhat+{\vec b}_\perp\right)  \right)
 \label{eq:particle_flux}\\
   %\nonumber\\
   %=& N \left(\frac{\bhat\times \vn\phi}{B}
   %+ \tau_e \frac{\bhat\times\vn n_e}{n_eB}
-  %+ \mu_e u_e^2\vec K_{\vn\times\bhat}
-  %+ u_e(\bhat + {\vec b}_\perp) \right), \\
+  %+ \mu_e u_{\parallel,e}^2\vec K_{\vn\times\bhat}
+  %+ u_{\parallel,e}(\bhat + {\vec b}_\perp) \right), \\
   \Lambda_{N} =&
   \nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N
 \\
@@ -1106,11 +1106,11 @@ the rotation vanishes under the divergence.
 We here also derive the particle flux \eqref{eq:particle_flux} through a flux surface
 \begin{align} \label{eq:radial_particle_flux}
  \vec j_{N}\cn v %=& N\left( \vec u_E + \vec u_C + \vec u_{\vn
- %B} + U \left(\bhat + {\vec b}_\perp\right)\right) \cn \psi_p \nonumber\\
+ %B} + U_\parallel \left(\bhat + {\vec b}_\perp\right)\right) \cn \psi_p \nonumber\\
  =&
-  \frac{\d v}{\d \psi_p} N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U^2\right)
+  \frac{\d v}{\d \psi_p} N\left[\frac{1}{B}[\psi, \psi_p]_\perp + \left(\tau + \mu U_\parallel^2\right)
    \mathcal K_{\vn\times\bhat}(\psi_p) + \tau  \mathcal K_{\vn B}(\psi_p) \right] \nonumber\\
- &+ NU\frac{\d v}{\d \psi_p}\left [\left( A_\parallel \mathcal
+ &+ NU_\parallel\frac{\d v}{\d \psi_p}\left [\left( A_\parallel \mathcal
  K_{\vn\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right) \right]
 \end{align}
 
@@ -1121,7 +1121,7 @@ The relevant terms in the output file are
 \midrule
     electrons & $n_e$ &
     jsneC\_tt &$ n_e ( \vec u_K + \vec u_C )\cn \psi_p$ \\
-    jsneA\_tt &$ n_e u_e \vec{ b}_\perp  \cn \psi_p$ &
+    jsneA\_tt &$ n_e u_{\parallel,e} \vec{ b}_\perp  \cn \psi_p$ &
     jsneE\_tt & $ n_e \vec u_E\cn\psi_p$ \\
     lneperp\_tt &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ &
     lneparallel\_tt &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
@@ -1129,7 +1129,7 @@ The relevant terms in the output file are
     jsdiae\_tt & $\tau_e \bhat \times \vn n_e \cn \psi_p /B$\\
     ions & $N_i$ &
     jsniC\_tt &$ N_i ( \vec u_K + \vec u_C )\cn \psi_p$ \\
-    jsniA\_tt &$ N_i U_i \vec{ b}_\perp  \cn \psi_p$ &
+    jsniA\_tt &$ N_i U_{\parallel,i} \vec{ b}_\perp  \cn \psi_p$ &
     jsniE\_tt & $ N_i \vec u^i_E\cn\psi_p$ \\
     lniperp\_tt &$ \Lambda_{\perp,N_i} = \nu_\perp \Delta_\perp N_i$ or $-\nu_\perp \Delta^2_\perp N_i$ &
     lniparallel\_tt &$ \Lambda_{\parallel,N_i} = \nu_\parallel \Delta_\parallel N_i$ \\
@@ -1154,19 +1154,19 @@ with ( $z_e=-1$ and $z_i=+1$) and $\vec u_E := {\bhat\times \vn\phi}/{B}$
   \mathcal{E}= & z_e\tau_e n_e \ln{(n_e)} +z_i\tau_i N_i\ln{(N_i)}
   +\frac{1}{2\beta}\left(\np A_\parallel\right)^2
    +  \frac{1}{2} z_i \mu_i N_i u_E^2  \nonumber\\
-   & +\frac{1}{2} z_e\mu_e  n_e u_e^2
-  +\frac{1}{2} z_i\mu_i  N_i U_i^2,\\
+   & +\frac{1}{2} z_e\mu_e  n_e u_{\parallel,e}^2
+  +\frac{1}{2} z_i\mu_i  N_i U_{\parallel,i}^2,\\
   \vec j_{\mathcal E} =& \sum_s z\left[
-  \left(\tau \ln N + \frac{1}{2}\mu U^2 + \psi \right)N\left(
-  \vec u_E + \vec u_C + \vec u_{K} +U\left(\bhat+{\vec b}_\perp\right)  \right) \right]
+  \left(\tau \ln N + \frac{1}{2}\mu U_\parallel^2 + \psi \right)N\left(
+  \vec u_E + \vec u_C + \vec u_{K} +U_\parallel\left(\bhat+{\vec b}_\perp\right)  \right) \right]
   \nonumber\\
-  &+ \sum_z z\left[\mu \tau NU^2\vec K_{\vn\times\bhat} + \tau NU \left(\bhat + {\vec b}_\perp\right)\right], \\
-  \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U^2 \right)
-  \left(\nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N\right)  +  \mu NU\left(\nu_\perp\Delta_\perp U + \nu_\parallel\Delta_\parallel U\right) \right]
+  &+ \sum_z z\left[\mu \tau NU_\parallel^2\vec K_{\vn\times\bhat} + \tau NU_\parallel \left(\bhat + {\vec b}_\perp\right)\right], \\
+  \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U_\parallel^2 \right)
+  \left(\nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N\right)  +  \mu NU_\parallel\left(\nu_\perp\Delta_\perp U_\parallel + \nu_\parallel\Delta_\parallel U_\parallel\right) \right]
 \nonumber \\
-  S_{\mathcal E} =&  \sum_s  z\left[ \left(\tau\left( 1+\ln{N}\right) +\psi + \frac{1}{2} \mu U^2 \right)S_{N}\right]
+  S_{\mathcal E} =&  \sum_s  z\left[ \left(\tau\left( 1+\ln{N}\right) +\psi + \frac{1}{2} \mu U_\parallel^2 \right)S_{N}\right]
 \nonumber \\
-  R_{\mathcal E} =&  -\eta_\parallel  \left[ n_e(U_i-u_e)\right]^2.
+  R_{\mathcal E} =&  -\eta_\parallel  \left[ n_e(U_{\parallel,i}-u_{\parallel,e})\right]^2.
 \end{align}
 where in the energy flux $\vec j_{\mathcal E}$
 we neglect terms  containing time derivatives
@@ -1180,9 +1180,9 @@ for the diffusion terms in the above equations.
 We have the energy flux through a flux surface
 \begin{align}
  \vec j_{\mathcal E}\cn v =&%\frac{\d v}{\d \psi_p} \vec j_{\mathcal E}\cn \psi_p  =
-\frac{\d v}{\d \psi_p}\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U^2 + \psi\right) \vec j_N\cn\psi_p
-+ z \mu\tau NU^2 \mathcal K_{\vn\times\bhat}(\psi_p) \nonumber\\
-&+ z \tau NU
+\frac{\d v}{\d \psi_p}\sum_s z\left (\tau\ln N + \frac{1}{2}\mu U_\parallel^2 + \psi\right) \vec j_N\cn\psi_p
++ z \mu\tau NU_\parallel^2 \mathcal K_{\vn\times\bhat}(\psi_p) \nonumber\\
+&+ z \tau NU_\parallel
  \left( A_\parallel \mathcal
  K_{\vn\times\bhat}(\psi_p) + \frac{1}{B}[\psi_p, A_\parallel]_\perp\right)
 \label{eq:energy_flux}
@@ -1196,34 +1196,34 @@ The relevant terms in the output file are
     nilnni &$ z_i\tau_i N_i \ln N_i$ \\
     aperp2 &$ (\np A_\parallel)^2/2/\beta$ \\
     ue2   &$z_i\mu_i N_i u_E^2 /2$ \\
-    neue2 &$ z_e\mu_e n_e u_e^2/2$ \\
-    niui2 &$ z_i\mu_i N_i U_i^2/2$ \\
-    see\_tt & $z_e(\tau_e (1+\ln n_e) + \phi + \frac{1}{2}\mu_e u_e^2) S_{n_e} $ \\
-    sei\_tt & $z_i(\tau_i (1+\ln N_i) + \psi + \frac{1}{2}\mu_i U_i^2) S_{N_i} $ \\
-    resistivity\_tt &-$\eta_\parallel n_e^2 (U_i-u_e)^2$ \\
-    jsee\_tt &$z_e(\tau_e \ln n_e + \mu_e u_e^2/2 + \phi)n_e(\vec u_E + \vec u_C + \vec u_K)\cn \psi_p
-        + z_e \tau_e n_e u_e^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
-    jsei\_tt &$z_i(\tau_i \ln N_i + \mu_i U_i^2/2 + \psi_i)N_i(\vec u_E^i + \vec u_C + \vec u_K)\cn \psi_p
-        + z_i \tau_i N_i U_i^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
-    jseea\_tt &$z_e(\tau_e \ln n_e + \mu_e u_e^2 + \phi)n_e \vec { b}_\perp\cn \psi_p
-        + z_e \tau_e n_e u_e \vec{ b}_\perp \cn \psi_p $ \\
-    jseia\_tt &$z_i(\tau_i \ln N_i + \mu_i U_i^2 + \psi_i)N_i \vec { b}_\perp\cn \psi_p
-        + z_i \tau_i N_i U_i \vec{ b}_\perp \cn \psi_p $ \\
-    leeperp\_tt &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_e \nu_\perp \Delta_\perp u_e$ \\
-    leiperp\_tt &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_i \nu_\perp \Delta_\perp U_i$ \\
-    leeparallel\_tt &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_e^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_e \nu_\parallel \Delta_\parallel u_e$ \\
-    leiparallel\_tt &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_i^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_i \nu_\parallel \Delta_\parallel U_i$ \\
+    neue2 &$ z_e\mu_e n_e u_{\parallel,e}^2/2$ \\
+    niui2 &$ z_i\mu_i N_i U_{\parallel,i}^2/2$ \\
+    see\_tt & $z_e(\tau_e (1+\ln n_e) + \phi + \frac{1}{2}\mu_e u_{\parallel,e}^2) S_{n_e} $ \\
+    sei\_tt & $z_i(\tau_i (1+\ln N_i) + \psi + \frac{1}{2}\mu_i U_{\parallel,i}^2) S_{N_i} $ \\
+    resistivity\_tt &-$\eta_\parallel n_e^2 (U_{\parallel,i}-u_{\parallel,e})^2$ \\
+    jsee\_tt &$z_e(\tau_e \ln n_e + \mu_e u_{\parallel,e}^2/2 + \phi)n_e(\vec u_E + \vec u_C + \vec u_K)\cn \psi_p
+        + z_e \tau_e n_e u_{\parallel,e}^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
+    jsei\_tt &$z_i(\tau_i \ln N_i + \mu_i U_{\parallel,i}^2/2 + \psi_i)N_i(\vec u_E^i + \vec u_C + \vec u_K)\cn \psi_p
+        + z_i \tau_i N_i U_{\parallel,i}^2 \vec K_{\vn\times\bhat}\cn \psi_p$ \\
+    jseea\_tt &$z_e(\tau_e \ln n_e + \mu_e u_{\parallel,e}^2 + \phi)n_e \vec { b}_\perp\cn \psi_p
+        + z_e \tau_e n_e u_{\parallel,e} \vec{ b}_\perp \cn \psi_p $ \\
+    jseia\_tt &$z_i(\tau_i \ln N_i + \mu_i U_{\parallel,i}^2 + \psi_i)N_i \vec { b}_\perp\cn \psi_p
+        + z_i \tau_i N_i U_{\parallel,i} \vec{ b}_\perp \cn \psi_p $ \\
+    leeperp\_tt &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_{\parallel,e}^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_{\parallel,e} \nu_\perp \Delta_\perp u_{\parallel,e}$ \\
+    leiperp\_tt &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_{\parallel,i}^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_{\parallel,i} \nu_\perp \Delta_\perp U_{\parallel,i}$ \\
+    leeparallel\_tt &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_{\parallel,e}^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_{\parallel,e} \nu_\parallel \Delta_\parallel u_{\parallel,e}$ \\
+    leiparallel\_tt &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_{\parallel,i}^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_{\parallel,i} \nu_\parallel \Delta_\parallel U_{\parallel,i}$ \\
 \bottomrule
 \end{longtable}
 
 \subsubsection{Toroidal ExB angular momentum equation} \label{sec:vorticity_eq}
-We integrate the polarisation equation over volume and derive by time. In the drift-ordering up to order $\mathcal O(\delta^3)$ we get
+We integrate the polarisation equation over volume, multiply by $\d \psi_p/\d v$ and derive by time. In the drift-ordering up to order $\mathcal O(\delta^3)$ we get
 \begin{align}
     &\partial_t \RA{\Omega} + \frac{\partial}{\partial v}\frac{\d v}{\d\psi_p}\RA{\vec j_\Omega\cn\psi_p} = -\RA{F_{L,\varphi}} + \RA{\mathcal S_\Omega} \label{eq:vorticity_average} \\
 \Omega &:= \mu_i N_i \left(\frac{\vn\psi_p\cn\phi}{B^2} + \tau_i \vn\ln N_i\cn\psi_p\right) \equiv \mu_i N_i(u_{E,\varphi} + u_{D,\varphi}) \\
 \vec j_{\Omega} &:= \Omega \vec u_E
-    - \left(\frac{1}{\beta} \vn\psi_p \cn A_\parallel +\frac{1}{2}\tau_i \vn\psi_p\cn  (N_iU_i)\right)\frac{\bhat\times\vn A_\parallel}{B} \\
-    F_{L,\varphi} &:=  -(z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) - (z_e\mu_e n_eu_e^2 + z_i\mu_i N_iU_i^2)\mathcal K_{\vn\times\bhat}(\psi_p) \\
+    - \left(\frac{1}{\beta} \vn\psi_p \cn A_\parallel +\frac{1}{2}\tau_i \vn\psi_p\cn  (N_iU_{\parallel,i})\right)\frac{\bhat\times\vn A_\parallel}{B} \\
+    F_{L,\varphi} &:=  -(z_e \tau_e n_e + z_i\tau_i N_i)\mathcal K(\psi_p) - (z_e\mu_e n_eu_{\parallel,e}^2 + z_i\mu_i N_iU_{\parallel,i}^2)\mathcal K_{\vn\times\bhat}(\psi_p) \\
     \mathcal S_\Omega &:= \mu_i S_{n_e} \frac{\vn\psi_p\cn \phi}{B^2} + \mu_i\tau_i\vn\psi_p\cn S_{n_e} \label{eq:em_source}
 \end{align}
 Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continuity equation to yield an equation only for the \ExB angular momentum. Again up to order $\mathcal O(\delta^3)$ in the drift ordering we obtain
@@ -1232,7 +1232,7 @@ Equation~\eqref{eq:vorticity_average} can be rewritten by inserting the continui
 &\partial_t \RA{\Omega_E} + \frac{\partial}{\partial v} \frac{\d v}{\d \psi_p}\RA{ \vec j_{\Omega_E}\cn\psi_p} = -\RA{F_{L,\varphi}}+ \RA{\mathcal S_{\Omega_E}} + \RA{\Lambda_{\Omega_E}} \label{eq:exb_average} \\
 \Omega_E &:= \mu_i N_i \frac{\vn\psi_p\cn\phi}{B^2} \equiv \mu_i N_i u_{E,\varphi} \\
 \vec j_{\Omega_E} &:= \Omega_E (\vec u_E + \vec u_D)
-    - \vn A_\parallel\cn\psi_p \left(\frac{1}{\beta} \frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \mu_i \tau_i N_iU_i\right) \\
+    - \vn A_\parallel\cn\psi_p \left(\frac{1}{\beta} \frac{\bhat\times\vn A_\parallel}{B} +\frac{1}{2} \bhat \times \vn \mu_i \tau_i N_iU_{\parallel,i}\right) \\
     \mathcal S_{\Omega_E} &:= \mu_i S_{n_e} \frac{\vn\psi_p\cn\phi}{B^2} \quad
     \Lambda_{\Omega_E} := \mu_i \Lambda_{n_e}\frac{\vn\psi_p\cn\phi}{B^2}
 \end{align}
@@ -1254,12 +1254,12 @@ In the output file we have
     jsoexbiUD\_tt &$\mu_i\tau_i \frac{\vn\psi_p\cn\phi}{B^2} \frac{\bhat\times\vn N_i\cn \psi_p}{B}$ &
     jsoexbeUD\_tt &$\mu_i\tau_i \frac{\vn\psi_p\cn\phi}{B^2} \frac{\bhat\times\vn n_e\cn \psi_p}{B}$ \\
     jsoapar\_tt &$ -\vn\psi_p\cn A_\parallel \frac{\bhat\times\vn A_\parallel\cn \psi_p}{B\beta}$ &
-    jsodiaApar\_tt & $ -\frac{1}{2}\tau_i \vn\psi_p\cn  (N_iU_i)\frac{\bhat\times\vn A_\parallel}{B}\cn\psi_p$ \\
-    jsoexbApar\_tt & $ -\frac{1}{2}\tau_i \bhat\times\vn  (N_iU_i)\cn\psi_p \vn A_\parallel\cn\psi_p$ &
+    jsodiaApar\_tt & $ -\frac{1}{2}\tau_i \vn\psi_p\cn  (N_iU_{\parallel,i})\frac{\bhat\times\vn A_\parallel}{B}\cn\psi_p$ \\
+    jsoexbApar\_tt & $ -\frac{1}{2}\tau_i \bhat\times\vn  (N_iU_{\parallel,i})\cn\psi_p \vn A_\parallel\cn\psi_p$ &
     socurve\_tt &$z_e\tau_e n_e \mathcal K(\psi_p)$ \\
     socurvi\_tt &$z_i\tau_i N_i \mathcal K(\psi_p)$ &
-    socurvkappae\_tt &$z_e\mu_e n_eu_e^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
-    socurvkappai\_tt &$z_i\mu_i N_iU_i^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ & \\
+    socurvkappae\_tt &$z_e\mu_e n_eu_{\parallel,e}^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ \\
+    socurvkappai\_tt &$z_i\mu_i N_iU_{\parallel,i}^2 \mathcal K_{\vn\times\bhat}(\psi_p)$ & \\
     sosne\_tt & $\mu_i S_{n_e} \vn\psi_p\cn\phi/B^2$ &
     sospi\_tt & $\mu_i \tau_i \vn\psi_p \cn S_{n_e}$\\
     loexbe\_tt & $ \mu_i \Lambda_{n_e} \vn\psi_p\cn\phi/B^2$ & \\
@@ -1270,19 +1270,19 @@ In the output file we have
 \subsubsection{Parallel momentum balance}
 The flux surface average over the parallel momentum equation under species summation  yields up to order $\mathcal O(\delta^3)$ in the drift-ordering
 \begin{align}
-  \frac{\partial}{\partial t}\RA{\mu_iN_iU_{i} }
+  \frac{\partial}{\partial t}\RA{\mu_iN_iU_{\parallel,i} }
     % \nonumber\\
-    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i \frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_sN_s + z_s\mu_s N_sU_s^2) b_{\perp}^{\;v}  }
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_{\parallel,i} \frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_sN_s + z_s\mu_s N_sU_{\parallel,s}^2) b_{\perp}^{\;v}  }
     \nonumber\\
-   = \sum_s\RA{-z_s\tau_s N_s\npar \ln B} + \mu_i \RA{ S_{N_i} U_i}
+   = \sum_s\RA{-z_s\tau_s N_s\npar \ln B} + \mu_i \RA{ S_{N_i} U_{\parallel,i}}
    \label{eq:parallel_momentum}
 \end{align}
 while the toroidal parallel angular momentum contribution reads up to order $\mathcal O(\delta^3)$
 \begin{align}\label{eq:parallel_momentum_direction}
-    \frac{\partial}{\partial t}  \RA{\mu_iN_iU_i b_\varphi}
-    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_i b_\varphi\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_s N_s + z_s\mu_sN_sU_s^2) b_\varphi b_{\perp}^{\;v} }
+    \frac{\partial}{\partial t}  \RA{\mu_iN_iU_{\parallel,i} b_\varphi}
+    + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_{\parallel,i} b_\varphi\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_s N_s + z_s\mu_sN_sU_{\parallel,s}^2) b_\varphi b_{\perp}^{\;v} }
     \nonumber\\
-   = \RA{F_{L,\varphi}} + \mu_i \RA{ S_{N_i} U_i b_\varphi}
+   = \RA{F_{L,\varphi}} + \mu_i \RA{ S_{N_i} U_{\parallel,i} b_\varphi}
 \end{align}
 
 The relevant terms in the output file are (the Lorentz force term is described in the previous subsection \ref{sec:vorticity_eq})
@@ -1291,18 +1291,18 @@ The relevant terms in the output file are (the Lorentz force term is described i
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} &
 \textbf{Name} &  \textbf{Equation}\\
 \midrule
-    neue &$n_e u_e$ &
-    niui &$\mu_i N_i U_i$ \\
-    neuebphi &$n_eu_eb_\varphi$ &
-    niuibphi &$\mu_i N_iU_ib_\varphi$ \\
-    jsparexbi\_tt       & $\mu_i N_iU_i(\bhat\times\vn\phi)\cn \psi_p/B$ &
-    jsparbphiexbi\_tt   & $\mu_i N_iU_ib_\varphi(\bhat\times\vn\phi)\cn \psi_p/B$ \\
-    jsparApar\_tt       & $\sum_s (z_s \tau_s N_s + z_s \mu_s N_s U_s^2)b_\perp^v$ &
-    jsparbphiApar\_tt   & $\sum_s (z_s \tau_s N_s + z_s \mu_s N_s U_s^2)b_\varphi b_\perp^v$ \\
+    neue &$n_e u_{\parallel,e}$ &
+    niui &$\mu_i N_i U_{\parallel,i}$ \\
+    neuebphi &$n_eu_{\parallel,e}b_\varphi$ &
+    niuibphi &$\mu_i N_iU_{\parallel,i}b_\varphi$ \\
+    jsparexbi\_tt       & $\mu_i N_iU_{\parallel,i}(\bhat\times\vn\phi)\cn \psi_p/B$ &
+    jsparbphiexbi\_tt   & $\mu_i N_iU_{\parallel,i}b_\varphi(\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    jsparApar\_tt       & $\sum_s (z_s \tau_s N_s + z_s \mu_s N_s U_{\parallel,s}^2)b_\perp^v$ &
+    jsparbphiApar\_tt   & $\sum_s (z_s \tau_s N_s + z_s \mu_s N_s U_{\parallel,s}^2)b_\varphi b_\perp^v$ \\
     sparmirrore\_tt & $-z_e\tau_en_e\npar \ln B$ &
     sparmirrori\_tt & $-z_i\tau_iN_i\npar \ln B$ \\
-    sparsni\_tt & $\mu_i S_{N_i} U_i$ &
-    sparsnibphi\_tt & $\mu_i S_{N_i} U_ib_\varphi $ \\
+    sparsni\_tt & $\mu_i S_{N_i} U_{\parallel,i}$ &
+    sparsnibphi\_tt & $\mu_i S_{N_i} U_{\parallel,i}b_\varphi $ \\
 \bottomrule
 \end{longtable}
 
@@ -1346,8 +1346,8 @@ In order to test the implementation we manufacture a solution to Eqs.~\eqref{eq:
 \begin{align*}
 n_e(R,Z,\varphi, t) &:= 1 + 0.5\sin(\pi(R-R_0))\sin(\pi Z)\sin(\varphi)\sin(\pi t) \\
 N_i(R,Z,\varphi, t) &:= n_e(R,Z,\varphi,t) = \gamma_{ N_i}  \\
-u_e(R,Z,\varphi, t) &:= \sin(2\pi(R-R_0))\sin(2\pi Z)\sin(2\varphi)\sin(2\pi t)/(3\sqrt{-\mu_e}) \\
-U_i(R,Z,\varphi, t) &:= \sqrt{-\mu_e}u_e(R,Z,\varphi,t) \\
+u_{\parallel,e}(R,Z,\varphi, t) &:= \sin(2\pi(R-R_0))\sin(2\pi Z)\sin(2\varphi)\sin(2\pi t)/(3\sqrt{-\mu_e}) \\
+U_{\parallel,i}(R,Z,\varphi, t) &:= \sqrt{-\mu_e}u_{\parallel,e}(R,Z,\varphi,t) \\
 \phi(R,Z,\varphi,t) &:= \sin(3\pi(R-R_0))\sin(3\pi Z)\sin(3\varphi)\sin(3\pi t)/5; \\
 \psi(R,Z,\varphi,t) &:= \phi(R,Z,\varphi, t) = \gamma_{\phi} \\
 A_\parallel( R,Z,\varphi,t) &:= \beta\sin(4\pi(R-R_0))\sin(4\pi Z)\sin(4\varphi)\sin(4\pi t)/4;
@@ -1376,7 +1376,7 @@ discontinuous Galerkin on structured grid
 Advection terms & direct DG & DG approximation with centered flux of derivatives \\
 Elliptic terms & local DG & The local DG approximation with centered flux \\
 Helmholtz and Elliptic matrix inversions & multigrid/ conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
-Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017}. The terms $\npar N$ and $\npar \phi$ in the velocity equation use a forward difference, while the term $\npar U$ in the
+Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017}. The terms $\npar N$ and $\npar \phi$ in the velocity equation use a forward difference, while the term $\npar U_\parallel$ in the
 density equation uses backward difference. This is to avoid a too wide stencil for the diverence of the current and increases stability for low resistivity. \\
 time & Multistep "Karniadakis" & \\
 \qquad explicit & Multistep "Karniadakis" & $3$rd order explicit\\
@@ -1457,7 +1457,7 @@ If you want to let the simulation run for a certain time instead just choose
 this parameter very large and let the simulation hit the time-limit.
 \\
 eps\_time   & float & 1e-7  & - & Tolerance for solver for implicit part in
-time-stepper (if too low, you'll see oscillations in $u_e$ and/or $\phi$)
+time-stepper (if too low, you'll see oscillations in $u_{\parallel,e}$ and/or $\phi$)
 \\
 rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper. (Ignored in Multistep)
 \\
@@ -1515,13 +1515,13 @@ symmetric & bool & false & false & If true, initialize all quantities symmetric
 in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used
 to construct the parallel derivatives and then overwritten to $N_z\equiv 1$.
 \\
-bc & dict & & & Boundary conditions (note that $A_\parallel$ has the same bc as $U$) \ldots\\
+bc & dict & & & Boundary conditions (note that $A_\parallel$ has the same bc as $U_\parallel$) \ldots\\
 \qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y
 for $n_e$ and $N_i$, DIR (density 1 on boundary) means both convective and
     diffusive outflow while NEU (gradient 0) means no outflow by diffusion
 \\
 \qquad velocity  & char[2] & [NEU,NEU] & - & boundary conditions in x and y for
-$u_e$ and $U_i$ and $A_\parallel$, DIR is in general not very stable, NEU works
+$u_{\parallel,e}$ and $U_{\parallel,i}$ and $A_\parallel$, DIR is in general not very stable, NEU works
 better\\
 \qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for
 $\phi$ and $\psi$, DIR means that the $v_{E,\perp}=0$ on the boundary (i.e. no
@@ -1627,8 +1627,8 @@ BZ               & Dataset & 3 (z,y,x) & Contravariant magnetic field component
 BP               & Dataset & 3 (z,y,x) & Contravariant magnetic field component $B^\varphi$ \\
 electrons        & Dataset & 4 (time, z, y, x) & electron density $n_e$ \\
 ions             & Dataset & 4 (time, z, y, x) & ion density $N_i$ \\
-Ue               & Dataset & 4 (time, z, y, x) & electron velocity $u_e$ \\
-Ui               & Dataset & 4 (time, z, y, x) & ion velocity $U_i$ \\
+Ue               & Dataset & 4 (time, z, y, x) & electron velocity $u_{\parallel,e}$ \\
+Ui               & Dataset & 4 (time, z, y, x) & ion velocity $U_{\parallel,i}$ \\
 potential        & Dataset & 4 (time, z, y, x) & electric potential $\phi$ \\
 induction        & Dataset & 4 (time, z, y, x) & parallel vector potential $A_\parallel$ \\
 X\_2d            & Dataset & 3 (time,y,x) & Selected plane $X(\varphi=0)$ \\
@@ -1650,9 +1650,9 @@ X and Y\_tt represent the quantities described in the tables in previous section
 \midrule
     vorticity &$-\Delta_\perp\phi$ &
     apar\_vorticity &$-\Delta_\perp A_\parallel$ \\
-    dssue & $\npar^2 u_e$&
-    dppue & $\partial_\varphi^2 u_e$\\
-    dpue2 & $(\partial_\varphi u_e)^2$&
+    dssue & $\npar^2 u_{\parallel,e}$&
+    dppue & $\partial_\varphi^2 u_{\parallel,e}$\\
+    dpue2 & $(\partial_\varphi u_{\parallel,e})^2$&
     lperpinv &$L_\perp^{-1} := |\vec\np n_e|/n_e$ \\
     perpaligned &$(\vec\np n_e)^2/n_e$ &
     lparallelinv &$L_\parallel^{-1} := |\npar n_e|/n_e$ \\
@@ -1793,15 +1793,15 @@ The program terminates usually caused by a NaN exception raised. However,
 the cause for the instability has to be determined inspecting the
 last output in the output file.
     \\
-\qquad large fieldaligned oscillations in $u_e$ paired with instability in the edge of the box
+\qquad large fieldaligned oscillations in $u_{\parallel,e}$ paired with instability in the edge of the box
 &
 Apply damping region
     \\
-\qquad Perpendicular grid oscillations in $u_e$ and $\Delta_\perp \phi$ in the damping region, symmetric in $\varphi$
+\qquad Perpendicular grid oscillations in $u_{\parallel,e}$ and $\Delta_\perp \phi$ in the damping region, symmetric in $\varphi$
 &
 Increase damping $alpha$, increase damping boundary, make the box larger/smaller
     \\
-\qquad Spike in $u_e$ shortly after simulation start
+\qquad Spike in $u_{\parallel,e}$ shortly after simulation start
 &
 Increase $\nu_\perp$, increase $N_x$, $N_y$, decrease perturbation amplitude
     \\
-- 
GitLab


From 3cfdea2cac24e64b250eb6d8e654273a386f84d9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 16 Sep 2020 11:56:13 +0200
Subject: [PATCH 316/540] Add parallel source output in feltordiag

---
 src/feltor/feltordiag.h | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index cceca4786..6f8b55a59 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -518,6 +518,13 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::copy( v.f.density_source(0), result);
         }
     },
+    {"spne_tt", "Parallel Source term for electron density (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 1., v.f.density(0), v.f.velocity(0), v.f.divb(), 0., result);
+            dg::blas1::pointwiseDot( 1., v.f.density(0),  v.f.dsU(0), 1., result);
+            dg::blas1::pointwiseDot( 1., v.f.velocity(0), v.f.dsN(0), 1., result);
+        }
+    },
     {"jsniE_tt", "Radial ion particle flux: ExB contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             // ExB Dot GradPsi
@@ -573,6 +580,13 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::copy( v.f.density_source(1), result);
         }
     },
+    {"spni_tt", "Parallel Source term for ion density (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 1., v.f.density(1), v.f.velocity(1), v.f.divb(), 0., result);
+            dg::blas1::pointwiseDot( 1., v.f.density(1),  v.f.dsU(1), 1., result);
+            dg::blas1::pointwiseDot( 1., v.f.velocity(1), v.f.dsN(1), 1., result);
+        }
+    },
     /// ------------------- Energy terms ------------------------//
     {"nelnne", "Entropy electrons", false,
         []( DVec& result, Variables& v ) {
-- 
GitLab


From f137d3dada681be6b7f3409e6f836248d24e95d7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 16 Sep 2020 23:56:15 +0200
Subject: [PATCH 317/540] Add volume output in ds_guenther_t

should be zero for div quantities, mx and my does not help with this
but centered seems to be much better than forward/backward
---
 inc/geometries/ds_guenther_t.cu | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/inc/geometries/ds_guenther_t.cu b/inc/geometries/ds_guenther_t.cu
index 77dca63e5..131ffbb45 100644
--- a/inc/geometries/ds_guenther_t.cu
+++ b/inc/geometries/ds_guenther_t.cu
@@ -59,7 +59,7 @@ int main( )
 
     ///##########################################################///
     std::cout << "# TEST Guenther (No Boundary conditions)!\n";
-    std::cout <<"Guenther:\n";
+    std::cout <<"Guenther rel_Error rel_Volume_integral(should be zero for div and Lap):\n";
     const dg::DVec vol3d = dg::create::volume( g3d);
     for( const auto& tuple :  names)
     {
@@ -68,10 +68,11 @@ int main( )
         const dg::DVec& solution = *std::get<1>(tuple)[1];
         callDS( ds, name, function, derivative, divb, max_iter,1e-8);
         double sol = dg::blas2::dot( vol3d, solution);
+        double vol = dg::blas1::dot( vol3d, derivative)/sqrt( dg::blas2::dot( vol3d, derivative));
         dg::blas1::axpby( 1., solution, -1., derivative);
         double norm = dg::blas2::dot( derivative, vol3d, derivative);
         std::cout <<"    "<<name<<":" <<std::setw(18-name.size())
-                  <<" "<<sqrt(norm/sol)<<"\n";
+                  <<" "<<sqrt(norm/sol)<<"  \t"<<vol<<"\n";
     }
     ///##########################################################///
     return 0;
-- 
GitLab


From d8017c15b583efb1b9a0d05b99191e5b327d48c9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Sep 2020 00:27:29 +0200
Subject: [PATCH 318/540] Add diamagnetic parallel term and par par source

hopefully the parallel terms add up to zero then
---
 src/feltor/feltor.tex   |  2 +-
 src/feltor/feltordiag.h | 28 ++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index e821bbbe8..2715e87ae 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1023,7 +1023,7 @@ two functions for which we have no boundary conditions
         -\mu \mathcal K_{\vn\times\bhat}(NU_\parallel^2)
         -\mu NU_\parallel^2\nc \vec{ \mathcal K_{\vn\times\bhat}}
         + \nu_\perp\Delta_\perp N + \nu_\parallel \Delta_\parallel N + S_N, \\
-    \frac{\partial}{\partial t} W =&
+    \frac{\partial}{\partial t} W_\parallel =&
         - \frac{1}{B}\left[\psi, U_\parallel\right]_{\perp}%& \nonumber\\
         - \frac{1}{\mu} \bar \npar \psi% \nonumber\\
         - \frac{1}{2}\bar \npar U_\parallel^2
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 6f8b55a59..d26b13dd1 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -1003,6 +1003,26 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( 1., result, v.tmp[0],v.f.bphi(), 0., result);
         }
     },
+    {"jspardiai_tt", "Parallel momentum radial flux by Diamagnetic velocity with ion density (Time average)", true,
+        []( DVec& result, Variables& v){
+            // DiaN Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradN(1), v.gradPsip, v.tmp[0]);
+            // DiaU Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradU(1), v.gradPsip, v.tmp[1]);
+
+            // Multiply everything
+            dg::blas1::pointwiseDot( v.p.mu[1]*v.p.tau[1], v.tmp[0], v.f.velocity(1), v.p.mu[1]*v.p.tau[1], v.tmp[1], v.f.density(1), 0., result);
+        }
+    },
+    {"jsparbphidiai_tt", "Parallel angular momentum radial flux by Diamagnetic velocity with ion density (Time average)", true,
+        []( DVec& result, Variables& v){
+            // bphi K Dot GradPsi
+            routines::dot( v.f.curv(), v.gradPsip, result);
+            dg::blas1::pointwiseDot( result, v.f.bphi(), result);
+            // Multiply everything
+            dg::blas1::pointwiseDot( v.p.mu[1]*v.p.tau[1], result, v.f.velocity(1), v.f.density(1), 0., result);
+        }
+    },
     {"jsparApar_tt", "Parallel momentum radial flux by magnetic flutter (Time average)", true,
         []( DVec& result, Variables& v){
             if( v.p.beta == 0)
@@ -1043,6 +1063,14 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// --------------------- Parallel momentum source terms ---------------------//
+    {"sparpar_tt", "Parallel Source for momentum source", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( v.f.velocity(1), v.f.velocity(1), v.tmp[1]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.tmp[1], v.f.divb(), 0., result);
+            dg::blas1::pointwiseDot( 0.5*v.p.mu[1], v.f.density(1),  v.f.velocity(1), v.f.dsU(1), 1., result);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[1], v.f.dsN(1), 1., result);
+        }
+    },
     {"sparsni_tt", "Parallel momentum source by density source", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.p.mu[1],
-- 
GitLab


From 19887836e600918fbae64f634d380aa94b25383a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Sep 2020 00:29:29 +0200
Subject: [PATCH 319/540] EXPERIMENTAL change foward/backward to centered ds

Hopefully this will significantly lower the parallel leakage term
at the presumable cost of stability
---
 src/feltor/feltor.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 185e399a8..702fd4ada 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -821,7 +821,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         //---------------------density--------------------------//
         //density: -Div ( NUb)
         m_ds_N.centered( y[0][i], m_dsN[i]);
-        m_ds_U.backward( fields[1][i], m_dsU[i]);
+        m_ds_U.centered( fields[1][i], m_dsU[i]);
         dg::blas1::pointwiseDot(-1., m_dsN[i], fields[1][i],
             -1., fields[0][i], m_dsU[i], 1., yp[0][i] );
         dg::blas1::pointwiseDot( -1., fields[0][i],fields[1][i],m_divb,
@@ -837,8 +837,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         m_ds_U.centered(-0.5, m_temp1, 1., yp[1][i]);
         // force terms: -tau/mu * ds lnN -1/mu * ds Phi
         // (These two terms converge slowly and require high z resolution)
-        m_ds_N.forward(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
-        m_ds_P.forward(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
+        m_ds_N.centered(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
+        m_ds_P.centered(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
         // diffusion: + nu_par Delta_par U
         dg::blas1::pointwiseDot(m_p.nu_parallel, m_divb, m_dsU[i],
                                 1., yp[1][i]);
-- 
GitLab


From 4c92108f779fdc50f54e8c6f95532d9fede9bac6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Sep 2020 12:09:31 +0200
Subject: [PATCH 320/540] Make multistage eps_pol an input parameter

---
 src/feltor/feltor.h                |  8 ++++----
 src/feltor/feltor.tex              |  9 +++++----
 src/feltor/implicit.h              |  6 +++---
 src/feltor/input/compass.json      |  4 ++--
 src/feltor/input/default.json      |  4 ++--
 src/feltor/input/manufactured.json |  2 +-
 src/feltor/parameters.h            | 15 +++++++++++----
 7 files changed, 28 insertions(+), 20 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 702fd4ada..7da3beaab 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -670,10 +670,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_phi(
     //----------Invert polarisation----------------------------//
     m_old_phi.extrapolate( time, m_phi[0]);
     std::vector<unsigned> number = m_multigrid.direct_solve(
-        m_multi_pol, m_phi[0], m_temp0, {m_p.eps_pol, 2000*m_p.eps_pol, 100*m_p.eps_pol});
+        m_multi_pol, m_phi[0], m_temp0, m_p.eps_pol);
     m_old_phi.update( time, m_phi[0]);
     if(  number[0] == m_multigrid.max_iter())
-        throw dg::Fail( m_p.eps_pol);
+        throw dg::Fail( m_p.eps_pol[0]);
 }
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
@@ -744,10 +744,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
         m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
 #endif //DG_MANUFACTURED
     std::vector<unsigned> number = m_multigrid.direct_solve(
-        m_multi_induction, m_apar, m_temp0, m_p.eps_pol);
+        m_multi_induction, m_apar, m_temp0, m_p.eps_pol[0]);
     m_old_apar.update( time, m_apar);
     if(  number[0] == m_multigrid.max_iter())
-        throw dg::Fail( m_p.eps_pol);
+        throw dg::Fail( m_p.eps_pol[0]);
     //----------Compute Derivatives----------------------------//
     dg::blas2::symv( m_dx_U, m_apar, m_dA[0]);
     dg::blas2::symv( m_dy_U, m_apar, m_dA[1]);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 2715e87ae..45363dad3 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1461,15 +1461,16 @@ time-stepper (if too low, you'll see oscillations in $u_{\parallel,e}$ and/or $\
 \\
 rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper. (Ignored in Multistep)
 \\
-eps\_pol    & float & 1e-6  & - &  Tolerance for residual of the inversion of polarisation and induction Eq. (should not be more than a factor 10 from eps\_time for $\beta\neq  0$ )
+stages      & integer & 3 & 3 & number of stages in multigrid, $2^{\text{stages-1}}$
+has to evenly divide both $N_x$ and $N_y$
+\\
+eps\_pol    & float[stages] & [1e-6,1,1]  & - &  The first number is the tolerance for residual of the inversion of polarisation and induction Eq.. The second number is a multiplicative factor for the accuracy on the second grid in a multigrid scheme, the third for the third grid and so on.  (i.e. $\eps_0\eps_i$ is the accuracy on the i-th grid)
+Tuning those factors is a major performance tuning oppourtunity!! For saturated turbulence the suggested values are [1e-6, 2000, 100].
 \\
 jumpfactor  & float & 1 & 1 & Jumpfactor $\in \left[0.01,1\right]$ in the local DG method for the elliptic terms. (Don't touch unless you know what you're doing.
 \\
 eps\_gamma  & float & 1e-6  & - & Tolerance for $\Gamma_1$
 \\
-stages      & integer & 3 & 3 & number of stages in multigrid, $2^{\text{stages-1}}$
-has to evenly divide both $N_x$ and $N_y$
-\\
 FCI & dict & & & Parameters for Flux coordinate independent approach
 \\
 \qquad refine     & integer[2] & [1,1] & [1,1] & refinement factor in FCI approach in R- and Z-direction.
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index a61b37136..a556e6488 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -113,7 +113,7 @@ struct ImplicitVelocity
         //m_induction.construct(  g,
         //    p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
         //m_induction.elliptic().set_chi( hh);
-        //m_invert.construct( m_temp, g.size(), p.eps_pol,1 );
+        //m_invert.construct( m_temp, g.size(), p.eps_pol[0],1 );
         //Multigrid setup
         m_multi_induction.resize(p.stages);
         m_multigrid.construct( g, p.stages);
@@ -166,10 +166,10 @@ struct ImplicitVelocity
             m_old_apar.extrapolate( m_apar);
             //dg::blas1::scal( m_apar, 0.);
             std::vector<unsigned> number = m_multigrid.direct_solve(
-                m_multi_induction, m_apar, m_temp, {m_p.eps_pol,m_p.eps_pol,m_p.eps_pol});
+                m_multi_induction, m_apar, m_temp, m_p.eps_pol[0]); //eps_pol[0] on all grids
             //m_old_apar.update( m_apar); //don't update here: makes the solver potentially unstable
             if(  number[0] == m_multigrid.max_iter())
-                throw dg::Fail( m_p.eps_pol);
+                throw dg::Fail( m_p.eps_pol[0]);
 
             //compute u_e and U_i from w_e, W_i and apar
             dg::blas1::axpby( 1., m_fields[1][0], -1./m_p.mu[0],
diff --git a/src/feltor/input/compass.json b/src/feltor/input/compass.json
index f450330a9..d88333828 100644
--- a/src/feltor/input/compass.json
+++ b/src/feltor/input/compass.json
@@ -14,10 +14,10 @@
     "inner_loop": 5, 
     "itstp": 500, 
     "maxout": 50, 
-    "eps_pol"    : 1e-6,
+    "stages"     : 3,
+    "eps_pol"    : [1e-6,1,1],
     "jumpfactor" : 1,
     "eps_gamma"  : 1e-6,
-    "stages"     : 3,
     "eps_time"   : 1e-10,
     "rtol"       : 1e-5,
     "mu"          : -0.000272121,
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index 6fe58fff3..d1a480c7b 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -14,10 +14,10 @@
     "inner_loop" : 2,
     "itstp"  : 2,
     "maxout" : 5,
-    "eps_pol"    : 1e-6,
+    "stages"     : 3,
+    "eps_pol"    : [1e-6,1,1],
     "jumpfactor" : 1,
     "eps_gamma"  : 1e-6,
-    "stages"     : 3,
     "eps_time"   : 1e-7,
     "rtol"       : 1e-4,
     "mu"          : -0.000272121,
diff --git a/src/feltor/input/manufactured.json b/src/feltor/input/manufactured.json
index cf1f77919..8172c7a44 100644
--- a/src/feltor/input/manufactured.json
+++ b/src/feltor/input/manufactured.json
@@ -14,7 +14,7 @@
     "inner_loop" : 2,
     "itstp"  : 2,
     "maxout" : 5,
-    "eps_pol"    : 1e-7,
+    "eps_pol"    : [1e-7,1,1],
     "jumpfactor" : 1,
     "eps_gamma"  : 1e-5,
     "stages"     : 3,
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 9adfb579a..b2f680786 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -18,7 +18,7 @@ struct Parameters
     unsigned itstp;
     unsigned maxout;
 
-    double eps_pol;
+    std::vector<double> eps_pol;
     double jfactor;
     double eps_gamma;
     double eps_time;
@@ -67,11 +67,16 @@ struct Parameters
         eps_time    = file::get( mode, js, "eps_time", 1e-10).asDouble();
         rtol        = file::get( mode, js, "rtol", 1e-5).asDouble();
 
-        eps_pol     = file::get( mode, js, "eps_pol", 1e-6).asDouble();
+        stages      = file::get( mode, js, "stages", 3).asUInt();
+        eps_pol.resize(stages);
+        for( unsigned i=0;i<stages; i++)
+        {
+            eps_pol[i] = file::get_idx( mode, js, "eps_pol", i, i==0 ? 1e-6 : 1).asDouble();
+            eps_pol[i]*=eps_pol[0];
+        }
         jfactor     = file::get( mode, js, "jumpfactor", 1).asDouble();
 
         eps_gamma   = file::get( mode, js, "eps_gamma", 1e-6).asDouble();
-        stages      = file::get( mode, js, "stages", 3).asUInt();
         mx          = file::get_idx( mode, js,"FCI","refine", 0u, 1).asUInt();
         my          = file::get_idx( mode, js,"FCI","refine", 1u, 1).asUInt();
         rk4eps      = file::get( mode, js,"FCI", "rk4eps", 1e-6).asDouble();
@@ -164,7 +169,7 @@ struct Parameters
             <<"     Ny = "<<Ny<<"\n"
             <<"     Nz = "<<Nz<<"\n"
             <<"     dt = "<<dt<<"\n"
-            <<"     Accuracy Polar CG:    "<<eps_pol<<"\n"
+            <<"     Accuracy Polar CG:    "<<eps_pol[0]<<"\n"
             <<"     Jump scale factor:    "<<jfactor<<"\n"
             <<"     Accuracy Gamma CG:    "<<eps_gamma<<"\n"
             <<"     Accuracy Time  CG:    "<<eps_time<<"\n"
@@ -172,6 +177,8 @@ struct Parameters
             <<"     Accuracy Fieldline    "<<rk4eps<<"\n"
             <<"     Periodify FCI         "<<std::boolalpha<< periodify<<"\n"
             <<"     Refined FCI           "<<mx<<" "<<my<<"\n";
+        for( unsigned i=1; i<stages; i++)
+            os <<"     Factors for Multigrid "<<i<<" "<<eps_pol[i]<<"\n";
         os << "Output parameters are: \n"
             <<"     n_out  =                 "<<n_out<<"\n"
             <<"     Nx_out =                 "<<Nx_out<<"\n"
-- 
GitLab


From cab10e7340b8b9722bc5d9dc790abd5e57b01950 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Sep 2020 14:16:58 +0200
Subject: [PATCH 321/540] Fix MPI_OUT in feltor_hpc

---
 src/feltor/feltor_hpc.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 733da6be7..420f07b75 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -201,7 +201,7 @@ int main( int argc, char* argv[])
         double psipO = mag.psip()( RO, ZO);
         double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
         double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
+        MPI_OUT std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
         Json::Value jsmod;
         jsmod["modifier"] = "heaviside";
         jsmod["psi0"] = damping_psi0p + damping_alphap/2.;
-- 
GitLab


From f7feafa616a025023ca260b9525fdb522ca04a0c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Sep 2020 15:20:07 +0200
Subject: [PATCH 322/540] Fix: read new eps_pol in parameters correctly

---
 src/feltor/parameters.h | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index b2f680786..62c1f529f 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -69,9 +69,10 @@ struct Parameters
 
         stages      = file::get( mode, js, "stages", 3).asUInt();
         eps_pol.resize(stages);
-        for( unsigned i=0;i<stages; i++)
+        eps_pol[0] = file::get_idx( mode, js, "eps_pol", 0, 1e-6).asDouble();
+        for( unsigned i=1;i<stages; i++)
         {
-            eps_pol[i] = file::get_idx( mode, js, "eps_pol", i, i==0 ? 1e-6 : 1).asDouble();
+            eps_pol[i] = file::get_idx( mode, js, "eps_pol", i, 1).asDouble();
             eps_pol[i]*=eps_pol[0];
         }
         jfactor     = file::get( mode, js, "jumpfactor", 1).asDouble();
-- 
GitLab


From 6c64a27ff9e980aba04059291ba567b1ce11f471 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Sep 2020 17:18:54 +0200
Subject: [PATCH 323/540] Add tcv.json input file

We need a damping region in the PFR as well else the
simulation always crashes
---
 src/feltor/input/tcv.json | 69 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 src/feltor/input/tcv.json

diff --git a/src/feltor/input/tcv.json b/src/feltor/input/tcv.json
new file mode 100644
index 000000000..3c108dbd8
--- /dev/null
+++ b/src/feltor/input/tcv.json
@@ -0,0 +1,69 @@
+{
+    "n"  : 3,
+    "Nx" : 56,
+    "Ny" : 96,
+    "Nz" : 8,
+    "dt" : 1e-1,
+    "compression" : [2,2],
+    "FCI":
+    {
+        "refine": [1,1],
+        "rk4eps": 1e-6,
+        "periodify": true
+    },
+    "inner_loop": 4,
+    "itstp": 2,
+    "maxout": 10,
+    "eps_pol"    : [1e-6,1,1],
+    "jumpfactor" : 1,
+    "eps_gamma"  : 1e-6,
+    "stages"     : 3,
+    "eps_time"   : 1e-10,
+    "rtol"       : 1e-5,
+    "mu"          : -0.000272121,
+    "tau"         : 0.0,
+    "beta"        : 1e-4,
+    "nu_perp"     : 1e-2,
+    "perp_diff"   : "hyperviscous",
+    "nu_parallel" : 1e3,
+    "resistivity" : 3.43e-6,
+    "curvmode"   : "toroidal",
+    "symmetric"  : false,
+    "bc" :
+    {
+        "density" : ["DIR", "DIR"],
+        "velocity": ["NEU", "NEU"],
+        "potential":["DIR", "DIR"]
+    },
+    "box" :
+    {
+        "scaleR" :  [1.30,1.10],
+        "scaleZ" :  [1.3,1.05]
+    },
+    "initne"     : "turbulence",
+    "initphi"    : "zero",
+    "amplitude" : 0.001,
+    "sigma"     : 2.0,
+    "posX"      : 0.6,
+    "posY"      : 0,
+    "sigma_z"   : 0.25,
+    "k_psi"     : 0,
+    "profile":
+    {
+        "amp": 0,
+        "alpha": 0.2
+    },
+    "source" :
+    {
+        "rate": 1e-2,
+        "type": "influx",
+        "boundary": 0.55,
+        "alpha" : 0.2
+    },
+    "damping":
+    {
+        "rate" : 1e-2,
+        "boundary": 1.05,
+        "alpha": 0.20
+    }
+}
-- 
GitLab


From fa7269a01802cafd865150a836309d0281daa656 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Thu, 17 Sep 2020 23:06:06 +0200
Subject: [PATCH 324/540] EXPERIMENTAL save two par derivatives on log N and U

preliminary test show absolutely no difference to the previous version
so we should be safe to choose the faster one
---
 src/feltor/feltor.h | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 7da3beaab..034e7b231 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -833,11 +833,13 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         dg::blas1::axpby( m_p.nu_parallel, m_dssN[i], 1., yp[0][i]);
         //---------------------velocity-------------------------//
         // Burgers term: -0.5 ds U^2
-        dg::blas1::pointwiseDot(fields[1][i], fields[1][i], m_temp1); //U^2
-        m_ds_U.centered(-0.5, m_temp1, 1., yp[1][i]);
+        //dg::blas1::pointwiseDot(fields[1][i], fields[1][i], m_temp1); //U^2
+        //m_ds_U.centered(-0.5, m_temp1, 1., yp[1][i]);
+            dg::blas1::pointwiseDot(-1., fields[1][i], m_dsU[i], 1., yp[1][i]); //U^2
         // force terms: -tau/mu * ds lnN -1/mu * ds Phi
         // (These two terms converge slowly and require high z resolution)
-        m_ds_N.centered(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
+        //m_ds_N.centered(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
+        dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_dsN[i], fields[0][i], 1., yp[1][i]);
         m_ds_P.centered(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
         // diffusion: + nu_par Delta_par U
         dg::blas1::pointwiseDot(m_p.nu_parallel, m_divb, m_dsU[i],
-- 
GitLab


From 19deb9696b90ea73802144b04d6a3e25d955de45 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Sep 2020 23:52:11 +0200
Subject: [PATCH 325/540] Add more output in parallel momentum equation

---
 src/feltor/feltor.h     | 34 +++++++++++++++++++++-------------
 src/feltor/feltor.tex   | 10 ++++++----
 src/feltor/feltordiag.h | 28 ++++++++++++++++++++++------
 3 files changed, 49 insertions(+), 23 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 034e7b231..892fd02b7 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -131,13 +131,13 @@ struct ComputePsi{
         GammaPhi = GammaPhi - 0.5*HdxPhi;
     }
 };
-struct ComputeLogN{
-    DG_DEVICE
-    void operator()( double tilde_n, double& npe, double& logn) const{
-        npe =  tilde_n + 1.;
-        logn =  log(npe);
-    }
-};
+//struct ComputeLogN{
+//    DG_DEVICE
+//    void operator()( double tilde_n, double& npe, double& logn) const{
+//        npe =  tilde_n + 1.;
+//        logn =  log(npe);
+//    }
+//};
 struct ComputeSource{
     DG_DEVICE
     void operator()( double& result, double tilde_n, double profne,
@@ -235,6 +235,9 @@ struct Explicit
     const Container & dsU (int i) const {
         return m_dsU[i];
     }
+    const Container & dsP (int i) const {
+        return m_dsP[i];
+    }
     const Container & dssN(int i) { //2nd fieldaligned derivative
         return m_dssN[i];
     }
@@ -368,7 +371,7 @@ struct Explicit
     Container m_vol3d;
 
     Container m_apar;
-    std::array<Container,2> m_phi, m_logn, m_dsN, m_dsU, m_dssN, m_dssU;
+    std::array<Container,2> m_phi, m_dsN, m_dsU, m_dsP, m_dssN, m_dssU;
     std::array<Container,3> m_dA;
     std::array<std::array<Container,3>,2> m_dP, m_dN, m_dU;
     std::array<std::array<Container,2>,2> m_fields, m_s; //fields, sources
@@ -564,7 +567,7 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     m_apar = m_temp0;
 
     m_phi[0] = m_phi[1] = m_temp0;
-    m_dssN = m_dssU = m_dsN = m_dsU =  m_logn = m_phi;
+    m_dssN = m_dssU = m_dsN = m_dsU = m_dsP = m_phi;
     m_dA[0] = m_dA[1] = m_dA[2] = m_temp0;
     m_dP[0] = m_dP[1] = m_dA;
     m_dN = m_dU = m_dP;
@@ -835,12 +838,14 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         // Burgers term: -0.5 ds U^2
         //dg::blas1::pointwiseDot(fields[1][i], fields[1][i], m_temp1); //U^2
         //m_ds_U.centered(-0.5, m_temp1, 1., yp[1][i]);
-            dg::blas1::pointwiseDot(-1., fields[1][i], m_dsU[i], 1., yp[1][i]); //U^2
+        dg::blas1::pointwiseDot(-1., fields[1][i], m_dsU[i], 1., yp[1][i]); //U^2
         // force terms: -tau/mu * ds lnN -1/mu * ds Phi
         // (These two terms converge slowly and require high z resolution)
-        //m_ds_N.centered(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
+        ////m_ds_N.centered(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
         dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_dsN[i], fields[0][i], 1., yp[1][i]);
-        m_ds_P.centered(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
+        //m_ds_P.centered(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
+        m_ds_P.centered( m_phi[i], m_dsP[i]);
+        dg::blas1::axpby(-1./m_p.mu[i], m_dsP[i], 1.0, yp[1][i]);
         // diffusion: + nu_par Delta_par U
         dg::blas1::pointwiseDot(m_p.nu_parallel, m_divb, m_dsU[i],
                                 1., yp[1][i]);
@@ -881,7 +886,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     timer.tic();
 
     // Transform n-1 to n and n to logn
-    dg::blas1::subroutine( routines::ComputeLogN(), y[0], m_fields[0], m_logn);
+    //dg::blas1::subroutine( routines::ComputeLogN(), y[0], m_fields[0], m_logn);
+
+    // Transform n-1 to n
+    dg::blas1::axpby( 1., y[0], 1., 1.,  m_fields[0]);
 
     // Compute Apar and m_U if necessary --- reads and updates m_fields[1]
     dg::blas1::copy( y[1], m_fields[1]);
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 45363dad3..fe49d96ed 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -744,15 +744,17 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
 \begin{align}
 \frac{\partial}{\partial t} N &+ \vec\nc\left( N \left(
     \vec u_E + \vec u_K + \vec u_{C} + U_\parallel\left(\bhat + {\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
-\mu N \frac{\partial}{\partial t} U_\parallel &+ \mu N \left(
+    \mu \frac{\partial}{\partial t} \left(N U_\parallel\right) &+ \mu \nc \left( NU_\parallel \left(
     \vec u_E + \vec u_K + \vec u_{C} + U_\parallel\left(\bhat + {\vec b}_\perp\right)
-    \right)\cn U_\parallel  \nonumber \\
+    \right)\right)  \nonumber \\
     &+ 2\mu \nc ( NU_\parallel \vec u_{\vn\times\bhat})
     -\mu NU_\parallel\nc \vec u_{\vn\times\bhat}
     + \mu NU_\parallel\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
-    &= -\tau \left(\bhat + {\vec b}_\perp\right)\cn N
+    =& -\tau \left(\bhat + {\vec b}_\perp\right)\cn N
     -N \left( \left(\bhat+{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right)
-    - \eta n_e^2(U_{\parallel,i}-u_{\parallel,e}) + \mu N\Lambda_U + \mu N S_U
+    - \eta n_e^2(U_{\parallel,i}-u_{\parallel,e})
+    \nonumber\\
+    &+ \mu N\left(\Lambda_U + S_U\right) + \mu U_\parallel \left(\Lambda_N + S_N\right)
 \label{}
 \end{align}
 with
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index d26b13dd1..2115c1c29 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -985,10 +985,7 @@ std::vector<Record> diagnostics2d_list = {
             routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
 
             // parallel momentum mu_iN_iU_i
-            dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), 0., v.tmp[0]);
-
-            // Multiply everything
-            dg::blas1::pointwiseDot( result, v.tmp[0], result);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), result, 0., result);
         }
     },
     {"jsparbphiexbi_tt", "Parallel angular momentum radial flux by ExB velocity with ion density (Time average)", true,
@@ -1063,6 +1060,7 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// --------------------- Parallel momentum source terms ---------------------//
+    //Not important
     {"sparpar_tt", "Parallel Source for momentum source", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.f.velocity(1), v.f.velocity(1), v.tmp[1]);
@@ -1086,12 +1084,30 @@ std::vector<Record> diagnostics2d_list = {
     /// --------------------- Mirror force term ---------------------------//
     {"sparmirrore_tt", "Mirror force term with electron density (Time average)", true,
         []( DVec& result, Variables& v){
-            dg::blas1::pointwiseDot( -v.p.tau[0], v.f.divb(), v.f.density(0), 0., result);
+            //dg::blas1::pointwiseDot( -v.p.tau[0], v.f.divb(), v.f.density(0), 0., result);
+            dg::blas1::axpby( -v.p.tau[0], v.f.dsN(0), 0., result);
         }
     },
     {"sparmirrori_tt", "Mirror force term with ion density (Time average)", true,
         []( DVec& result, Variables& v){
-            dg::blas1::pointwiseDot( v.p.tau[1], v.f.divb(), v.f.density(1), 0., result);
+            //dg::blas1::pointwiseDot( v.p.tau[1], v.f.divb(), v.f.density(1), 0., result);
+            dg::blas1::axpby( v.p.tau[1], v.f.dsN(1), 0., result);
+        }
+    },
+    {"sparphie_tt", "Electric force in electron momentum density (Time average)", true,
+        []( DVec& result, Variables& v){
+            dg::blas1::pointwiseDot( -1., v.f.dsP(0), v.f.density(0), 0., result);
+        }
+    },
+    {"sparphii_tt", "Electric force term in ion momentum density (Time average)", true,
+        []( DVec& result, Variables& v){
+            dg::blas1::pointwiseDot( -1., v.f.dsP(1), v.f.density(1), 0., result);
+        }
+    },
+    {"friction_tt", "Friction force in momentum density (Time average)", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::axpby( 1., v.f.velocity(1), -1., v.f.velocity(0), result);
+            dg::blas1::pointwiseDot( -v.p.eta, result, v.f.density(0), v.f.density(0), 0, result);
         }
     },
     /// --------------------- Lorentz force terms ---------------------------//
-- 
GitLab


From 73b53a9d8e3571f6b7920bf03110d44d02d726bb Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 18 Sep 2020 13:49:16 +0200
Subject: [PATCH 326/540] FIX sign error in mirror force

---
 src/feltor/feltordiag.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 2115c1c29..d1e17f879 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -1085,13 +1085,13 @@ std::vector<Record> diagnostics2d_list = {
     {"sparmirrore_tt", "Mirror force term with electron density (Time average)", true,
         []( DVec& result, Variables& v){
             //dg::blas1::pointwiseDot( -v.p.tau[0], v.f.divb(), v.f.density(0), 0., result);
-            dg::blas1::axpby( -v.p.tau[0], v.f.dsN(0), 0., result);
+            dg::blas1::axpby( v.p.tau[0], v.f.dsN(0), 0., result);
         }
     },
     {"sparmirrori_tt", "Mirror force term with ion density (Time average)", true,
         []( DVec& result, Variables& v){
             //dg::blas1::pointwiseDot( v.p.tau[1], v.f.divb(), v.f.density(1), 0., result);
-            dg::blas1::axpby( v.p.tau[1], v.f.dsN(1), 0., result);
+            dg::blas1::axpby( -v.p.tau[1], v.f.dsN(1), 0., result);
         }
     },
     {"sparphie_tt", "Electric force in electron momentum density (Time average)", true,
-- 
GitLab


From 41110494b5051580d527c811d714a6d713548282 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 18 Sep 2020 16:27:49 +0200
Subject: [PATCH 327/540] Add all terms in parallel ion momentum density

even the diffusive ones, it must be one of those
---
 src/feltor/feltor.h     |  4 ++--
 src/feltor/feltordiag.h | 50 ++++++++++++++++++++++++++++++++++++-----
 2 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 892fd02b7..6a7d98a0e 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -888,8 +888,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     // Transform n-1 to n and n to logn
     //dg::blas1::subroutine( routines::ComputeLogN(), y[0], m_fields[0], m_logn);
 
-    // Transform n-1 to n
-    dg::blas1::axpby( 1., y[0], 1., 1.,  m_fields[0]);
+    // Transform n-1 to n (this computes both n_e and N_i)
+    dg::blas1::transform( y[0], m_fields[0], dg::PLUS<double>(+1));
 
     // Compute Apar and m_U if necessary --- reads and updates m_fields[1]
     dg::blas1::copy( y[1], m_fields[1]);
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index d1e17f879..1db670aba 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -979,10 +979,11 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// --------------------- Parallel momentum flux terms ---------------------//
-    {"jsparexbi_tt", "Parallel momentum radial flux by ExB velocity with ion density (Time average)", true,
+    // In equation the electron potential is used!
+    {"jsparexbi_tt", "Parallel momentum radial flux by ExB velocity with ion potential (Time average)", true,
         []( DVec& result, Variables& v){
             // ExB Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(1), v.gradPsip, result);
 
             // parallel momentum mu_iN_iU_i
             dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), result, 0., result);
@@ -991,7 +992,7 @@ std::vector<Record> diagnostics2d_list = {
     {"jsparbphiexbi_tt", "Parallel angular momentum radial flux by ExB velocity with ion density (Time average)", true,
         []( DVec& result, Variables& v){
             // ExB Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(1), v.gradPsip, result);
 
             // parallel momentum mu_iN_iU_i
             dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), 0., v.tmp[0]);
@@ -1011,6 +1012,14 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( v.p.mu[1]*v.p.tau[1], v.tmp[0], v.f.velocity(1), v.p.mu[1]*v.p.tau[1], v.tmp[1], v.f.density(1), 0., result);
         }
     },
+    {"jsparkappai_tt", "Parallel momentum radial flux by curvature velocity (Time average)", true,
+        []( DVec& result, Variables& v){
+            routines::dot( v.f.curvKappa(), v.gradPsip, v.tmp[0]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), v.tmp[0], 0., v.tmp[0]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.f.velocity(1), v.f.velocity(1), v.tmp[0], 0., v.tmp[1]);
+            dg::blas1::axpbypgz( 2.*v.p.tau[1], v.tmp[0], +1., v.tmp[1], 0., result);
+        }
+    },
     {"jsparbphidiai_tt", "Parallel angular momentum radial flux by Diamagnetic velocity with ion density (Time average)", true,
         []( DVec& result, Variables& v){
             // bphi K Dot GradPsi
@@ -1060,8 +1069,8 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// --------------------- Parallel momentum source terms ---------------------//
-    //Not important
-    {"sparpar_tt", "Parallel Source for momentum source", true,
+    //Not so important
+    {"sparpar_tt", "Parallel Source for parallel momentum", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.f.velocity(1), v.f.velocity(1), v.tmp[1]);
             dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.tmp[1], v.f.divb(), 0., result);
@@ -1069,6 +1078,17 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[1], v.f.dsN(1), 1., result);
         }
     },
+    {"spardivKappa_tt", "Divergence Kappa Source for parallel momentum", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( -v.p.mu[1]*v.p.tau[1], v.f.density(1), v.f.velocity(1), v.f.divCurvKappa(), 0., result);
+        }
+    },
+    {"sparKappaphi_tt", "Kappa Phi Source for parallel momentum", true,
+        []( DVec& result, Variables& v ) {
+            routines::dot( v.f.curvKappa(), v.gradPsip, result);
+            dg::blas1::pointwiseDot( -v.p.mu[1], v.f.density(1), v.f.velocity(1), result, 0., result);
+        }
+    },
     {"sparsni_tt", "Parallel momentum source by density source", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.p.mu[1],
@@ -1081,6 +1101,25 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.density_source(1), v.f.velocity(1), v.f.bphi(), 0., result);
         }
     },
+    {"lparpar_tt", "Parallel momentum dissipation by parallel diffusion", true,
+        []( DVec& result, Variables& v ) {
+            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(1),
+                                     0., v.tmp[0]);
+            dg::blas1::axpby( 1., v.f.dssN(1), 1., v.tmp[0]);
+            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(1),
+                                     0., v.tmp[1]);
+            dg::blas1::axpby( 1., v.f.dssU(1), 1., v.tmp[1]);
+            dg::blas1::pointwiseDot( v.p.nu_parallel, v.tmp[0], v.f.velocity(1), v.p.nu_parallel, v.tmp[1], v.f.density(1), 0., result);
+        }
+    },
+    //remove if it does not contribute since it might be expensive
+    {"lparperp_tt", "Parallel momentum dissipation by perp diffusion", true,
+        []( DVec& result, Variables& v ) {
+            v.f.compute_diffusive_lapMperpN( v.f.density(1), result, v.tmp[0]);
+            v.f.compute_diffusive_lapMperpU( v.f.velocity(1), result, v.tmp[1]);
+            dg::blas1::pointwiseDot( -v.p.nu_perp, v.tmp[0], v.f.velocity(1), -v.p.nu_perp, v.tmp[1], v.f.density(1), 0., result);
+        }
+    },
     /// --------------------- Mirror force term ---------------------------//
     {"sparmirrore_tt", "Mirror force term with electron density (Time average)", true,
         []( DVec& result, Variables& v){
@@ -1094,6 +1133,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::axpby( -v.p.tau[1], v.f.dsN(1), 0., result);
         }
     },
+    //electric force balance usually well-fulfilled
     {"sparphie_tt", "Electric force in electron momentum density (Time average)", true,
         []( DVec& result, Variables& v){
             dg::blas1::pointwiseDot( -1., v.f.dsP(0), v.f.density(0), 0., result);
-- 
GitLab


From ee9b85b80f52b07a3b7030eea17db39b168e1b92 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 18 Sep 2020 17:30:11 +0200
Subject: [PATCH 328/540] Added PolynomialRectangle function

---
 inc/dg/functors.h | 38 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index cbea4ac11..0a31ca49e 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -938,6 +938,44 @@ struct PolynomialHeaviside {
     double x0, a;
     int m_s;
 };
+/**
+ * @brief An approximation to the Rectangle function using polynomials
+     \f[ \begin{cases}
+     0 \text{ if } x < x_l-a_l \\
+        ((16 a_l^3 - 29 a_l^2 (x - x_l) + 20 a_l (x - x_l)^2 - 5 (x - x_l)^3) (a_l + x -
+   x_l)^4)/(32 a_l^7) \text{ if } |x-x_l| < a_l \\
+        1  \text{ if } x_l + a_l < x < x_r-a_r \\
+        ((16 a_r^3 - 29 a_r^2 (x - x_r) + 20 a_r (x - x_r)^2 - 5 (x - x_r)^3) (a_r + x -
+   x_l)^4)/(32 a_r^7) \text{ if } |x-x_r| < a_r \\
+   0 \text{ if } x > x_r + a_r
+     \end{cases}\f]
+
+     Basically just the product of two PolynomialHeaviside functions
+
+     This function is 3 times continuously differentiable, takes the value 0.5 at xl and xr and
+     has a transition width a_l on both sides of xl and a width a_r on both sides of xr.
+ */
+struct PolynomialRectangle {
+    /**
+     * @brief Construct with xb, width and sign
+     *
+     * @param xl left boundary value
+     * @param al left transition width (must be != 0)
+     * @param xr right boundary value
+     * @param ar right transition width (must be != 0)
+     */
+    PolynomialRectangle(double xl, double al, double xr, double ar) :
+        m_hl( xl, al, +1), m_hr( xr, ar, -1) {
+        assert( xl < xr && "left boundary must be left of right boundary");
+    }
+    DG_DEVICE
+    double operator() (double x)const
+    {
+        return m_hl(x)*m_hr(x);
+    }
+    private:
+    PolynomialHeaviside m_hl, m_hr;
+};
 
 /**
  * @brief The integral of PolynomialHeaviside approximates xH(x)
-- 
GitLab


From a84501526a9d129750a46b5cab5631d9c423249c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sat, 19 Sep 2020 23:45:00 +0200
Subject: [PATCH 329/540] Improve parallel momentum diffusion and source

make momentum source zero and diffusion conservative
---
 src/feltor/feltor.h     | 30 ++++++++++++-------
 src/feltor/feltor.tex   | 25 +++++++++++-----
 src/feltor/feltordiag.h | 64 +++++++++++++++++++++--------------------
 3 files changed, 71 insertions(+), 48 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 6a7d98a0e..605efb6d3 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -831,26 +831,29 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
             1.,yp[0][i]);
         //density: + nu_par Delta_par N
         dg::blas1::pointwiseDot( m_p.nu_parallel, m_divb, m_dsN[i],
-                                 1., yp[0][i]);
+                                 0., m_temp0);
         m_ds_N.dss( y[0][i], m_dssN[i]);
-        dg::blas1::axpby( m_p.nu_parallel, m_dssN[i], 1., yp[0][i]);
+        dg::blas1::axpby( m_p.nu_parallel, m_dssN[i], 1., m_temp0);//nu_par Delta_par N
+        //Add to rhs, we again need it further down
+        dg::blas1::apxby( 1., m_temp0, 1., yp[0][i]);
         //---------------------velocity-------------------------//
         // Burgers term: -0.5 ds U^2
         //dg::blas1::pointwiseDot(fields[1][i], fields[1][i], m_temp1); //U^2
         //m_ds_U.centered(-0.5, m_temp1, 1., yp[1][i]);
-        dg::blas1::pointwiseDot(-1., fields[1][i], m_dsU[i], 1., yp[1][i]); //U^2
+        dg::blas1::pointwiseDot(-1., fields[1][i], m_dsU[i], 1., yp[1][i]); //-U ds U
         // force terms: -tau/mu * ds lnN -1/mu * ds Phi
-        // (These two terms converge slowly and require high z resolution)
         ////m_ds_N.centered(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
         dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_dsN[i], fields[0][i], 1., yp[1][i]);
         //m_ds_P.centered(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
         m_ds_P.centered( m_phi[i], m_dsP[i]);
         dg::blas1::axpby(-1./m_p.mu[i], m_dsP[i], 1.0, yp[1][i]);
-        // diffusion: + nu_par Delta_par U
+        // diffusion: + nu_par Delta_par U/N - nu_par U Delta_par N/ N
         dg::blas1::pointwiseDot(m_p.nu_parallel, m_divb, m_dsU[i],
-                                1., yp[1][i]);
+                                0., m_temp1);
         m_ds_U.dss( fields[1][i], m_dssU[i]);
-        dg::blas1::axpby( m_p.nu_parallel, m_dssU[i], 1., yp[1][i]);
+        dg::blas1::axpby( m_p.nu_parallel, m_dssU[i], 1., m_temp1); //nu_par Delta_par U
+        dg::blas1::pointwiseDot( -1., fields[1][i], m_temp0, 1., m_temp1); //
+        dg::blas1::pointwiseDivide( 1., m_temp1, fields[0][i], 1., yp[1][i]);
     }
 }
 
@@ -930,9 +933,6 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
             dg::blas1::axpby( m_omega_source, m_source, 0., m_s[0][0]);
         // -w_d ~n
         dg::blas1::pointwiseDot( -m_omega_damping, m_damping, y[0][0], 1.,  m_s[0][0]);
-        // - w_d U
-        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][0], 0.,  m_s[1][0]);
-        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][1], 0.,  m_s[1][1]);
         //compute FLR corrections S_N = (1-0.5*mu*tau*Lap)*S_n
         dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
         dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
@@ -940,6 +940,16 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         dg::blas1::pointwiseDot( m_p.mu[1], m_s[0][0], m_binv, m_binv, 0., m_temp0);
         m_lapperpP.multiply_sigma( 1., m_temp0, m_phi[0], 1., m_s[0][1]);
 
+        // S_U = - w_d U
+        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][0], 0.,  m_s[1][0]);
+        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][1], 0.,  m_s[1][1]);
+        // S_U += - U S_N/N
+        dg::blas1::pointwiseDot( -1.,  m_fields[1][0],  m_s[0][0], 0., m_temp0);
+        dg::blas1::pointwiseDot( -1.,  m_fields[1][1],  m_s[0][1], 0., m_temp1);
+        dg::blas1::pointwiseDivide( 1.,  m_temp0,  m_fields[0][0], 1., m_s[1][0]);
+        dg::blas1::pointwiseDivide( 1.,  m_temp1,  m_fields[0][1], 1., m_s[1][1]);
+
+        //Add all to the right hand side
         dg::blas1::axpby( 1., m_s, 1.0, yp);
     }
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index fe49d96ed..fe9bfad39 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -823,9 +823,12 @@ For numerical stabilisation we choose:
 \begin{align}
 \Lambda_{n_e,\parallel} &= \nu_\parallel \Delta_\parallel n_e &
 \Lambda_{N_i,\parallel} &= \nu_\parallel \Delta_\parallel N_i \\
-\Lambda_{u_e,\parallel} &= \nu_\parallel \Delta_\parallel u_{\parallel,e} &
-\Lambda_{U_i,\parallel} &= \nu_\parallel \Delta_\parallel U_{\parallel,i}
+\Lambda_{u_e,\parallel} &= \nu_\parallel \left( -u_{e,\parallel} \Delta_\parallel n_e + \Delta_\parallel u_{\parallel,e}\right)/n_e &
+\Lambda_{U_i,\parallel} &= \nu_\parallel \left( -U_{i,\parallel} \Delta_\parallel N_i + \Delta_\parallel U_{\parallel,i}\right)/N_i &
 \end{align}
+Note that the parallel velocity diffusion is designed such that the parallel momentum
+diffusion is $\Lambda_{NU} = \nu_\parallel \Delta_\parallel U_\parallel$, which
+is important in order to conserve parallel momentum inside the flux surfaces.
 Similarly, for the perpendicular dissipation we apply viscous or hyperviscous terms.
 \begin{align}\label{eq:perpdiffNT}
  \Lambda_{n_e,\perp} &=  \nu_\perp \Delta_\perp n_e \text{ or } -\nu_\perp \Delta_\perp^2 n_e&
@@ -1007,9 +1010,11 @@ For both electrons and ions we choose
 \begin{align}
     S^d_{n_e}(R,Z,\varphi,t) &:= -\omega_d (n_e-1)\Theta_{\alpha_p/2}\left(\rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2}\right)\\
     S^d_{N_i}(R,Z,\varphi,t) &= \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S^d_{n_e} -\nc\left( \frac{\mu_i S^d_{n_e}}{B^2}\np \phi\right)\\
-S^d_U(R,Z,\varphi, t)& := -\omega_d U_\parallel \Theta_{\alpha_p/2}\left(  \rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2} \right)
+    S^d_U(R,Z,\varphi, t)& := -\omega_d U_\parallel \Theta_{\alpha_p/2}\left(  \rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2} \right) - \frac{U_\parallel}{N} S_N
 \end{align}
 \end{subequations}
+The last term is there to avoid introducing an unintentional parallel momentum source through the
+density sources.
 
 \subsection{Implemented form}
 The form that we implement avoids derivatives on the product of
@@ -1378,8 +1383,14 @@ discontinuous Galerkin on structured grid
 Advection terms & direct DG & DG approximation with centered flux of derivatives \\
 Elliptic terms & local DG & The local DG approximation with centered flux \\
 Helmholtz and Elliptic matrix inversions & multigrid/ conjugate gradient & Use previous two solutions to extrapolate initial guess and $1/\chi$ as preconditioner \\
-Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017}. The terms $\npar N$ and $\npar \phi$ in the velocity equation use a forward difference, while the term $\npar U_\parallel$ in the
-density equation uses backward difference. This is to avoid a too wide stencil for the diverence of the current and increases stability for low resistivity. \\
+Parallel derivatives & regular  FCI & cf.~\cite{Held2016,Stegmeir2017}.
+All terms use the direct centered difference, which turns out is best
+at keeping the numerical flux-surface leakage in $\RA{\nc ( \bhat NU_\parallel)}$ to a minimum, even
+though it is not exactly zero like in the adjoint discretizations, which in turn are unusable because they do not reliably converge.
+There seems to be no benefit in using the grid refinement technique except when field-aligning the initial condition.
+%The terms $\npar N$ and $\npar \phi$ in the velocity equation use a forward difference, while the term $\npar U_\parallel$ in the
+%density equation uses backward difference. This is to avoid a too wide stencil for the diverence of the current and increases stability for low resistivity.
+\\
 time & Multistep "Karniadakis" & \\
 \qquad explicit & Multistep "Karniadakis" & $3$rd order explicit\\
 \qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion and Resistive terms. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
@@ -1654,8 +1665,8 @@ X and Y\_tt represent the quantities described in the tables in previous section
     vorticity &$-\Delta_\perp\phi$ &
     apar\_vorticity &$-\Delta_\perp A_\parallel$ \\
     dssue & $\npar^2 u_{\parallel,e}$&
-    dppue & $\partial_\varphi^2 u_{\parallel,e}$\\
-    dpue2 & $(\partial_\varphi u_{\parallel,e})^2$&
+    %dppue & $\partial_\varphi^2 u_{\parallel,e}$\\
+    %dpue2 & $(\partial_\varphi u_{\parallel,e})^2$&
     lperpinv &$L_\perp^{-1} := |\vec\np n_e|/n_e$ \\
     perpaligned &$(\vec\np n_e)^2/n_e$ &
     lparallelinv &$L_\parallel^{-1} := |\npar n_e|/n_e$ \\
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 1db670aba..35c5e6b0b 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -399,16 +399,16 @@ std::vector<Record> diagnostics2d_list = {
              dg::blas1::copy(v.f.dssU(0), result);
         }
     },
-    {"dppue", "2nd varphi derivative of electron velocity", false,
-        []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.compute_dppU(0), result);
-        }
-    },
-    {"dpue2", "1st varphi derivative squared of electron velocity", false,
-        []( DVec& result, Variables& v ) {
-             dg::blas1::pointwiseDot(v.f.gradU(0)[2], v.f.gradU(0)[2], result);
-        }
-    },
+    //{"dppue", "2nd varphi derivative of electron velocity", false,
+    //    []( DVec& result, Variables& v ) {
+    //         dg::blas1::copy(v.f.compute_dppU(0), result);
+    //    }
+    //},
+    //{"dpue2", "1st varphi derivative squared of electron velocity", false,
+    //    []( DVec& result, Variables& v ) {
+    //         dg::blas1::pointwiseDot(v.f.gradU(0)[2], v.f.gradU(0)[2], result);
+    //    }
+    //},
     {"lperpinv", "Perpendicular density gradient length scale", false,
         []( DVec& result, Variables& v ) {
             const std::array<DVec, 3>& dN = v.f.gradN(0);
@@ -966,12 +966,6 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.density(1), v.f.velocity(1), result);
         }
     },
-    {"neuebphi", "Product of neue and covariant phi component of magnetic field unit vector", false,
-        []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( 1.,
-                v.f.density(0), v.f.velocity(0), v.f.bphi(), 0., result);
-        }
-    },
     {"niuibphi", "Product of NiUi and covariant phi component of magnetic field unit vector", false,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1.,
@@ -979,11 +973,10 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// --------------------- Parallel momentum flux terms ---------------------//
-    // In equation the electron potential is used!
-    {"jsparexbi_tt", "Parallel momentum radial flux by ExB velocity with ion potential (Time average)", true,
+    {"jsparexbi_tt", "Parallel momentum radial flux by ExB velocity with electron potential (Time average)", true,
         []( DVec& result, Variables& v){
             // ExB Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.f.gradP(1), v.gradPsip, result);
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
 
             // parallel momentum mu_iN_iU_i
             dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), result, 0., result);
@@ -1029,6 +1022,15 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( v.p.mu[1]*v.p.tau[1], result, v.f.velocity(1), v.f.density(1), 0., result);
         }
     },
+    {"jsparbphikappai_tt", "Parallel angular momentum radial flux by curvature velocity (Time average)", true,
+        []( DVec& result, Variables& v){
+            routines::dot( v.f.curvKappa(), v.gradPsip, v.tmp[0]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.f.velocity(1), v.tmp[0], 0., v.tmp[0]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.f.velocity(1), v.f.velocity(1), v.tmp[0], 0., v.tmp[1]);
+            dg::blas1::axpbypgz( 2.*v.p.tau[1], v.tmp[0], +1., v.tmp[1], 0., result);
+            dg::blas1::pointwiseDot( result, v.f.bphi(), result);
+        }
+    },
     {"jsparApar_tt", "Parallel momentum radial flux by magnetic flutter (Time average)", true,
         []( DVec& result, Variables& v){
             if( v.p.beta == 0)
@@ -1069,7 +1071,7 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// --------------------- Parallel momentum source terms ---------------------//
-    //Not so important
+    //Not so important (and probably not accurate)
     {"sparpar_tt", "Parallel Source for parallel momentum", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.f.velocity(1), v.f.velocity(1), v.tmp[1]);
@@ -1078,21 +1080,25 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[1], v.f.dsN(1), 1., result);
         }
     },
+    //not so important
     {"spardivKappa_tt", "Divergence Kappa Source for parallel momentum", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( -v.p.mu[1]*v.p.tau[1], v.f.density(1), v.f.velocity(1), v.f.divCurvKappa(), 0., result);
         }
     },
+    //not so important
     {"sparKappaphi_tt", "Kappa Phi Source for parallel momentum", true,
         []( DVec& result, Variables& v ) {
-            routines::dot( v.f.curvKappa(), v.gradPsip, result);
+            routines::dot( v.f.curvKappa(), v.f.gradP(1), result);
             dg::blas1::pointwiseDot( -v.p.mu[1], v.f.density(1), v.f.velocity(1), result, 0., result);
         }
     },
-    {"sparsni_tt", "Parallel momentum source by density source", true,
+    // should be zero in new implementation
+    {"sparsni_tt", "Parallel momentum source by density and velocity sources", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.p.mu[1],
-                v.f.density_source(1), v.f.velocity(1), 0., result);
+                v.f.density_source(1), v.f.velocity(1),
+                v.p.mu[1], v.f.velocity_source(1), v.f.density(1), 0., result);
         }
     },
     {"sparsnibphi_tt", "Parallel angular momentum source by density source", true,
@@ -1101,18 +1107,14 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.density_source(1), v.f.velocity(1), v.f.bphi(), 0., result);
         }
     },
+    //should be zero
     {"lparpar_tt", "Parallel momentum dissipation by parallel diffusion", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(1),
-                                     0., v.tmp[0]);
-            dg::blas1::axpby( 1., v.f.dssN(1), 1., v.tmp[0]);
-            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(1),
-                                     0., v.tmp[1]);
-            dg::blas1::axpby( 1., v.f.dssU(1), 1., v.tmp[1]);
-            dg::blas1::pointwiseDot( v.p.nu_parallel, v.tmp[0], v.f.velocity(1), v.p.nu_parallel, v.tmp[1], v.f.density(1), 0., result);
+            dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsU(1),
+                                     0., result);
+            dg::blas1::axpby( v.p.nu_parallel, v.f.dssU(1), 1., result);
         }
     },
-    //remove if it does not contribute since it might be expensive
     {"lparperp_tt", "Parallel momentum dissipation by perp diffusion", true,
         []( DVec& result, Variables& v ) {
             v.f.compute_diffusive_lapMperpN( v.f.density(1), result, v.tmp[0]);
-- 
GitLab


From 65752f6294539a1ffdd87adece344e3940110b5c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 20 Sep 2020 00:39:21 +0200
Subject: [PATCH 330/540] Add derive to Extrapolate object

in order to derive Apar in diagnostics
---
 inc/dg/cg.h             | 10 ++++++++++
 src/feltor/feltor.h     |  6 +++++-
 src/feltor/feltordiag.h |  5 ++++-
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index d069534bf..76d3fd7da 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -372,6 +372,16 @@ struct Extrapolation
         }
     }
 
+    /**
+    * @brief Backard difference formula to get time derivative \f$ (x_0-x_1)/(t_0-t_1)\f$
+    * @param dot_x (write only) contains derived value on output
+    * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
+    */
+    template<class ContainerType0>
+    void derive( ContainerType0& dot_x) const{
+        dg::blas1::axpbypgz( 1./(m_t[0]-m_t[1]), m_x[0], 1./(m_t[0]-m_t[1]), m_x[1], 0., dot_x);
+    }
+
     /**
     * @brief Extrapolate value
     * @param new_x (write only) contains extrapolated value on output ( may alias the tail)
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 605efb6d3..074cd0611 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -252,6 +252,10 @@ struct Explicit
         dg::blas2::symv( m_dy_N, m_s[0][i], gradS[1]);
         if(!m_p.symmetric)dg::blas2::symv( m_dz, m_s[0][i], gradS[2]);
     }
+    const Container& compute_dot_induction( Container& tmp) const {
+        m_old_apar.derive( tmp);
+    }
+    //maybe better give these a temporary
     const Container & compute_dppN(int i) { //2nd varphi derivative
         dg::blas2::symv( m_dz, m_fields[0][i], m_temp0);
         dg::blas2::symv( m_dz, m_temp0, m_temp1);
@@ -835,7 +839,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         m_ds_N.dss( y[0][i], m_dssN[i]);
         dg::blas1::axpby( m_p.nu_parallel, m_dssN[i], 1., m_temp0);//nu_par Delta_par N
         //Add to rhs, we again need it further down
-        dg::blas1::apxby( 1., m_temp0, 1., yp[0][i]);
+        dg::blas1::axpby( 1., m_temp0, 1., yp[0][i]);
         //---------------------velocity-------------------------//
         // Burgers term: -0.5 ds U^2
         //dg::blas1::pointwiseDot(fields[1][i], fields[1][i], m_temp1); //U^2
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 35c5e6b0b..239951f1c 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -1101,10 +1101,12 @@ std::vector<Record> diagnostics2d_list = {
                 v.p.mu[1], v.f.velocity_source(1), v.f.density(1), 0., result);
         }
     },
-    {"sparsnibphi_tt", "Parallel angular momentum source by density source", true,
+    {"sparsnibphi_tt", "Parallel angular momentum source by density and velocity sources", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.p.mu[1],
                 v.f.density_source(1), v.f.velocity(1), v.f.bphi(), 0., result);
+            dg::blas1::pointwiseDot( v.p.mu[1],
+                v.f.velocity_source(1), v.f.density(1), v.f.bphi(), 1., result);
         }
     },
     //should be zero
@@ -1141,6 +1143,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( -1., v.f.dsP(0), v.f.density(0), 0., result);
         }
     },
+    //These two should be almost the same
     {"sparphii_tt", "Electric force term in ion momentum density (Time average)", true,
         []( DVec& result, Variables& v){
             dg::blas1::pointwiseDot( -1., v.f.dsP(1), v.f.density(1), 0., result);
-- 
GitLab


From b7b5f404f29ba9bfed851c056aa2df5117363a83 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 20 Sep 2020 22:01:06 +0200
Subject: [PATCH 331/540] Add b_perp terms in parallel e- force balance

also document, by now we have about 100 output quantities,
which effectively doubles the filesize compared to just outputting
the 3d quantities
---
 src/feltor/feltor.h     |  2 +-
 src/feltor/feltor.tex   | 34 ++++++++++++++++++++++++++++++----
 src/feltor/feltordiag.h | 25 ++++++++++++++++++++-----
 3 files changed, 51 insertions(+), 10 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 074cd0611..6aae00957 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -252,7 +252,7 @@ struct Explicit
         dg::blas2::symv( m_dy_N, m_s[0][i], gradS[1]);
         if(!m_p.symmetric)dg::blas2::symv( m_dz, m_s[0][i], gradS[2]);
     }
-    const Container& compute_dot_induction( Container& tmp) const {
+    void compute_dot_induction( Container& tmp) const {
         m_old_apar.derive( tmp);
     }
     //maybe better give these a temporary
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index fe9bfad39..e1f9be433 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -646,7 +646,7 @@ with $\alpha_j = \frac{j}{N_\varphi}$ and $w(\varphi)$ a linear weight function
 
 
 Now, an interesting question is, what happens if we are trying to apply the above
-results to a function that is not field-aligned like $B(R,Z)$ of $\vec K\cdot \nabla\psi_p$ for instance? For those functions $\bar f_\mathrm{old}$ actually yields the exact
+results to a function that is not field-aligned like $B(R,Z)$ of $\vec K\cn\psi_p$ for instance? For those functions $\bar f_\mathrm{old}$ actually yields the exact
 result, while the convolution is an approximation.
 Here, we have to test.
 Typically, the functions that we use are slowly varying in $R$ and $Z$ and
@@ -1042,7 +1042,7 @@ two functions for which we have no boundary conditions
         -2\tau U_\parallel\mathcal K_{\vn\times\bhat}(\ln N)
         - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_{\parallel,i} - u_{\parallel,e}) \nonumber\\&
         + \nu_\perp\Delta_\perp U_\parallel
-        + \nu_\parallel \Delta_\parallel U_\parallel
+        + \frac{\nu_\parallel}{N} \Delta_\parallel U_\parallel - \frac{\nu_\parallel}{N} U_\parallel \Delta_\parallel U_\parallel
         + S_U,
         \label{eq:EgyrofluidU} \\
         W_\parallel&:= \left( U_\parallel + \frac{A_\parallel}{\mu}\right)
@@ -1304,15 +1304,41 @@ The relevant terms in the output file are (the Lorentz force term is described i
     niuibphi &$\mu_i N_iU_{\parallel,i}b_\varphi$ \\
     jsparexbi\_tt       & $\mu_i N_iU_{\parallel,i}(\bhat\times\vn\phi)\cn \psi_p/B$ &
     jsparbphiexbi\_tt   & $\mu_i N_iU_{\parallel,i}b_\varphi(\bhat\times\vn\phi)\cn \psi_p/B$ \\
+    jspardiai\_tt       & $\mu_i \tau_i N_iU_{\parallel,i}\vec K\cn\psi_p$ &
+    jsparbphdiai\_tt   & $\mu_i \tau_i N_iU_{\parallel,i}b_\varphi\vec K\cn\psi_p$ \\
+    jsparkappai\_tt       & $\mu_i N_iU_{\parallel,i} ( \mu_i U_{\parallel,i}^2 + 2\tau_i)\vec K_{\vn\times \bhat}\cn\psi_p$ &
+    jsparbphikappai\_tt       & $\mu_i N_iU_{\parallel,i}b_\varphi ( \mu_i U_{\parallel,i}^2 + 2\tau_i)\vec K_{\vn\times \bhat}\cn\psi_p$ \\
     jsparApar\_tt       & $\sum_s (z_s \tau_s N_s + z_s \mu_s N_s U_{\parallel,s}^2)b_\perp^v$ &
     jsparbphiApar\_tt   & $\sum_s (z_s \tau_s N_s + z_s \mu_s N_s U_{\parallel,s}^2)b_\varphi b_\perp^v$ \\
     sparmirrore\_tt & $-z_e\tau_en_e\npar \ln B$ &
     sparmirrori\_tt & $-z_i\tau_iN_i\npar \ln B$ \\
-    sparsni\_tt & $\mu_i S_{N_i} U_{\parallel,i}$ &
-    sparsnibphi\_tt & $\mu_i S_{N_i} U_{\parallel,i}b_\varphi $ \\
+    sparsni\_tt & $\mu_i S_{N_i} U_{\parallel,i} + \mu_i S_{U_i} N_i $ &
+    sparsnibphi\_tt & $\mu_i S_{N_i} U_{\parallel,i}b_\varphi + \mu_i S_{U,i} N_i b_\varphi $ \\
+    lparpar\_tt   & $\nu_\parallel \Delta_\parallel U_{\parallel,i}$ &
+    lparperp\_tt & $-\nu_\perp U_{\parallel,i} \Delta_\perp^2 N_i - \nu_\perp N_i\Delta_\perp^2 U_{\parallel,i} $ \\
 \bottomrule
 \end{longtable}
 
+\subsubsection{Parallel electron force balance}
+We gather the dominant terms in the electron momentum equation (neglecting all terms as $\mu_e=0$). This leaves the parallel force balance
+\begin{align}
+    -(\bhat + \vec b_\perp) \cn n_e +n_e\left( \left( \bhat + \vec b_\perp \right) \cn \phi + \frac{\partial A_\parallel}{\partial t} \right) +\eta n_e^2 \left( U_{\parallel,i} - u_{\parallel,e}\right) = 0
+\end{align}
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Equation} &
+\textbf{Name} &  \textbf{Equation}\\
+\midrule
+    sparphie\_tt & $n_e\npar \phi$ &
+    friction\_tt & $ \eta n_e^2(U_{\parallel,i}-u_{\parallel,e})$ \\
+    sparmirrore\_tt & $-\npar n_e$ &
+    sparmirrorAe\_tt & $\vn A_\parallel \times \bhat \cn n_e /B$ \\
+    sparphiAe\_tt & $n_e \vn A_\parallel \times \bhat \cn \phi /B$ &
+    spardotAe\_tt & $ n_e \partial A_\parallel /\partial t$ \\
+\bottomrule
+\end{longtable}
+
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 
 
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 239951f1c..bc1b5c0a1 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -346,7 +346,7 @@ std::vector<Record_static> diagnostics2d_static_list = {
         }
     }
 };
-// and here are all the 2d outputs we want to produce
+// and here are all the 2d outputs we want to produce (currently ~ 100)
 std::vector<Record> diagnostics2d_list = {
     {"electrons", "Electron density", false,
         []( DVec& result, Variables& v ) {
@@ -1131,6 +1131,11 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::axpby( v.p.tau[0], v.f.dsN(0), 0., result);
         }
     },
+    {"sparmirrorAe_tt", "Apar Mirror force term with electron density (Time average)", true,
+        []( DVec& result, Variables& v){
+            routines::jacobian( v.f.gradA() , v.f.bhatgB(), v.f.gradN(0), result);
+        }
+    },
     {"sparmirrori_tt", "Mirror force term with ion density (Time average)", true,
         []( DVec& result, Variables& v){
             //dg::blas1::pointwiseDot( v.p.tau[1], v.f.divb(), v.f.density(1), 0., result);
@@ -1140,7 +1145,19 @@ std::vector<Record> diagnostics2d_list = {
     //electric force balance usually well-fulfilled
     {"sparphie_tt", "Electric force in electron momentum density (Time average)", true,
         []( DVec& result, Variables& v){
-            dg::blas1::pointwiseDot( -1., v.f.dsP(0), v.f.density(0), 0., result);
+            dg::blas1::pointwiseDot( 1., v.f.dsP(0), v.f.density(0), 0., result);
+        }
+    },
+    {"sparphiAe_tt", "Apar Electric force in electron momentum density (Time average)", true,
+        []( DVec& result, Variables& v){
+            routines::jacobian( v.f.gradA() , v.f.bhatgB(), v.f.gradP(0), result);
+            dg::blas1::pointwiseDot( v.f.density(0), result, result);
+        }
+    },
+    {"spardotAe_tt", "Apar Electric force in electron momentum density (Time average)", true,
+        []( DVec& result, Variables& v){
+            v.f.compute_dot_induction( result);
+            dg::blas1::pointwiseDot( v.f.density(0), result, result);
         }
     },
     //These two should be almost the same
@@ -1152,7 +1169,7 @@ std::vector<Record> diagnostics2d_list = {
     {"friction_tt", "Friction force in momentum density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::axpby( 1., v.f.velocity(1), -1., v.f.velocity(0), result);
-            dg::blas1::pointwiseDot( -v.p.eta, result, v.f.density(0), v.f.density(0), 0, result);
+            dg::blas1::pointwiseDot( v.p.eta, result, v.f.density(0), v.f.density(0), 0, result);
         }
     },
     /// --------------------- Lorentz force terms ---------------------------//
@@ -1193,8 +1210,6 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( v.f.density_source(0), v.hoo, result);
         }
     },
-
-
 };
 
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%END DIAGNOSTICS LIST%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-- 
GitLab


From 68d5306ffabd085722cb6d28f46e4204ee9eaf7d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 21 Sep 2020 00:38:38 +0200
Subject: [PATCH 332/540] Change implementation of Field modification

without changing its meaning
---
 inc/geometries/modified.h   | 151 ++++++++++++++
 inc/geometries/polynomial.h | 136 +------------
 inc/geometries/solovev.h    | 378 ++++++++++--------------------------
 3 files changed, 257 insertions(+), 408 deletions(-)
 create mode 100644 inc/geometries/modified.h

diff --git a/inc/geometries/modified.h b/inc/geometries/modified.h
new file mode 100644
index 000000000..0822e11a2
--- /dev/null
+++ b/inc/geometries/modified.h
@@ -0,0 +1,151 @@
+#pragma once
+
+#include <iostream>
+#include <cmath>
+#include <vector>
+
+#include "dg/blas.h"
+
+#include "dg/topology/functions.h"
+#include "dg/functors.h"
+#include "magnetic_field.h"
+
+
+/*!@file
+ *
+ * Modified MagneticField objects
+ */
+namespace dg
+{
+namespace geo
+{
+namespace mod
+{
+
+struct Psip: public aCylindricalFunctor<Psip>
+{
+    Psip( std::function<double(double,double)> psip, double psi0, double alpha, double sign = -1) :
+        m_ipoly( psi0, alpha, sign), m_psip(psip)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        return m_ipoly( psip);
+    }
+    private:
+    dg::IPolynomialHeaviside m_ipoly;
+    std::function<double(double,double)> m_psip;
+};
+struct PsipR: public aCylindricalFunctor<PsipR>
+{
+    PsipR( std::function<double(double,double)> psip, std::function<double(double,double)> psipR, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_psip(psip), m_psipR(psipR)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip  = m_psip(R,Z);
+        double psipR = m_psipR(R,Z);
+        return psipR*m_poly( psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    std::function<double(double,double)> m_psip, m_psipR;
+};
+struct PsipZ: public aCylindricalFunctor<PsipZ>
+{
+    PsipZ( std::function<double(double,double)> psip, std::function<double(double,double)> psipZ, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_psip(psip), m_psipZ(psipZ)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        double psipZ = m_psipZ(R,Z);
+        return psipZ*m_poly( psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    std::function<double(double,double)> m_psip, m_psipZ;
+};
+
+struct PsipZZ: public aCylindricalFunctor<PsipZZ>
+{
+    PsipZZ( std::function<double(double,double)> psip, std::function<double(double,double)> psipZ, std::function<double(double,double)> psipZZ, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(psip), m_psipZ(psipZ), m_psipZZ(psipZZ)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        double psipZ = m_psipZ(R,Z);
+        double psipZZ = m_psipZZ(R,Z);
+        return psipZZ*m_poly( psip) + psipZ*psipZ*m_dpoly(psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    dg::DPolynomialHeaviside m_dpoly;
+    std::function<double(double,double)> m_psip, m_psipZ, m_psipZZ;
+};
+
+struct PsipRR: public aCylindricalFunctor<PsipRR>
+{
+    PsipRR( std::function<double(double,double)> psip, std::function<double(double,double)> psipR, std::function<double(double,double)> psipRR, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(psip), m_psipR(psipR), m_psipRR(psipRR)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        double psipR = m_psipR(R,Z);
+        double psipRR = m_psipRR(R,Z);
+        return psipRR*m_poly( psip) + psipR*psipR*m_dpoly(psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    dg::DPolynomialHeaviside m_dpoly;
+    std::function<double(double,double)> m_psip, m_psipR, m_psipRR;
+};
+struct PsipRZ: public aCylindricalFunctor<PsipRZ>
+{
+    PsipRZ( std::function<double(double,double)> psip, std::function<double(double,double)> psipR, std::function<double(double,double)> psipZ, std::function<double(double,double)> psipRZ, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(psip), m_psipR(psipR), m_psipZ(psipZ), m_psipRZ(psipRZ)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        double psipR = m_psipR(R,Z);
+        double psipZ = m_psipZ(R,Z);
+        double psipRZ = m_psipRZ(R,Z);
+        return psipRZ*m_poly( psip) + psipR*psipZ*m_dpoly(psip);
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    dg::DPolynomialHeaviside m_dpoly;
+    std::function<double(double,double)> m_psip, m_psipR, m_psipZ, m_psipRZ;
+};
+
+static inline dg::geo::CylindricalFunctorsLvl2 createPsip( const CylindricalFunctorsLvl2& psip,
+    double psi0, double alpha, double sign = -1)
+{
+    return CylindricalFunctorsLvl2(
+            mod::Psip(psip.f(), psi0, alpha, sign),
+            mod::PsipR(psip.f(), psip.dfx(), psi0, alpha, sign),
+            mod::PsipZ(psip.f(), psip.dfy(), psi0, alpha, sign),
+            mod::PsipRR(psip.f(), psip.dfx(), psip.dfxx(), psi0, alpha, sign),
+            mod::PsipRZ(psip.f(), psip.dfx(), psip.dfy(), psip.dfxy(), psi0, alpha, sign),
+            mod::PsipZZ(psip.f(), psip.dfy(), psip.dfyy(), psi0, alpha, sign));
+}
+
+} //namespace mod
+
+//Create heaviside modification, does not modify Ipol
+static inline dg::geo::TokamakMagneticField createModifiedField(
+    const dg::geo::TokamakMagneticField& in, double psi0, double alpha, double sign = -1)
+{
+    const MagneticFieldParameters inp = in.params();
+    MagneticFieldParameters params = { inp.a(), inp.elongation(), inp.triangularity(),
+            inp.getEquilibrium(), modifier::heaviside, inp.getForm()};
+    return TokamakMagneticField( in.R0(), mod::createPsip(in.get_psip(), psi0,
+        alpha, sign), in.get_ipol(), params);
+}
+
+
+} //namespace geo
+} //namespace dg
diff --git a/inc/geometries/polynomial.h b/inc/geometries/polynomial.h
index ff5ec0fb8..bbc21223c 100644
--- a/inc/geometries/polynomial.h
+++ b/inc/geometries/polynomial.h
@@ -8,6 +8,7 @@
 
 #include "dg/topology/functions.h"
 #include "dg/functors.h"
+#include "modified.h"
 #include "polynomial_parameters.h"
 #include "magnetic_field.h"
 
@@ -184,136 +185,6 @@ static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp)
 
 ///@}
 
-///@cond
-namespace mod
-{
-
-struct Psip: public aCylindricalFunctor<Psip>
-{
-    Psip( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_ipoly( psi0, alpha, sign), m_psip(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        return m_ipoly( psip);
-    }
-    private:
-    dg::IPolynomialHeaviside m_ipoly;
-    polynomial::Psip m_psip;
-};
-struct PsipR: public aCylindricalFunctor<PsipR>
-{
-    PsipR( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_psip(gp), m_psipR(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipR = m_psipR(R,Z);
-        return psipR*m_poly( psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    polynomial::Psip m_psip;
-    polynomial::PsipR m_psipR;
-};
-struct PsipZ: public aCylindricalFunctor<PsipZ>
-{
-    PsipZ( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_psip(gp), m_psipZ(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipZ = m_psipZ(R,Z);
-        return psipZ*m_poly( psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    polynomial::Psip m_psip;
-    polynomial::PsipZ m_psipZ;
-};
-
-struct PsipZZ: public aCylindricalFunctor<PsipZZ>
-{
-    PsipZZ( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipZ(gp), m_psipZZ(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipZ = m_psipZ(R,Z);
-        double psipZZ = m_psipZZ(R,Z);
-        return psipZZ*m_poly( psip) + psipZ*psipZ*m_dpoly(psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    dg::DPolynomialHeaviside m_dpoly;
-    polynomial::Psip m_psip;
-    polynomial::PsipZ m_psipZ;
-    polynomial::PsipZZ m_psipZZ;
-};
-struct PsipRR: public aCylindricalFunctor<PsipRR>
-{
-    PsipRR( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipR(gp), m_psipRR(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipR = m_psipR(R,Z);
-        double psipRR = m_psipRR(R,Z);
-        return psipRR*m_poly( psip) + psipR*psipR*m_dpoly(psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    dg::DPolynomialHeaviside m_dpoly;
-    polynomial::Psip m_psip;
-    polynomial::PsipR m_psipR;
-    polynomial::PsipRR m_psipRR;
-};
-struct PsipRZ: public aCylindricalFunctor<PsipRZ>
-{
-    PsipRZ( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipR(gp), m_psipZ(gp), m_psipRZ(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipR = m_psipR(R,Z);
-        double psipZ = m_psipZ(R,Z);
-        double psipRZ = m_psipRZ(R,Z);
-        return psipRZ*m_poly( psip) + psipR*psipZ*m_dpoly(psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    dg::DPolynomialHeaviside m_dpoly;
-    polynomial::Psip m_psip;
-    polynomial::PsipR m_psipR;
-    polynomial::PsipZ m_psipZ;
-    polynomial::PsipRZ m_psipRZ;
-};
-
-static inline dg::geo::CylindricalFunctorsLvl2 createPsip( Parameters gp,
-    double psi0, double alpha, double sign = -1)
-{
-    return CylindricalFunctorsLvl2( Psip(gp, psi0, alpha, sign), PsipR(gp,
-    psi0, alpha, sign), PsipZ(gp, psi0, alpha, sign), PsipRR(gp, psi0, alpha,
-    sign), PsipRZ(gp, psi0, alpha, sign), PsipZZ(gp, psi0, alpha, sign));
-}
-static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp,
-    double psi0, double alpha, double sign = -1)
-{
-    return CylindricalFunctorsLvl1( Constant( gp.pi), Constant( 0), Constant( 0));
-}
-
-} //namespace mod
-///@endcond
-
-///////////////////////////////////////introduce fields into polynomial namespace
-
-
 } //namespace polynomial
 
 /**
@@ -354,8 +225,9 @@ static inline dg::geo::TokamakMagneticField createModifiedPolynomialField(
 {
     MagneticFieldParameters params( gp.a, gp.elongation, gp.triangularity,
             equilibrium::polynomial, modifier::heaviside, str2form.at( gp.form));
-    return TokamakMagneticField( gp.R_0, polynomial::mod::createPsip(gp, psi0,
-    alpha, sign), polynomial::mod::createIpol(gp, psi0, alpha, sign), params);
+    return TokamakMagneticField( gp.R_0,
+            mod::createPsip( polynomial::createPsip(gp), psi0, alpha, sign),
+        polynomial::createIpol( gp), params);
 }
 
 } //namespace geo
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index d268cf23c..3fd8232a5 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -10,6 +10,7 @@
 #include "dg/functors.h"
 #include "solovev_parameters.h"
 #include "magnetic_field.h"
+#include "modified.h"
 
 
 /*!@file
@@ -64,32 +65,32 @@ struct Psip: public aCylindricalFunctor<Psip>
      *
      * @param gp geometric parameters
      */
-    Psip( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) {}
+    Psip( Parameters gp ): m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_c(gp.c) {}
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,Zn5,Zn6,lgRn;
-        Rn = R/R_0_; Rn2 = Rn*Rn; Rn4 = Rn2*Rn2;
-        Zn = Z/R_0_; Zn2 = Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2; Zn5 = Zn3*Zn2; Zn6 = Zn3*Zn3;
+        Rn = R/m_R0; Rn2 = Rn*Rn; Rn4 = Rn2*Rn2;
+        Zn = Z/m_R0; Zn2 = Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2; Zn5 = Zn3*Zn2; Zn6 = Zn3*Zn3;
         lgRn= log(Rn);
-        return   R_0_*pp_*( Rn4/8.+ A_ * ( 1./2.* Rn2* lgRn-(Rn4)/8.)
-                      + c_[0]  //c_[0] entspricht c_1
-              + c_[1]  *Rn2
-              + c_[2]  *(Zn2 - Rn2 * lgRn )
-              + c_[3]  *(Rn4 - 4.* Rn2*Zn2 )
-              + c_[4]  *(3.* Rn4 * lgRn  -9.*Rn2*Zn2 -12.* Rn2*Zn2 * lgRn + 2.*Zn4)
-              + c_[5]  *(Rn4*Rn2-12.* Rn4*Zn2 +8.* Rn2 *Zn4 )
-              + c_[6]  *(-15.*Rn4*Rn2 * lgRn + 75.* Rn4 *Zn2 + 180.* Rn4*Zn2 * lgRn
+        return   m_R0*m_pp*( Rn4/8.+ m_A * ( 1./2.* Rn2* lgRn-(Rn4)/8.)
+                      + m_c[0]  //m_c[0] entspricht c_1
+              + m_c[1]  *Rn2
+              + m_c[2]  *(Zn2 - Rn2 * lgRn )
+              + m_c[3]  *(Rn4 - 4.* Rn2*Zn2 )
+              + m_c[4]  *(3.* Rn4 * lgRn  -9.*Rn2*Zn2 -12.* Rn2*Zn2 * lgRn + 2.*Zn4)
+              + m_c[5]  *(Rn4*Rn2-12.* Rn4*Zn2 +8.* Rn2 *Zn4 )
+              + m_c[6]  *(-15.*Rn4*Rn2 * lgRn + 75.* Rn4 *Zn2 + 180.* Rn4*Zn2 * lgRn
                          -140.*Rn2*Zn4 - 120.* Rn2*Zn4 *lgRn + 8.* Zn6 )
-              + c_[7]  *Zn
-              + c_[8]  *Rn2*Zn
-                      + c_[9] *(Zn2*Zn - 3.* Rn2*Zn * lgRn)
-              + c_[10] *( 3. * Rn4*Zn - 4. * Rn2*Zn3)
-              + c_[11] *(-45.* Rn4*Zn + 60.* Rn4*Zn* lgRn - 80.* Rn2*Zn3* lgRn + 8. * Zn5)
+              + m_c[7]  *Zn
+              + m_c[8]  *Rn2*Zn
+                      + m_c[9] *(Zn2*Zn - 3.* Rn2*Zn * lgRn)
+              + m_c[10] *( 3. * Rn4*Zn - 4. * Rn2*Zn3)
+              + m_c[11] *(-45.* Rn4*Zn + 60.* Rn4*Zn* lgRn - 80.* Rn2*Zn3* lgRn + 8. * Zn5)
                       );
     }
   private:
-    double R_0_, A_, pp_;
-    std::vector<double> c_;
+    double m_R0, m_A, m_pp;
+    std::vector<double> m_c;
 };
 
 /**
@@ -113,26 +114,26 @@ struct Psip: public aCylindricalFunctor<Psip>
 struct PsipR: public aCylindricalFunctor<PsipR>
 {
     ///@copydoc Psip::Psip()
-    PsipR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) {}
+    PsipR( Parameters gp ): m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_c(gp.c) {}
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn3,Rn5,Zn,Zn2,Zn3,Zn4,lgRn;
-        Rn = R/R_0_; Rn2 = Rn*Rn; Rn3 = Rn2*Rn;  Rn5 = Rn3*Rn2;
-        Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
+        Rn = R/m_R0; Rn2 = Rn*Rn; Rn3 = Rn2*Rn;  Rn5 = Rn3*Rn2;
+        Zn = Z/m_R0; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
         lgRn= log(Rn);
-        return   pp_*(Rn3/2. + (Rn/2. - Rn3/2. + Rn*lgRn)* A_ +
-        2.* Rn* c_[1] + (-Rn - 2.* Rn*lgRn)* c_[2] + (4.*Rn3 - 8.* Rn *Zn2)* c_[3] +
-        (3. *Rn3 - 30.* Rn *Zn2 + 12. *Rn3*lgRn -  24.* Rn *Zn2*lgRn)* c_[4]
-        + (6 *Rn5 - 48 *Rn3 *Zn2 + 16.* Rn *Zn4)*c_[5]
+        return   m_pp*(Rn3/2. + (Rn/2. - Rn3/2. + Rn*lgRn)* m_A +
+        2.* Rn* m_c[1] + (-Rn - 2.* Rn*lgRn)* m_c[2] + (4.*Rn3 - 8.* Rn *Zn2)* m_c[3] +
+        (3. *Rn3 - 30.* Rn *Zn2 + 12. *Rn3*lgRn -  24.* Rn *Zn2*lgRn)* m_c[4]
+        + (6 *Rn5 - 48 *Rn3 *Zn2 + 16.* Rn *Zn4)*m_c[5]
         + (-15. *Rn5 + 480. *Rn3 *Zn2 - 400.* Rn *Zn4 - 90. *Rn5*lgRn +
-            720. *Rn3 *Zn2*lgRn - 240.* Rn *Zn4*lgRn)* c_[6] +
-        2.* Rn *Zn *c_[8] + (-3. *Rn *Zn - 6.* Rn* Zn*lgRn)* c_[9] + (12. *Rn3* Zn - 8.* Rn *Zn3)* c_[10] + (-120. *Rn3* Zn - 80.* Rn *Zn3 + 240. *Rn3* Zn*lgRn -
-            160.* Rn *Zn3*lgRn) *c_[11]
+            720. *Rn3 *Zn2*lgRn - 240.* Rn *Zn4*lgRn)* m_c[6] +
+        2.* Rn *Zn *m_c[8] + (-3. *Rn *Zn - 6.* Rn* Zn*lgRn)* m_c[9] + (12. *Rn3* Zn - 8.* Rn *Zn3)* m_c[10] + (-120. *Rn3* Zn - 80.* Rn *Zn3 + 240. *Rn3* Zn*lgRn -
+            160.* Rn *Zn3*lgRn) *m_c[11]
           );
     }
   private:
-    double R_0_, A_, pp_;
-    std::vector<double> c_;
+    double m_R0, m_A, m_pp;
+    std::vector<double> m_c;
 };
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R}^2}\f]
@@ -154,24 +155,24 @@ struct PsipR: public aCylindricalFunctor<PsipR>
 struct PsipRR: public aCylindricalFunctor<PsipRR>
 {
     ///@copydoc Psip::Psip()
-    PsipRR( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) {}
+    PsipRR( Parameters gp ): m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_c(gp.c) {}
     double do_compute(double R, double Z) const
     {
        double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,lgRn;
-       Rn = R/R_0_; Rn2 = Rn*Rn;  Rn4 = Rn2*Rn2;
-       Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
+       Rn = R/m_R0; Rn2 = Rn*Rn;  Rn4 = Rn2*Rn2;
+       Zn = Z/m_R0; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
        lgRn= log(Rn);
-       return   pp_/R_0_*( (3.* Rn2)/2. + (3./2. - (3. *Rn2)/2. +lgRn) *A_ +  2.* c_[1] + (-3. - 2.*lgRn)* c_[2] + (12. *Rn2 - 8. *Zn2) *c_[3] +
-         (21. *Rn2 - 54. *Zn2 + 36. *Rn2*lgRn - 24. *Zn2*lgRn)* c_[4]
-         + (30. *Rn4 - 144. *Rn2 *Zn2 + 16.*Zn4)*c_[5] + (-165. *Rn4 + 2160. *Rn2 *Zn2 - 640. *Zn4 - 450. *Rn4*lgRn +
-      2160. *Rn2 *Zn2*lgRn - 240. *Zn4*lgRn)* c_[6] +
-      2.* Zn* c_[8] + (-9. *Zn - 6.* Zn*lgRn) *c_[9]
- +   (36. *Rn2* Zn - 8. *Zn3) *c_[10]
- +   (-120. *Rn2* Zn - 240. *Zn3 + 720. *Rn2* Zn*lgRn - 160. *Zn3*lgRn)* c_[11]);
+       return   m_pp/m_R0*( (3.* Rn2)/2. + (3./2. - (3. *Rn2)/2. +lgRn) *m_A +  2.* m_c[1] + (-3. - 2.*lgRn)* m_c[2] + (12. *Rn2 - 8. *Zn2) *m_c[3] +
+         (21. *Rn2 - 54. *Zn2 + 36. *Rn2*lgRn - 24. *Zn2*lgRn)* m_c[4]
+         + (30. *Rn4 - 144. *Rn2 *Zn2 + 16.*Zn4)*m_c[5] + (-165. *Rn4 + 2160. *Rn2 *Zn2 - 640. *Zn4 - 450. *Rn4*lgRn +
+      2160. *Rn2 *Zn2*lgRn - 240. *Zn4*lgRn)* m_c[6] +
+      2.* Zn* m_c[8] + (-9. *Zn - 6.* Zn*lgRn) *m_c[9]
+ +   (36. *Rn2* Zn - 8. *Zn3) *m_c[10]
+ +   (-120. *Rn2* Zn - 240. *Zn3 + 720. *Rn2* Zn*lgRn - 160. *Zn3*lgRn)* m_c[11]);
     }
   private:
-    double R_0_, A_, pp_;
-    std::vector<double> c_;
+    double m_R0, m_A, m_pp;
+    std::vector<double> m_c;
 };
 /**
  * @brief \f[\frac{\partial \hat{\psi}_p }{ \partial \hat{Z}}\f]
@@ -190,29 +191,29 @@ struct PsipRR: public aCylindricalFunctor<PsipRR>
 struct PsipZ: public aCylindricalFunctor<PsipZ>
 {
     ///@copydoc Psip::Psip()
-    PsipZ( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) { }
+    PsipZ( Parameters gp ): m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_c(gp.c) { }
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,Zn5,lgRn;
-        Rn = R/R_0_; Rn2 = Rn*Rn;  Rn4 = Rn2*Rn2;
-        Zn = Z/R_0_; Zn2 = Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2; Zn5 = Zn3*Zn2;
+        Rn = R/m_R0; Rn2 = Rn*Rn;  Rn4 = Rn2*Rn2;
+        Zn = Z/m_R0; Zn2 = Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2; Zn5 = Zn3*Zn2;
         lgRn= log(Rn);
 
-        return   pp_*(2.* Zn* c_[2]
-            -  8. *Rn2* Zn* c_[3] +
-              ((-18.)*Rn2 *Zn + 8. *Zn3 - 24. *Rn2* Zn*lgRn) *c_[4]
-            + ((-24.) *Rn4* Zn + 32. *Rn2 *Zn3)* c_[5]
-            + (150. *Rn4* Zn - 560. *Rn2 *Zn3 + 48. *Zn5 + 360. *Rn4* Zn*lgRn - 480. *Rn2 *Zn3*lgRn)* c_[6]
-            + c_[7]
-            + Rn2 * c_[8]
-            + (3. *Zn2 - 3. *Rn2*lgRn)* c_[9]
-            + (3. *Rn4 - 12. *Rn2 *Zn2) *c_[10]
-            + ((-45.)*Rn4 + 40. *Zn4 + 60. *Rn4*lgRn -  240. *Rn2 *Zn2*lgRn)* c_[11]);
+        return   m_pp*(2.* Zn* m_c[2]
+            -  8. *Rn2* Zn* m_c[3] +
+              ((-18.)*Rn2 *Zn + 8. *Zn3 - 24. *Rn2* Zn*lgRn) *m_c[4]
+            + ((-24.) *Rn4* Zn + 32. *Rn2 *Zn3)* m_c[5]
+            + (150. *Rn4* Zn - 560. *Rn2 *Zn3 + 48. *Zn5 + 360. *Rn4* Zn*lgRn - 480. *Rn2 *Zn3*lgRn)* m_c[6]
+            + m_c[7]
+            + Rn2 * m_c[8]
+            + (3. *Zn2 - 3. *Rn2*lgRn)* m_c[9]
+            + (3. *Rn4 - 12. *Rn2 *Zn2) *m_c[10]
+            + ((-45.)*Rn4 + 40. *Zn4 + 60. *Rn4*lgRn -  240. *Rn2 *Zn2*lgRn)* m_c[11]);
 
     }
   private:
-    double R_0_, A_, pp_;
-    std::vector<double> c_;
+    double m_R0, m_A, m_pp;
+    std::vector<double> m_c;
 };
 /**
  * @brief \f[ \frac{\partial^2  \hat{\psi}_p }{ \partial \hat{Z}^2}\f]
@@ -228,19 +229,19 @@ struct PsipZ: public aCylindricalFunctor<PsipZ>
 struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 {
     ///@copydoc Psip::Psip()
-    PsipZZ( Parameters gp): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) { }
+    PsipZZ( Parameters gp): m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_c(gp.c) { }
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn4,Zn,Zn2,Zn3,Zn4,lgRn;
-        Rn = R/R_0_; Rn2 = Rn*Rn; Rn4 = Rn2*Rn2;
-        Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
+        Rn = R/m_R0; Rn2 = Rn*Rn; Rn4 = Rn2*Rn2;
+        Zn = Z/m_R0; Zn2 =Zn*Zn; Zn3 = Zn2*Zn; Zn4 = Zn2*Zn2;
         lgRn= log(Rn);
-        return   pp_/R_0_*( 2.* c_[2] - 8. *Rn2* c_[3] + (-18. *Rn2 + 24. *Zn2 - 24. *Rn2*lgRn) *c_[4] + (-24.*Rn4 + 96. *Rn2 *Zn2) *c_[5]
-        + (150. *Rn4 - 1680. *Rn2 *Zn2 + 240. *Zn4 + 360. *Rn4*lgRn - 1440. *Rn2 *Zn2*lgRn)* c_[6] + 6.* Zn* c_[9] -  24. *Rn2 *Zn *c_[10] + (160. *Zn3 - 480. *Rn2* Zn*lgRn) *c_[11]);
+        return   m_pp/m_R0*( 2.* m_c[2] - 8. *Rn2* m_c[3] + (-18. *Rn2 + 24. *Zn2 - 24. *Rn2*lgRn) *m_c[4] + (-24.*Rn4 + 96. *Rn2 *Zn2) *m_c[5]
+        + (150. *Rn4 - 1680. *Rn2 *Zn2 + 240. *Zn4 + 360. *Rn4*lgRn - 1440. *Rn2 *Zn2*lgRn)* m_c[6] + 6.* Zn* m_c[9] -  24. *Rn2 *Zn *m_c[10] + (160. *Zn3 - 480. *Rn2* Zn*lgRn) *m_c[11]);
     }
   private:
-    double R_0_, A_, pp_;
-    std::vector<double> c_;
+    double m_R0, m_A, m_pp;
+    std::vector<double> m_c;
 };
 /**
  * @brief  \f[\frac{\partial^2  \hat{\psi}_p }{ \partial \hat{R} \partial\hat{Z}}\f]
@@ -258,22 +259,22 @@ struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 struct PsipRZ: public aCylindricalFunctor<PsipRZ>
 {
     ///@copydoc Psip::Psip()
-    PsipRZ( Parameters gp ): R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), c_(gp.c) { }
+    PsipRZ( Parameters gp ): m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_c(gp.c) { }
     double do_compute(double R, double Z) const
     {
         double Rn,Rn2,Rn3,Zn,Zn2,Zn3,lgRn;
-        Rn = R/R_0_; Rn2 = Rn*Rn; Rn3 = Rn2*Rn;
-        Zn = Z/R_0_; Zn2 =Zn*Zn; Zn3 = Zn2*Zn;
+        Rn = R/m_R0; Rn2 = Rn*Rn; Rn3 = Rn2*Rn;
+        Zn = Z/m_R0; Zn2 =Zn*Zn; Zn3 = Zn2*Zn;
         lgRn= log(Rn);
-        return   pp_/R_0_*(
-              -16.* Rn* Zn* c_[3] + (-60.* Rn* Zn - 48.* Rn* Zn*lgRn)* c_[4] + (-96. *Rn3* Zn + 64.*Rn *Zn3)* c_[5]
-            + (960. *Rn3 *Zn - 1600.* Rn *Zn3 + 1440. *Rn3* Zn*lgRn - 960. *Rn *Zn3*lgRn) *c_[6] +  2.* Rn* c_[8] + (-3.* Rn - 6.* Rn*lgRn)* c_[9]
-            + (12. *Rn3 - 24.* Rn *Zn2) *c_[10] + (-120. *Rn3 - 240. *Rn *Zn2 + 240. *Rn3*lgRn -   480.* Rn *Zn2*lgRn)* c_[11]
+        return   m_pp/m_R0*(
+              -16.* Rn* Zn* m_c[3] + (-60.* Rn* Zn - 48.* Rn* Zn*lgRn)* m_c[4] + (-96. *Rn3* Zn + 64.*Rn *Zn3)* m_c[5]
+            + (960. *Rn3 *Zn - 1600.* Rn *Zn3 + 1440. *Rn3* Zn*lgRn - 960. *Rn *Zn3*lgRn) *m_c[6] +  2.* Rn* m_c[8] + (-3.* Rn - 6.* Rn*lgRn)* m_c[9]
+            + (12. *Rn3 - 24.* Rn *Zn2) *m_c[10] + (-120. *Rn3 - 240. *Rn *Zn2 + 240. *Rn3*lgRn -   480.* Rn *Zn2*lgRn)* m_c[11]
                  );
     }
   private:
-    double R_0_, A_, pp_;
-    std::vector<double> c_;
+    double m_R0, m_A, m_pp;
+    std::vector<double> m_c;
 };
 
 /**
@@ -284,17 +285,17 @@ struct PsipRZ: public aCylindricalFunctor<PsipRZ>
 struct Ipol: public aCylindricalFunctor<Ipol>
 {
     ///@copydoc Psip::Psip()
-    Ipol(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp) {
+    Ipol( Parameters gp, std::function<double(double,double)> psip ):  m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_pi(gp.pi), m_psip(psip) {
         if( gp.pp == 0.)
-            pp_ = 1.; //safety measure to avoid divide by zero errors
+            m_pp = 1.; //safety measure to avoid divide by zero errors
     }
     double do_compute(double R, double Z) const
     {
-        return pi_*sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.);
+        return m_pi*sqrt(-2.*m_A* m_psip(R,Z) /m_R0/m_pp + 1.);
     }
   private:
-    double R_0_, A_, pp_, pi_;
-    Psip psip_;
+    double m_R0, m_A, m_pp, m_pi;
+    std::function<double(double,double)> m_psip;
 };
 /**
  * @brief \f[\hat I_R\f]
@@ -302,18 +303,18 @@ struct Ipol: public aCylindricalFunctor<Ipol>
 struct IpolR: public aCylindricalFunctor<IpolR>
 {
     ///@copydoc Psip::Psip()
-    IpolR(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp), psipR_(gp) {
+    IpolR(  Parameters gp, std::function<double(double,double)> psip, std::function<double(double,double)> psipR ):
+        m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_pi(gp.pi), m_psip(psip), m_psipR(psipR) {
         if( gp.pp == 0.)
-            pp_ = 1.; //safety measure to avoid divide by zero errors
+            m_pp = 1.; //safety measure to avoid divide by zero errors
     }
     double do_compute(double R, double Z) const
     {
-        return -pi_/sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.)*(A_*psipR_(R,Z)/R_0_/pp_);
+        return -m_pi/sqrt(-2.*m_A* m_psip(R,Z) /m_R0/m_pp + 1.)*(m_A*m_psipR(R,Z)/m_R0/m_pp);
     }
   private:
-    double R_0_, A_, pp_, pi_;
-    Psip psip_;
-    PsipR psipR_;
+    double m_R0, m_A, m_pp, m_pi;
+    std::function<double(double,double)> m_psip, m_psipR;
 };
 /**
  * @brief \f[\hat I_Z\f]
@@ -321,213 +322,36 @@ struct IpolR: public aCylindricalFunctor<IpolR>
 struct IpolZ: public aCylindricalFunctor<IpolZ>
 {
     ///@copydoc Psip::Psip()
-    IpolZ(  Parameters gp ):  R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp), psipZ_(gp) {
+    IpolZ(  Parameters gp, std::function<double(double,double)> psip, std::function<double(double,double)> psipZ ):
+        m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_pi(gp.pi), m_psip(psip), m_psipZ(psipZ) {
         if( gp.pp == 0.)
-            pp_ = 1.; //safety measure to avoid divide by zero errors
+            m_pp = 1.; //safety measure to avoid divide by zero errors
     }
     double do_compute(double R, double Z) const
     {
-        return -pi_/sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.)*(A_*psipZ_(R,Z)/R_0_/pp_);
+        return -m_pi/sqrt(-2.*m_A* m_psip(R,Z) /m_R0/m_pp + 1.)*(m_A*m_psipZ(R,Z)/m_R0/m_pp);
     }
   private:
-    double R_0_, A_, pp_, pi_;
-    Psip psip_;
-    PsipZ psipZ_;
+    double m_R0, m_A, m_pp, m_pi;
+    std::function<double(double,double)> m_psip, m_psipZ;
 };
 
-static inline dg::geo::CylindricalFunctorsLvl2 createPsip( Parameters gp)
+static inline dg::geo::CylindricalFunctorsLvl2 createPsip( const Parameters& gp)
 {
     return CylindricalFunctorsLvl2( Psip(gp), PsipR(gp), PsipZ(gp),
         PsipRR(gp), PsipRZ(gp), PsipZZ(gp));
 }
-static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp)
+static inline dg::geo::CylindricalFunctorsLvl1 createIpol( const Parameters& gp, const CylindricalFunctorsLvl1& psip)
 {
-    return CylindricalFunctorsLvl1( Ipol(gp), IpolR(gp), IpolZ(gp));
+    return CylindricalFunctorsLvl1(
+            solovev::Ipol(gp, psip.f()),
+            solovev::IpolR(gp,psip.f(), psip.dfx()),
+            solovev::IpolZ(gp,psip.f(), psip.dfy()));
 }
 
 ///@}
-
-///@cond
-namespace mod
-{
-
-struct Psip: public aCylindricalFunctor<Psip>
-{
-    Psip( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_ipoly( psi0, alpha, sign), m_psip(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        return m_ipoly( psip);
-    }
-    private:
-    dg::IPolynomialHeaviside m_ipoly;
-    solovev::Psip m_psip;
-};
-struct PsipR: public aCylindricalFunctor<PsipR>
-{
-    PsipR( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_psip(gp), m_psipR(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipR = m_psipR(R,Z);
-        return psipR*m_poly( psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    solovev::Psip m_psip;
-    solovev::PsipR m_psipR;
-};
-struct PsipZ: public aCylindricalFunctor<PsipZ>
-{
-    PsipZ( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_psip(gp), m_psipZ(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipZ = m_psipZ(R,Z);
-        return psipZ*m_poly( psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    solovev::Psip m_psip;
-    solovev::PsipZ m_psipZ;
-};
-
-struct PsipZZ: public aCylindricalFunctor<PsipZZ>
-{
-    PsipZZ( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipZ(gp), m_psipZZ(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipZ = m_psipZ(R,Z);
-        double psipZZ = m_psipZZ(R,Z);
-        return psipZZ*m_poly( psip) + psipZ*psipZ*m_dpoly(psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    dg::DPolynomialHeaviside m_dpoly;
-    solovev::Psip m_psip;
-    solovev::PsipZ m_psipZ;
-    solovev::PsipZZ m_psipZZ;
-};
-struct PsipRR: public aCylindricalFunctor<PsipRR>
-{
-    PsipRR( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipR(gp), m_psipRR(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipR = m_psipR(R,Z);
-        double psipRR = m_psipRR(R,Z);
-        return psipRR*m_poly( psip) + psipR*psipR*m_dpoly(psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    dg::DPolynomialHeaviside m_dpoly;
-    solovev::Psip m_psip;
-    solovev::PsipR m_psipR;
-    solovev::PsipRR m_psipRR;
-};
-struct PsipRZ: public aCylindricalFunctor<PsipRZ>
-{
-    PsipRZ( Parameters gp, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(gp), m_psipR(gp), m_psipZ(gp), m_psipRZ(gp)
-    { }
-    double do_compute(double R, double Z) const
-    {
-        double psip = m_psip(R,Z);
-        double psipR = m_psipR(R,Z);
-        double psipZ = m_psipZ(R,Z);
-        double psipRZ = m_psipRZ(R,Z);
-        return psipRZ*m_poly( psip) + psipR*psipZ*m_dpoly(psip);
-    }
-    private:
-    dg::PolynomialHeaviside m_poly;
-    dg::DPolynomialHeaviside m_dpoly;
-    solovev::Psip m_psip;
-    solovev::PsipR m_psipR;
-    solovev::PsipZ m_psipZ;
-    solovev::PsipRZ m_psipRZ;
-};
-
-struct Ipol: public aCylindricalFunctor<Ipol>
-{
-    Ipol(  Parameters gp, double psi0, double alpha, double sign = -1):
-        R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp, psi0, alpha, sign) {
-            if( gp.pp == 0.)
-                pp_ = 1.; //safety measure to avoid divide by zero errors
-    }
-    double do_compute(double R, double Z) const
-    {
-        return pi_*sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.);
-    }
-  private:
-    double R_0_, A_, pp_, pi_;
-    mod::Psip psip_;
-};
-struct IpolR: public aCylindricalFunctor<IpolR>
-{
-    IpolR(  Parameters gp, double psi0, double alpha, double sign = -1 ):
-        R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi), psip_(gp, psi0, alpha,
-        sign), psipR_(gp, psi0, alpha, sign) {
-            if( gp.pp == 0.)
-                pp_ = 1.; //safety measure to avoid divide by zero errors
-    }
-    double do_compute(double R, double Z) const
-    {
-        return -pi_/sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.)*(A_*psipR_(R,Z)/R_0_/pp_);
-    }
-  private:
-    double R_0_, A_, pp_, pi_;
-    mod::Psip psip_;
-    mod::PsipR psipR_;
-};
-struct IpolZ: public aCylindricalFunctor<IpolZ>
-{
-    IpolZ(  Parameters gp, double psi0, double alpha, double sign = -1 ):
-        R_0_(gp.R_0), A_(gp.A), pp_(gp.pp), pi_(gp.pi),
-        psip_(gp, psi0, alpha, sign), psipZ_(gp, psi0, alpha, sign) {
-            if( gp.pp == 0.)
-                pp_ = 1.; //safety measure to avoid divide by zero errors
-    }
-    double do_compute(double R, double Z) const
-    {
-        return -pi_/sqrt(-2.*A_* psip_(R,Z) /R_0_/pp_ + 1.)*(A_*psipZ_(R,Z)/R_0_/pp_);
-    }
-  private:
-    double R_0_, A_, pp_, pi_;
-    mod::Psip psip_;
-    mod::PsipZ psipZ_;
-};
-
-static inline dg::geo::CylindricalFunctorsLvl2 createPsip( Parameters gp,
-    double psi0, double alpha, double sign = -1)
-{
-    return CylindricalFunctorsLvl2( Psip(gp, psi0, alpha, sign), PsipR(gp,
-    psi0, alpha, sign), PsipZ(gp, psi0, alpha, sign), PsipRR(gp, psi0, alpha,
-    sign), PsipRZ(gp, psi0, alpha, sign), PsipZZ(gp, psi0, alpha, sign));
-}
-static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp,
-    double psi0, double alpha, double sign = -1)
-{
-    return CylindricalFunctorsLvl1( Ipol(gp, psi0, alpha, sign), IpolR(gp,
-    psi0, alpha, sign), IpolZ(gp, psi0, alpha, sign));
-}
-
-} //namespace mod
-///@endcond
-
 ///////////////////////////////////////introduce fields into solovev namespace
 
-
 } //namespace solovev
 
 /**
@@ -544,7 +368,7 @@ static inline dg::geo::TokamakMagneticField createSolovevField(
     MagneticFieldParameters params = { gp.a, gp.elongation, gp.triangularity,
             equilibrium::solovev, modifier::none, str2form.at( gp.form)};
     return TokamakMagneticField( gp.R_0, solovev::createPsip(gp),
-        solovev::createIpol(gp), params);
+        solovev::createIpol(gp, solovev::createPsip(gp)), params);
 }
 /**
  * @brief Create a modified Solovev Magnetic field
@@ -568,8 +392,10 @@ static inline dg::geo::TokamakMagneticField createModifiedSolovevField(
 {
     MagneticFieldParameters params = { gp.a, gp.elongation, gp.triangularity,
             equilibrium::solovev, modifier::heaviside, str2form.at( gp.form)};
-    return TokamakMagneticField( gp.R_0, solovev::mod::createPsip(gp, psi0,
-    alpha, sign), solovev::mod::createIpol(gp, psi0, alpha, sign), params);
+    return TokamakMagneticField( gp.R_0,
+            mod::createPsip( solovev::createPsip(gp), psi0, alpha, sign),
+        solovev::createIpol( gp, mod::createPsip( solovev::createPsip(gp), psi0, alpha, sign)),
+        params);
 }
 
 } //namespace geo
-- 
GitLab


From 89a3ead7ceb773ac0ade168da36761905298efdb Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 21 Sep 2020 01:03:01 +0200
Subject: [PATCH 333/540] Change form to description

It is just a better name, naming is everything
---
 inc/geometries/geometry_diag.cu               | 14 ++++-----
 inc/geometries/geometry_params.json           |  2 +-
 inc/geometries/geometry_params_Xpoint.json    |  4 ++-
 .../geometry_params_Xpoint_harmonic.json      |  2 +-
 .../geometry_params_Xpoint_taylor.json        |  2 +-
 .../geometry_params_circ_Iconst_largeR0.json  |  2 +-
 inc/geometries/guenther.h                     |  2 +-
 inc/geometries/magnetic_field.h               | 30 +++++++++----------
 inc/geometries/modified.h                     |  2 +-
 inc/geometries/polynomial.h                   |  4 +--
 inc/geometries/polynomial_parameters.h        |  8 ++---
 inc/geometries/solovev.h                      |  4 +--
 inc/geometries/solovev_parameters.h           | 14 ++++-----
 inc/geometries/taylor.h                       |  2 +-
 inc/geometries/toroidal.h                     |  4 +--
 src/feltor/geometry/compass.json              |  2 +-
 src/feltor/geometry/geometry_params.json      |  2 +-
 .../geometry/geometry_paramsXpoint.json       |  2 +-
 src/feltor/geometry/tcv.json                  |  4 +--
 src/feltor/geometry/torpex.json               |  2 +-
 src/feltor/init.h                             |  2 +-
 src/heat/geometry/geometry_params.json        |  2 +-
 src/heat/geometry/guenther_params.json        |  2 +-
 23 files changed, 58 insertions(+), 56 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 798e4eb43..61cfb947d 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -132,9 +132,9 @@ int main( int argc, char* argv[])
     double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
     double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
 
-    dg::geo::form mag_form = mag.params().getForm();
+    dg::geo::description mag_description = mag.params().getDescription();
     double psipO = -1.;
-    if( mag_form == dg::geo::form::standardX || mag_form == dg::geo::form::standardO )
+    if( mag_description == dg::geo::description::standardX || mag_description == dg::geo::description::standardO )
     {
         //Find O-point
         double RO = mag.R0(), ZO = 0.;
@@ -241,14 +241,14 @@ int main( int argc, char* argv[])
     /// -------  Elements for fsa on X-point grid ----------------
     double psipmax = dg::blas1::reduce( psipog2d, 0., thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
     std::unique_ptr<dg::geo::CurvilinearGridX2d> gX2d;
-    if( mag_form == dg::geo::form::standardX)
+    if( mag_description == dg::geo::description::standardX)
     {
         std::cout << "Generate X-point flux-aligned grid ... \n";
         double RX = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
         double ZX = -1.1*mag.params().elongation()*mag.params().a();
         dg::geo::findXpoint( mag.get_psip(), RX, ZX);
         double psipX = mag.psip()(RX, ZX);
-        std::cout << "Found X-point at "<<RX<<" "<<ZX<<" with Psip = "<<psipX<<std::endl;
+        std::cout << "X-point found at "<<RX<<" "<<ZX<<" with Psip = "<<psipX<<std::endl;
         if( fabs(psipX ) > 1e-10)
         {
             std::cerr << " Psip at X-point is not zero. Unable to construct grid\n";
@@ -309,7 +309,7 @@ int main( int argc, char* argv[])
     /// --------- More flux labels --------------------------------
     dg::Grid1d grid1d(psipO<psipmax ? psipO : psipmax,
             psipO<psipmax ? psipmax : psipO, npsi ,Npsi,dg::DIR_NEU); //inner value is always zero
-    if( mag_form != dg::geo::form::none && mag_form != dg::geo::form::centeredX)
+    if( mag_description != dg::geo::description::none && mag_description != dg::geo::description::centeredX)
     {
         dg::HVec rho = dg::evaluate( dg::cooX1d, grid1d);
         dg::blas1::axpby( -1./psipO, rho, +1., 1., rho); //transform psi to rho
@@ -318,7 +318,7 @@ int main( int argc, char* argv[])
         dg::blas1::transform( rho, rho, dg::SQRT<double>());
         map1d.emplace_back("rho_p", rho,
             "Alternative flux label rho_p = Sqrt[-psi/psimin + 1]");
-        if( mag_form == dg::geo::form::standardX || mag_form == dg::geo::form::standardO)
+        if( mag_description == dg::geo::description::standardX || mag_description == dg::geo::description::standardO)
         {
             dg::geo::SafetyFactor qprof( mag);
             dg::HVec qprofile = dg::evaluate( qprof, grid1d);
@@ -369,7 +369,7 @@ int main( int argc, char* argv[])
             pair.first.data(), pair.second.size(), pair.second.data());
 
     int dim1d_ids[1], dim2d_ids[2], dim3d_ids[3] ;
-    if( mag_form == dg::geo::form::standardX)
+    if( mag_description == dg::geo::description::standardX)
     {
         int dim_idsX[2] = {0,0};
         err = file::define_dimensions( ncid, dim_idsX, gX2d->grid(), {"eta", "zeta"} );
diff --git a/inc/geometries/geometry_params.json b/inc/geometries/geometry_params.json
index 856130505..c3785593f 100644
--- a/inc/geometries/geometry_params.json
+++ b/inc/geometries/geometry_params.json
@@ -23,7 +23,7 @@
     "elongation"         : 1.00,
     "triangularity"      : 0.0,
     "equilibrium"        : "solovev",
-    "form"               : "standardO"
+    "description"               : "standardO"
 }
 //@ ----------------------------------------------------------------
 
diff --git a/inc/geometries/geometry_params_Xpoint.json b/inc/geometries/geometry_params_Xpoint.json
index c1a2c2a9e..e944e759e 100644
--- a/inc/geometries/geometry_params_Xpoint.json
+++ b/inc/geometries/geometry_params_Xpoint.json
@@ -5,6 +5,8 @@
 {
     //----------------------Solovev coefficients---------------
     "A" : 0.0,
+    "PP": 1,
+    "PI": 1,
     "c" :[  0.07350114445500399706283007092406934834526,
            -0.08662417436317227513877947632069712210813,
            -0.1463931543401102620740934776490506239925,
@@ -23,7 +25,7 @@
     "elongation"         : 1.75,
     "triangularity"      : 0.47,
     "equilibrium"  : "solovev",
-    "form" : "standardX"
+    "description" : "standardX"
 }
 //@ ----------------------------------------------------------
 
diff --git a/inc/geometries/geometry_params_Xpoint_harmonic.json b/inc/geometries/geometry_params_Xpoint_harmonic.json
index 217e4af0b..e619ec418 100644
--- a/inc/geometries/geometry_params_Xpoint_harmonic.json
+++ b/inc/geometries/geometry_params_Xpoint_harmonic.json
@@ -24,7 +24,7 @@
     "elongation"         : 1.75,
     "triangularity"      : 0.47,
     "equilibrium" : "solovev",
-    "form" : "standardX"
+    "description" : "standardX"
 }
 //@ ----------------------------------------------------------
 
diff --git a/inc/geometries/geometry_params_Xpoint_taylor.json b/inc/geometries/geometry_params_Xpoint_taylor.json
index 66c407bba..f92ba8a7a 100644
--- a/inc/geometries/geometry_params_Xpoint_taylor.json
+++ b/inc/geometries/geometry_params_Xpoint_taylor.json
@@ -5,7 +5,7 @@
  *          ----------------------------------------------------------- */
 {
     "equilibrium" : "taylor", //taylor state
-    "form"               : "standardX"
+    "description"               : "standardX"
     //----------------------Taylor coefficients---------------
     "c" :[-0.730004529864617913443309878090712685057289011596696572774289301646,
     -0.013752279867251353918502943858642437963550235876487492144645446786828705,
diff --git a/inc/geometries/geometry_params_circ_Iconst_largeR0.json b/inc/geometries/geometry_params_circ_Iconst_largeR0.json
index cb16bfea2..d14f3f419 100644
--- a/inc/geometries/geometry_params_circ_Iconst_largeR0.json
+++ b/inc/geometries/geometry_params_circ_Iconst_largeR0.json
@@ -24,6 +24,6 @@
 	"elongation" : 1,
 	"triangularity" : 0,
 	"equilibrium" : "solovev",
-    "form": "standardO",
+    "description": "standardO",
 	"inverseaspectratio" : 0.016666666666666666
 }
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index 3a8db6e12..ba96b824f 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -160,7 +160,7 @@ static inline CylindricalFunctorsLvl1 createIpol( double I_0)
 static inline dg::geo::TokamakMagneticField createGuentherField( double R_0, double I_0)
 {
     MagneticFieldParameters params = { 1., 1., 0.,
-            equilibrium::guenther, modifier::none, form::square};
+            equilibrium::guenther, modifier::none, description::square};
     return TokamakMagneticField( R_0, guenther::createPsip(R_0), guenther::createIpol(I_0), params);
 }
 } //namespace geo
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 87d88238f..e5e9cf89a 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -38,7 +38,7 @@ enum class modifier
     heaviside //!< Psip is dampened to a constant outside a critical value
 };
 ///@brief How flux function looks like. Decider on whether and what flux aligned grid to construct
-enum class form
+enum class description
 {
     standardO, //!< closed flux surfaces centered around an O-point located near (R_0, 0); flux-aligned grids can be constructed
     standardX, //!< closed flux surfaces centered around an O-point located near (R_0, 0) and bordered by a separatrix with a single X-point; flux-aligned X-grids can be constructed
@@ -59,12 +59,12 @@ static const std::map<std::string, modifier> str2modifier{
     {"none", modifier::none},
     {"heaviside", modifier::heaviside}
 };
-static const std::map<std::string, form> str2form{
-    {"standardO", form::standardO},
-    {"standardX", form::standardX},
-    {"square", form::square},
-    {"none", form::none},
-    {"centeredX", form::centeredX}
+static const std::map<std::string, description> str2description{
+    {"standardO", description::standardO},
+    {"standardX", description::standardX},
+    {"square", description::square},
+    {"none", description::none},
+    {"centeredX", description::centeredX}
 };
 ///@endcond
 
@@ -78,7 +78,7 @@ static const std::map<std::string, form> str2form{
  *
  * For example it is very hard to automatically detect if the construction
  * of a flux aligned X-grid is possible, but for a human it is very easy.
- * Here we give the \c form specifier that can be used in an if-else statement.
+ * Here we give the \c description specifier that can be used in an if-else statement.
  */
 struct MagneticFieldParameters
 {
@@ -89,7 +89,7 @@ struct MagneticFieldParameters
         m_a = 1, m_elongation = 1, m_triangularity = 0;
         m_equilibrium = equilibrium::toroidal;
         m_modifier = modifier::none;
-        m_form = form::none;
+        m_description = description::none;
     }
     /**
      * @brief Constructor
@@ -99,14 +99,14 @@ struct MagneticFieldParameters
      * @param triangularity (R_0 - R_X) /a;  The purpose of this parameter is to find the approximate location of R_X (if an X-point is present, Z_X is given by elongation) the exact location can be computed by the \c findXpoint function
      * @param equ the way the flux function is computed
      * @param mod the way the flux function is modified
-     * @param frm human readable descriptor of how the flux function looks
+     * @param des human readable descriptor of how the flux function looks
      */
     MagneticFieldParameters( double a, double elongation, double triangularity,
-            equilibrium equ, modifier mod, form frm): m_a(a),
+            equilibrium equ, modifier mod, description des): m_a(a),
         m_elongation(elongation),
         m_triangularity( triangularity),
         m_equilibrium( equ),
-        m_modifier(mod), m_form( frm){}
+        m_modifier(mod), m_description( des){}
  //!< The minor radius; the purpose of this parameter is not to be exact but to serve as a refernce of how to setup the size of a simulation box
     double a() const{return m_a;}
  //!< (maximum Z - minimum Z of lcfs)/2a; 1 for a circle; the purpose of this parameter is not to be exact but more to be a reference of how to setup the aspect ratio of a simulation box
@@ -118,20 +118,20 @@ struct MagneticFieldParameters
  //!<  the way the flux function is modified
     modifier getModifier() const{return m_modifier;}
  //!< human readable descriptor of how the flux function looks
-    form getForm() const{return m_form;}
+    description getDescription() const{return m_description;}
     private:
     double m_a,
            m_elongation,
            m_triangularity;
     equilibrium m_equilibrium;
     modifier m_modifier;
-    form m_form;
+    description m_description;
 };
 
 /**
 * @brief A tokamak field as given by R0, Psi and Ipol plus Meta-data like shape and equilibrium
 
- This is the representation of toroidally axisymmetric magnetic fields that can be modeled in the form
+ This is the representation of toroidally axisymmetric magnetic fields that can be modeled in the description
  \f$
  \vec B(R,Z,\varphi) = \frac{R_0}{R} \left( I(\psi_p) \hat e_\varphi + \nabla \psi_p \times \hat e_\varphi\right)
  \f$
diff --git a/inc/geometries/modified.h b/inc/geometries/modified.h
index 0822e11a2..66110a4e1 100644
--- a/inc/geometries/modified.h
+++ b/inc/geometries/modified.h
@@ -141,7 +141,7 @@ static inline dg::geo::TokamakMagneticField createModifiedField(
 {
     const MagneticFieldParameters inp = in.params();
     MagneticFieldParameters params = { inp.a(), inp.elongation(), inp.triangularity(),
-            inp.getEquilibrium(), modifier::heaviside, inp.getForm()};
+            inp.getEquilibrium(), modifier::heaviside, inp.getDescription()};
     return TokamakMagneticField( in.R0(), mod::createPsip(in.get_psip(), psi0,
         alpha, sign), in.get_ipol(), params);
 }
diff --git a/inc/geometries/polynomial.h b/inc/geometries/polynomial.h
index bbc21223c..731af50c6 100644
--- a/inc/geometries/polynomial.h
+++ b/inc/geometries/polynomial.h
@@ -199,7 +199,7 @@ static inline dg::geo::TokamakMagneticField createPolynomialField(
     dg::geo::polynomial::Parameters gp)
 {
     MagneticFieldParameters params( gp.a, gp.elongation, gp.triangularity,
-            equilibrium::polynomial, modifier::none, str2form.at( gp.form));
+            equilibrium::polynomial, modifier::none, str2description.at( gp.description));
     return TokamakMagneticField( gp.R_0, polynomial::createPsip(gp),
         polynomial::createIpol(gp), params);
 }
@@ -224,7 +224,7 @@ static inline dg::geo::TokamakMagneticField createModifiedPolynomialField(
     dg::geo::polynomial::Parameters gp, double psi0, double alpha, double sign = -1)
 {
     MagneticFieldParameters params( gp.a, gp.elongation, gp.triangularity,
-            equilibrium::polynomial, modifier::heaviside, str2form.at( gp.form));
+            equilibrium::polynomial, modifier::heaviside, str2description.at( gp.description));
     return TokamakMagneticField( gp.R_0,
             mod::createPsip( polynomial::createPsip(gp), psi0, alpha, sign),
         polynomial::createIpol( gp), params);
diff --git a/inc/geometries/polynomial_parameters.h b/inc/geometries/polynomial_parameters.h
index ab5606555..2892fe64d 100644
--- a/inc/geometries/polynomial_parameters.h
+++ b/inc/geometries/polynomial_parameters.h
@@ -30,7 +30,7 @@ struct Parameters
     unsigned M, //!< number of coefficients in R
              N; //!< number of coefficients in Z
     std::vector<double> c;  //!< M*N coefficients for the polynomial equilibrium, \c c[i*N+j] corresponds to R^i Z^j;
-    std::string form;
+    std::string description;
 #ifdef JSONCPP_VERSION_STRING
     /**
      * @brief Construct from Json dataset
@@ -54,7 +54,7 @@ struct Parameters
         a  = R_0*file::get( mode, js, "inverseaspectratio", 0.).asDouble();
         elongation=file::get( mode, js, "elongation", 1.).asDouble();
         triangularity=file::get( mode, js, "triangularity", 0.).asDouble();
-        form = file::get( mode, js, "form", "standardX").asString();
+        description = file::get( mode, js, "description", "standardX").asString();
     }
     /**
      * @brief Put values into a json string
@@ -75,7 +75,7 @@ struct Parameters
         js["elongation"] = elongation;
         js["triangularity"] = triangularity;
         js[ "equilibrium"] = "polynomial";
-        js[ "form"] = form;
+        js[ "description"] = description;
         return js;
     }
 #endif // JSONCPP_VERSION_STRING
@@ -103,7 +103,7 @@ struct Parameters
         os  <<" R0            = "<<R_0<<"\n"
             <<" a             = "<<a<<"\n"
             <<" epsilon_a     = "<<a/R_0<<"\n"
-            <<" form          = "<<form<<"\n"
+            <<" description   = "<<description<<"\n"
             <<" elongation    = "<<elongation<<"\n"
             <<" triangularity = "<<triangularity<<"\n";
         os << std::flush;
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 3fd8232a5..8e26dd764 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -366,7 +366,7 @@ static inline dg::geo::TokamakMagneticField createSolovevField(
     dg::geo::solovev::Parameters gp)
 {
     MagneticFieldParameters params = { gp.a, gp.elongation, gp.triangularity,
-            equilibrium::solovev, modifier::none, str2form.at( gp.form)};
+            equilibrium::solovev, modifier::none, str2description.at( gp.description)};
     return TokamakMagneticField( gp.R_0, solovev::createPsip(gp),
         solovev::createIpol(gp, solovev::createPsip(gp)), params);
 }
@@ -391,7 +391,7 @@ static inline dg::geo::TokamakMagneticField createModifiedSolovevField(
     dg::geo::solovev::Parameters gp, double psi0, double alpha, double sign = -1)
 {
     MagneticFieldParameters params = { gp.a, gp.elongation, gp.triangularity,
-            equilibrium::solovev, modifier::heaviside, str2form.at( gp.form)};
+            equilibrium::solovev, modifier::heaviside, str2description.at( gp.description)};
     return TokamakMagneticField( gp.R_0,
             mod::createPsip( solovev::createPsip(gp), psi0, alpha, sign),
         solovev::createIpol( gp, mod::createPsip( solovev::createPsip(gp), psi0, alpha, sign)),
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index e84833070..be01241a2 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -29,7 +29,7 @@ struct Parameters
            elongation, //!< elongation of the magnetic surfaces
            triangularity; //!< triangularity of the magnetic surfaces
     std::vector<double> c;  //!< 12 coefficients for the solovev equilibrium;
-    std::string form;
+    std::string description;
 #ifdef JSONCPP_VERSION_STRING
     /**
      * @brief Construct from Json dataset
@@ -53,15 +53,15 @@ struct Parameters
         elongation=file::get( mode, js, "elongation", 1.).asDouble();
         triangularity=file::get( mode, js, "triangularity", 0.).asDouble();
         try{
-            form = file::get( file::error::is_throw, js, "form", "standardX").asString();
+            description = file::get( file::error::is_throw, js, "description", "standardX").asString();
         } catch ( std::exception& err)
         {
             if( isToroidal())
-                form = "none";
+                description = "none";
             else if( !hasXpoint())
-                form = "standardO";
+                description = "standardO";
             else
-                form = "standardX";
+                description = "standardX";
         }
     }
     /**
@@ -82,7 +82,7 @@ struct Parameters
         js["elongation"] = elongation;
         js["triangularity"] = triangularity;
         js[ "equilibrium"] = "solovev";
-        js[ "form"] = form;
+        js[ "description"] = description;
         return js;
     }
 #endif // JSONCPP_VERSION_STRING
@@ -125,7 +125,7 @@ struct Parameters
         os  <<" R0            = "<<R_0<<"\n"
             <<" a             = "<<a<<"\n"
             <<" epsilon_a     = "<<a/R_0<<"\n"
-            <<" form          = "<<form<<"\n"
+            <<" description   = "<<description<<"\n"
             <<" elongation    = "<<elongation<<"\n"
             <<" triangularity = "<<triangularity<<"\n";
         os << std::flush;
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index f40839f10..7b1f6adb0 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -317,7 +317,7 @@ static inline CylindricalFunctorsLvl1 createIpol( solovev::Parameters gp)
 static inline dg::geo::TokamakMagneticField createTaylorField( dg::geo::solovev::Parameters gp)
 {
     MagneticFieldParameters params = { gp.a, gp.elongation, gp.triangularity,
-            equilibrium::solovev, modifier::none, str2form.at( gp.form)};
+            equilibrium::solovev, modifier::none, str2description.at( gp.description)};
     return TokamakMagneticField( gp.R_0, dg::geo::taylor::createPsip(gp), dg::geo::taylor::createIpol(gp), params);
 }
 } //namespace geo
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 38e2a7023..2d3b0e53f 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -108,7 +108,7 @@ static inline CylindricalFunctorsLvl1 createIpol( double I0 )
 static inline dg::geo::TokamakMagneticField createToroidalField( double R0)
 {
     MagneticFieldParameters params = { 1., 1., 0.,
-            equilibrium::circular, modifier::none, form::none};
+            equilibrium::circular, modifier::none, description::none};
     return TokamakMagneticField( R0, toroidal::createPsip(), toroidal::createIpol(), params);
 }
 /**
@@ -123,7 +123,7 @@ static inline dg::geo::TokamakMagneticField createToroidalField( double R0)
 static inline dg::geo::TokamakMagneticField createCircularField( double R0, double I0)
 {
     MagneticFieldParameters params = { 1., 1., 0.,
-            equilibrium::circular, modifier::none, form::standardO};
+            equilibrium::circular, modifier::none, description::standardO};
     return TokamakMagneticField( R0, circular::createPsip(R0), circular::createIpol(I0), params);
 }
 
diff --git a/src/feltor/geometry/compass.json b/src/feltor/geometry/compass.json
index 4f13af462..5917756c5 100644
--- a/src/feltor/geometry/compass.json
+++ b/src/feltor/geometry/compass.json
@@ -20,7 +20,7 @@
    	],
    	"elongation": 1.6594,
    	"equilibrium": "solovev",
-    "form" : "standardX",
+    "description" : "standardX",
    	"inverseaspectratio": 0.2857142857142857,
    	"triangularity": 0.4
 }
diff --git a/src/feltor/geometry/geometry_params.json b/src/feltor/geometry/geometry_params.json
index 2ffc090ac..e0dae4cdb 100644
--- a/src/feltor/geometry/geometry_params.json
+++ b/src/feltor/geometry/geometry_params.json
@@ -24,7 +24,7 @@
 	],
 	"elongation" : 1,
 	"equilibrium" : "solovev",
-    "form" : "standardO",
+    "description" : "standardO",
 	"inverseaspectratio" : 0.1666666666666667,
 	"triangularity" : 0
 }
diff --git a/src/feltor/geometry/geometry_paramsXpoint.json b/src/feltor/geometry/geometry_paramsXpoint.json
index febe3e4e3..d5e74ead2 100644
--- a/src/feltor/geometry/geometry_paramsXpoint.json
+++ b/src/feltor/geometry/geometry_paramsXpoint.json
@@ -24,7 +24,7 @@
 	],
 	"elongation" : 1.7,
 	"equilibrium" : "solovev",
-	"form" : "standardX",
+	"description" : "standardX",
 	"inverseaspectratio" : 0.16666666666666669,
 	"triangularity" : 0.33000000000000002
 }
diff --git a/src/feltor/geometry/tcv.json b/src/feltor/geometry/tcv.json
index 7da3be901..caf907e68 100644
--- a/src/feltor/geometry/tcv.json
+++ b/src/feltor/geometry/tcv.json
@@ -4,7 +4,7 @@
 	"PI" : -1.0,
 	"PP" : -1.0,
 	"R_0" : 906.38,
-	"c" : 
+	"c" :
 	[
 		-1.1279927930530271,
 		-1.0842606353509106,
@@ -45,7 +45,7 @@
 	],
 	"elongation" : 1.5,
 	"equilibrium" : "polynomial",
-	"form" : "standardX",
+	"description" : "standardX",
 	"inverseaspectratio" : 0.27593818984547458,
 	"triangularity" : 0.40000000000000002
 }
diff --git a/src/feltor/geometry/torpex.json b/src/feltor/geometry/torpex.json
index f655d9cfe..e467fd45b 100644
--- a/src/feltor/geometry/torpex.json
+++ b/src/feltor/geometry/torpex.json
@@ -13,7 +13,7 @@
     "PI": -1,
     "R_0": 500,
     "equilibrium" : "solovev", 
-    "form" : "centeredX",
+    "description" : "centeredX",
     "elongation": 1,
     "triangularity": 0,
     "inverseaspectratio": 0.2
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 2988e622a..15873a793 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -54,7 +54,7 @@ HVec xpoint_damping(const Geometry& grid,
     const dg::geo::TokamakMagneticField& mag )
 {
     HVec xpoint_damping = dg::evaluate( dg::one, grid);
-    if( mag.params().getForm() == dg::geo::form::standardX)
+    if( mag.params().getDescription() == dg::geo::description::standardX)
     {
         double RX = mag.R0() - 1.1*mag.params().triangularity()*mag.params().a();
         double ZX = -1.1*mag.params().elongation()*mag.params().a();
diff --git a/src/heat/geometry/geometry_params.json b/src/heat/geometry/geometry_params.json
index cf604c462..1e815eb24 100644
--- a/src/heat/geometry/geometry_params.json
+++ b/src/heat/geometry/geometry_params.json
@@ -23,7 +23,7 @@
     "elongation"         : 1.75,
     "triangularity"      : 0.47,
     "equilibrium" : "solovev",
-    "form" : "standardX"
+    "description" : "standardX"
 }
 //@ ----------------------------------------------------------
 
diff --git a/src/heat/geometry/guenther_params.json b/src/heat/geometry/guenther_params.json
index c1ba5b170..0177f4e7f 100644
--- a/src/heat/geometry/guenther_params.json
+++ b/src/heat/geometry/guenther_params.json
@@ -3,7 +3,7 @@
     "I_0"               :  20.,
     "R_0"               :  10.,
     "a"                 :  1,
-    "form"              : "square"
+    "description"              : "square"
 }
 
 
-- 
GitLab


From 613c6849a716d86f283f9ea564a7356cbe7a30bd Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 21 Sep 2020 15:38:19 +0200
Subject: [PATCH 334/540] Fix derivative in Extrapolate

---
 inc/dg/cg.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 76d3fd7da..af8792059 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -379,7 +379,7 @@ struct Extrapolation
     */
     template<class ContainerType0>
     void derive( ContainerType0& dot_x) const{
-        dg::blas1::axpbypgz( 1./(m_t[0]-m_t[1]), m_x[0], 1./(m_t[0]-m_t[1]), m_x[1], 0., dot_x);
+        dg::blas1::axpbypgz( 1./(m_t[0]-m_t[1]), m_x[0], -1./(m_t[0]-m_t[1]), m_x[1], 0., dot_x);
     }
 
     /**
-- 
GitLab


From e94120b4cd81d51c11eb99605a942deeeaf5efca Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 21 Sep 2020 21:17:25 +0200
Subject: [PATCH 335/540] Add missing source terms in documentation

---
 src/feltor/feltor.tex | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index e1f9be433..484df0851 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -135,6 +135,7 @@ With the help of the metric elements we get a well behaved volume element \(\sqr
 The cylindrical coordinate basis vectors are mutually orthogonal to each other.
 
 \subsection{Solov'ev equilbrium}\label{sec:solovev}
+%Document polynomial field as well
 In cylindrical coordinates the general axisymmetric  magnetic field can be written as (dimensionless)
 \begin{align}
  \vec{B} &= \frac{R_0}{R}\left[I(\psi_p) \ehat_{\varphi} + \frac{\partial
@@ -1281,7 +1282,7 @@ The flux surface average over the parallel momentum equation under species summa
     % \nonumber\\
     + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_{\parallel,i} \frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_sN_s + z_s\mu_s N_sU_{\parallel,s}^2) b_{\perp}^{\;v}  }
     \nonumber\\
-   = \sum_s\RA{-z_s\tau_s N_s\npar \ln B} + \mu_i \RA{ S_{N_i} U_{\parallel,i}}
+   = \sum_s\RA{-z_s\tau_s N_s\npar \ln B} + \mu_i \RA{ S_{N_i} U_{\parallel,i} + N_i S_{U_\parallel}}
    \label{eq:parallel_momentum}
 \end{align}
 while the toroidal parallel angular momentum contribution reads up to order $\mathcal O(\delta^3)$
@@ -1289,7 +1290,7 @@ while the toroidal parallel angular momentum contribution reads up to order $\ma
     \frac{\partial}{\partial t}  \RA{\mu_iN_iU_{\parallel,i} b_\varphi}
     + \frac{\partial}{\partial v} \frac{\d v}{\d\psi_p} \RA{\mu_iN_iU_{\parallel,i} b_\varphi\frac{\bhat\times\vn\phi}{B}\cn\psi_p + \sum_s (z_s\tau_s N_s + z_s\mu_sN_sU_{\parallel,s}^2) b_\varphi b_{\perp}^{\;v} }
     \nonumber\\
-   = \RA{F_{L,\varphi}} + \mu_i \RA{ S_{N_i} U_{\parallel,i} b_\varphi}
+    = \RA{F_{L,\varphi}} + \mu_i \RA{ (S_{N_i} U_{\parallel,i} + N_i S_{U_\parallel}) b_\varphi}
 \end{align}
 
 The relevant terms in the output file are (the Lorentz force term is described in the previous subsection \ref{sec:vorticity_eq})
@@ -1619,6 +1620,7 @@ magnetic field and damping is ignored.\\
 \bottomrule
 \end{longtable}
 \subsection{Geometry file structure} \label{sec:geometry_file}
+%Document description and equilibrium fields and polynomial field
 File format: json
 
 %%This is a booktabs table
-- 
GitLab


From d78abf53b07fc90d5685a782eeb9cb391a1d10ee Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 21 Sep 2020 22:54:45 +0200
Subject: [PATCH 336/540] Fix sign in parallel A force in feltordiag

---
 src/feltor/feltor.tex   | 2 +-
 src/feltor/feltordiag.h | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 484df0851..9aab87066 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1333,7 +1333,7 @@ We gather the dominant terms in the electron momentum equation (neglecting all t
     sparphie\_tt & $n_e\npar \phi$ &
     friction\_tt & $ \eta n_e^2(U_{\parallel,i}-u_{\parallel,e})$ \\
     sparmirrore\_tt & $-\npar n_e$ &
-    sparmirrorAe\_tt & $\vn A_\parallel \times \bhat \cn n_e /B$ \\
+    sparmirrorAe\_tt & $-\vn A_\parallel \times \bhat \cn n_e /B$ \\
     sparphiAe\_tt & $n_e \vn A_\parallel \times \bhat \cn \phi /B$ &
     spardotAe\_tt & $ n_e \partial A_\parallel /\partial t$ \\
 \bottomrule
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index bc1b5c0a1..8d98d68fe 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -1134,6 +1134,7 @@ std::vector<Record> diagnostics2d_list = {
     {"sparmirrorAe_tt", "Apar Mirror force term with electron density (Time average)", true,
         []( DVec& result, Variables& v){
             routines::jacobian( v.f.gradA() , v.f.bhatgB(), v.f.gradN(0), result);
+            dg::blas1::scal( result, v.p.tau[0]);
         }
     },
     {"sparmirrori_tt", "Mirror force term with ion density (Time average)", true,
-- 
GitLab


From f23af43a038954d235793bd697252d96fece6b0a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 22 Sep 2020 10:14:24 +0200
Subject: [PATCH 337/540] Generate a sol_pfr damping mechanism

now we need to test this in geometry_diag and feltor
---
 inc/geometries/magnetic_field.h |   6 +-
 inc/geometries/make_field.h     |  91 +++++++++++++++++++++-------
 inc/geometries/modified.h       | 102 +++++++++++++++++++++-----------
 inc/geometries/polynomial.h     |  26 --------
 inc/geometries/solovev.h        |   4 +-
 5 files changed, 143 insertions(+), 86 deletions(-)

diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index e5e9cf89a..3f9fc4795 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -35,7 +35,8 @@ enum class equilibrium
 enum class modifier
 {
     none, //!< no modification
-    heaviside //!< Psip is dampened to a constant outside a critical value
+    heaviside, //!< Psip is dampened to a constant outside a critical value
+    sol_pfr //!< Psip is dampened in the SOL and PFR regions but not in the closed field line region
 };
 ///@brief How flux function looks like. Decider on whether and what flux aligned grid to construct
 enum class description
@@ -57,7 +58,8 @@ static const std::map<std::string, equilibrium> str2equilibrium{
 };
 static const std::map<std::string, modifier> str2modifier{
     {"none", modifier::none},
-    {"heaviside", modifier::heaviside}
+    {"heaviside", modifier::heaviside},
+    {"sol_pfr", modifier::sol_pfr}
 };
 static const std::map<std::string, description> str2description{
     {"standardO", description::standardO},
diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
index 60b40c00a..d4695559d 100644
--- a/inc/geometries/make_field.h
+++ b/inc/geometries/make_field.h
@@ -10,7 +10,7 @@
 namespace dg{
 namespace geo{
 
-TokamakMagneticField createMagneticField( Json::Value js, file::error mode)
+static inline TokamakMagneticField createMagneticField( Json::Value js, file::error mode)
 {
     std::string e = file::get( mode, js, "equilibrium", "solovev" ).asString();
     equilibrium equi = str2equilibrium.at( e);
@@ -44,46 +44,97 @@ TokamakMagneticField createMagneticField( Json::Value js, file::error mode)
         }
     }
 }
-TokamakMagneticField createModifiedField( Json::Value js, Json::Value jsmod, file::error mode)
+
+/**
+ * @brief Modify Magnetic Field above or below certain Psi values according to given parameters
+ *
+ * We modify psi above or below certain Psip values to a constant using the
+ * \c dg::IPolynomialHeaviside function (an approximation to the integrated Heaviside
+ * function with width alpha), i.e. we replace psi with IPolynomialHeaviside(psi).
+ * This subsequently modifies all derivatives of psi and the poloidal
+ * current in this region.
+ * @param in Magnetic field to change
+ * @param psi0 boundary value where psi is modified to a constant psi0
+ * @param alpha radius of the transition region where the modification acts (smaller is quicker)
+ * @param sign determines which side of Psi to dampen (negative or positive, forwarded to \c dg::IPolynomialHeaviside)
+ * @param ZX Here you can give a Z value (of the X-point).
+ * @param side  The modification will only happen on either the upper (positive) or lower (negative) side of ZX in Z.
+ * @note Per default the dampening happens everywhere
+ * @return A magnetic field object
+ * @ingroup geom
+ */
+static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Value jsmod, file::error mode)
 {
     std::string e = file::get( mode, js, "equilibrium", "solovev" ).asString();
     equilibrium equi = str2equilibrium.at( e);
     std::string m = file::get( mode, jsmod, "modifier", "heaviside" ).asString();
     modifier mod = str2modifier.at( m);
+    std::string d = file::get( mode, js, "description", "standardX" ).asString();
+    description desc = str2description.at( d);
+    TokamakMagneticField mag = createMagneticField( js, mode);
+    const MagneticFieldParameters& inp = mag.params();
+    MagneticFieldParameters mod_params{ inp.a(), inp.elongation(),
+        inp.triangularity(), inp.getEquilibrium(), mod, inp.getDescription()};
+    CylindricalFunctorsLvl2 mod_psip;
     switch (mod) {
         default:
         {
-            return createMagneticField( js, mode);
+            return mag;
         }
         case modifier::heaviside:
         {
             double psi0 = file::get( mode, jsmod, "psi0", 0. ).asDouble();
             double alpha = file::get( mode, jsmod, "alpha", 0. ).asDouble();
             double sign = file::get( mode, jsmod, "sign", -1. ).asDouble();
-            switch( equi){
-                case equilibrium::polynomial:
+            mod_psip = mod::createPsip( mod::everywhere, mag.get_psip(), psi0, alpha, sign);
+            break;
+        }
+        case modifier::sol_pfr:
+        {
+            CylindricalFunctorsLvl2 mod0_psip;
+            double psi0 = file::get_idx( mode, jsmod, "psi0",0, 0. ).asDouble();
+            double alpha0 = file::get_idx( mode, jsmod, "alpha",0, 0. ).asDouble();
+            double sign0 = file::get_idx( mode, jsmod, "sign",0, -1. ).asDouble();
+            double psi1 = file::get_idx( mode, jsmod, "psi0",1, 0. ).asDouble();
+            double alpha1 = file::get_idx( mode, jsmod, "alpha",1, 0. ).asDouble();
+            double sign1 = file::get_idx( mode, jsmod, "sign", 1, +1. ).asDouble();
+            switch( desc){
+                case description::standardX:
                 {
-                    polynomial::Parameters gp( js, mode);
-                    return createModifiedPolynomialField( gp, psi0, alpha, sign );
+                    //we can find the X-point
+                    double RX = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
+                    double ZX = -1.1*mag.params().elongation()*mag.params().a();
+                    dg::geo::findXpoint( mag.get_psip(), RX, ZX);
+                    mod0_psip = mod::createPsip(
+                            mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
+                    mod_psip = mod::createPsip(
+                            mod::HeavisideZ( ZX, -1), mod0_psip, psi1, alpha1, sign1);
+                    break;
                 }
-                case equilibrium::solovev:
+                default:
                 {
-                    solovev::Parameters gp( js, mode);
-                    return createModifiedSolovevField( gp, psi0, alpha, sign );
+                    mod0_psip = mod::createPsip(
+                            mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
+                    mod_psip = mod::createPsip(
+                            mod::everywhere, mod0_psip, psi1, alpha1, sign1);
+                    break;
                 }
-                default:
-                    std::stringstream message;
-                    message << "*** "<<e<<" has no modification implemented!";
-                    if( file::error::is_throw == mode)
-                        throw std::runtime_error( message.str());
-                    else if (file::error::is_warning == mode)
-                        std::cerr <<"WARNING "<< message.str()<<"\n";
-                    else
-                        ;
-                    return createMagneticField( js, mode);
             }
         }
     }
+    switch( equi){
+        case equilibrium::solovev:
+        {
+            solovev::Parameters gp( js, mode);
+            return TokamakMagneticField( gp.R_0,mod_psip,
+                solovev::createIpol( gp, mod_psip), mod_params);
+        }
+        default:
+        {
+            return TokamakMagneticField( mag.R0(), mod_psip,
+                    mag.get_ipol(), mod_params);
+        }
+    }
 }
 } //namespace geo
 }//namespace dg
diff --git a/inc/geometries/modified.h b/inc/geometries/modified.h
index 66110a4e1..97ff6d612 100644
--- a/inc/geometries/modified.h
+++ b/inc/geometries/modified.h
@@ -21,91 +21,112 @@ namespace geo
 {
 namespace mod
 {
+    //modify with a polynomial Heaviside function
 
 struct Psip: public aCylindricalFunctor<Psip>
 {
-    Psip( std::function<double(double,double)> psip, double psi0, double alpha, double sign = -1) :
-        m_ipoly( psi0, alpha, sign), m_psip(psip)
+    Psip( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, double psi0, double alpha, double sign = -1) :
+        m_ipoly( psi0, alpha, sign), m_psip(psip), m_pred(predicate)
     { }
     double do_compute(double R, double Z) const
     {
         double psip = m_psip(R,Z);
-        return m_ipoly( psip);
+        if( m_pred( R,Z))
+            return m_ipoly( psip);
+        else
+            return psip;
     }
     private:
     dg::IPolynomialHeaviside m_ipoly;
     std::function<double(double,double)> m_psip;
+    std::function<bool(double,double)> m_pred;
 };
 struct PsipR: public aCylindricalFunctor<PsipR>
 {
-    PsipR( std::function<double(double,double)> psip, std::function<double(double,double)> psipR, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_psip(psip), m_psipR(psipR)
+    PsipR( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, std::function<double(double,double)> psipR, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_psip(psip), m_psipR(psipR), m_pred(predicate)
     { }
     double do_compute(double R, double Z) const
     {
         double psip  = m_psip(R,Z);
         double psipR = m_psipR(R,Z);
-        return psipR*m_poly( psip);
+        if( m_pred( R,Z))
+            return psipR*m_poly( psip);
+        else
+            return psipR;
     }
     private:
     dg::PolynomialHeaviside m_poly;
     std::function<double(double,double)> m_psip, m_psipR;
+    std::function<bool(double,double)> m_pred;
 };
 struct PsipZ: public aCylindricalFunctor<PsipZ>
 {
-    PsipZ( std::function<double(double,double)> psip, std::function<double(double,double)> psipZ, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_psip(psip), m_psipZ(psipZ)
+    PsipZ( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, std::function<double(double,double)> psipZ, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_psip(psip), m_psipZ(psipZ), m_pred(predicate)
     { }
     double do_compute(double R, double Z) const
     {
         double psip = m_psip(R,Z);
         double psipZ = m_psipZ(R,Z);
-        return psipZ*m_poly( psip);
+        if( m_pred( R,Z))
+            return psipZ*m_poly( psip);
+        else
+            return psipZ;
     }
     private:
     dg::PolynomialHeaviside m_poly;
     std::function<double(double,double)> m_psip, m_psipZ;
+    std::function<bool(double,double)> m_pred;
 };
 
 struct PsipZZ: public aCylindricalFunctor<PsipZZ>
 {
-    PsipZZ( std::function<double(double,double)> psip, std::function<double(double,double)> psipZ, std::function<double(double,double)> psipZZ, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(psip), m_psipZ(psipZ), m_psipZZ(psipZZ)
+    PsipZZ( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, std::function<double(double,double)> psipZ, std::function<double(double,double)> psipZZ, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(psip), m_psipZ(psipZ), m_psipZZ(psipZZ), m_pred(predicate)
     { }
     double do_compute(double R, double Z) const
     {
         double psip = m_psip(R,Z);
         double psipZ = m_psipZ(R,Z);
         double psipZZ = m_psipZZ(R,Z);
-        return psipZZ*m_poly( psip) + psipZ*psipZ*m_dpoly(psip);
+        if( m_pred( R,Z))
+            return psipZZ*m_poly( psip) + psipZ*psipZ*m_dpoly(psip);
+        else
+            return psipZZ;
     }
     private:
     dg::PolynomialHeaviside m_poly;
     dg::DPolynomialHeaviside m_dpoly;
     std::function<double(double,double)> m_psip, m_psipZ, m_psipZZ;
+    std::function<bool(double,double)> m_pred;
 };
 
 struct PsipRR: public aCylindricalFunctor<PsipRR>
 {
-    PsipRR( std::function<double(double,double)> psip, std::function<double(double,double)> psipR, std::function<double(double,double)> psipRR, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(psip), m_psipR(psipR), m_psipRR(psipRR)
+    PsipRR( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, std::function<double(double,double)> psipR, std::function<double(double,double)> psipRR, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(psip), m_psipR(psipR), m_psipRR(psipRR), m_pred(predicate)
     { }
     double do_compute(double R, double Z) const
     {
         double psip = m_psip(R,Z);
         double psipR = m_psipR(R,Z);
         double psipRR = m_psipRR(R,Z);
-        return psipRR*m_poly( psip) + psipR*psipR*m_dpoly(psip);
+        if( m_pred( R,Z))
+            return psipRR*m_poly( psip) + psipR*psipR*m_dpoly(psip);
+        else
+            return psipRR;
     }
     private:
     dg::PolynomialHeaviside m_poly;
     dg::DPolynomialHeaviside m_dpoly;
     std::function<double(double,double)> m_psip, m_psipR, m_psipRR;
+    std::function<bool(double,double)> m_pred;
 };
 struct PsipRZ: public aCylindricalFunctor<PsipRZ>
 {
-    PsipRZ( std::function<double(double,double)> psip, std::function<double(double,double)> psipR, std::function<double(double,double)> psipZ, std::function<double(double,double)> psipRZ, double psi0, double alpha, double sign = -1) :
-        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(psip), m_psipR(psipR), m_psipZ(psipZ), m_psipRZ(psipRZ)
+    PsipRZ( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, std::function<double(double,double)> psipR, std::function<double(double,double)> psipZ, std::function<double(double,double)> psipRZ, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_dpoly( psi0, alpha, sign), m_psip(psip), m_psipR(psipR), m_psipZ(psipZ), m_psipRZ(psipRZ), m_pred(predicate)
     { }
     double do_compute(double R, double Z) const
     {
@@ -113,39 +134,48 @@ struct PsipRZ: public aCylindricalFunctor<PsipRZ>
         double psipR = m_psipR(R,Z);
         double psipZ = m_psipZ(R,Z);
         double psipRZ = m_psipRZ(R,Z);
-        return psipRZ*m_poly( psip) + psipR*psipZ*m_dpoly(psip);
+        if( m_pred( R,Z))
+            return psipRZ*m_poly( psip) + psipR*psipZ*m_dpoly(psip);
+        else
+            return psipRZ;
     }
     private:
     dg::PolynomialHeaviside m_poly;
     dg::DPolynomialHeaviside m_dpoly;
     std::function<double(double,double)> m_psip, m_psipR, m_psipZ, m_psipRZ;
+    std::function<bool(double,double)> m_pred;
 };
 
-static inline dg::geo::CylindricalFunctorsLvl2 createPsip( const CylindricalFunctorsLvl2& psip,
+static inline dg::geo::CylindricalFunctorsLvl2 createPsip(
+        const std::function<bool(double,double)> predicate,
+        const CylindricalFunctorsLvl2& psip,
     double psi0, double alpha, double sign = -1)
 {
     return CylindricalFunctorsLvl2(
-            mod::Psip(psip.f(), psi0, alpha, sign),
-            mod::PsipR(psip.f(), psip.dfx(), psi0, alpha, sign),
-            mod::PsipZ(psip.f(), psip.dfy(), psi0, alpha, sign),
-            mod::PsipRR(psip.f(), psip.dfx(), psip.dfxx(), psi0, alpha, sign),
-            mod::PsipRZ(psip.f(), psip.dfx(), psip.dfy(), psip.dfxy(), psi0, alpha, sign),
-            mod::PsipZZ(psip.f(), psip.dfy(), psip.dfyy(), psi0, alpha, sign));
+            mod::Psip(predicate,psip.f(), psi0, alpha, sign),
+            mod::PsipR(predicate,psip.f(), psip.dfx(), psi0, alpha, sign),
+            mod::PsipZ(predicate,psip.f(), psip.dfy(), psi0, alpha, sign),
+            mod::PsipRR(predicate,psip.f(), psip.dfx(), psip.dfxx(), psi0, alpha, sign),
+            mod::PsipRZ(predicate,psip.f(), psip.dfx(), psip.dfy(), psip.dfxy(), psi0, alpha, sign),
+            mod::PsipZZ(predicate,psip.f(), psip.dfy(), psip.dfyy(), psi0, alpha, sign));
 }
 
-} //namespace mod
+//some possible predicates
 
-//Create heaviside modification, does not modify Ipol
-static inline dg::geo::TokamakMagneticField createModifiedField(
-    const dg::geo::TokamakMagneticField& in, double psi0, double alpha, double sign = -1)
-{
-    const MagneticFieldParameters inp = in.params();
-    MagneticFieldParameters params = { inp.a(), inp.elongation(), inp.triangularity(),
-            inp.getEquilibrium(), modifier::heaviside, inp.getDescription()};
-    return TokamakMagneticField( in.R0(), mod::createPsip(in.get_psip(), psi0,
-        alpha, sign), in.get_ipol(), params);
-}
+static bool everywhere( double R, double Z){return true;}
+struct HeavisideZ{
+    HeavisideZ( double Z_X, int side): m_ZX( Z_X), m_side(side) {}
+    bool operator()(double R, double Z){
+        if( Z < m_ZX && m_side <= 0) return true;
+        if( Z >= m_ZX && m_side > 0) return true;
+        return false;
+    }
+    private:
+    double m_ZX;
+    int m_side;
+};
 
+} //namespace mod
 
 } //namespace geo
 } //namespace dg
diff --git a/inc/geometries/polynomial.h b/inc/geometries/polynomial.h
index 731af50c6..70aaa59fa 100644
--- a/inc/geometries/polynomial.h
+++ b/inc/geometries/polynomial.h
@@ -203,32 +203,6 @@ static inline dg::geo::TokamakMagneticField createPolynomialField(
     return TokamakMagneticField( gp.R_0, polynomial::createPsip(gp),
         polynomial::createIpol(gp), params);
 }
-/**
- * @brief Create a modified Polynomial Magnetic field
- *
- * Based on \c dg::geo::polynomial::mod::Psip(gp) and
- * \c dg::geo::polynomial::mod::Ipol(gp)
- * We modify psi above a certain value to a constant using the
- * \c dg::IPolynomialHeaviside function (an approximation to the integrated Heaviside
- * function with width alpha), i.e. we replace psi with IPolynomialHeaviside(psi).
- * This subsequently modifies all derivatives of psi and the poloidal
- * current.
- * @param gp Polynomial parameters
- * @param psi0 boundary value where psi is modified to a constant psi0
- * @param alpha radius of the transition region where the modification acts (smaller is quicker)
- * @param sign determines which side of Psi to dampen (negative or positive, forwarded to \c dg::IPolynomialHeaviside)
- * @return A magnetic field object
- * @ingroup geom
- */
-static inline dg::geo::TokamakMagneticField createModifiedPolynomialField(
-    dg::geo::polynomial::Parameters gp, double psi0, double alpha, double sign = -1)
-{
-    MagneticFieldParameters params( gp.a, gp.elongation, gp.triangularity,
-            equilibrium::polynomial, modifier::heaviside, str2description.at( gp.description));
-    return TokamakMagneticField( gp.R_0,
-            mod::createPsip( polynomial::createPsip(gp), psi0, alpha, sign),
-        polynomial::createIpol( gp), params);
-}
 
 } //namespace geo
 } //namespace dg
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 8e26dd764..897db7dd2 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -393,8 +393,8 @@ static inline dg::geo::TokamakMagneticField createModifiedSolovevField(
     MagneticFieldParameters params = { gp.a, gp.elongation, gp.triangularity,
             equilibrium::solovev, modifier::heaviside, str2description.at( gp.description)};
     return TokamakMagneticField( gp.R_0,
-            mod::createPsip( solovev::createPsip(gp), psi0, alpha, sign),
-        solovev::createIpol( gp, mod::createPsip( solovev::createPsip(gp), psi0, alpha, sign)),
+            mod::createPsip( mod::everywhere, solovev::createPsip(gp), psi0, alpha, sign),
+        solovev::createIpol( gp, mod::createPsip( mod::everywhere, solovev::createPsip(gp), psi0, alpha, sign)),
         params);
 }
 
-- 
GitLab


From 0c58a3467b428c14c4b8385c93c7d697b69a6392 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 22 Sep 2020 16:39:44 +0200
Subject: [PATCH 338/540] Add wall_damping and transition to make_field

now we can safely let make_field do all normalizations
but the thing is not documented yet and probably should be
carefully documented at some point in the near future
---
 inc/geometries/fluxfunctions.h             | 17 +++---
 inc/geometries/geometry_diag.cu            | 29 ++-------
 inc/geometries/geometry_params_Xpoint.json |  4 +-
 inc/geometries/make_field.h                | 70 ++++++++++++++++++----
 inc/geometries/modified.h                  | 50 ++++++++++++++++
 src/feltor/feltor.cu                       | 26 ++------
 src/feltor/feltor.tex                      |  5 +-
 src/feltor/feltor_hpc.cu                   | 22 +------
 src/feltor/feltordiag.cu                   | 14 +----
 src/feltor/feltordiag.h                    |  6 ++
 src/feltor/init.h                          | 13 ----
 src/feltor/parameters.h                    |  8 +--
 12 files changed, 146 insertions(+), 118 deletions(-)

diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index d9e5b18c5..3d7495853 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -277,18 +277,21 @@ static inline int findCriticalPoint( const CylindricalFunctorsLvl2& psi, double&
     X[0] = RC, X[1] = ZC;
     double eps = 1e10, eps_old= 2e10;
     unsigned counter = 0; //safety measure to avoid deadlock
-    double Dinv = 0., psipRR  = 0., psipZZ = 0., psipRZ = 0.;
+    double psipRZ = psi.dfxy()(X[0], X[1]);
+    double psipRR = psi.dfxx()(X[0], X[1]), psipZZ = psi.dfyy()(X[0],X[1]);
+    double psipR  = psi.dfx()(X[0], X[1]), psipZ = psi.dfy()(X[0], X[1]);
+    double Dinv = 1./(psipZZ*psipRR - psipRZ*psipRZ);
     while( (eps < eps_old || eps > 1e-7) && eps > 1e-10 && counter < 100)
     {
-        X_OLD = X; eps= eps_old;
-        psipRZ = psi.dfxy()(X[0], X[1]);
-        psipRR = psi.dfxx()(X[0], X[1]), psipZZ = psi.dfyy()(X[0],X[1]);
-        double psipR  = psi.dfx()(X[0], X[1]), psipZ = psi.dfy()(X[0], X[1]);
-        Dinv = 1./(psipZZ*psipRR - psipRZ*psipRZ);
         XN[0] = X[0] - Dinv*(psipZZ*psipR - psipRZ*psipZ);
         XN[1] = X[1] - Dinv*(-psipRZ*psipR + psipRR*psipZ);
         XN.swap(X);
         eps = sqrt( (X[0]-X_OLD[0])*(X[0]-X_OLD[0]) + (X[1]-X_OLD[1])*(X[1]-X_OLD[1]));
+        X_OLD = X; eps_old= eps;
+        psipRZ = psi.dfxy()(X[0], X[1]);
+        psipRR = psi.dfxx()(X[0], X[1]), psipZZ = psi.dfyy()(X[0],X[1]);
+        psipR  = psi.dfx()(X[0], X[1]), psipZ = psi.dfy()(X[0], X[1]);
+        Dinv = 1./(psipZZ*psipRR - psipRZ*psipRZ);
         counter++;
     }
     if ( counter >= 100 || std::isnan( Dinv) )
@@ -296,7 +299,7 @@ static inline int findCriticalPoint( const CylindricalFunctorsLvl2& psi, double&
     RC = X[0], ZC = X[1];
     if( Dinv > 0 &&  psipRR > 0)
         return 1; //local minimum
-    if( Dinv < 0 &&  psipRR < 0)
+    if( Dinv > 0 &&  psipRR < 0)
         return 2; //local maximum
     //if( Dinv < 0)
     return 3; //saddle point
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 61cfb947d..e3b5a5124 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -27,7 +27,7 @@ struct Parameters
     double boxscaleZm, boxscaleZp;
     double amp, k_psi, nprofileamp;
     double sigma, posX, posY;
-    double damping_boundary, source_alpha, damping_alpha, source_boundary;
+    double source_alpha, source_boundary;
     double profile_alpha;
     Parameters( const Json::Value& js){
         n = js.get("n",3).asUInt();
@@ -46,8 +46,6 @@ struct Parameters
         sigma = js.get("sigma", 10).asDouble();
         posX = js.get("posX", 0.5).asDouble();
         posY = js.get("posY", 0.5).asDouble();
-        damping_boundary = js["damping"].get("boundary", 1.2).asDouble();
-        damping_alpha = js["damping"].get("alpha", 0.1).asDouble();
         source_alpha = js["source"].get("alpha", 0.5).asDouble();
         source_boundary = js["source"].get("boundary", 0.5).asDouble();
     }
@@ -65,8 +63,6 @@ struct Parameters
             <<" boxscaleZp    = "<<boxscaleZp<<"\n"
             <<" source bound  = "<<source_boundary<<"\n"
             <<" source alpha  = "<<source_alpha<<"\n"
-            <<" damping bound = "<<damping_boundary<<"\n"
-            <<" damping alpha = "<<damping_alpha<<"\n"
             <<" amp           = "<<amp<<"\n"
             <<" k_psi         = "<<k_psi<<"\n"
             <<" nprofileamp   = "<<nprofileamp<<"\n"
@@ -121,8 +117,8 @@ int main( int argc, char* argv[])
     const Parameters p(input_js);
     p.display( std::cout);
     //Test coefficients
-    dg::geo::TokamakMagneticField mag_origin = dg::geo::createMagneticField(geom_js, file::error::is_throw);
-    dg::geo::TokamakMagneticField mag = mag_origin;
+    dg::geo::CylindricalFunctor damping, transition;
+    dg::geo::TokamakMagneticField mag = dg::geo::createModifiedField(geom_js, input_js, file::error::is_throw, damping, transition);
     std::string input = input_js.toStyledString();
     std::string geom = geom_js.toStyledString();
     unsigned n, Nx, Ny, Nz;
@@ -147,18 +143,6 @@ int main( int argc, char* argv[])
             std::cout << " (maximum)"<<std::endl;
         double psip0 = mag.psip()(mag.R0(), 0);
         std::cout << "psip( R_0, 0) = "<<psip0<<"\n";
-        if( p.damping_alpha > 0.)
-        {
-            double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
-            double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-            std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
-            Json::Value jsmod;
-            jsmod["modifier"] = "heaviside";
-            jsmod["psi0"] = damping_psi0p + damping_alphap/2.;
-            jsmod["alpha"] = fabs( damping_alphap/2.);
-            jsmod["sign"] = ((psipO>0)-(psipO<0));
-            mag = dg::geo::createModifiedField(geom_js, jsmod, file::error::is_throw);
-        }
     }
 
 
@@ -223,11 +207,8 @@ int main( int argc, char* argv[])
         {"SourceProfile", "A source profile", dg::compose( dg::PolynomialHeaviside(
                     p.source_boundary-p.source_alpha/2., p.source_alpha/2., -1 ),
                 dg::geo::RhoP(mag))},
-        {"ProfileDamping", "Density profile damping", dg::compose(dg::PolynomialHeaviside(
-            1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)) },
-        {"MagneticTransition", "The region where the magnetic field is modified", dg::compose(dg::DPolynomialHeaviside(
-            p.damping_boundary+p.damping_alpha/2.,
-            p.damping_alpha/2., +1 ), dg::geo::RhoP(mag_origin))},
+        {"ProfileDamping", "Density profile damping", damping },
+        {"MagneticTransition", "The region where the magnetic field is modified", transition},
         {"Nprofile", "A flux aligned profile", dg::compose( dg::LinearX( p.nprofileamp/mag.psip()(mag.R0(),0.), p.nprofileamp ), mag.psip())},
         {"Delta", "A flux aligned Gaussian peak", dg::compose( dg::GaussianX( psipO*0.2, 0.1, 1./(sqrt(2.*M_PI)*0.1)), mag.psip())},
         {"TanhDamping", "A flux aligned Heaviside with Tanh Damping", dg::compose( dg::TanhProfX( -3*p.source_alpha, p.source_alpha, -1), mag.psip())},
diff --git a/inc/geometries/geometry_params_Xpoint.json b/inc/geometries/geometry_params_Xpoint.json
index e944e759e..dcf20ca3b 100644
--- a/inc/geometries/geometry_params_Xpoint.json
+++ b/inc/geometries/geometry_params_Xpoint.json
@@ -5,8 +5,8 @@
 {
     //----------------------Solovev coefficients---------------
     "A" : 0.0,
-    "PP": 1,
-    "PI": 1,
+    "PP": -1,
+    "PI": -1,
     "c" :[  0.07350114445500399706283007092406934834526,
            -0.08662417436317227513877947632069712210813,
            -0.1463931543401102620740934776490506239925,
diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
index d4695559d..cb9fa19a2 100644
--- a/inc/geometries/make_field.h
+++ b/inc/geometries/make_field.h
@@ -63,11 +63,11 @@ static inline TokamakMagneticField createMagneticField( Json::Value js, file::er
  * @return A magnetic field object
  * @ingroup geom
  */
-static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Value jsmod, file::error mode)
+static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Value jsmod, file::error mode, CylindricalFunctor& damping, CylindricalFunctor& transition)
 {
     std::string e = file::get( mode, js, "equilibrium", "solovev" ).asString();
     equilibrium equi = str2equilibrium.at( e);
-    std::string m = file::get( mode, jsmod, "modifier", "heaviside" ).asString();
+    std::string m = file::get( mode, jsmod, "damping", "modifier", "heaviside" ).asString();
     modifier mod = str2modifier.at( m);
     std::string d = file::get( mode, js, "description", "standardX" ).asString();
     description desc = str2description.at( d);
@@ -77,27 +77,44 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
         inp.triangularity(), inp.getEquilibrium(), mod, inp.getDescription()};
     CylindricalFunctorsLvl2 mod_psip;
     switch (mod) {
-        default:
+        default: //none
         {
+            damping = mod::DampingRegion( mod::nowhere, mag.psip(), 0, 0, 0);
+            transition = mod::MagneticTransition( mod::nowhere, mag.psip(), 0, 0, 0);
             return mag;
         }
         case modifier::heaviside:
         {
-            double psi0 = file::get( mode, jsmod, "psi0", 0. ).asDouble();
-            double alpha = file::get( mode, jsmod, "alpha", 0. ).asDouble();
-            double sign = file::get( mode, jsmod, "sign", -1. ).asDouble();
+            double psi0 = file::get( mode, jsmod, "damping", "boundary", 1.1 ).asDouble();
+            double alpha = file::get( mode, jsmod, "damping", "alpha", 0.2 ).asDouble();
+            double sign = -1;
+            if( desc == description::standardX)
+            {
+                double RO=mag.R0(), ZO=0.;
+                dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+                double psipO = mag.psip()( RO, ZO);
+                double damping_psi0p = (1.-psi0*psi0)*psipO;
+                double damping_alphap = -(2.*psi0+alpha)*alpha*psipO;
+                //std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
+                psi0 = damping_psi0p + damping_alphap/2.;
+                alpha = fabs( damping_alphap/2.);
+                sign = ((psipO>0)-(psipO<0));
+            }
+            else
+                sign = file::get( mode, jsmod, "damping", "sign", -1. ).asDouble();
+
             mod_psip = mod::createPsip( mod::everywhere, mag.get_psip(), psi0, alpha, sign);
+            damping = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha, -sign);
+            transition = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha, sign);
             break;
         }
         case modifier::sol_pfr:
         {
             CylindricalFunctorsLvl2 mod0_psip;
-            double psi0 = file::get_idx( mode, jsmod, "psi0",0, 0. ).asDouble();
-            double alpha0 = file::get_idx( mode, jsmod, "alpha",0, 0. ).asDouble();
-            double sign0 = file::get_idx( mode, jsmod, "sign",0, -1. ).asDouble();
-            double psi1 = file::get_idx( mode, jsmod, "psi0",1, 0. ).asDouble();
-            double alpha1 = file::get_idx( mode, jsmod, "alpha",1, 0. ).asDouble();
-            double sign1 = file::get_idx( mode, jsmod, "sign", 1, +1. ).asDouble();
+            double psi0 = file::get_idx( mode, jsmod, "damping", "boundary",0, 1.1 ).asDouble();
+            double alpha0 = file::get_idx( mode, jsmod, "damping", "alpha",0, 0.2 ).asDouble();
+            double psi1 = file::get_idx( mode, jsmod, "damping", "boundary",1, 0.97 ).asDouble();
+            double alpha1 = file::get_idx( mode, jsmod, "damping", "alpha",1, 0.2 ).asDouble();
             switch( desc){
                 case description::standardX:
                 {
@@ -105,18 +122,46 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
                     double RX = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
                     double ZX = -1.1*mag.params().elongation()*mag.params().a();
                     dg::geo::findXpoint( mag.get_psip(), RX, ZX);
+                    //and the X-point
+                    double RO=mag.R0(), ZO=0.;
+                    dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+                    double psipO = mag.psip()( RO, ZO);
+                    double damping_psi0p = (1.-psi0*psi0)*psipO;
+                    double damping_alpha0p = -(2.*psi0+alpha0)*alpha0*psipO;
+                    double damping_psi1p = (1.-psi1*psi1)*psipO;
+                    double damping_alpha1p = -(2.*psi1+alpha1)*alpha1*psipO;
+                    psi0 = damping_psi0p + damping_alpha0p/2.;
+                    psi1 = damping_psi1p - damping_alpha1p/2.;
+                    alpha0 = fabs( damping_alpha0p/2.);
+                    alpha1 = fabs( damping_alpha1p/2.);
+                    double sign0 = ((psipO>0)-(psipO<0));
+                    double sign1 = -sign0;
                     mod0_psip = mod::createPsip(
                             mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
                     mod_psip = mod::createPsip(
                             mod::HeavisideZ( ZX, -1), mod0_psip, psi1, alpha1, sign1);
+                    CylindricalFunctor damping0 = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha0, -sign0);
+                    CylindricalFunctor transition0 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
+                    CylindricalFunctor damping1 = mod::DampingRegion( mod::HeavisideZ(ZX, -1), mag.psip(), psi1, alpha1, -sign1);
+                    CylindricalFunctor transition1 = mod::MagneticTransition( mod::HeavisideZ(ZX, -1), mag.psip(), psi1, alpha1, sign1);
+                    damping = mod::Combine( damping0, damping1);
+                    transition = mod::Combine( transition0, transition1);
                     break;
                 }
                 default:
                 {
+                    double sign0 = file::get_idx( mode, jsmod, "damping", "sign",0, -1. ).asDouble();
+                    double sign1 = file::get_idx( mode, jsmod, "damping", "sign", 1, +1. ).asDouble();
                     mod0_psip = mod::createPsip(
                             mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
                     mod_psip = mod::createPsip(
                             mod::everywhere, mod0_psip, psi1, alpha1, sign1);
+                    CylindricalFunctor damping0 = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
+                    CylindricalFunctor transition0 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
+                    CylindricalFunctor damping1 = mod::DampingRegion( mod::everywhere, mag.psip(), psi1, alpha1, sign1);
+                    CylindricalFunctor transition1 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi1, alpha1, sign1);
+                    damping = mod::Combine( damping0, damping1);
+                    transition = mod::Combine( transition0, transition1);
                     break;
                 }
             }
@@ -136,5 +181,6 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
         }
     }
 }
+
 } //namespace geo
 }//namespace dg
diff --git a/inc/geometries/modified.h b/inc/geometries/modified.h
index 97ff6d612..809461052 100644
--- a/inc/geometries/modified.h
+++ b/inc/geometries/modified.h
@@ -159,9 +159,59 @@ static inline dg::geo::CylindricalFunctorsLvl2 createPsip(
             mod::PsipRZ(predicate,psip.f(), psip.dfx(), psip.dfy(), psip.dfxy(), psi0, alpha, sign),
             mod::PsipZZ(predicate,psip.f(), psip.dfy(), psip.dfyy(), psi0, alpha, sign));
 }
+struct DampingRegion : public aCylindricalFunctor<DampingRegion>
+{
+    DampingRegion( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, double psi0, double alpha, double sign = -1) :
+        m_poly( psi0, alpha, sign), m_psip(psip), m_pred(predicate)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        if( m_pred( R,Z))
+            return m_poly( psip);
+        else
+            return 0;
+    }
+    private:
+    dg::PolynomialHeaviside m_poly;
+    std::function<double(double,double)> m_psip;
+    std::function<bool(double,double)> m_pred;
+};
+struct MagneticTransition : public aCylindricalFunctor<MagneticTransition>
+{
+    MagneticTransition( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, double psi0, double alpha, double sign = -1) :
+        m_dpoly( psi0, alpha, sign), m_psip(psip), m_pred(predicate)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double psip = m_psip(R,Z);
+        if( m_pred( R,Z))
+            return m_dpoly( psip);
+        else
+            return 0;
+    }
+    private:
+    dg::DPolynomialHeaviside m_dpoly;
+    std::function<double(double,double)> m_psip;
+    std::function<bool(double,double)> m_pred;
+};
+//combine damping and transition regions by adding them up
+struct Combine : public aCylindricalFunctor<Combine>
+{
+    Combine( std::function<double(double,double)> fct1, std::function<double(double,double)> fct2) :
+        m_fct1(fct1), m_fct2(fct2)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        return m_fct1(R,Z)+m_fct2(R,Z);
+    }
+    private:
+    std::function<double(double,double)> m_fct1, m_fct2;
+};
 
 //some possible predicates
 
+static bool nowhere( double R, double Z){return false;}
 static bool everywhere( double R, double Z){return true;}
 struct HeavisideZ{
     HeavisideZ( double Z_X, int side): m_ZX( Z_X), m_side(side) {}
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index ce517ccf7..a6c523685 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -60,8 +60,9 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     std::cout << gs.toStyledString() << std::endl;
     dg::geo::TokamakMagneticField mag;
+    dg::geo::CylindricalFunctor damping, transition;
     try{
-        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
+        mag = dg::geo::createModifiedField(gs, js, file::error::is_throw, damping, transition);
     }catch(std::runtime_error& e)
     {
         std::cerr << "ERROR in geometry file "<<geomfile<<std::endl;
@@ -76,24 +77,8 @@ int main( int argc, char* argv[])
     //Make grid
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
-    //Wall damping has to be constructed before modification (!)
-    HVec damping_profile = dg::evaluate( dg::zero, grid);
-    if( p.damping_alpha > 0.)
-    {
-        damping_profile = feltor::wall_damping( grid, p, mag);
-        double RO=mag.R0(), ZO=0.;
-        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
-        double psipO = mag.psip()( RO, ZO);
-        double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
-        double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
-        Json::Value jsmod;
-        jsmod["modifier"] = "heaviside";
-        jsmod["psi0"] = damping_psi0p + damping_alphap/2.;
-        jsmod["alpha"] = fabs( damping_alphap/2.);
-        jsmod["sign"] = ((psipO>0)-(psipO<0));
-        mag = dg::geo::createModifiedField(gs, jsmod, file::error::is_warning);
-    }
+
+    HVec damping_profile = dg::pullback( damping, grid);
     if( p.periodify)
         mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
 
@@ -116,9 +101,6 @@ int main( int argc, char* argv[])
     feltor::Variables var = {
         feltor, p,mag, gradPsip, gradPsip, hoo
     };
-
-
-
     /////////////////////The initial field///////////////////////////////////////////
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 9aab87066..e01128752 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1610,10 +1610,11 @@ in the density Eq.~\eqref{eq:density_profile} (with $\rho_{p,b}=0$ and source pr
 small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes, must not be zero)
 \\
 damping & dict & & & magnetic and density damping region \\
+\qquad modifier & string & "sol\_pfr" & "sol\_pfr" & One of ``none'', ``heaviside'' or ``sol\_pfr'' \\
 \qquad rate & float & 0    & 0   & Friction coefficient $\omega_d$ in density and velocity damping Eq.~\eqref{eq:velocity_source} \\
-\qquad boundary & float & 0.2  & 1.2 & Modification region boundary $\rho_{p,b}$: yields $\psi_0 = (1-\rho_{p,b}^2)\psi_{p,O}$ in Eq.~\eqref{eq:modified_psip}.
+\qquad boundary & float[2] & [1.2,0.8]  & [1.2,0.8] & Modification region boundary $\rho_{p,b}$: yields $\psi_0 = (1-\rho_{p,b}^2)\psi_{p,O}$ in Eq.~\eqref{eq:modified_psip}.
 \\
-\qquad alpha   & float & 0.25 & 0 & Transition width $\alpha_p$: yields
+\qquad alpha   & float[2] & [0.25,0.25] & 0 & Transition width $\alpha_p$: yields
 $\alpha=-2\rho_{p,b}\alpha_p+\alpha_p^2)\psi_{p,O}$ for the Heaviside in the modified
 $\psi_p$ function \eqref{eq:modified_psip}. If zero, then we do not modify the
 magnetic field and damping is ignored.\\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 420f07b75..20450b51a 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -147,8 +147,9 @@ int main( int argc, char* argv[])
     std::string input = js.toStyledString(), geom = gs.toStyledString();
     MPI_OUT std::cout << geom << std::endl;
     dg::geo::TokamakMagneticField mag;
+    dg::geo::CylindricalFunctor damping, transition;
     try{
-        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
+        mag = dg::geo::createModifiedField(gs, js, file::error::is_throw, damping, transition);
     }catch(std::runtime_error& e)
     {
         std::cerr << "ERROR in geometry file "<<argv[2]<<std::endl;
@@ -191,24 +192,7 @@ int main( int argc, char* argv[])
     unsigned local_size2d = g2d_out_ptr->size();
 #endif
 
-    //Wall damping has to be constructed before modification (!)
-    HVec damping_profile = dg::evaluate( dg::zero, grid);
-    if( p.damping_alpha > 0.)
-    {
-        damping_profile = feltor::wall_damping( grid, p, mag);
-        double RO=mag.R0(), ZO=0.;
-        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
-        double psipO = mag.psip()( RO, ZO);
-        double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
-        double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        MPI_OUT std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
-        Json::Value jsmod;
-        jsmod["modifier"] = "heaviside";
-        jsmod["psi0"] = damping_psi0p + damping_alphap/2.;
-        jsmod["alpha"] = fabs( damping_alphap/2.);
-        jsmod["sign"] = ((psipO>0)-(psipO<0));
-        mag = dg::geo::createModifiedField(gs, jsmod, file::error::is_warning);
-    }
+    HVec damping_profile = dg::pullback( damping, grid);
     if( p.periodify)
         mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
 
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 62b010ccf..b9b87287d 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -95,17 +95,9 @@ int main( int argc, char* argv[])
     Geometry g3d_fine( Rmin, Rmax, Zmin, Zmax, 0., 2.*M_PI,
         p.n_out, p.Nx_out, p.Ny_out, FACTOR*p.Nz, p.bcxN, p.bcyN, dg::PER);
 
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    double RO=mag.R0(), ZO=0.;
-    dg::geo::findOpoint( mag.get_psip(), RO, ZO);
-    const double psipO = mag.psip()( RO, ZO);
-    if( p.damping_alpha > 0.)
-    {
-        double damping_psi0 = (1.-p.damping_boundary*p.damping_boundary)*psipO;
-        double damping_alpha = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0+damping_alpha/2.,
-                fabs(p.damping_alpha/2.), ((psipO>0)-(psipO<0)));
-    }
+    dg::geo::CylindricalFunctor damping, transition;
+    dg::geo::TokamakMagneticField mag =
+        dg::geo::createModifiedField(gs, js, file::error::is_warning, damping, transition);
     dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
     // Construct weights and temporaries
 
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 8d98d68fe..3c56f43d1 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -344,6 +344,12 @@ std::vector<Record_static> diagnostics2d_static_list = {
         []( HVec& result, Variables& v, Geometry& grid){
             result = v.f.bphi();
         }
+    },
+    {"NormGradPsip", "Norm of gradient of Psip",
+        []( HVec& result, Variables& v, Geometry& grid){
+            result = dg::pullback(
+                dg::geo::SquareNorm( dg::geo::createGradPsip(v.mag), dg::geo::createGradPsip(v.mag)), grid);
+        }
     }
 };
 // and here are all the 2d outputs we want to produce (currently ~ 100)
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 15873a793..a9fdcca7b 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -126,19 +126,6 @@ void init_ni(
 };
 }//namespace detail
 
-//for wall shadow
-HVec wall_damping(const Geometry& grid,
-    const feltor::Parameters& p,
-    const dg::geo::TokamakMagneticField& mag )
-{
-    if( p.source_alpha == 0)
-        throw dg::Error(dg::Message()<< "Invalid parameter: damping alpha must not be 0\n");
-    HVec wall_damping = dg::pullback(dg::compose( dg::PolynomialHeaviside(
-        p.damping_boundary+p.damping_alpha/2., p.damping_alpha/2., +1),
-                dg::geo::RhoP(mag)), grid);
-    return wall_damping;
-}
-
 /* The purpose of this file is to provide an interface for custom initial conditions and
  * source profiles.  Just add your own to the relevant map below.
  */
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 62c1f529f..a36c5f12b 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -40,8 +40,8 @@ struct Parameters
     double k_psi;
 
     double source_rate, damping_rate;
-    double damping_alpha, source_alpha, profile_alpha;
-    double source_boundary, damping_boundary;
+    double source_alpha, profile_alpha;
+    double source_boundary;
     double nprofamp;
     double boxscaleRm, boxscaleRp;
     double boxscaleZm, boxscaleZp;
@@ -110,8 +110,6 @@ struct Parameters
         source_boundary = file::get( mode, js, "source", "boundary", 0.5).asDouble();
         source_alpha    = file::get( mode, js, "source", "alpha", 0.2).asDouble();
         damping_rate = file::get( mode, js, "damping", "rate", 0.).asDouble();
-        damping_alpha= file::get( mode, js, "damping", "alpha", 0.).asDouble();
-        damping_boundary = file::get( mode, js, "damping", "boundary", 1.2).asDouble();
 
         bcxN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 0, "").asString());
         bcyN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 1, "").asString());
@@ -157,8 +155,6 @@ struct Parameters
             <<"     source_alpha:                 "<<source_alpha<<"\n"
             <<"     source_type:                  "<<source_type<<"\n"
             <<"     damping_rate:                 "<<damping_rate<<"\n"
-            <<"     damping_boundary:             "<<damping_boundary<<"\n"
-            <<"     damping_alpha:                "<<damping_alpha<<"\n"
             <<"     density profile amplitude:    "<<nprofamp<<"\n"
             <<"     boxscale R+:                  "<<boxscaleRp<<"\n"
             <<"     boxscale R-:                  "<<boxscaleRm<<"\n"
-- 
GitLab


From b1bd6a59d9750ee3f70018659757925679246890 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 22 Sep 2020 16:44:08 +0200
Subject: [PATCH 339/540] Fix missing O-point in feltordiag

---
 src/feltor/feltordiag.cu | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index b9b87287d..e45951174 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -125,6 +125,10 @@ int main( int argc, char* argv[])
     double Z_X = -1.1*gp.elongation*gp.a;
     dg::geo::findXpoint( mag.get_psip(), R_X, Z_X);
     dg::geo::CylindricalSymmTensorLvl1 monitor_chi = dg::geo::make_Xconst_monitor( mag.get_psip(), R_X, Z_X) ;
+    double R_O = gp.R_0, Z_O = 0;
+    dg::geo::findOpoint( mag.get_psip(), R_O, Z_O);
+    double psipO = mag.psip()(R_O, Z_O);
+
     dg::geo::SeparatrixOrthogonal generator(mag.get_psip(), monitor_chi, psipO, R_X, Z_X, mag.R0(), 0, 0, false);
     double fx_0 = 1./8.;
     double psipmax = dg::blas1::reduce( psipog2d, 0. ,thrust::maximum<double>()); //DEPENDS ON GRID RESOLUTION!!
-- 
GitLab


From 40411b47cf37b59c1fc976b1b0cf8d8a40d49bea Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 22 Sep 2020 17:20:57 +0200
Subject: [PATCH 340/540] Add modifier to input json files

---
 src/feltor/geometry/tcv.json       |  2 +-
 src/feltor/input/compass.json      |  3 ++-
 src/feltor/input/default.json      |  1 +
 src/feltor/input/manufactured.json |  1 +
 src/feltor/input/tcv.json          | 11 ++++++-----
 5 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/feltor/geometry/tcv.json b/src/feltor/geometry/tcv.json
index caf907e68..2d80a35fd 100644
--- a/src/feltor/geometry/tcv.json
+++ b/src/feltor/geometry/tcv.json
@@ -2,7 +2,7 @@
 	"M" : 6,
 	"N" : 6,
 	"PI" : -1.0,
-	"PP" : -1.0,
+	"PP" : -0.8,
 	"R_0" : 906.38,
 	"c" :
 	[
diff --git a/src/feltor/input/compass.json b/src/feltor/input/compass.json
index d88333828..b727c8fa6 100644
--- a/src/feltor/input/compass.json
+++ b/src/feltor/input/compass.json
@@ -53,7 +53,7 @@
         "amp": 0,
         "alpha": 0.2
     },
-    "source" : 
+    "source" :
     {
         "rate": 2e-3,
         "type": "influx",
@@ -62,6 +62,7 @@
     },
     "damping":
     {
+        "modifier": "heaviside",
         "rate" : 1e-2,
         "boundary": 1.1,
         "alpha": 0.25
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index d1a480c7b..b8e7407f6 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -62,6 +62,7 @@
     },
     "damping":
     {
+        "modifier": "heaviside",
         "rate": 1e-2,
         "boundary": 1.1,
         "alpha": 0.2
diff --git a/src/feltor/input/manufactured.json b/src/feltor/input/manufactured.json
index 8172c7a44..47de7611d 100644
--- a/src/feltor/input/manufactured.json
+++ b/src/feltor/input/manufactured.json
@@ -62,6 +62,7 @@
     },
     "damping":
     {
+        "modifier": "heaviside",
         "rate": 1e-2,
         "boundary": 1.1,
         "alpha": 0.1
diff --git a/src/feltor/input/tcv.json b/src/feltor/input/tcv.json
index 3c108dbd8..3fa7b91f0 100644
--- a/src/feltor/input/tcv.json
+++ b/src/feltor/input/tcv.json
@@ -2,7 +2,7 @@
     "n"  : 3,
     "Nx" : 56,
     "Ny" : 96,
-    "Nz" : 8,
+    "Nz" : 16,
     "dt" : 1e-1,
     "compression" : [2,2],
     "FCI":
@@ -37,8 +37,8 @@
     },
     "box" :
     {
-        "scaleR" :  [1.30,1.10],
-        "scaleZ" :  [1.3,1.05]
+        "scaleR" :  [1.3,1.10],
+        "scaleZ" :  [1.35,1.05]
     },
     "initne"     : "turbulence",
     "initphi"    : "zero",
@@ -62,8 +62,9 @@
     },
     "damping":
     {
+        "modifier": "sol_pfr",
         "rate" : 1e-2,
-        "boundary": 1.05,
-        "alpha": 0.20
+        "boundary": [1.03,0.99],
+        "alpha": [0.20, 0.1]
     }
 }
-- 
GitLab


From 625128fc28ae2ec0866e59bf950e299387053abe Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 24 Sep 2020 16:56:07 +0200
Subject: [PATCH 341/540] Add improved tcv geometry file

---
 src/feltor/feltor.tex        |   9 +--
 src/feltor/geometry/tcv.json | 110 ++++++++++++++++++++++-------------
 2 files changed, 74 insertions(+), 45 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index e01128752..698654c17 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1514,7 +1514,7 @@ eps\_gamma  & float & 1e-6  & - & Tolerance for $\Gamma_1$
 FCI & dict & & & Parameters for Flux coordinate independent approach
 \\
 \qquad refine     & integer[2] & [1,1] & [1,1] & refinement factor in FCI approach in R- and Z-direction.
-We use [1,1], higher values take more time.
+We use [1,1], higher values take more time, but possibly stabilize the simulation.
 \\
 \qquad rk4eps     & float & 1e-6 & 1e-6 & Accuracy of fieldline integrator in FCI. The default is reasonable.
 \\
@@ -1842,7 +1842,7 @@ Apply damping region
     \\
 \qquad Perpendicular grid oscillations in $u_{\parallel,e}$ and $\Delta_\perp \phi$ in the damping region, symmetric in $\varphi$
 &
-Increase damping $alpha$, increase damping boundary, make the box larger/smaller
+Increase damping $alpha$, increase damping boundary, make the box larger/smaller, increasing DS refinement might help.
     \\
 \qquad Spike in $u_{\parallel,e}$ shortly after simulation start
 &
@@ -1850,11 +1850,12 @@ Increase $\nu_\perp$, increase $N_x$, $N_y$, decrease perturbation amplitude
     \\
 \qquad Grid oscillations far away from the edge
 &
-Probably caused by the perpendicular transport that goes unstable. Increase $\nu_\perp$ and/or $N_x$, $N_y$
+Probably caused by the perpendicular transport that goes unstable. Increase $\nu_\perp$ and/or $N_x$, $N_y$. Increasing DS refinement might also help.
 \\
 \qquad Oscillations where fieldlines intersect the wall
 &
-Caused by boundary conditions in FCI method and necessarily underresolved toroidal direction. Increase $N_z$, decrease $N_x$, $N_y$ or better decrease $q$ value by decreasing $\mathcal P_\psi$ in geometry input file
+Caused by boundary conditions in FCI method and necessarily underresolved toroidal direction.
+Increase $\nu_\parallel$, $N_z$, decrease $N_x$, $N_y$ or better decrease $q$ value by decreasing $\mathcal P_\psi$ in geometry input file
 \\
 \bottomrule
 \end{longtable}
diff --git a/src/feltor/geometry/tcv.json b/src/feltor/geometry/tcv.json
index 2d80a35fd..d97b6bb2a 100644
--- a/src/feltor/geometry/tcv.json
+++ b/src/feltor/geometry/tcv.json
@@ -1,51 +1,79 @@
 {
-	"M" : 6,
-	"N" : 6,
+	"M" : 8,
+	"N" : 8,
 	"PI" : -1.0,
-	"PP" : -0.8,
+	"PP" : -1.0,
 	"R_0" : 906.38,
-	"c" :
+	"c" : 
 	[
-		-1.1279927930530271,
-		-1.0842606353509106,
-		27.536407585400283,
-		20.08700802501453,
-		-116.02565935574977,
-		-120.66010077293852,
-		7.3031256199572727,
-		5.818210372553227,
-		-154.98851467644269,
-		-102.29730950402626,
-		622.85031890258006,
-		604.78771516793313,
-		-19.034940230887859,
-		-11.402531138983106,
-		348.40241798842959,
-		198.3558695933294,
-		-1325.376409836957,
-		-1179.8289027290296,
-		24.223202203614758,
-		10.268162419571157,
-		-388.13523888449845,
-		-183.31357499053175,
-		1393.7746630606455,
-		1117.3199053627043,
-		-14.796032650579809,
-		-4.2178443141145641,
-		212.35696838126282,
-		80.01889289206099,
-		-720.56519556796411,
-		-509.77624096800616,
-		3.4558698698433301,
-		0.61537891729523686,
-		-45.450863728406709,
-		-12.955518132950145,
-		146.06735359643812,
-		88.726490458428032
+		-0.96689843290517163,
+		3.0863312163153722,
+		12.704740056639602,
+		-16.289618948500596,
+		-58.773804550893978,
+		20.737434141967192,
+		60.767622887618082,
+		-13.412090215973649,
+		1.5030934710614168,
+		-18.606153187942841,
+		-51.268594063161892,
+		90.330580961827934,
+		334.62239761589331,
+		-96.337218693331863,
+		-369.4426118842149,
+		69.12132156851527,
+		15.089237799045774,
+		46.531264842892817,
+		9.6257646606458742,
+		-198.43750312580366,
+		-691.82430062538606,
+		124.64936082323587,
+		881.41320105570958,
+		-102.82161109684944,
+		-64.873821524138094,
+		-59.150028431025312,
+		271.70847921693229,
+		184.66804234468091,
+		536.96159830784757,
+		161.10088844376682,
+		-1052.663871350825,
+		-127.5935630486964,
+		110.50941213620901,
+		36.5700397200184,
+		-590.5793476953678,
+		-0.33870147588947325,
+		109.19788144885672,
+		-674.85982303083813,
+		678.28677388707547,
+		619.60740183752432,
+		-95.509528907130814,
+		-5.7996956604640708,
+		547.72824001431275,
+		-137.13693489888882,
+		-416.85358515640866,
+		823.49445765649568,
+		-266.57840051970788,
+		-820.6589698292122,
+		41.456100988675544,
+		-4.1783800265033957,
+		-241.62418252372751,
+		100.43158338485145,
+		225.86069590166665,
+		-455.01502774212986,
+		88.284228523021198,
+		482.97613261346356,
+		-7.1845137221775053,
+		1.5432441372005783,
+		41.427186303824904,
+		-23.298757210641259,
+		-38.340751331596628,
+		96.640541117435689,
+		-20.966404661902423,
+		-107.84602283590218
 	],
+	"description" : "standardX",
 	"elongation" : 1.5,
 	"equilibrium" : "polynomial",
-	"description" : "standardX",
 	"inverseaspectratio" : 0.27593818984547458,
 	"triangularity" : 0.40000000000000002
 }
-- 
GitLab


From 7c90fc1b5b2218df5456d0834e2102b4dfc99b0b Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 28 Sep 2020 12:31:49 +0200
Subject: [PATCH 342/540] Update m100 make file

a step so that we do not have to install jsoncpp manually any more
---
 config/m100.mk | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/config/m100.mk b/config/m100.mk
index 81e734d1b..0cd5cee07 100644
--- a/config/m100.mk
+++ b/config/m100.mk
@@ -3,10 +3,6 @@ ifeq ($(strip $(HPC_SYSTEM)),m100)
 #MPICC=mpixlC  #mpi compiler
 #CFLAGS=-Wall -std=c++1y -DWITHOUT_VCL -mcpu=power9 -qstrict# -mavx -mfma #flags for CC
 #OMPFLAG=-qsmp=omp
-#OPT=-O3 # optimization flags for host code
-#NVCC=nvcc #CUDA compiler
-#NVCCARCH=-arch sm_70 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
-#NVCCFLAGS= -std=c++14 -Xcompiler "-mcpu=power9 -Wall"# -mavx -mfma" #flags for NVCC
 CFLAGS=-Wall -std=c++14 -DWITHOUT_VCL -mcpu=power9 # -mavx -mfma #flags for CC
 OPT=-O3 # optimization flags for host code
 NVCC=nvcc #CUDA compiler
@@ -14,8 +10,18 @@ NVCCARCH=-arch sm_70 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppre
 NVCCFLAGS= -std=c++14 -Xcompiler "-mcpu=power9 -Wall"# -mavx -mfma" #flags for NVCC
 
 INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC) -I$(JSONCPP_INC)
-JSONLIB=-L$(JSONCPP_LIB) -ljsoncpp
-#JSONLIB=-L$(HOME)/include/json/../../lib -ljsoncpp_static # json library for input parameters
+JSONLIB=-L$(JSONCPP_LIB) -ljsoncpp # json library for input parameters
 LIBS    =-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
 LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
 endif
+#########################Modules to load ##################
+#module load cuda
+#module load gnu
+#module load spectrum_mpi
+#module load binutils/2.34
+#
+#module load zlib/1.2.11--gnu--8.4.0 
+#module load szip/2.1.1--gnu--8.4.0
+#module load hdf5/1.12.0--gnu--8.4.0
+#module load netcdf/4.7.3--gnu--8.4.0
+#module load jsoncpp/1.9.3--spectrum_mpi--10.3.1--binary
-- 
GitLab


From e0082e3fcf46759b5cdede8f31561de13e496c75 Mon Sep 17 00:00:00 2001
From: Matthias Wiesenberger <mattwi@fysik.dtu.dk>
Date: Mon, 28 Sep 2020 12:36:07 +0200
Subject: [PATCH 343/540] Update marconi.mk config file

also with jsoncpp. At some point we should update the README
accordingly of the new standard way to do it
---
 config/marconi.mk | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/config/marconi.mk b/config/marconi.mk
index e85083490..d2cfa1ad9 100644
--- a/config/marconi.mk
+++ b/config/marconi.mk
@@ -8,22 +8,18 @@ OMPFLAG=-qopenmp
 CFLAGS=-Wall -std=c++14 -restrict -fp-model precise -fimf-arch-consistency=true #-mfma  #flags for CC
 
 INCLUDE += -I$(HOME)/include # cusp, thrust
-INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
+INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC) -I$(JSONCPP_INC)
+JSONLIB=-L$(JSONCPP_LIB) -ljsoncpp
 LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
 LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
 endif
-#Using GNU compiler
-#ifeq ($(strip $(HPC_SYSTEM)),marconi)
-#INCLUDE += -I$(HOME)/include # cusp, thrust
-#INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
-#LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
-#LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
-#endif
 #############################modules to load in .bashrc#######################
-#module load profile/base
-#module load intel/pe-xe-2017--binary
-#module load intelmpi/2017--binary
+#module load profile/advanced
+#module load intel/pe-xe-2018--binary
+#module load intelmpi/2018--binary
 #module load szip/2.1--gnu--6.1.0
 #module load zlib/1.2.8--gnu--6.1.0
-#module load hdf5/1.8.17--intelmpi--2017--binary
-#module load netcdf/4.4.1--intelmpi--2017--binary
+#module load hdf5/1.10.4--intel--pe-xe-2018--binary
+#module load netcdf/4.4.1--intel--pe-xe-2018--binary
+#module load gnu/8.3.0
+#module load jsoncpp/1.9.3--gnu--8.3.0
-- 
GitLab


From 50dcc16eb8beb7a11ebbb0cd8419d235c7722d9a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 28 Sep 2020 21:09:35 +0200
Subject: [PATCH 344/540] Move Horner2d to dg/functors

---
 inc/dg/functors.h           | 256 +++++++++++++++++++++---------------
 inc/geometries/polynomial.h |  36 +----
 2 files changed, 152 insertions(+), 140 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 0a31ca49e..818033e0c 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -23,7 +23,7 @@ namespace dg
 
 /**
  * @brief Absolute maximum
- * \f[ f(x,y) = \max(|x|,|y|)\f]
+ * \f$ f(x,y) = \max(|x|,|y|)\f$
  *
  */
 template <class T = double>
@@ -47,7 +47,7 @@ DG_DEVICE
 };
 /**
  * @brief Absolute minimum
- * \f[ f(x,y) = \min(|x|,|y|)\f]
+ * \f$ f(x,y) = \min(|x|,|y|)\f$
  */
 template <class T = double>
 struct AbsMin
@@ -70,15 +70,15 @@ DG_DEVICE
 };
 
 /**
- * @brief Functor returning a 2d Gaussian
- * \f[
+ * @brief Functor returning a 2d %Gaussian
+ * \f$
    f(x,y) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}\right)}
-   \f]
+   \f$
  */
 struct Gaussian
 {
     /**
-     * @brief Functor returning a Gaussian
+     * @brief Functor returning a %Gaussian
      *
      * @param x0 x-center-coordinate
      * @param y0 y-center-coordinate
@@ -92,7 +92,7 @@ struct Gaussian
             assert( m_sigma_y != 0  &&  "sigma_y must not be 0 in Gaussian");
     }
     /**
-     * @brief Return the value of the Gaussian
+     * @brief Return the value of the %Gaussian
      *
      * \f[
        f(x,y) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}\right)}
@@ -110,7 +110,7 @@ struct Gaussian
                           (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
     /**
-     * @brief Return the value of the Gaussian
+     * @brief Return the value of the %Gaussian
      * \f[
        f(x,y,z) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
        \f]
@@ -132,12 +132,12 @@ struct Gaussian
 
 /**
  * @brief A bump that drops to zero and is infinitely continuously differentiable
- * \f[
+ * \f$
    f(x,y) = \begin{cases}
    Ae^{1 + \left(\frac{(x-x_0)^2}{\sigma_x^2} + \frac{(y-y_0)^2}{\sigma_y^2} - 1\right)^{-1}} \text{ if } \frac{(x-x_0)^2}{\sigma_x^2} + \frac{(y-y_0)^2}{\sigma_y^2} < 1\\
    0 \text{ else}
    \end{cases}
-   \f]
+   \f$
  */
 struct Cauchy
 {
@@ -210,15 +210,15 @@ struct Cauchy
 };
 
 /**
-* @brief The 3d Gaussian
-* \f[
+* @brief The 3d %Gaussian
+* \f$
 f(x,y,z) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2} + \frac{(z-z_0)^2}{2\sigma_z^2}\right)}
-\f]
+\f$
 */
 struct Gaussian3d
 {
     /**
-     * @brief Functor returning a Gaussian
+     * @brief Functor returning a %Gaussian
      *
      * @param x0 x-center-coordinate
      * @param y0 y-center-coordinate
@@ -235,7 +235,7 @@ struct Gaussian3d
             assert( m_sigma_z != 0  &&  "sigma_z must be !=0 in Gaussian3d");
     }
     /**
-     * @brief Return a 2d Gaussian
+     * @brief Return a 2d %Gaussian
      *
      * \f[
        f(x,y) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
@@ -253,7 +253,7 @@ struct Gaussian3d
                           (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
     /**
-     * @brief Return the value of the Gaussian
+     * @brief Return the value of the %Gaussian
      *
      * \f[
        f(x,y) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}+\frac{(z-z_0)^2}{2\sigma_z^2})}
@@ -277,15 +277,15 @@ struct Gaussian3d
 
 };
 /**
- * @brief A Gaussian in x-direction
- * \f[
+ * @brief A %Gaussian in x-direction
+ * \f$
    f(x,y) = Ae^{-\frac{(x-x_0)^2}{2\sigma_x^2} }
-   \f]
+   \f$
  */
 struct GaussianX
 {
     /**
-     * @brief A Gaussian in x
+     * @brief A %Gaussian in x
      *
      * @param x0 x-center-coordinate
      * @param sigma_x x - variance (must be !=0)
@@ -315,10 +315,10 @@ struct GaussianX
 
 };
 /**
- * @brief A Gaussian in y-direction
- * \f[
+ * @brief A %Gaussian in y-direction
+ * \f$
    f(x,y) = Ae^{-\frac{(y-y_0)^2}{2\sigma_y^2}}
-   \f]
+   \f$
  */
 struct GaussianY
 {
@@ -354,10 +354,10 @@ struct GaussianY
 
 };
 /**
- * @brief A Gaussian in z-direction
- * \f[
+ * @brief A %Gaussian in z-direction
+ * \f$
    f(x,y,z) = Ae^{-\frac{(z-z_0)^2}{2\sigma_z^2}}
-   \f]
+   \f$
  */
 struct GaussianZ
 {
@@ -410,7 +410,7 @@ struct GaussianZ
 };
 /**
  * @brief Island function
- * \f[ f(x,y) = \lambda \ln{(\cosh{(x/\lambda) } +\epsilon \cos(y/\lambda)) } \f]
+ * \f$ f(x,y) = \lambda \ln{(\cosh{(x/\lambda) } +\epsilon \cos(y/\lambda)) } \f$
  */
 struct IslandXY
 {
@@ -437,7 +437,7 @@ struct IslandXY
 };
 /**
  * @brief A sin prof in x and y-direction
- * \f[ f(x,y) =B+ A \sin(k_x x) \sin(k_y y) \f]
+ * \f$ f(x,y) =B+ A \sin(k_x x) \sin(k_y y) \f$
  */
 struct SinXSinY
 {
@@ -465,7 +465,7 @@ struct SinXSinY
 };
 /**
  * @brief A cos prof in x and y-direction
- * \f[ f(x,y) =B+ A \cos(k_x x) \cos(k_y y) \f]
+ * \f$ f(x,y) =B+ A \cos(k_x x) \cos(k_y y) \f$
  */
 struct CosXCosY
 {
@@ -493,7 +493,7 @@ struct CosXCosY
 };
 /**
  * @brief A sin prof in x- and cos prof in  y-direction
- * \f[ f(x,y) =B+ A \sin(k_x x) \cos(k_y y) \f]
+ * \f$ f(x,y) =B+ A \sin(k_x x) \cos(k_y y) \f$
  */
 struct SinXCosY
 {
@@ -521,7 +521,7 @@ struct SinXCosY
 };
 /**
  * @brief A sin prof in x-direction
- * \f[ f(x) = f(x,y) = f(x,y,z) =B+ A \sin(k_x x) \f]
+ * \f$ f(x) = f(x,y) = f(x,y,z) =B+ A \sin(k_x x) \f$
  */
 struct SinX
 {
@@ -544,7 +544,7 @@ struct SinX
 };
 /**
  * @brief A sin prof in y-direction
- * \f[ f(x,y) =B+ A \sin(k_y y) \f]
+ * \f$ f(x,y) =B+ A \sin(k_y y) \f$
  */
 struct SinY
 {
@@ -563,7 +563,7 @@ struct SinY
 };
 /**
  * @brief A sin prof in x-direction
- * \f[ f(x,y) =B+ A \cos(k_y y) \f]
+ * \f$ f(x,y) =B+ A \cos(k_y y) \f$
  */
 struct CosY
 {
@@ -582,7 +582,7 @@ struct CosY
 };
 /**
  * @brief Inverse cosh profile
- * \f[ f(x,y) =A/\cosh^2(k_x x) \f]
+ * \f$ f(x,y) =A/\cosh^2(k_x x) \f$
  */
 struct InvCoshXsq
 {
@@ -604,7 +604,7 @@ struct InvCoshXsq
 };
 /**
  * @brief Sin prof in x-direction
- * \f[ f(x) = f(x,y) = f(x,y,z) = B + A(1 - \sin(k_xx )) \f]
+ * \f$ f(x) = f(x,y) = f(x,y,z) = B + A(1 - \sin(k_xx )) \f$
  */
 struct SinProfX
 {
@@ -627,7 +627,7 @@ struct SinProfX
 };
 /**
  * @brief Exp prof in x-direction
- * \f[ f(x) = f(x,y) = f(x,y,z) = B + A\exp(-x/L_n) \f]
+ * \f$ f(x) = f(x,y) = f(x,y,z) = B + A\exp(-x/L_n) \f$
  */
 struct ExpProfX
 {
@@ -653,7 +653,7 @@ struct ExpProfX
 
 /**
  * @brief The linear interpolation polynomial
- * \f[ f(x) = y_1\frac{x-x_0}{x_1-x_0} + y_0\frac{x-x_1}{x_0-x_1}\f]
+ * \f$ f(x) = y_1\frac{x-x_0}{x_1-x_0} + y_0\frac{x-x_1}{x_0-x_1}\f$
  */
 struct Line{
     Line(double x0, double y0, double x1, double y1) :
@@ -667,7 +667,7 @@ struct Line{
 
 /**
  * @brief A linear function in x-direction
- * \f[ f(x) = f(x,y) = f(x,y,z) = ax+b \f]
+ * \f$ f(x) = f(x,y) = f(x,y,z) = ax+b \f$
  */
 struct LinearX
 {
@@ -689,7 +689,7 @@ struct LinearX
 };
 /**
  * @brief A linear polynomial in y-direction
- * \f[ f(x,y) = f(x,y,z) = ay+b \f]
+ * \f$ f(x,y) = f(x,y,z) = ay+b \f$
  */
 struct LinearY
 {
@@ -709,7 +709,7 @@ struct LinearY
 };
 /**
  * @brief A linear function in z-direction
- * \f[ f(x,y,z) = az+b \f]
+ * \f$ f(x,y,z) = az+b \f$
  */
 struct LinearZ
 {
@@ -727,11 +727,11 @@ struct LinearZ
 };
 
 /**
- * @brief Zero outside psimax and inside psimin, otherwise 1
-     \f[ \begin{cases}
+ * @brief
+     \f$ \begin{cases}
         1  \text{ if } \psi_{\min} < \psi < \psi_{\max}\\
         0  \text{ else}
-     \end{cases}\f]
+     \end{cases}\f$ Zero outside psimax and inside psimin, otherwise 1
  */
 struct Iris
 {
@@ -748,11 +748,11 @@ struct Iris
     double m_psimin, m_psimax;
 };
 /**
- * @brief Zero outside psimax, otherwise 1
-     \f[ \begin{cases}
+ * @brief
+     \f$ \begin{cases}
         0  \text{ if } \psi > \psi_{\max} \\
         1  \text{ else}
-     \end{cases}\f]
+     \end{cases}\f$ Zero outside psimax, otherwise 1
  */
 struct Pupil
 {
@@ -768,11 +768,11 @@ struct Pupil
     double psimax_;
 };
 /**
- * @brief Psi inside psimax and psimax outside psimax
-     \f[ \begin{cases}
+ * @brief
+     \f$ \begin{cases}
         \psi_{\max}  \text{ if } \psi > \psi_{\max} \\
         \psi \text{ else}
-     \end{cases}\f]
+     \end{cases}\f$ Psi inside psimax and psimax outside psimax
  */
 struct PsiPupil
 {
@@ -788,11 +788,11 @@ struct PsiPupil
     double psimax_;
 };
 /**
- * @brief Zero up to xb, then one
-     \f[ \begin{cases}
+ * @brief
+     \f$ \begin{cases}
         0  \text{ if } x < x_b \\
         1  \text{ else}
-     \end{cases}\f]
+     \end{cases}\f$ Zero up to xb, then one
   @note the 1 is inclusive i.e if x==x_b the functor always returns 1
  */
 struct Heaviside
@@ -822,7 +822,7 @@ struct Heaviside
 
 
 /**
- * @brief \f[ \sqrt{ (x-x_0)^2 + (y-y_0)^2} \f]
+ * @brief \f$ \sqrt{ (x-x_0)^2 + (y-y_0)^2} \f$
  */
 struct Distance
 {
@@ -836,13 +836,15 @@ struct Distance
 };
 
 /**
- * @brief One up to \c psimax, then a Gaussian down to zero
-     \f[ \begin{cases}
+ * @brief
+     \f$ \begin{cases}
  1 \text{ if } \psi < \psi_{\max}\\
  0 \text{ if } \psi > (\psi_{\max} + 4\alpha) \\
  \exp\left( - \frac{(\psi - \psi_{\max})^2}{2\alpha^2}\right), \text{ else}
  \end{cases}
-   \f]
+   \f$
+
+   One up to \c psimax, then a %Gaussian down to zero
  */
 struct GaussianDamping
 {
@@ -862,7 +864,7 @@ struct GaussianDamping
 };
 /**
  * @brief An approximation to Heaviside using tanh
- * \f[ f(x) = B + 0.5 A(1+ \text{sign} \tanh((x-x_b)/\alpha ) ) \f]
+ * \f$ f(x) = B + 0.5 A(1+ \text{sign} \tanh((x-x_b)/\alpha ) ) \f$
  */
 struct TanhProfX {
     /**
@@ -898,14 +900,52 @@ struct TanhProfX {
 };
 
 /**
- * @brief An approximation to Heaviside using polynomials
-     \f[ \begin{cases}
+ * @brief \f$ \sum_{i=0}^{M-1} \sum_{j=0}^{N-1} c_{iN+j} x^i y^j  \f$
+ *
+ * Evaluated using [Horner's method](https://en.wikipedia.org/wiki/Horner%27s_method)
+ */
+struct Horner2d
+{
+    ///Initialize 1 coefficient to 1
+    Horner2d(): m_c( 1, 1), m_M(1), m_N(1){}
+
+    /**
+     * @brief Initialize coefficients and dimensions
+     *
+     * @param c vector of size MN containing coefficientc c (accessed as c[i*N+j] i.e. y-direction is contiguous)
+     * @param M number of polynomials in x
+     * @param N number of polynomials in y
+     */
+    Horner2d( std::vector<double> c, unsigned M, unsigned N): m_c(c), m_M(M), m_N(N){}
+    double operator()( double x, double y) const
+    {
+        std::vector<double> cx( m_M);
+        for( unsigned i=0; i<m_M; i++)
+            cx[i] = horner( &m_c[i*m_N], m_N, y);
+        return horner( &cx[0], m_M, x);
+    }
+    private:
+    double horner( const double * c, unsigned M, double x) const
+    {
+        double b = c[M-1];
+        for( unsigned i=0; i<M-1; i++)
+            b = c[M-2-i] + b*x;
+        return b;
+    }
+    std::vector<double> m_c;
+    unsigned m_M, m_N;
+};
+
+
+/**
+ * @brief \f$ \begin{cases}
      0 \text{ if } x < x_b-a \\
         ((16 a^3 - 29 a^2 (x - x_b) + 20 a (x - x_b)^2 - 5 (x - x_b)^3) (a + x -
    x_b)^4)/(32 a^7) \text{ if } |x-x_b| < a \\
         1  \text{ if } x > x_b + a
-     \end{cases}\f]
+     \end{cases}\f$
 
+ An approximation to Heaviside using polynomials.
      This function is 3 times continuously differentiable, takes the value 0.5 at xb and
      has a transition width a on both sides of xb.
  */
@@ -939,8 +979,8 @@ struct PolynomialHeaviside {
     int m_s;
 };
 /**
- * @brief An approximation to the Rectangle function using polynomials
-     \f[ \begin{cases}
+ * @brief
+     \f$ \begin{cases}
      0 \text{ if } x < x_l-a_l \\
         ((16 a_l^3 - 29 a_l^2 (x - x_l) + 20 a_l (x - x_l)^2 - 5 (x - x_l)^3) (a_l + x -
    x_l)^4)/(32 a_l^7) \text{ if } |x-x_l| < a_l \\
@@ -948,8 +988,9 @@ struct PolynomialHeaviside {
         ((16 a_r^3 - 29 a_r^2 (x - x_r) + 20 a_r (x - x_r)^2 - 5 (x - x_r)^3) (a_r + x -
    x_l)^4)/(32 a_r^7) \text{ if } |x-x_r| < a_r \\
    0 \text{ if } x > x_r + a_r
-     \end{cases}\f]
+     \end{cases}\f$
 
+ An approximation to the Rectangle function using polynomials
      Basically just the product of two PolynomialHeaviside functions
 
      This function is 3 times continuously differentiable, takes the value 0.5 at xl and xr and
@@ -978,13 +1019,13 @@ struct PolynomialRectangle {
 };
 
 /**
- * @brief The integral of PolynomialHeaviside approximates xH(x)
-     \f[ \begin{cases}
+ * @brief \f$ \begin{cases}
      x_b \text{ if } x < x_b-a \\
      x_b + ((35 a^3 - 47 a^2 (x - x_b) + 25 a (x - x_b)^2 - 5 (x - x_b)^3) (a + x - x_b)^5)/(256 a^7)
         \text{ if } |x-x_b| < a \\
         x  \text{ if } x > x_b + a
-     \end{cases}\f]
+     \end{cases}\f$
+ The integral of PolynomialHeaviside approximates xH(x)
 
      This function is 4 times continuously differentiable,
      has a transition width \c a on both sides of \c xb, where it transitions from the
@@ -1024,12 +1065,12 @@ struct IPolynomialHeaviside {
 };
 
 /**
- * @brief The derivative of PolynomialHeaviside approximates delta(x)
-     \f[ \begin{cases}
+ * @brief \f$ \begin{cases}
      0 \text{ if } x < x_b-a || x > x_b+a \\
      (35 (a + x - x_b)^3 (a - x + x_b)^3)/(32 a^7)
         \text{ if } |x-x_b| < a
-     \end{cases}\f]
+     \end{cases}\f$
+     The derivative of PolynomialHeaviside approximates delta(x)
 
      This function is 2 times continuously differentiable, is symmetric around \c xb
      and has a width \c a on both sides of \c x0.
@@ -1062,7 +1103,7 @@ struct DPolynomialHeaviside {
 
 
 /**
- * @brief Exponential \f[ f(x) = A \exp(\lambda x)\f]
+ * @brief Exponential \f$ f(x) = A \exp(\lambda x)\f$
  *
  * @tparam T value-type
  */
@@ -1092,8 +1133,8 @@ struct EXP
     T m_amp, m_lambda;
 };
 /**
- * @brief natural logarithm
- * \f[ f(x) = \ln(x)\f]
+ * @brief
+ * \f$ f(x) = \ln(x)\f$ natural logarithm
  *
  * @tparam T value-type
  */
@@ -1115,7 +1156,7 @@ struct LN
 };
 /**
  * @brief Square root
- * \f[ f(x) = \sqrt{x}\f]
+ * \f$ f(x) = \sqrt{x}\f$
  *
  * @tparam T value-type
  */
@@ -1139,12 +1180,12 @@ struct SQRT
 
 /**
  * @brief Minmod function
- \f[ f(x_1, x_2, x_3) = \begin{cases}
+ \f$ f(x_1, x_2, x_3) = \begin{cases}
          \min(x_1, x_2, x_3) \text{ for } x_1, x_2, x_3 >0 \\
          \max(x_1, x_2, x_3) \text{ for } x_1, x_2, x_3 <0 \\
          0 \text{ else}
  \end{cases}
- \f]
+ \f$
  *
  * might be useful for flux limiter schemes
  * @tparam T value-type
@@ -1190,9 +1231,10 @@ struct MinMod
 };
 
 /**
- * @brief Add a constant value
- * \f[ f(x) = x + c\f]
+ * @brief
+ * \f$ f(x) = x + c\f$
  *
+ * Add a constant value
  * @tparam T value type
  */
 template <class T = double>
@@ -1218,9 +1260,10 @@ struct PLUS
 };
 
 /**
- * @brief %Invert the given value
- * \f[ f(x) = 1/x \f]
+ * @brief
+ * \f$ f(x) = 1/x \f$
  *
+ * %Invert the given value
  * @tparam T value type
  */
 template <class T = double>
@@ -1238,7 +1281,7 @@ struct INVERT
 
 /**
  * @brief returns (positive) modulo
- * \f[ f(x) = x\mod m\f]
+ * \f$ f(x) = x\mod m\f$
  *
  * @tparam T value type
  */
@@ -1269,7 +1312,7 @@ struct MOD
 };
 /**
  * @brief absolute value
- * \f[ f(x) = |x|\f]
+ * \f$ f(x) = |x|\f$
  *
  * @tparam T value type
  */
@@ -1288,11 +1331,11 @@ struct ABS
 };
 /**
  * @brief returns positive values
- \f[ f(x) = \begin{cases}
+ \f$ f(x) = \begin{cases}
          x \text{ for } x>0 \\
          0 \text{ else}
  \end{cases}
- \f]
+ \f$
  *
  * @tparam T value type
  */
@@ -1314,8 +1357,7 @@ struct POSVALUE
 };
 
 /**
- * @brief Return a constant
- * \f[ f(x) = c\f]
+ * @brief \f$ f(x) = c\f$
  */
 struct CONSTANT
 {
@@ -1337,8 +1379,7 @@ struct CONSTANT
 };
 
 /**
- * @brief Return one
- * \f[ f(x) = 1\f]
+ * @brief \f$ f(x) = 1\f$
  */
 struct ONE
 {
@@ -1350,8 +1391,7 @@ struct ONE
     double operator()(double x, double y, double z)const{return 1.;}
 };
 /**
- * @brief Return zero
- * \f[ f(x) = 0\f]
+ * @brief \f$ f(x) = 0\f$ Return 0
  */
 struct ZERO
 {
@@ -1365,10 +1405,10 @@ struct ZERO
 
 /**
  * @brief Functor returning a Lamb dipole
- \f[ f(x,y) = \begin{cases} 2\lambda U J_1(\lambda r) / J_0(\gamma)\cos(\theta) \text{ for } r<R \\
+ \f$ f(x,y) = \begin{cases} 2\lambda U J_1(\lambda r) / J_0(\gamma)\cos(\theta) \text{ for } r<R \\
          0 \text{ else}
          \end{cases}
- \f]
+ \f$
 
  with \f$ r = \sqrt{(x-x_0)^2 + (y-y_0)^2}\f$, \f$
  \theta = \arctan_2( (y-y_), (x-x_0))\f$,
@@ -1440,15 +1480,16 @@ struct Lamb
 };
 
 /**
- * @brief Return a 2d vortex function
-       \f[f(x,y) =\begin{cases}
+ * @brief
+       \f$f(x,y) =\begin{cases}
        \frac{u_d}{1.2965125} \left(
        r\left(1+\frac{\beta_i^2}{g_i^2}\right)
        - R \frac{\beta_i^2}{g_i^2} \frac{J_1(g_ir/R)}{J_1(g_i)}\right)\cos(\theta) \text{ if } r < R \\
       \frac{u_d}{1.2965125} R \frac{K_1(\beta_i {r}/{R})}{K_1(\beta)} \cos(\theta) \text{ else }
       \end{cases}
-      \f]
+      \f$
 
+      Return a 2d vortex function
      * where \f$ i\in \{0,1,2\}\f$ is the mode number and r and \f$\theta\f$ are poloidal coordinates
  with \f$ r = \sqrt{(x-x_0)^2 + (y-y_0)^2}\f$, \f$ \theta = \arctan_2( (y-y_), (x-x_0))\f$,
         \f$ g_0 = 3.831896621 \f$,
@@ -1482,15 +1523,15 @@ struct Vortex
         b_[2] = 0.07071067810 ;
     }
     /**
-     * @brief Evaluate the vortex
-     *
-       \f[f(x,y) =\begin{cases}
+     * @brief \f$f(x,y) =\begin{cases}
        \frac{u_d}{1.2965125} \left(
        r\left(1+\frac{\beta_i^2}{g_i^2}\right)
        - R \frac{\beta_i^2}{g_i^2} \frac{J_1(g_ir/R)}{J_1(g_i)}\right)\cos(\theta) \text{ if } r < R \\
       \frac{u_d}{1.2965125} R \frac{K_1(\beta_i {r}/{R})}{K_1(\beta)} \cos(\theta) \text{ else }
       \end{cases}
-      \f]
+      \f$
+
+      Evaluate the vortex
      * where \f$ i\in \{0,1,2\}\f$ is the mode number and r and \f$\theta\f$ are poloidal coordinates
      * @param x value
      * @param y value
@@ -1517,15 +1558,15 @@ struct Vortex
         return u_d * R_* bessk1(beta*r/R_)/bessk1(beta)*cos(theta)/norm;
     }
     /**
-     * @brief Evaluate the vortex modulated by a sine wave in z
-     *
-       \f[f(x,y,z) =\cos(k_z z)\begin{cases}
+     * @brief \f$f(x,y,z) =\cos(k_z z)\begin{cases}
        \frac{u_d}{1.2965125} \left(
        r\left(1+\frac{\beta_i^2}{g_i^2}\right)
        - R \frac{\beta_i^2}{g_i^2} \frac{J_1(g_ir/R)}{J_1(g_i)}\right)\cos(\theta) \text{ if } r < R \\
       \frac{u_d}{1.2965125} R \frac{K_1(\beta_i {r}/{R})}{K_1(\beta)} \cos(\theta) \text{ else }
       \end{cases}
-      \f]
+      \f$
+
+      Evaluate the vortex modulated by a sine wave in z
      * where \f$ i\in \{0,1,2\}\f$ is the mode number and r and \f$\theta\f$ are poloidal coordinates
      * @param x value
      * @param y value
@@ -1591,10 +1632,10 @@ struct Vortex
 };
 
 /**
-* @brief A random bath in the RZ plane
-*
-\f[f(R,Z) = A B \sum_\vec{k} \sqrt{E_k} \alpha_k \cos{\left(k \kappa_k + \theta_k \right)}
-\f]
+* @brief \f$f(R,Z) = A B \sum_\vec{k} \sqrt{E_k} \alpha_k \cos{\left(k \kappa_k + \theta_k \right)}
+\f$
+
+* A random bath in the R-Z plane
 * with \f[ B := \sqrt{\frac{2}{N_{k_R} N_{k_Z}}} \\
         k:=\sqrt{k_R^2 + k_Z^2} \\
         k_R:=2 \pi \left( i -N_{k_R}/2\right)/N_{k_R} \\
@@ -1879,8 +1920,7 @@ struct Histogram2D
 
 
 /**
- * @brief Check for NaN
- * \f[ f(x) = std::isnan(x) \f]
+ * @brief Check for NaN: \f$ f(x) = std::isnan(x) \f$
  */
 template <class T>
 struct ISNAN
diff --git a/inc/geometries/polynomial.h b/inc/geometries/polynomial.h
index 70aaa59fa..cf86535a2 100644
--- a/inc/geometries/polynomial.h
+++ b/inc/geometries/polynomial.h
@@ -21,33 +21,6 @@ namespace dg
 {
 namespace geo
 {
-
-/**
- * @brief \f[ \sum_{i=0}^{M-1} \sum_{j=0}^{N-1} c_{i*N+j} x^i y^j  \f]
- */
-struct Horner2d
-{
-    Horner2d(): m_c( 1, 1), m_M(1), m_N(1){}
-    Horner2d( std::vector<double> c, unsigned M, unsigned N): m_c(c), m_M(M), m_N(N){}
-    double operator()( double x, double y) const
-    {
-        std::vector<double> cx( m_M);
-        for( unsigned i=0; i<m_M; i++)
-            cx[i] = horner( &m_c[i*m_N], m_N, y);
-        return horner( &cx[0], m_M, x);
-    }
-    private:
-    double horner( const double * c, unsigned M, double x) const
-    {
-        double b = c[M-1];
-        for( unsigned i=0; i<M-1; i++)
-            b = c[M-2-i] + b*x;
-        return b;
-    }
-    std::vector<double> m_c;
-    unsigned m_M, m_N;
-};
-
 /**
  * @brief Contains a polynomial approximation type flux function
  */
@@ -57,11 +30,10 @@ namespace polynomial
 ///@{
 
 /**
- * @brief \f[ \hat{\psi}_p  \f]
- *
- * \f[ \hat{\psi}_p(R,Z) =
-      \hat{R}_0P_{\psi}\Bigg\{ \sum_i \sum_j c_{i*N+j} x^i y^j \Bigg\}
-      \Bigg\} \f]
+ * @brief \f$ \psi_p(R,Z) =
+      R_0P_{\psi}\Bigg\{ \sum_i \sum_j c_{i*N+j} \bar R^i \bar Z^j \Bigg\}
+      \f$
+
       with \f$ \bar R := \frac{ R}{R_0} \f$ and \f$\bar Z := \frac{Z}{R_0}\f$
  *
  */
-- 
GitLab


From cd764d542897a0c9048e787d0220be8911ad9f49 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 28 Sep 2020 21:10:02 +0200
Subject: [PATCH 345/540] Add documentation of make_field and modifiedField

---
 inc/geometries/geometries_doc.h |  1 +
 inc/geometries/make_field.h     | 38 +++++++++++++++++++-------
 inc/geometries/solovev.h        | 17 +++++++++---
 src/feltor/feltor.tex           | 48 ++++++++++++++++++++++++++-------
 4 files changed, 83 insertions(+), 21 deletions(-)

diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index a565723bd..86c98641e 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -13,6 +13,7 @@
       @defgroup geom 3.1 New flux functions and derivatives
       @{
         @defgroup solovev The solovev magnetic field
+        @defgroup polynomial The polynomial magnetic field
         @defgroup taylor The Taylor state magnetic field
         @defgroup guenther The Guenther magnetic field
         @defgroup toroidal The Purely Toroidal magnetic field
diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
index cb9fa19a2..f8985f9a3 100644
--- a/inc/geometries/make_field.h
+++ b/inc/geometries/make_field.h
@@ -1,15 +1,30 @@
+#ifdef JSONCPP_VERSION_STRING
 #include "magnetic_field.h"
 #include "solovev.h"
 #include "guenther.h"
 #include "polynomial.h"
 #include "toroidal.h"
-//#include "taylor.h" //only if boost is included
 #include <dg/file/json_utilities.h>
 
-
 namespace dg{
 namespace geo{
 
+/**
+ * @brief Create a Magnetic field based on the given parameters
+ *
+ * This function abstracts the Magnetic field generation. It reads an
+ * input Json file that tells this function via the "equilibrium" parameter which
+ * field to generate and which parameters to expect in the file, for example if
+ * "equilibrium" reads "toroidal", then only on additional parameter "R_0" is
+ * read from the file and a field is constructed
+ * @param js Has to contain "equilibrium" which is converted dg::geo::equilibrium,
+ * i.e. "solovev", "polynomial", .... After that the respective parameters are created,
+ * for example if "solovev", then the dg::geo::solovev::Parameters( js, mode) is called and forwarded to dg::geo::createSolovevField(gp); similar for the rest
+ * @param mode signifies what to do if an error occurs
+ * @return A magnetic field object
+ * @ingroup geom
+ * @attention This function is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
+ */
 static inline TokamakMagneticField createMagneticField( Json::Value js, file::error mode)
 {
     std::string e = file::get( mode, js, "equilibrium", "solovev" ).asString();
@@ -53,15 +68,19 @@ static inline TokamakMagneticField createMagneticField( Json::Value js, file::er
  * function with width alpha), i.e. we replace psi with IPolynomialHeaviside(psi).
  * This subsequently modifies all derivatives of psi and the poloidal
  * current in this region.
- * @param in Magnetic field to change
- * @param psi0 boundary value where psi is modified to a constant psi0
- * @param alpha radius of the transition region where the modification acts (smaller is quicker)
- * @param sign determines which side of Psi to dampen (negative or positive, forwarded to \c dg::IPolynomialHeaviside)
- * @param ZX Here you can give a Z value (of the X-point).
- * @param side  The modification will only happen on either the upper (positive) or lower (negative) side of ZX in Z.
- * @note Per default the dampening happens everywhere
+ * @param js forwarded to dg::geo::createMagneticField
+ * @param jsmod must contain the field "damping": "modifier" which has one of the values "none", "heaviside" then
+ *  "damping": "boundary" value where psi is modified to a constant psi0
+ * "damping": "alpha" radius of the transition region where the modification acts (smaller is quicker) or "sol_pfr", then "
+ *  "damping": "boundary" and
+ * "damping": "alpha" must be arrays of size 2 to indicate values for the SOL and the PFR respectively
+ * @param mode Determines behaviour in case of an error
+ * @param damping On output contains the region where the damping is applied
+ * @param transition On output contains the region where the transition of Psip to a constant value occurs
+ * @note Per default the dampening happens nowhere
  * @return A magnetic field object
  * @ingroup geom
+ * @attention This function is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
  */
 static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Value jsmod, file::error mode, CylindricalFunctor& damping, CylindricalFunctor& transition)
 {
@@ -184,3 +203,4 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
 
 } //namespace geo
 }//namespace dg
+#endif //JSONCPP_VERSION_STRING
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 897db7dd2..815c56c36 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -284,7 +284,12 @@ struct PsipRZ: public aCylindricalFunctor<PsipRZ>
  */
 struct Ipol: public aCylindricalFunctor<Ipol>
 {
-    ///@copydoc Psip::Psip()
+    /**
+     * @brief Construct from given geometric parameters
+     *
+     * @param gp geometric parameters (for R_0, A, PP and PI)
+     * @param psip the flux function to use
+     */
     Ipol( Parameters gp, std::function<double(double,double)> psip ):  m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_pi(gp.pi), m_psip(psip) {
         if( gp.pp == 0.)
             m_pp = 1.; //safety measure to avoid divide by zero errors
@@ -302,7 +307,10 @@ struct Ipol: public aCylindricalFunctor<Ipol>
  */
 struct IpolR: public aCylindricalFunctor<IpolR>
 {
-    ///@copydoc Psip::Psip()
+    /**
+     * @copydoc Ipol::Ipol()
+     * @param psipR the R-derivative of the flux function to use
+     */
     IpolR(  Parameters gp, std::function<double(double,double)> psip, std::function<double(double,double)> psipR ):
         m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_pi(gp.pi), m_psip(psip), m_psipR(psipR) {
         if( gp.pp == 0.)
@@ -321,7 +329,10 @@ struct IpolR: public aCylindricalFunctor<IpolR>
  */
 struct IpolZ: public aCylindricalFunctor<IpolZ>
 {
-    ///@copydoc Psip::Psip()
+    /**
+     * @copydoc Ipol::Ipol()
+     * @param psipZ the Z-derivative of the flux function to use
+     */
     IpolZ(  Parameters gp, std::function<double(double,double)> psip, std::function<double(double,double)> psipZ ):
         m_R0(gp.R_0), m_A(gp.A), m_pp(gp.pp), m_pi(gp.pi), m_psip(psip), m_psipZ(psipZ) {
         if( gp.pp == 0.)
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 698654c17..8997fb935 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -235,7 +235,18 @@ $\bar{\psi}_{p11}=3 \bar{Z}\bar{R}^4 - 4\bar{Z}^3\bar{R}^2$\\
    & \\
 \bottomrule
 \end{longtable}
-Since Eq.~\eqref{eq:solovev} is given analytically we can numerically evaluate $\psi_p$ and $I$
+
+As an alternative and in order to better fit experimental equilibria we offer
+a polynomial expansion of the magnetic flux function
+\begin{subequations}
+\label{eq:polynomial}
+\begin{align}
+    \psi_p(R,Z) &= \mathcal P_\psi R_0\sum_{i=0}^{N_R-1}\sum_{j=0}^{N_Z-1} c_{ij}\bar R^i\bar Z^j\\
+   I(\psi_p) &= \mathcal P_I
+\end{align}
+\end{subequations}
+where the number of polynomial coefficients $N_R$ and $N_Z$ can be freely chosen.
+Since Eqs.~\eqref{eq:solovev} and \eqref{eq:polynomial} are given analytically we can numerically evaluate $\psi_p$ and $I$
 and all their derivatives
 at arbitrary points to machine precision, which is simple to implement and fast to execute.
 This translates to an exact representation of the magnetic field and related
@@ -243,9 +254,9 @@ quantities like the curvature operators in code. In particular,
 the X-point and O-point can be determined to machine
 precision via a few Newton iterations.
 
-The choice of the coefficients \(c_{i}\) and \(A\) determines the actual form
+The choice of the coefficients \(c_{i}\) and \(A\), respectively $c_{ij}$ determines the actual form
 of the magnetic field.
-Eq.~\eqref{eq:solovev} can for example represent single and asymmetric double X-point configurations, force-free states,
+We can for example represent single and asymmetric double X-point configurations, force-free states,
 field reversed configurations and low and high beta tokamak equilibria.
 $R_0$ appears as an artificial scaling factor
 (note here that a change in $\rho_s$ changes $R_0$ but not the form or size of
@@ -1621,21 +1632,40 @@ magnetic field and damping is ignored.\\
 \bottomrule
 \end{longtable}
 \subsection{Geometry file structure} \label{sec:geometry_file}
-%Document description and equilibrium fields and polynomial field
 File format: json
 
+The file structure of the geometry file depends on which expansion for $\psi_p$ is chosen Eq.~\eqref{eq:solovev} or Eq.~\eqref{eq:polynomial}.
 %%This is a booktabs table
+A solovev magnetic field equilibrium
 \begin{longtable}{llll>{\RaggedRight}p{7cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
     A      & float & 0 &  0 & Solovev parameter in Eq.~\eqref{eq:solovev} \\
     c      & float[12] &  - & - & Solovev coefficients in Eq.~\eqref{eq:solovev} \\
-    PP     & float & 1 &  1 & Prefactor $\mathcal P_\psi$ for $\psi_p$ in Eq.~\eqref{eq:solovev} \\
-    PI     & float & 1 &  1 & Prefactor $\mathcal P_I$ for $I$ in Eq.~\eqref{eq:solovev} \\
-    R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ in Eq.~\eqref{eq:solovev} (This is the only geometry quantity to change if $\rho_s$ changes)\\
+\bottomrule
+\end{longtable}
+A polynomial magnetic field equilibrium
+\begin{longtable}{llll>{\RaggedRight}p{7cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
+    M      & float & 1 &  1 & Number of polynomial coefficients in $R$ in Eq.~\eqref{eq:polynomial} \\
+    N      & float & 1 &  1 & Number of polynomial coefficients in $Z$ in Eq.~\eqref{eq:polynomial} \\
+    c      & float[MN] &  - & - & Polynomial coefficients in Eq.~\eqref{eq:polynomial} \\
+\bottomrule
+\end{longtable}
+In addition both files must contain
+\begin{longtable}{llll>{\RaggedRight}p{7cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
+    PP     & float & 1 &  1 & Prefactor $\mathcal P_\psi$ for $\psi_p$ \\
+    PI     & float & 1 &  1 & Prefactor $\mathcal P_I$ for $I$ \\
+    R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ (This is the only geometry quantity to change if $\rho_s$ changes)\\
     elongation    & float & 1 & - & Elongation $e$, used in determining the box size Eq.~\eqref{eq:box} and the initial guess for the location of the X-point $Z_X = -1.1 ea$ \\
     triangularity & float & 0 & - & Triangularity $\delta$, used in the initial guess for the location of the X-point $R_X = R_0-1.1\delta a$ \\
     inverseaspectratio & float & 0.16667 & - & minor to major radius $a/R_0$ (used to compute $a$ from $R_0$) \\
+    equilibrium & string & polynomial & solovev & Tells the magnetic field generation which type of expansion to use for the flux function \\
+    description & string & standardX & standardX & Tells the magnetic field modifier
+    where to look for the SOL and PFR regions \\
 \bottomrule
 \end{longtable}
 
@@ -1838,7 +1868,7 @@ last output in the output file.
     \\
 \qquad large fieldaligned oscillations in $u_{\parallel,e}$ paired with instability in the edge of the box
 &
-Apply damping region
+Increase parallel diffusion $nu_\parallel$, apply damping region
     \\
 \qquad Perpendicular grid oscillations in $u_{\parallel,e}$ and $\Delta_\perp \phi$ in the damping region, symmetric in $\varphi$
 &
@@ -1855,7 +1885,7 @@ Probably caused by the perpendicular transport that goes unstable. Increase $\nu
 \qquad Oscillations where fieldlines intersect the wall
 &
 Caused by boundary conditions in FCI method and necessarily underresolved toroidal direction.
-Increase $\nu_\parallel$, $N_z$, decrease $N_x$, $N_y$ or better decrease $q$ value by decreasing $\mathcal P_\psi$ in geometry input file
+Increase $\nu_\parallel$, $N_z$, decrease $N_x$, $N_y$ or decrease $q$ value by decreasing $\mathcal P_\psi$ in geometry input file
 \\
 \bottomrule
 \end{longtable}
-- 
GitLab


From 7534a90bad37b82ab0ef7149638466e8468b46f6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 1 Oct 2020 22:39:19 +0200
Subject: [PATCH 346/540] Rename divergence of parallel current

d for divergence
---
 src/feltor/feltor.tex   | 6 +++++-
 src/feltor/feltordiag.h | 4 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 8997fb935..52d0ee1a6 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1070,7 +1070,7 @@ and
     -\nc\left( \frac{N_i}{B^2}\np \phi \right) &= \Gamma_{1,i} N_i - n_e, \quad\quad
     \Gamma_{1,i}^{-1} = 1-\frac{1}{2}\tau_i\mu_i \Delta_\perp \\
     \psi_e = \phi, \quad \psi_i &= \Gamma_{1,i}\phi -\frac{\mu_i}{2}\frac{(\np\phi)^2}{B^2} \\
-    \left(\frac{\beta}{\mu_i}N_i - \frac{\beta}{\mu_{\parallel,e}}n_e-\Delta_\perp\right)
+    \left(\frac{\beta}{\mu_i}N_i - \frac{\beta}{\mu_e}n_e-\Delta_\perp\right)
     A_\parallel &= \beta\left(N_iW_i-n_e w_e\right)
   \end{align}
 \end{subequations}
@@ -1146,6 +1146,8 @@ The relevant terms in the output file are
     lneparallel\_tt &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
     sne\_tt & $S_{n_e}$ &
     jsdiae\_tt & $\tau_e \bhat \times \vn n_e \cn \psi_p /B$\\
+    dnepar\_tt & $\nc (\bhat n_e u_{e,\parallel}$) &
+      & \\
     ions & $N_i$ &
     jsniC\_tt &$ N_i ( \vec u_K + \vec u_C )\cn \psi_p$ \\
     jsniA\_tt &$ N_i U_{\parallel,i} \vec{ b}_\perp  \cn \psi_p$ &
@@ -1154,6 +1156,8 @@ The relevant terms in the output file are
     lniparallel\_tt &$ \Lambda_{\parallel,N_i} = \nu_\parallel \Delta_\parallel N_i$ \\
     sni\_tt & $S_{N_i}$ &
     jsdiai\_tt & $\tau_i \bhat \times \vn N_i \cn \psi_p /B$\\
+    dnipar\_tt & $\nc (\bhat N_i U_{i,\parallel}$) &
+      & \\
 \bottomrule
 \end{longtable}
 
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 3c56f43d1..2b171a3b5 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -524,7 +524,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::copy( v.f.density_source(0), result);
         }
     },
-    {"spne_tt", "Parallel Source term for electron density (Time average)", true,
+    {"dnepar_tt", "Divergence of Parallel velocity term for electron density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.density(0), v.f.velocity(0), v.f.divb(), 0., result);
             dg::blas1::pointwiseDot( 1., v.f.density(0),  v.f.dsU(0), 1., result);
@@ -586,7 +586,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::copy( v.f.density_source(1), result);
         }
     },
-    {"spni_tt", "Parallel Source term for ion density (Time average)", true,
+    {"dnipar_tt", "Divergence of Parallel velocity term in ion density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.density(1), v.f.velocity(1), v.f.divb(), 0., result);
             dg::blas1::pointwiseDot( 1., v.f.density(1),  v.f.dsU(1), 1., result);
-- 
GitLab


From b31ec76be6c6610e7a5d634b073fa5ae5aab73ee Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 5 Oct 2020 21:53:41 +0200
Subject: [PATCH 347/540] Modify feltor equations parallel viscosity in doc

this only changes documentation not yet the code
---
 doc/related_pages/references.bib |  10 +++
 src/feltor/feltor.tex            | 148 ++++++++++++++++---------------
 2 files changed, 87 insertions(+), 71 deletions(-)

diff --git a/doc/related_pages/references.bib b/doc/related_pages/references.bib
index 8fd055d5a..356006e70 100644
--- a/doc/related_pages/references.bib
+++ b/doc/related_pages/references.bib
@@ -30,6 +30,16 @@
   Volume                   = {21},
   Doi                      = {10.1063/1.4881466},
 }
+@Article{Braginskii1965,
+  Title                    = {Transport processes in a plasma},
+  Author                   = {Braginskii, S.I.},
+  Journal                  = {Rev. Plasma Phys.},
+  Year                     = {21965},
+  Number                   = {},
+  Pages                    = {205},
+  Volume                   = {1},
+  Url                      = {http://people.hao.ucar.edu/judge/homepage/PHSX515/fall2012/Braginskii1965.pdf}
+}
 
 @Article{Cockburn2001runge,
   Title                    = {{Runge--Kutta discontinuous Galerkin methods for convection-dominated problems}},
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 52d0ee1a6..d413a3bb8 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -740,7 +740,6 @@ We then need to point mirror Eq.~\eqref{eq:modified_psip} at $\psi_{p,b}+\frac{\
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
 \subsection{Conservative form}
-%MW: don't we have a momentum source in the form we currently give?
 We scale all spatial lengths by $\rho_s = \sqrt{T_e m_i}/(eB_0)$ and time by the ion gyro-frequency $\Omega_0 = eB_0/m_i$.
 The magnetic field is scaled with $B_0$, densities with $n_0$ and the parallel velocity is scaled with $c_s = \sqrt{T_e/m_i}$.
 The potential is scaled with $\hat \phi = e/T_e$ and the vector potential with
@@ -764,7 +763,7 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
     + \mu NU_\parallel\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
     =& -\tau \left(\bhat + {\vec b}_\perp\right)\cn N
     -N \left( \left(\bhat+{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right)
-    - \eta n_e^2(U_{\parallel,i}-u_{\parallel,e})
+    - \eta n_e^2(U_{\parallel,i}-u_{\parallel,e}) + \mu \nu_\parallel \Delta_\parallel U
     \nonumber\\
     &+ \mu N\left(\Lambda_U + S_U\right) + \mu U_\parallel \left(\Lambda_N + S_N\right)
 \label{}
@@ -797,71 +796,52 @@ potential $A_\parallel$.
 We have the continuity equation for the electron density \(n_e\) and the ion gyro-centre
 density \(N_i\) and the momentum conservation equation for
 the parallel electron velocity \(u_{\parallel,e}\) and the parallel ion gyro-centre velocity \(U_{\parallel,i}\)~\cite{WiesenbergerPhD, HeldPhD}.
-\subsection{ Scale invariance}
-\subsubsection{Sign reversals of the magnetic field}\label{sec:field_reversal}
-If we change the direction of the magnetic field vector $\bhat$, we immediately see that all perpendicular
-drifts and $U_\parallel\bhat$ change directions. On the other side, the diffusive and resistive terms remain unchanged.
-Without resistivity and diffusion a change in direction of the magnetic field thus corresponds to
-a time reversal $t\rightarrow t'=-t$.
-In the code $\bhat$ changes sign by using both $-\mathcal P_\psi$ and $-\mathcal P_I$.
-
-Also note that changing the sign of the magnetic field only in the parallel derivatives $\npar \rightarrow -\npar$ does not
-have any effect. This can be seen by simply renormalizing $U_\parallel'=-U_\parallel$. This reverts the equations back to the original equations.
-\subsubsection{Scaling of density}
-If $N, U_\parallel, \phi, A_\parallel$ are a solution to the model equations
-then so are $N'=\alpha N$, $U_\parallel'=U_\parallel$, $\phi'=\phi$ and $A_\parallel'=A_\parallel$ with the changed parameters $S_N' = \alpha S_N$, $\eta' = \eta/\alpha$ and $ \beta' = \beta/\alpha$. If $N$
-has a Dirichlet boundary condition, then $N'$ satisfies a correspondingly scaled boundary condition.
-
-
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsection{Diffusive terms}\label{sec:dissres}
-We define with
-$\eta_\parallel := \frac{0.51 m_e \nu_{ei}}{n_e e^2}$ and $\nu_{ei} = \sqrt{2} z^2 e^4 \ln \Lambda/ (12\pi^{3/2} \sqrt{m_e} \epsilon_0^2) n_e /T_e^{3/2}$
+Since the gyro-fluid derivation does not include collisional terms we
+copied the parallel resistive and viscous terms from the Braginskii fluid equations~\cite{Braginskii1965}.
+The electron-ion and ion-ion collision frequencies are given by
+$\nu_{ei} = \sqrt{2} z^2 e^4 \ln \Lambda/ (12\pi^{3/2} \sqrt{m_e} \epsilon_0^2) n_e /T_e^{3/2}$, $\nu_{ee} = \nu_{ei}/\sqrt{2}$
+and
+$\nu_{ii} =  z^4 e^4 \ln \Lambda/ (12\pi^{3/2} \sqrt{m_i} \epsilon_0^2) n_i /T_i^{3/2}$.
+We define with the parallel Spitzer resistivity
+$\eta_\parallel := \frac{0.51 m_e \nu_{ei}}{n_e e^2}$ and the parallel electron and ion viscosities
+$\mu_{\parallel,e}:=0.73\frac{n_eT_e}{\nu_{ei}}$ and $\mu_{\parallel,i} = 0.96\frac{n_iT_i}{\nu_{ii}}$~\cite{Braginskii1965}
+\begin{subequations}
 \begin{align}
-  \eta:=\frac{en_0\eta_\parallel}{B_0} = 8.45\cdot 10^{-5}\ln \lambda \left(\frac{n_0}{10^{19}\text{m}^3}\right) \left(\frac{T_e}{\text{eV}}\right)^{-3/2} \left(\frac{B_0}{\text{T}}\right)^{-1},
-    \label{eq:resistivity}
+    \eta&:=\frac{en_0\eta_\parallel}{B_0} = \frac{\nu_{ei,0}}{\Omega_{ce}}= 8.45\cdot 10^{-5}\ln \lambda \left(\frac{n_0}{10^{19}\text{m}^3}\right) \left(\frac{T_e}{\text{eV}}\right)^{-3/2} \left(\frac{B_0}{\text{T}}\right)^{-1}, \\
+    \nu_{\parallel,e}&:=\frac{\mu_{\parallel,e,0}}{m_e n_0\rho_s^2\Omega_{ci}} = 0.73 \frac{\Omega_{ce}}{\nu_{ei,0}} = \frac{0.73}{\eta} \\
+    \nu_{\parallel,i}&:=\frac{\mu_{\parallel,i,0}}{m_i n_0 \rho_s^2\Omega_{ci}} = 0.96 \frac{\Omega_{ci}}{\nu_{ii,0}} = \sqrt{\frac{m_e}{m_i}} \frac{1.36}{\eta}
 \end{align}
-with $\ln \lambda \approx 10$.
+    \label{eq:resistivity}
+\end{subequations}
+with $\ln \lambda \approx 10$ and $T_e=T_i$ for the purpose of computing the diffusive coefficients.
  The approximate Spitzer current \(J_{\parallel,s}:= n_e \left(U_{\parallel,i} - u_{\parallel,e}\right)\)
  determines the parallel resistive terms to $R_\parallel:= n_e\eta J_{\parallel,s}$.
 
-The dissipative terms can be decomposed into perpendicular and parallel components
+ For the perpendicular terms we use ad-hoc numerical diffusion terms for numerical
+ stabilisation.
 \begin{align}
- \Lambda_{n_e} &= \Lambda_{n_e,\perp}+\Lambda_{n_e,\parallel}, &
- \Lambda_{N_i} &= \Lambda_{N_i,\perp}+\Lambda_{N_i,\parallel},\\
- \Lambda_{u_e} &= \Lambda_{u_e,\perp}+\Lambda_{u_e,\parallel},&
- \Lambda_{U_i} &= \Lambda_{U_i,\perp}+\Lambda_{U_i,\parallel}.
-\end{align}
-For numerical stabilisation we choose:
-\begin{align}
-\Lambda_{n_e,\parallel} &= \nu_\parallel \Delta_\parallel n_e &
-\Lambda_{N_i,\parallel} &= \nu_\parallel \Delta_\parallel N_i \\
-\Lambda_{u_e,\parallel} &= \nu_\parallel \left( -u_{e,\parallel} \Delta_\parallel n_e + \Delta_\parallel u_{\parallel,e}\right)/n_e &
-\Lambda_{U_i,\parallel} &= \nu_\parallel \left( -U_{i,\parallel} \Delta_\parallel N_i + \Delta_\parallel U_{\parallel,i}\right)/N_i &
-\end{align}
-Note that the parallel velocity diffusion is designed such that the parallel momentum
-diffusion is $\Lambda_{NU} = \nu_\parallel \Delta_\parallel U_\parallel$, which
-is important in order to conserve parallel momentum inside the flux surfaces.
-Similarly, for the perpendicular dissipation we apply viscous or hyperviscous terms.
-\begin{align}\label{eq:perpdiffNT}
- \Lambda_{n_e,\perp} &=  \nu_\perp \Delta_\perp n_e \text{ or } -\nu_\perp \Delta_\perp^2 n_e&
- \Lambda_{N_i,\perp} &=  \nu_\perp \Delta_\perp N_i \text{ or } -\nu_\perp \Delta_\perp^2 N_i & \\
- \Lambda_{u_e,\perp} &=  \nu_\perp \Delta_\perp u_{\parallel,e} \text{ or } -\nu_\perp \Delta_\perp^2 u_{\parallel,e} &
- \Lambda_{U_i,\perp} &=  \nu_\perp \Delta_\perp U_{\parallel,i} \text{ or } -\nu_\perp \Delta_\perp^2 U_{\parallel,i}
+\label{eq:perpdiffNT}
+ \Lambda_{n_e} &=  \nu_\perp \Delta_\perp n_e \text{ or } -\nu_\perp \Delta_\perp^2 n_e&
+ \Lambda_{N_i} &=  \nu_\perp \Delta_\perp N_i \text{ or } -\nu_\perp \Delta_\perp^2 N_i & \\
+ \Lambda_{u_e} &=  \nu_\perp \Delta_\perp u_{\parallel,e} \text{ or } -\nu_\perp \Delta_\perp^2 u_{\parallel,e} &
+ \Lambda_{U_i} &=  \nu_\perp \Delta_\perp U_{\parallel,i} \text{ or } -\nu_\perp \Delta_\perp^2 U_{\parallel,i}
 \end{align}
 Here the mass diffusion coefficient coincides with the viscous coefficient, hence we fixed the Schmidt number \(\mathit{Sc}_\parallel:= \frac{\nu_U}{\nu_N}\) to unity.
-The drift-fluid corresponding diffusion gives an order-of-magnitude estimate for $\nu_\perp$.
-We have with $\nu_{ii0} = z^4e^4\ln \Lambda/ (12\pi^{3/2} \sqrt{m_i} \epsilon_0^2) n_{i0} /T_{i0}^{3/2}$ and choose $D_i = \rho_i^2 \nu_{ii}$ and $T_{i0} = T_{e0}$.
-By dividing by $\rho_s^2 \Omega_{ci}$ we arrive at $\nu_\perp = m_i \nu_{ii0}/eB_0$.
-Together with neoclassical corrections we then have
+
+We do not derive resistive drifts in the gyro-fluid approach.
+The drift-fluid corresponding resistive drift gives an order-of-magnitude estimate for $\nu_\perp$.
+We have  $D_i = \rho_i^2 \nu_{ii}$ and $T_{i0} = T_{e0}$.
+By dividing by $\rho_s^2 \Omega_{ci}$ we arrive at $\nu_\perp = \nu_{ii0}/\Omega_{ci}$.
 \begin{align}
 \nu_\perp =
-5\cdot 10^{-3} \left(1+\frac{R}{a}q_{95}\right) \ln \lambda
+5\cdot 10^{-3} \ln \lambda
 \left(\frac{n_0}{10^{19}\text{m}^3}\right)
 \left(\frac{T_e}{\text{eV}}\right)^{-3/2}
 \left(\frac{B_0}{\text{T}}\right)^{-1}
 \left(\frac{m_i}{m_H}\right)^{1/2},
 \end{align}
-where we use the major radius $R_0$ and minor radius $a$ and the safety factor $q_{95}$~\cite{Madsen2016}.
 
 \subsection{Boundary and initial conditions}
 We define the simulation box as
@@ -1041,7 +1021,7 @@ two functions for which we have no boundary conditions
         - N \mathcal K(\psi)
         -\mu \mathcal K_{\vn\times\bhat}(NU_\parallel^2)
         -\mu NU_\parallel^2\nc \vec{ \mathcal K_{\vn\times\bhat}}
-        + \nu_\perp\Delta_\perp N + \nu_\parallel \Delta_\parallel N + S_N, \\
+        - \nu_\perp\Delta_\perp^2 N + S_N, \\
     \frac{\partial}{\partial t} W_\parallel =&
         - \frac{1}{B}\left[\psi, U_\parallel\right]_{\perp}%& \nonumber\\
         - \frac{1}{\mu} \bar \npar \psi% \nonumber\\
@@ -1052,10 +1032,10 @@ two functions for which we have no boundary conditions
         -\tau U_\parallel\nc\vec{ \mathcal K_{\vn\times\bhat}}\nonumber\\&
         - \left(2\tau + {\mu}U_\parallel^2\right) \mathcal K_{\vn\times\bhat} (U_\parallel)
         -2\tau U_\parallel\mathcal K_{\vn\times\bhat}(\ln N)
-        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_{\parallel,i} - u_{\parallel,e}) \nonumber\\&
-        + \nu_\perp\Delta_\perp U_\parallel
-        + \frac{\nu_\parallel}{N} \Delta_\parallel U_\parallel - \frac{\nu_\parallel}{N} U_\parallel \Delta_\parallel U_\parallel
-        + S_U,
+        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_{\parallel,i} - u_{\parallel,e})+ \frac{\nu_\parallel}{N} \Delta_\parallel U_\parallel
+ \nonumber\\&
+        - \nu_\perp\Delta_\perp^2 U_\parallel
+                + S_U,
         \label{eq:EgyrofluidU} \\
         W_\parallel&:= \left( U_\parallel + \frac{A_\parallel}{\mu}\right)
     \end{align}
@@ -1090,6 +1070,27 @@ In the output file we have
     induction &$A_\parallel$ & \\
 \bottomrule
 \end{longtable}
+\subsection{ Scale invariance}
+\subsubsection{Sign reversals of the magnetic field}\label{sec:field_reversal}
+If we change the direction of the magnetic field vector $\bhat$, we immediately see that all perpendicular
+drifts and $U_\parallel\bhat$ change directions. On the other side, the diffusive and resistive terms remain unchanged.
+Without resistivity and diffusion a change in direction of the magnetic field thus corresponds to
+a time reversal $t\rightarrow t'=-t$.
+In the code $\bhat$ changes sign by using both $-\mathcal P_\psi$ and $-\mathcal P_I$.
+
+Also note that changing the sign of the magnetic field only in the parallel derivatives $\npar \rightarrow -\npar$ does not
+have any effect. This can be seen by simply renormalizing $U_\parallel'=-U_\parallel$. This reverts the equations back to the original equations.
+\subsubsection{Scaling of density}
+If $N, U_\parallel, \phi, A_\parallel$ are a solution to the model equations
+then so are $N'=\alpha N$, $U_\parallel'=U_\parallel$, $\phi'=\phi$ and $A_\parallel'=A_\parallel$ with the changed parameters $S_N' = \alpha S_N$, $\eta' = \eta/\alpha$ and $ \beta' = \beta/\alpha$. If $N$
+has a Dirichlet boundary condition, then $N'$ satisfies a correspondingly scaled boundary condition.
+\subsubsection{Helicity}
+The standard helicity of the magnetic field would be a right handed screw,
+where the magnetic field and the toroidal plasma current point in the clockwise
+direction if the tokamak is seen from above.
+The helicity should however not have any influence on the plasma dynamics as
+the tokamak viewed in the mirror has the reversed helicity.
+
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
 The density equations directly yield the particle conservation
@@ -1110,7 +1111,7 @@ The terms of the particle conservation thus read
   %+ \mu_e u_{\parallel,e}^2\vec K_{\vn\times\bhat}
   %+ u_{\parallel,e}(\bhat + {\vec b}_\perp) \right), \\
   \Lambda_{N} =&
-  \nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N
+  \nu_\perp\Delta_\perp N% + \nu_\parallel\Delta_\parallel N
 \\
   S_{N} =&  S_{N}
 \end{align}
@@ -1143,17 +1144,19 @@ The relevant terms in the output file are
     jsneA\_tt &$ n_e u_{\parallel,e} \vec{ b}_\perp  \cn \psi_p$ &
     jsneE\_tt & $ n_e \vec u_E\cn\psi_p$ \\
     lneperp\_tt &$ \Lambda_{\perp,n_e} = \nu_\perp \Delta_\perp n_e$ or $-\nu_\perp \Delta^2_\perp n_e$ &
-    lneparallel\_tt &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
+    & \\
+    %lneparallel\_tt &$ \Lambda_{\parallel,n_e} = \nu_\parallel \Delta_\parallel n_e$ \\
     sne\_tt & $S_{n_e}$ &
     jsdiae\_tt & $\tau_e \bhat \times \vn n_e \cn \psi_p /B$\\
     dnepar\_tt & $\nc (\bhat n_e u_{e,\parallel}$) &
-      & \\
+    & \\
     ions & $N_i$ &
     jsniC\_tt &$ N_i ( \vec u_K + \vec u_C )\cn \psi_p$ \\
     jsniA\_tt &$ N_i U_{\parallel,i} \vec{ b}_\perp  \cn \psi_p$ &
     jsniE\_tt & $ N_i \vec u^i_E\cn\psi_p$ \\
     lniperp\_tt &$ \Lambda_{\perp,N_i} = \nu_\perp \Delta_\perp N_i$ or $-\nu_\perp \Delta^2_\perp N_i$ &
-    lniparallel\_tt &$ \Lambda_{\parallel,N_i} = \nu_\parallel \Delta_\parallel N_i$ \\
+    & \\
+    %lniparallel\_tt &$ \Lambda_{\parallel,N_i} = \nu_\parallel \Delta_\parallel N_i$ \\
     sni\_tt & $S_{N_i}$ &
     jsdiai\_tt & $\tau_i \bhat \times \vn N_i \cn \psi_p /B$\\
     dnipar\_tt & $\nc (\bhat N_i U_{i,\parallel}$) &
@@ -1185,7 +1188,7 @@ with ( $z_e=-1$ and $z_i=+1$) and $\vec u_E := {\bhat\times \vn\phi}/{B}$
   \nonumber\\
   &+ \sum_z z\left[\mu \tau NU_\parallel^2\vec K_{\vn\times\bhat} + \tau NU_\parallel \left(\bhat + {\vec b}_\perp\right)\right], \\
   \Lambda_{\mathcal E} =&  \sum_s z\left[\left( \tau\left( 1+\ln{N}\right) + \psi + \frac{1}{2} \mu U_\parallel^2 \right)
-  \left(\nu_\perp\Delta_\perp N + \nu_\parallel\Delta_\parallel N\right)  +  \mu NU_\parallel\left(\nu_\perp\Delta_\perp U_\parallel + \nu_\parallel\Delta_\parallel U_\parallel\right) \right]
+  \left(\nu_\perp\Delta_\perp N \right)  +  \mu NU_\parallel\left(\nu_\perp\Delta_\perp U_\parallel + \nu_\parallel\Delta_\parallel U_\parallel\right) \right]
 \nonumber \\
   S_{\mathcal E} =&  \sum_s  z\left[ \left(\tau\left( 1+\ln{N}\right) +\psi + \frac{1}{2} \mu U_\parallel^2 \right)S_{N}\right]
 \nonumber \\
@@ -1234,8 +1237,11 @@ The relevant terms in the output file are
         + z_i \tau_i N_i U_{\parallel,i} \vec{ b}_\perp \cn \psi_p $ \\
     leeperp\_tt &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_{\parallel,e}^2/2) \nu_\perp \Delta_\perp n_e + z_e\mu_e n_e u_{\parallel,e} \nu_\perp \Delta_\perp u_{\parallel,e}$ \\
     leiperp\_tt &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_{\parallel,i}^2/2) \nu_\perp \Delta_\perp N_i + z_i\mu_i N_i U_{\parallel,i} \nu_\perp \Delta_\perp U_{\parallel,i}$ \\
-    leeparallel\_tt &$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_{\parallel,e}^2/2) \nu_\parallel \Delta_\parallel n_e + z_e\mu_e n_e u_{\parallel,e} \nu_\parallel \Delta_\parallel u_{\parallel,e}$ \\
-    leiparallel\_tt &$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_{\parallel,i}^2/2) \nu_\parallel \Delta_\parallel N_i + z_i\mu_i N_i U_{\parallel,i} \nu_\parallel \Delta_\parallel U_{\parallel,i}$ \\
+    leeparallel\_tt & %$z_e(\tau_e(1+\ln n_e) + \phi + \mu_eu_{\parallel,e}^2/2) \nu_\parallel \Delta_\parallel n_e +$
+    $z_e\mu_e n_e u_{\parallel,e} \nu_{\parallel,e} \Delta_\parallel u_{\parallel,e}$ \\
+    leiparallel\_tt & %$z_i(\tau_i(1+\ln N_i) + \psi_i + \mu_iU_{\parallel,i}^2/2) \nu_\parallel \Delta_\parallel N_i + $
+    $z_i\mu_i N_i
+    U_{\parallel,i} \nu_{\parallel,i} \Delta_\parallel U_{\parallel,i}$ \\
 \bottomrule
 \end{longtable}
 
@@ -1330,7 +1336,7 @@ The relevant terms in the output file are (the Lorentz force term is described i
     sparmirrori\_tt & $-z_i\tau_iN_i\npar \ln B$ \\
     sparsni\_tt & $\mu_i S_{N_i} U_{\parallel,i} + \mu_i S_{U_i} N_i $ &
     sparsnibphi\_tt & $\mu_i S_{N_i} U_{\parallel,i}b_\varphi + \mu_i S_{U,i} N_i b_\varphi $ \\
-    lparpar\_tt   & $\nu_\parallel \Delta_\parallel U_{\parallel,i}$ &
+    lparpar\_tt   & $\nu_{\parallel,i} \Delta_\parallel U_{\parallel,i}$ &
     lparperp\_tt & $-\nu_\perp U_{\parallel,i} \Delta_\perp^2 N_i - \nu_\perp N_i\Delta_\perp^2 U_{\parallel,i} $ \\
 \bottomrule
 \end{longtable}
@@ -1552,11 +1558,11 @@ perp\_diff & string & "viscous" & "viscous" & "viscous": $\Lambda_\perp\propto
 \nu_\perp\Delta_\perp$ , "hyperviscous": $\Lambda_\perp \propto
 -\nu_\perp\Delta_\perp^2$
 \\
-nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$
-(dimensional analysis reveals there can be a factor $(R_0/\rho_s)^2$ between
-$\nu_\perp $n and $\nu_\parallel$ for $\nu_\parallel$ to become relevant for
-the dynamics)
-\\
+%nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$
+%(dimensional analysis reveals there can be a factor $(R_0/\rho_s)^2$ between
+%$\nu_\perp $n and $\nu_\parallel$ for $\nu_\parallel$ to become relevant for
+%the dynamics)
+%\\
 resistivity & float &1e-4  & - & parallel resistivity parameter Eq.~\eqref{eq:resistivity}
 \\
 curvmode  & string & "low beta"  & "toroidal" &
@@ -1872,7 +1878,7 @@ last output in the output file.
     \\
 \qquad large fieldaligned oscillations in $u_{\parallel,e}$ paired with instability in the edge of the box
 &
-Increase parallel diffusion $nu_\parallel$, apply damping region
+Apply damping region
     \\
 \qquad Perpendicular grid oscillations in $u_{\parallel,e}$ and $\Delta_\perp \phi$ in the damping region, symmetric in $\varphi$
 &
@@ -1889,7 +1895,7 @@ Probably caused by the perpendicular transport that goes unstable. Increase $\nu
 \qquad Oscillations where fieldlines intersect the wall
 &
 Caused by boundary conditions in FCI method and necessarily underresolved toroidal direction.
-Increase $\nu_\parallel$, $N_z$, decrease $N_x$, $N_y$ or decrease $q$ value by decreasing $\mathcal P_\psi$ in geometry input file
+Increase $N_z$, decrease $N_x$, $N_y$ or decrease $q$ value by decreasing $\mathcal P_\psi$ in geometry input file
 \\
 \bottomrule
 \end{longtable}
-- 
GitLab


From e32c59bfe17289dd35dfdab9588fc7ee10f3a027 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 5 Oct 2020 22:14:10 +0200
Subject: [PATCH 348/540] Reflect changes on viscosity in code

- implement nu_para in parameters and delete from input file
- delete in density equation
- delete from energy and mass conservation
- delete dssN in feltor.h
---
 src/feltor/feltor.h             | 31 +++++++++---------
 src/feltor/feltor.tex           |  5 ++-
 src/feltor/feltordiag.h         | 57 +++++++++++++++++----------------
 src/feltor/interpolate_in_3d.cu |  1 -
 src/feltor/parameters.h         | 13 +++++---
 5 files changed, 59 insertions(+), 48 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 6aae00957..15f1d66f9 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -238,9 +238,9 @@ struct Explicit
     const Container & dsP (int i) const {
         return m_dsP[i];
     }
-    const Container & dssN(int i) { //2nd fieldaligned derivative
-        return m_dssN[i];
-    }
+    //const Container & dssN(int i) { //2nd fieldaligned derivative
+    //    return m_dssN[i];
+    //}
     const Container & dssU(int i) {
         return m_dssU[i];
     }
@@ -375,7 +375,7 @@ struct Explicit
     Container m_vol3d;
 
     Container m_apar;
-    std::array<Container,2> m_phi, m_dsN, m_dsU, m_dsP, m_dssN, m_dssU;
+    std::array<Container,2> m_phi, m_dsN, m_dsU, m_dsP, m_dssU;// m_dssN;
     std::array<Container,3> m_dA;
     std::array<std::array<Container,3>,2> m_dP, m_dN, m_dU;
     std::array<std::array<Container,2>,2> m_fields, m_s; //fields, sources
@@ -571,7 +571,8 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     m_apar = m_temp0;
 
     m_phi[0] = m_phi[1] = m_temp0;
-    m_dssN = m_dssU = m_dsN = m_dsU = m_dsP = m_phi;
+    //m_dssN =
+    m_dssU = m_dsN = m_dsU = m_dsP = m_phi;
     m_dA[0] = m_dA[1] = m_dA[2] = m_temp0;
     m_dP[0] = m_dP[1] = m_dA;
     m_dN = m_dU = m_dP;
@@ -833,13 +834,13 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
             -1., fields[0][i], m_dsU[i], 1., yp[0][i] );
         dg::blas1::pointwiseDot( -1., fields[0][i],fields[1][i],m_divb,
             1.,yp[0][i]);
-        //density: + nu_par Delta_par N
-        dg::blas1::pointwiseDot( m_p.nu_parallel, m_divb, m_dsN[i],
-                                 0., m_temp0);
-        m_ds_N.dss( y[0][i], m_dssN[i]);
-        dg::blas1::axpby( m_p.nu_parallel, m_dssN[i], 1., m_temp0);//nu_par Delta_par N
-        //Add to rhs, we again need it further down
-        dg::blas1::axpby( 1., m_temp0, 1., yp[0][i]);
+        //////////density: + nu_par Delta_par N
+        ////////dg::blas1::pointwiseDot( m_p.nu_parallel, m_divb, m_dsN[i],
+        ////////                         0., m_temp0);
+        ////////m_ds_N.dss( y[0][i], m_dssN[i]);
+        ////////dg::blas1::axpby( m_p.nu_parallel, m_dssN[i], 1., m_temp0);//nu_par Delta_par N
+        //////////Add to rhs, we again need it further down
+        ////////dg::blas1::axpby( 1., m_temp0, 1., yp[0][i]);
         //---------------------velocity-------------------------//
         // Burgers term: -0.5 ds U^2
         //dg::blas1::pointwiseDot(fields[1][i], fields[1][i], m_temp1); //U^2
@@ -852,11 +853,11 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         m_ds_P.centered( m_phi[i], m_dsP[i]);
         dg::blas1::axpby(-1./m_p.mu[i], m_dsP[i], 1.0, yp[1][i]);
         // diffusion: + nu_par Delta_par U/N - nu_par U Delta_par N/ N
-        dg::blas1::pointwiseDot(m_p.nu_parallel, m_divb, m_dsU[i],
+        dg::blas1::pointwiseDot(m_p.nu_parallel[i], m_divb, m_dsU[i],
                                 0., m_temp1);
         m_ds_U.dss( fields[1][i], m_dssU[i]);
-        dg::blas1::axpby( m_p.nu_parallel, m_dssU[i], 1., m_temp1); //nu_par Delta_par U
-        dg::blas1::pointwiseDot( -1., fields[1][i], m_temp0, 1., m_temp1); //
+        dg::blas1::axpby( m_p.nu_parallel[i], m_dssU[i], 1., m_temp1); //nu_par Delta_par U
+        //////dg::blas1::pointwiseDot( -1., fields[1][i], m_temp0, 1., m_temp1);
         dg::blas1::pointwiseDivide( 1., m_temp1, fields[0][i], 1., yp[1][i]);
     }
 }
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index d413a3bb8..23738ae7c 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -815,7 +815,8 @@ $\mu_{\parallel,e}:=0.73\frac{n_eT_e}{\nu_{ei}}$ and $\mu_{\parallel,i} = 0.96\f
 \end{align}
     \label{eq:resistivity}
 \end{subequations}
-with $\ln \lambda \approx 10$ and $T_e=T_i$ for the purpose of computing the diffusive coefficients.
+with $\ln \lambda \approx 10$ and $T_e=T_i$ for the purpose of computing the diffusive coefficients. Note that $\nu_\parallel/N$ represents a kinematic viscosity and is a factor $\sqrt{|\mu_e|}$ smaller for ions than for electrons.
+The dynamic viscosity $\mu\nu_\parallel$ is larger for ions than for electrons.
  The approximate Spitzer current \(J_{\parallel,s}:= n_e \left(U_{\parallel,i} - u_{\parallel,e}\right)\)
  determines the parallel resistive terms to $R_\parallel:= n_e\eta J_{\parallel,s}$.
 
@@ -1166,6 +1167,7 @@ The relevant terms in the output file are
 
 
 
+Note that the parallel divergences vanish exactly under a flux-surface average. This can serve as a numerical test of our implementation.
 \subsubsection{Energy theorem}
 The terms of the energy theorem are
 \begin{align} \label{eq:energy_theorem}
@@ -1340,6 +1342,7 @@ The relevant terms in the output file are (the Lorentz force term is described i
     lparperp\_tt & $-\nu_\perp U_{\parallel,i} \Delta_\perp^2 N_i - \nu_\perp N_i\Delta_\perp^2 U_{\parallel,i} $ \\
 \bottomrule
 \end{longtable}
+Note that the parallel viscosity term vanishes exactly under the flux-surface average. This can serve as a numerical test.
 
 \subsubsection{Parallel electron force balance}
 We gather the dominant terms in the electron momentum equation (neglecting all terms as $\mu_e=0$). This leaves the parallel force balance
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 2b171a3b5..e695ab97e 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -512,13 +512,13 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
-    {"lneparallel_tt", "Parallel electron diffusion (Time average)", true,
-        []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(0),
-                                     0., result);
-            dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., result);
-        }
-    },
+    //{"lneparallel_tt", "Parallel electron diffusion (Time average)", true,
+    //    []( DVec& result, Variables& v ) {
+    //        dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(0),
+    //                                 0., result);
+    //        dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., result);
+    //    }
+    //},
     {"sne_tt", "Source term for electron density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::copy( v.f.density_source(0), result);
@@ -574,13 +574,13 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::scal( result, -v.p.nu_perp);
         }
     },
-    {"lniparallel_tt", "Parallel ion diffusion (Time average)", true,
-        []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(1),
-                                     0., result);
-            dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(1), 1., result);
-        }
-    },
+    //{"lniparallel_tt", "Parallel ion diffusion (Time average)", true,
+    //    []( DVec& result, Variables& v ) {
+    //        dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(1),
+    //                                 0., result);
+    //        dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(1), 1., result);
+    //    }
+    //},
     {"sni_tt", "Source term for ion density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::copy( v.f.density_source(1), result);
@@ -748,9 +748,10 @@ std::vector<Record> diagnostics2d_list = {
     },
     {"leeparallel_tt", "Parallel electron energy dissipation (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(0),
-                                     0., v.tmp[0]);
-            dg::blas1::axpby( 1., v.f.dssN(0), 1., v.tmp[0]);
+            //dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(0),
+            //                         0., v.tmp[0]);
+            //dg::blas1::axpby( 1., v.f.dssN(0), 1., v.tmp[0]);
+            dg::blas1::copy(0., v.tmp[0]);
             dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(0),
                                      0., v.tmp[1]);
             dg::blas1::axpby( 1., v.f.dssU(0), 1., v.tmp[1]);
@@ -759,14 +760,15 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.density(0), v.f.velocity(0), v.f.potential(0),
                 v.tmp[0], v.tmp[1]
             );
-            dg::blas1::scal( result, v.p.nu_parallel);
+            dg::blas1::scal( result, v.p.nu_parallel[0]);
         }
     },
     {"leiparallel_tt", "Parallel ion energy dissipation (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(1),
-                                     0., v.tmp[0]);
-            dg::blas1::axpby( 1., v.f.dssN(1), 1., v.tmp[0]);
+            //dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(1),
+            //                         0., v.tmp[0]);
+            //dg::blas1::axpby( 1., v.f.dssN(1), 1., v.tmp[0]);
+            dg::blas1::copy(0., v.tmp[0]);
             dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(1),
                                      0., v.tmp[1]);
             dg::blas1::axpby( 1., v.f.dssU(1), 1., v.tmp[1]);
@@ -775,7 +777,7 @@ std::vector<Record> diagnostics2d_list = {
                 v.f.density(1), v.f.velocity(1), v.f.potential(1),
                 v.tmp[0], v.tmp[1]
             );
-            dg::blas1::scal( result, v.p.nu_parallel);
+            dg::blas1::scal( result, v.p.nu_parallel[1]);
         }
     },
     /// ------------------------ Vorticity terms ---------------------------//
@@ -950,9 +952,10 @@ std::vector<Record> diagnostics2d_list = {
 
             v.f.compute_diffusive_lapMperpN( v.f.density(0), v.tmp[0], v.tmp[1]);
             dg::blas1::scal( v.tmp[1], -v.p.nu_perp);
-            dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(0),
-                                     0., v.tmp[2]);
-            dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., v.tmp[2]);
+            //dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(0),
+            //                         0., v.tmp[2]);
+            //dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., v.tmp[2]);
+            dg::blas1::copy( 0., v.tmp[2]);
             dg::blas1::axpby( 1., v.tmp[1], 1., v.tmp[2]); //Lambda_ne
             dg::blas1::pointwiseDot( v.tmp[2], result, result);
 
@@ -1118,9 +1121,9 @@ std::vector<Record> diagnostics2d_list = {
     //should be zero
     {"lparpar_tt", "Parallel momentum dissipation by parallel diffusion", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsU(1),
+            dg::blas1::pointwiseDot( v.p.nu_parallel[1], v.f.divb(), v.f.dsU(1),
                                      0., result);
-            dg::blas1::axpby( v.p.nu_parallel, v.f.dssU(1), 1., result);
+            dg::blas1::axpby( v.p.nu_parallel[1], v.f.dssU(1), 1., result);
         }
     },
     {"lparperp_tt", "Parallel momentum dissipation by perp diffusion", true,
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index 6364aa1a5..2fac50913 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -223,7 +223,6 @@ int main( int argc, char* argv[])
             {
                 err = nc_get_vara_double( ncid_in, dataID,
                     start3d, count3d_in, transferH_in.data());
-                //2. Compute fsa and output fsa
                 transferH_out = fieldaligned.interpolate_from_coarse_grid(
                     g3d_in, transferH_in);
                 dg::blas2::symv( interpolate_in_2d, transferH_out, transferH);
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index a36c5f12b..38dab5585 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -29,8 +29,9 @@ struct Parameters
 
     std::array<double,2> mu; // mu[0] = mu_e, m[1] = mu_i
     std::array<double,2> tau; // tau[0] = -1, tau[1] = tau_i
+    std::array<double,2> nu_parallel;
 
-    double nu_perp, nu_parallel;
+    double nu_perp;
     double eta, beta;
 
     double amp;
@@ -88,10 +89,13 @@ struct Parameters
         tau[0]      = -1.;
         tau[1]      = file::get( mode, js, "tau", 0.).asDouble();
         beta        = file::get( mode, js, "beta", 0.).asDouble();
+        eta         = file::get( mode, js, "resistivity", 0.).asDouble();
         nu_perp     = file::get( mode, js, "nu_perp", 0.).asDouble();
         perp_diff   = file::get( mode, js, "perp_diff", "viscous").asString();
-        nu_parallel = file::get( mode, js, "nu_parallel", 0.).asDouble();
-        eta         = file::get( mode, js, "resistivity", 0.).asDouble();
+        //nu_parallel = file::get( mode, js, "nu_parallel", 0.).asDouble();
+        //Init after reading in eta and mu[0]
+        nu_parallel[0] = 0.73/eta;
+        nu_parallel[1] = sqrt(fabs(mu[0]))*1.36/eta;
 
         initne      = file::get( mode, js, "initne", "blob").asString();
         initphi     = file::get( mode, js, "initphi", "zero").asString();
@@ -137,7 +141,8 @@ struct Parameters
             <<"     perp. Viscosity   = "<<perp_diff<<"\n"
             <<"     par. Resistivity  = "<<eta<<"\n"
             <<"     beta              = "<<beta<<"\n"
-            <<"     par. Viscosity    = "<<nu_parallel<<"\n"
+            <<"     par. Viscosity e  = "<<nu_parallel[0]<<"\n"
+            <<"     par. Viscosity i  = "<<nu_parallel[1]<<"\n"
             <<"     curvature mode    = "<<curvmode<<"\n"
             <<"     Symmetry in phi   = "<<std::boolalpha<<symmetric<<"\n";
         os  <<"Initial parameters are: \n"
-- 
GitLab


From 7445b9e0ce93e48c501c2322208820e74b085390 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 5 Oct 2020 23:43:11 +0200
Subject: [PATCH 349/540] Adapt compass input and geometry

---
 src/feltor/geometry/compass.json |  4 ++--
 src/feltor/input/compass.json    | 13 ++++++-------
 2 files changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/feltor/geometry/compass.json b/src/feltor/geometry/compass.json
index 5917756c5..9608a108e 100644
--- a/src/feltor/geometry/compass.json
+++ b/src/feltor/geometry/compass.json
@@ -1,8 +1,8 @@
 //Compass geometry file T_e = 200eV, B = 0.8T, Deuterium plasma
 {
     "A": 0,
-   	"R_0": 2.191566859511476e2,
-   	"PP": 0.8,
+   	"R_0": 219.23,
+   	"PP": 1,
    	"PI": 1,
    	"c":[
    	    0.10587834884347505,
diff --git a/src/feltor/input/compass.json b/src/feltor/input/compass.json
index b727c8fa6..a3ab20f77 100644
--- a/src/feltor/input/compass.json
+++ b/src/feltor/input/compass.json
@@ -11,9 +11,9 @@
         "rk4eps": 1e-6,
         "periodify": true
     },
-    "inner_loop": 5, 
-    "itstp": 500, 
-    "maxout": 50, 
+    "inner_loop": 5,
+    "itstp": 500,
+    "maxout": 50,
     "stages"     : 3,
     "eps_pol"    : [1e-6,1,1],
     "jumpfactor" : 1,
@@ -25,7 +25,6 @@
     "beta"        : 0e-4,
     "nu_perp"     : 2e-3,
     "perp_diff"   : "hyperviscous",
-    "nu_parallel" : 4e2,
     "resistivity" : 1e-4,
     "curvmode"   : "toroidal",
     "symmetric"  : false,
@@ -62,9 +61,9 @@
     },
     "damping":
     {
-        "modifier": "heaviside",
+        "modifier": "sol_pfr",
         "rate" : 1e-2,
-        "boundary": 1.1,
-        "alpha": 0.25
+        "boundary": [1.1,0.988],
+        "alpha": [0.25,0.15]
     }
 }
-- 
GitLab


From eaf9b484accd40fef9ee1eacc6a334ab4a9d86e9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 6 Oct 2020 16:55:34 +0200
Subject: [PATCH 350/540] Feltordiag refuses to overwrite file

and documentation now features tcolorbox package
---
 doc/related_pages/header.tex |  1 +
 src/feltor/feltor.tex        | 17 ++++++++++++++---
 src/feltor/feltordiag.cu     |  3 +--
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/doc/related_pages/header.tex b/doc/related_pages/header.tex
index 7a6ec1a99..ff0d9902c 100644
--- a/doc/related_pages/header.tex
+++ b/doc/related_pages/header.tex
@@ -24,6 +24,7 @@
 \usepackage[table]{xcolor} % for alternating colors
 %\rowcolors{2}{gray!25}{white} %%% Use this line in front of longtable
 \renewcommand\arraystretch{1.3}
+\usepackage{tcolorbox}
 \usepackage{doi}
 \usepackage[sort,square,numbers]{natbib}
 \bibliographystyle{abbrvnat}
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 23738ae7c..109894b8b 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1765,8 +1765,12 @@ are all taken from the regular input files. This means that the user must take c
 with the paramters in the existing \texttt{initial.nc} file. Also note that we try to discourage
 appending new results to an exisiting file directly,
 because if for some reason the cluster crashes and the file is corrupted
-the whole simulation is lost. It is safer to just merge files afterwards with for example\\
-\texttt{ncrcat output1.nc output2.nc output.nc}
+the whole simulation is lost.
+\begin{tcolorbox}[title=Note]
+It is safer to just merge files afterwards with\\
+\texttt{ncrcat output1.nc output2.nc output.nc}\\
+from the \texttt{nco} package
+\end{tcolorbox}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{Diagnostics}\label{sec:diagnostics}
 \texttt{feltor/src/feltor/feltordiag.cu}
@@ -1776,6 +1780,11 @@ Compilation\\
 Usage \\
 \texttt{./feltordiag input0.nc ... inputN.nc output.nc} \\
 
+\begin{tcolorbox}[title=Note]
+\texttt{feltordiag} refuses to overwrite existing files in order to protect against data loss in case of accidental spelling
+errors or other careless mistakes.
+\end{tcolorbox}
+
 Output file format: netcdf-4/hdf5, Conventions: CF-1.7; A \textit{coordinate variable (Coord. Var.)} is a Dataset with the same name as a dimension.
 
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
@@ -1807,9 +1816,11 @@ Z\_ifs\_norm     & Dataset & 1 (time) & Volume integrated square flux surface av
 \bottomrule
 \end{longtable}
 where Z $\in$ \{X, Y\_tt\}
-Note that feltoridag converts all $jsX$ quantities into $jvX$
+\begin{tcolorbox}[title=Note]
+\texttt{feltoridag} converts all $jsX$ quantities into $jvX$
 by multiplying $\d v/\d \psi_p$
 in the sense that $\vec j\cn v  = \vec j \cn \psi_p \d v/\d\psi_p$.
+\end{tcolorbox}
 The parameters used for the X-point flux-aligned grid construction are $f_x = 1/8$, $f_y = 0$, $n_\psi = 3$, $N_\zeta = 64$ and $N_\eta = 640$ and the constant monitor metric.
 
 We also have a useful geometry diagnostic program:
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index e45951174..0f1015413 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -55,8 +55,7 @@ int main( int argc, char* argv[])
 
     //-----------------Create Netcdf output file with attributes----------//
     int ncid_out;
-    //maybe we should issue a warning if file exists?
-    err = nc_create(argv[argc-1],NC_NETCDF4|NC_CLOBBER, &ncid_out);
+    err = nc_create(argv[argc-1],NC_NETCDF4|NC_NOCLOBBER, &ncid_out);
 
     /// Set global attributes
     std::map<std::string, std::string> att;
-- 
GitLab


From 8d5741ca3d43330f06c986d0d30f0ff56aed15c0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 7 Oct 2020 14:15:00 +0200
Subject: [PATCH 351/540] Add Hyper-elliptic convergence test

---
 inc/dg/elliptic2d_b.cu | 38 ++++++++++++++++++++++++++++----------
 src/feltor/feltor.tex  |  2 +-
 2 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 2a1eaa893..31574a713 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -67,6 +67,7 @@ int main()
 
     //std::cout << "Create Polarisation object and set chi!\n";
     {
+    std::cout << "Centered Elliptic Multigrid\n";
     //! [multigrid]
     dg::Timer t;
     t.tic();
@@ -107,8 +108,16 @@ int main()
     std::cout << " "<<err << "\t"<<res.i<<"\n";
     }
 
+    dg::DMatrix DX = dg::create::dx( grid);
+    dg::blas2::gemv( DX, x, error);
+    dg::blas1::axpby( 1.,derivati,-1., error);
+    double err = dg::blas2::dot( w2d, error);
+    const double norm_der = dg::blas2::dot( w2d, derivati);
+    std::cout << "L2 Norm of relative error in derivative is "<<std::setprecision(16)<< sqrt( err/norm_der)<<std::endl;
+    //derivative converges with p-1, for p = 1 with 1/2
 
     {
+        std::cout << "Forward Elliptic\n";
     x = temp;
     //![invert]
     //create an Elliptic object without volume form (not normed)
@@ -121,7 +130,7 @@ int main()
     dg::Invert<dg::DVec > invert_fw( x, n*n*Nx*Ny, eps);
 
     //invert the elliptic equation
-    std::cout << " "<< invert_fw( pol_forward, x, b, w2d, v2d, chi_inv);
+    invert_fw( pol_forward, x, b, w2d, v2d, chi_inv);
 
     //compute the error (solution contains analytic solution
     dg::blas1::axpby( 1.,x,-1., solution, error);
@@ -135,28 +144,37 @@ int main()
     }
 
     {
-        //try the compute_in_2d handle of Elliptic3d
+        std::cout << "Compute 2d handle of Elliptic3d\n";
 	    dg::CartesianGrid3d grid( 0, lx, 0, ly, 0,1,n, Nx, Ny, 1, bcx, bcy, dg::PER);
 		dg::Elliptic3d<dg::CartesianGrid3d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
         pol_backward.set_compute_in_2d(true);
 		pol_backward.set_chi( chi);
 		x = temp;
 		dg::Invert<dg::DVec > invert_bw( x, n*n*Nx*Ny, eps);
-		std::cout << " "<< invert_bw( pol_backward, x, b, w2d, v2d, chi_inv);
+		invert_bw( pol_backward, x, b, w2d, v2d, chi_inv);
 		dg::blas1::axpby( 1.,x,-1., solution, error);
 		double err = dg::blas2::dot( w2d, error);
         err = sqrt( err/norm); res.d = err;
         std::cout << " "<<err << "\t"<<res.i<<std::endl;
     }
+    {
+        //try the Hyperelliptic operator
+        std::cout << "HyperElliptic operator\n";
+	    dg::CartesianGrid2d grid( 0, lx, 0, ly,n, Nx, Ny, bcx, bcy);
+		dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol_backward( grid, dg::not_normed, dg::backward, jfactor);
+		x = temp;
+		dg::Invert<dg::DVec > invert( x, n*n*Nx*Ny, eps);
+        chi = temp;
+		x = temp;
+		invert( pol_backward, chi, solution);
+		invert( pol_backward, x, chi);
+		dg::blas1::axpby( 1.,x,-0.25, solution, error);
+		double err = dg::blas2::dot( w2d, error);
+        err = sqrt( err/norm); res.d = err;
+        std::cout << " "<<err << "\t"<<res.i<<std::endl;
+    }
 
 
-    dg::DMatrix DX = dg::create::dx( grid);
-    dg::blas2::gemv( DX, x, error);
-    dg::blas1::axpby( 1.,derivati,-1., error);
-    double err = dg::blas2::dot( w2d, error);
-    const double norm_der = dg::blas2::dot( w2d, derivati);
-    std::cout << "L2 Norm of relative error in derivative is "<<std::setprecision(16)<< sqrt( err/norm_der)<<std::endl;
-    //derivative converges with p-1, for p = 1 with 1/2
 
     return 0;
 }
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 109894b8b..a2fbc7aec 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1052,7 +1052,7 @@ and
     \Gamma_{1,i}^{-1} = 1-\frac{1}{2}\tau_i\mu_i \Delta_\perp \\
     \psi_e = \phi, \quad \psi_i &= \Gamma_{1,i}\phi -\frac{\mu_i}{2}\frac{(\np\phi)^2}{B^2} \\
     \left(\frac{\beta}{\mu_i}N_i - \frac{\beta}{\mu_e}n_e-\Delta_\perp\right)
-    A_\parallel &= \beta\left(N_iW_i-n_e w_e\right)
+    A_\parallel &= \beta\left(N_iW_{\parallel,i}-n_e w_{\parallel,e}\right)
   \end{align}
 \end{subequations}
 Note that the negative signs make the operators in Eqs.~\eqref{eq:elliptic} positive definite.
-- 
GitLab


From 67139bdffeaeeb9503477cd9cc09a73570604c12 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 7 Oct 2020 18:23:10 +0200
Subject: [PATCH 352/540] Update derive method of Extrapolation

with more formulas, now we can hopefully use it to implement
time derivative of A parallel in feltor equations
---
 inc/dg/cg.h | 62 ++++++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 8 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index af8792059..38a5bb20a 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -261,18 +261,21 @@ unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const Con
 /**
 * @brief Extrapolate based on up to three past solutions
 *
+* This class constructs an interpolating polynomial through up to three given points
+* and evaluates its value or its derivative at a new point.
+*
 * The intention of this class is to provide an initial guess for iterative solvers
 * based on past solutions:
  \f[ x_{init} = \alpha_0 x_0 + \alpha_{-1}x_{-1} + \alpha_{-2} x_{-2}\f]
  where the indices indicate the current (0) and past (negative) solutions.
- Choose between 1 (constant), 2 (linear) or 3 (parabola) extrapolation.
- The user can choose to provide a time value \c t_i associated with the \c x_i, which
+ Choose between 1 (constant), 2 (linear) or 3 (parabola) extrapolation.  The user can choose to provide a time value \c t_i associated with the \c x_i, which
  are then used to compute the coefficients \c alpha_i (using Lagrange interpolation).
  Otherwise an equidistant distribution is assumed.
 *
 * @note Since extrapolation with higher order polynomials is so prone to oscillations
 * anything higher than linear rarely leads to anything useful. So best stick to
 * constant or linear extrapolation
+* @note The derivative of the interpolating polynomial at a new point reduces to familiar finite difference formulas
 * @copydoc hide_ContainerType
 * @ingroup invert
 * @sa https://en.wikipedia.org/wiki/Extrapolation
@@ -344,7 +347,9 @@ struct Extrapolation
 
     /**
     * @brief Extrapolate value to given time
-    * @param t time to which to extrapolate
+    *
+    * Construt and evaluate the interpolating polynomial at a given point
+    * @param t time to which to extrapolate (or at which interpolating polynomial is evaluated)
     * @param new_x (write only) contains extrapolated value on output ( may alias the tail)
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
     */
@@ -358,7 +363,7 @@ struct Extrapolation
                      break;
             case(3): {
                 value_type f0 = (t-m_t[1])*(t-m_t[2])/(m_t[0]-m_t[1])/(m_t[0]-m_t[2]);
-                value_type f1 = (t-m_t[0])*(t-m_t[2])/(m_t[1]-m_t[0])/(m_t[1]-m_t[2]);
+                value_type f1 =-(t-m_t[0])*(t-m_t[2])/(m_t[0]-m_t[1])/(m_t[1]-m_t[2]);
                 value_type f2 = (t-m_t[0])*(t-m_t[1])/(m_t[2]-m_t[0])/(m_t[2]-m_t[1]);
                 dg::blas1::evaluate( new_x, dg::equals(), dg::PairSum(),
                         f0, m_x[0], f1, m_x[1], f2, m_x[2]);
@@ -373,17 +378,40 @@ struct Extrapolation
     }
 
     /**
-    * @brief Backard difference formula to get time derivative \f$ (x_0-x_1)/(t_0-t_1)\f$
+    * @brief Evaluate first derivative of interpolating polynomial
+    *
+    * @param t time at which derivative of interpolating polynomial is evaluated
     * @param dot_x (write only) contains derived value on output
+    * @note For equidistant values this is equivalent to computing the backward difference formula
+    * @attention If number==1, the result is 0 (derivative of a constant
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
     */
     template<class ContainerType0>
-    void derive( ContainerType0& dot_x) const{
-        dg::blas1::axpbypgz( 1./(m_t[0]-m_t[1]), m_x[0], -1./(m_t[0]-m_t[1]), m_x[1], 0., dot_x);
+    void derive( value_type t, ContainerType0& dot_x) const{
+        switch(m_number)
+        {
+            case(0):
+                     break;
+            case(1): dg::blas1::copy( 0, dot_x);
+                     break;
+            case(3): {
+                value_type f0 =-(-2.*t+m_t[1]+m_t[2])/(m_t[0]-m_t[1])/(m_t[0]-m_t[2]);
+                value_type f1 = (-2.*t+m_t[0]+m_t[2])/(m_t[0]-m_t[1])/(m_t[1]-m_t[2]);
+                value_type f2 =-(2.*t+m_t[0]+m_t[1])/(m_t[2]-m_t[0])/(m_t[2]-m_t[1]);
+                dg::blas1::evaluate( dot_x, dg::equals(), dg::PairSum(),
+                        f0, m_x[0], f1, m_x[1], f2, m_x[2]);
+                 break;
+            }
+            default: {
+                value_type f0 = 1./(m_t[0]-m_t[1]);
+                value_type f1 = 1./(m_t[1]-m_t[0]);
+                dg::blas1::axpby( f0, m_x[0], f1, m_x[1], dot_x);
+            }
+        }
     }
 
     /**
-    * @brief Extrapolate value
+    * @brief Extrapolate value (equidistant version)
     * @param new_x (write only) contains extrapolated value on output ( may alias the tail)
     * @note Assumes that extrapolation time equals last inserted time+1
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
@@ -393,6 +421,24 @@ struct Extrapolation
         value_type t = m_t[0] +1.;
         extrapolate( t, new_x);
     }
+    /**
+    * @brief Derive value (equidistant version)
+    * @param dot_x (write only) contains derived value on output
+    * @note Assumes that extrapolation time equals t0 + (t0 - t1)
+    * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
+    */
+    template<class ContainerType0>
+    void derive( ContainerType0& dot_x) const{
+        if ( 0 == m_number)
+            return;
+        if ( 1 == m_number)
+        {
+            dg::blas1::copy( 0, dot_x);
+            return;
+        }
+        value_type t = 2*m_t[0] - m_t[1];
+        derive( t, dot_x);
+    }
 
 
     /**
-- 
GitLab


From f4372b781330ef87d4ff15d4550f248c09c146cc Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 8 Oct 2020 00:01:42 +0200
Subject: [PATCH 353/540] Add a discussion and benchmark of swap and rotate

It really is not so much of an issue thanks to move assignments
so don't hesitate to use std::rotate or even std::swap in
favor of member swaps
---
 inc/dg/backend/memory.h     |  8 ++++++--
 inc/dg/backend/mpi_vector.h |  9 +++++++++
 inc/dg/blas_b.cu            | 16 ++++++++++++++++
 inc/dg/blas_mpib.cu         | 16 ++++++++++++++++
 inc/dg/chebyshev.h          |  9 ++++++---
 inc/dg/multistep.h          | 12 ++++--------
 inc/dg/runge_kutta.h        |  5 ++++-
 7 files changed, 61 insertions(+), 14 deletions(-)

diff --git a/inc/dg/backend/memory.h b/inc/dg/backend/memory.h
index 838d552ab..0e27b28e0 100644
--- a/inc/dg/backend/memory.h
+++ b/inc/dg/backend/memory.h
@@ -69,10 +69,14 @@ struct ClonePtr
     *
     * This follows the discussion in
     * https://stackoverflow.com/questions/5695548/public-friend-swap-member-function
-    * Use like
+    * @note the std library does call this function via unqualified calls in
+    * many algorithms, for example in std::iter_swap, it is just std::swap that
+    * will not call it directly. Most of the time this is not an issue because
+    * we have move assignments now, but if you do want to enable free swap
+    * functions like these use:
     * @code
     * using std::swap;
-    * swap( cloneptr1, cloneptr2);
+    * swap(a,b);
     * @endcode
     * @param first first instance
     * @param second second instance
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 680f39497..e3ea6f93e 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -106,6 +106,15 @@ struct MPI_Vector
     MPI_Comm m_comm, m_comm128, m_comm128Reduce;
 
 };
+///@cond
+//free function as required by the std to be swappable
+//https://en.cppreference.com/w/cpp/named_req/Swappable
+//even though with move assignments std::swap also works as fast
+template<class container>
+void swap( MPI_Vector<container>& a, MPI_Vector<container>& b){
+    a.swap(b);
+}
+///@endcond
 
 ///@addtogroup dispatch
 ///@{
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 788b490b7..953f5ce2e 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -253,5 +253,21 @@ int main()
         dg::blas1::axpby( 1., test_serial, 2., test_serial);//warm up
     t.toc();
     std::cout<<"serial axpby took                " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    std::cout << "\nUse of std::rotate and swap calls ( should not take any time)\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        std::rotate( x.rbegin(), x.rbegin()+1, x.rend()); //calls free swap functions
+    t.toc();
+    std::cout<<"Rotation        took             " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        std::swap( x[0], y[0]); //does not call free swap functions but uses move assignments which is just as fast
+    t.toc();
+    std::cout<<"std::sawp       took             " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        std::iter_swap( x.begin(), x.end()); //calls free swap functions
+    t.toc();
+    std::cout<<"Swap            took             " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
     return 0;
 }
diff --git a/inc/dg/blas_mpib.cu b/inc/dg/blas_mpib.cu
index 8df427f20..3cd097431 100644
--- a/inc/dg/blas_mpib.cu
+++ b/inc/dg/blas_mpib.cu
@@ -226,6 +226,22 @@ int main( int argc, char* argv[])
         norm += dg::blas2::dot( x, w2d, y);
     t.toc();
     if(rank==0)std::cout<<"DOT2(x,w,y) took                 " <<t.diff()/multi<<"s\t"<<3*gbytes*multi/t.diff()<<"GB/s\n"; //DOT should be faster than axpby since it is only loading vectors and not writing them
+    if(rank==0)std::cout << "\nUse of std::rotate and swap calls ( should not take any time)\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        std::rotate( x.rbegin(), x.rbegin()+1, x.rend()); //calls free swap functions
+    t.toc();
+    if(rank==0)std::cout<<"Rotation        took             " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        std::swap( x[0], y[0]); //does not call free swap functions but uses move assignments which is just as fast
+    t.toc();
+    if(rank==0)std::cout<<"std::swap       took             " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
+    t.tic();
+    for( int i=0; i<multi; i++)
+        std::iter_swap( x.begin(), x.end()); //calls free swap functions
+    t.toc();
+    if(rank==0)std::cout<<"Swap            took             " <<t.diff()/multi<<"s\t"<<gbytes*multi/t.diff()<<"GB/s\n";
 
     MPI_Finalize();
     return 0;
diff --git a/inc/dg/chebyshev.h b/inc/dg/chebyshev.h
index 10dbe0ad5..3c08c2fd6 100644
--- a/inc/dg/chebyshev.h
+++ b/inc/dg/chebyshev.h
@@ -127,7 +127,8 @@ class ChebyshevIteration
                              2.*rhok/delta,  b,
                             -2.*rhok/delta,  m_ax
                             );
-            x.swap(m_xm1);
+            using std::swap;
+            swap( x, m_xm1);
             rhokm1 = rhok;
         }
     }
@@ -190,7 +191,8 @@ class ChebyshevIteration
                              2.*rhok/delta,  m_z,
                             -rhok*rhokm1,    m_xm1
                             );
-            x.swap(m_xm1);
+            using std::swap;
+            swap( x, m_xm1);
             rhokm1 = rhok;
         }
     }
@@ -306,7 +308,8 @@ struct ModifiedChebyshevPreconditioner
             dg::blas1::axpby( 2., m_ax, -1., m_z2, m_z2); //T_k
             c_k *= (sqrt( m_ev_min/m_ev_max) - 1.)/(sqrt(m_ev_min/m_ev_max)+1);
             dg::blas1::axpby( c_k, m_z2, 1., y);
-            m_z1.swap(m_z2);
+            using std::swap;
+            swap(m_z1,m_z2);
         }
     }
     private:
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 9843d655a..548315cdb 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -146,8 +146,7 @@ void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType&
     for( unsigned i=0; i<m_k; i++)
         blas1::axpby( m_dt*m_ab[i], m_f[i], 1., m_u);
     //permute m_f[k-1]  to be the new m_f[0]
-    for( unsigned i=m_k-1; i>0; i--)
-        m_f[i-1].swap( m_f[i]);
+    std::rotate( m_f.rbegin(), m_f.rbegin()+1, m_f.rend());
     blas1::copy( m_u, u);
     t = m_tu = m_tu + m_dt;
     f( m_tu, m_u, m_f[0]); //evaluate f at new point
@@ -326,11 +325,8 @@ void Karniadakis<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, valu
     blas1::axpbypgz( m_dt*b[0], m_f[0], m_dt*b[1], m_f[1], m_dt*b[2], m_f[2]);
     blas1::axpbypgz( a[0], m_u[0], a[1], m_u[1], a[2], m_u[2]);
     //permute m_f[2], m_u[2]  to be the new m_f[0], m_u[0]
-    for( unsigned i=2; i>0; i--)
-    {
-        m_f[i-1].swap( m_f[i]);
-        m_u[i-1].swap( m_u[i]);
-    }
+    std::rotate( m_f.rbegin(), m_f.rbegin()+1, m_f.rend());
+    std::rotate( m_u.rbegin(), m_u.rbegin()+1, m_u.rend());
     blas1::axpby( 1., m_f[0], 1., m_u[0]);
     //compute implicit part
     value_type alpha[2] = {2., -1.};
@@ -465,7 +461,7 @@ void BDF<ContainerType, SolverType>::step(RHS& rhs, value_type& t, container_typ
         dg::blas1::axpby( alpha[0], m_u[0], alpha[1],  m_u[1], u);
     else
         dg::blas1::copy( m_u[0], u);
-    std::rotate(m_u.rbegin(), m_u.rbegin() + 1, m_u.rend()); //Rotate 1 to the right
+    std::rotate(m_u.rbegin(), m_u.rbegin() + 1, m_u.rend()); //Rotate 1 to the right (note the reverse iterator here!)
     m_solver.solve( -m_dt*m_beta, rhs, t, u, m_f);
     dg::blas1::copy( u, m_u[0]);
 }
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index ddda8b199..ae9ee3e59 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -229,7 +229,10 @@ void ERKStep<ContainerType>::step( RHS& f, value_type t0, const ContainerType& u
     if(!m_rk.isFsal() )
         f(t1,u1,m_k[0]);
     else
-        m_k[s-1].swap(m_k[0]);
+    {
+        using std::swap;
+        swap( m_k[0], m_k[s-1]); //enable free swap functions
+    }
 }
 
 
-- 
GitLab


From 847ea649a91921fb7a77246cfb8b811f361456f7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 8 Oct 2020 00:12:47 +0200
Subject: [PATCH 354/540] Test Extrapolation object

and fix a bug in the derive formula
This just shows once again that everything needs to be tested
---
 inc/dg/cg.h      | 34 +++++++++++++++-------------------
 inc/dg/cg2d_t.cu | 29 +++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+), 19 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 38a5bb20a..7d762557d 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -259,10 +259,10 @@ unsigned CG< ContainerType>::operator()( Matrix& A, ContainerType0& x, const Con
 ///@endcond
 
 /**
-* @brief Extrapolate based on up to three past solutions
+* @brief Extrapolate a polynomial passing through up to three points
 *
 * This class constructs an interpolating polynomial through up to three given points
-* and evaluates its value or its derivative at a new point.
+* and evaluates its value or its derivative at a new point. The points can be updated to get a new polynomial.
 *
 * The intention of this class is to provide an initial guess for iterative solvers
 * based on past solutions:
@@ -380,10 +380,13 @@ struct Extrapolation
     /**
     * @brief Evaluate first derivative of interpolating polynomial
     *
+    * Equivalent to constructing the interpolating polynomial, deriving it once
+    * and then evaluating it at the required point
     * @param t time at which derivative of interpolating polynomial is evaluated
     * @param dot_x (write only) contains derived value on output
-    * @note For equidistant values this is equivalent to computing the backward difference formula
-    * @attention If number==1, the result is 0 (derivative of a constant
+    * @note If t is chosen as the latest time of update t0, then the result coincides
+    * with the backward difference formula of order  \c number
+    * @attention If number==1, the result is 0 (derivative of a constant)
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
     */
     template<class ContainerType0>
@@ -397,10 +400,10 @@ struct Extrapolation
             case(3): {
                 value_type f0 =-(-2.*t+m_t[1]+m_t[2])/(m_t[0]-m_t[1])/(m_t[0]-m_t[2]);
                 value_type f1 = (-2.*t+m_t[0]+m_t[2])/(m_t[0]-m_t[1])/(m_t[1]-m_t[2]);
-                value_type f2 =-(2.*t+m_t[0]+m_t[1])/(m_t[2]-m_t[0])/(m_t[2]-m_t[1]);
+                value_type f2 =-(-2.*t+m_t[0]+m_t[1])/(m_t[2]-m_t[0])/(m_t[2]-m_t[1]);
                 dg::blas1::evaluate( dot_x, dg::equals(), dg::PairSum(),
                         f0, m_x[0], f1, m_x[1], f2, m_x[2]);
-                 break;
+                break;
             }
             default: {
                 value_type f0 = 1./(m_t[0]-m_t[1]);
@@ -418,26 +421,22 @@ struct Extrapolation
     */
     template<class ContainerType0>
     void extrapolate( ContainerType0& new_x) const{
+        if ( 0 == m_number)
+            return;
         value_type t = m_t[0] +1.;
         extrapolate( t, new_x);
     }
     /**
     * @brief Derive value (equidistant version)
     * @param dot_x (write only) contains derived value on output
-    * @note Assumes that extrapolation time equals t0 + (t0 - t1)
+    * @note Assumes that time equals t0 such that a backward difference formula will be evaluated
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
     */
     template<class ContainerType0>
     void derive( ContainerType0& dot_x) const{
         if ( 0 == m_number)
             return;
-        if ( 1 == m_number)
-        {
-            dg::blas1::copy( 0, dot_x);
-            return;
-        }
-        value_type t = 2*m_t[0] - m_t[1];
-        derive( t, dot_x);
+        derive( m_t[0], dot_x);
     }
 
 
@@ -458,11 +457,8 @@ struct Extrapolation
                 return;
             }
         //push out last value (keep track of what is oldest value
-        for (unsigned u=m_number-1; u>0; u--)
-        {
-            std::swap( m_t[u], m_t[u-1]);
-            m_x[u].swap( m_x[u-1]);
-        }
+        std::rotate( m_x.rbegin(), m_x.rbegin()+1, m_x.rend());
+        std::rotate( m_t.rbegin(), m_t.rbegin()+1, m_t.rend());
         m_t[0] = t_new;
         blas1::copy( new_entry, m_x[0]);
     }
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index bd16ec1de..120ebe99e 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -158,6 +158,35 @@ int main()
         res.d = sqrt(dg::blas2::dot( w2d, resi));
         std::cout << "L2 Norm of Residuum is        " << res.d<<"\n\n";
     }
+    // Test Extrapolation object
+    dg::Extrapolation<double> extra(3);
+    extra.update( 0, 0);
+    extra.update( 1, 1);
+    extra.update( 3, 9);
+    extra.update( 2, 4);
+    double value;
+    extra.extrapolate( 5, value);
+    std::cout << "Extrapolated value is "<<value<< " (25)\n";
+    extra.derive( 4, value);
+    std::cout << "Derived value is "<<value<< " (8)\n";
+    extra.set_number(2);
+    extra.update( 0, 0);
+    extra.update( 1, 1);
+    extra.update( 3, 9);
+    extra.update( 2, 4);
+    extra.extrapolate( 5, value);
+    std::cout << "Linear Extrapolated value is "<<value<< " (19)\n";
+    extra.derive( 4, value);
+    std::cout << "Linear Derived value is "<<value<< " (5)\n";
+    extra.set_number(1);
+    extra.update( 0, 0);
+    extra.update( 1, 1);
+    extra.update( 3, 9);
+    extra.update( 2, 4);
+    extra.extrapolate( 5, value);
+    std::cout << "Monomial Extrapolated value is "<<value<< " (4)\n";
+    extra.derive( 4, value);
+    std::cout << "Monomial Derived value is "<<value<< " (0)\n";
 
 
     return 0;
-- 
GitLab


From d346ee10d79fe2830f1388a572620fecffca09c0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 8 Oct 2020 13:33:29 +0200
Subject: [PATCH 355/540] Update the manufactured solution

the Aparallel terms still are not very nicely testable but
at least without it the perp terms seem to converge
---
 src/feltor/feltor.h                |   32 +-
 src/feltor/feltor.tex              |    8 +-
 src/feltor/implicit.h              |   21 +-
 src/feltor/input/manufactured.json |    9 +-
 src/feltor/manufactured.cu         |   14 +-
 src/feltor/manufactured.h          | 2012 ++++++----------------------
 6 files changed, 436 insertions(+), 1660 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 15f1d66f9..7480a589c 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -658,7 +658,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_phi(
         dg::blas1::copy( y[1], m_temp1);
         dg::blas1::evaluate( m_temp1, dg::plus_equals(), manufactured::SGammaNi{
             m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-            m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
+            m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,time);
         std::vector<unsigned> numberG = m_multigrid.direct_solve(
             m_multi_invgammaN, m_temp0, m_temp1, m_p.eps_gamma);
 #else
@@ -673,7 +673,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_phi(
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( m_temp0, dg::plus_equals(), manufactured::SPhie{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-        m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
+        m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,time);
 #endif //DG_MANUFACTURED
     //----------Invert polarisation----------------------------//
     m_old_phi.extrapolate( time, m_phi[0]);
@@ -697,7 +697,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
         dg::blas1::copy( m_phi[0], m_temp0);
         dg::blas1::evaluate( m_temp0, dg::plus_equals(), manufactured::SGammaPhie{
             m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-            m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
+            m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,time);
         std::vector<unsigned> number = m_multigrid.direct_solve(
             m_multi_invgammaP, m_phi[1], m_temp0, m_p.eps_gamma);
 #else
@@ -719,7 +719,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( m_phi[1], dg::plus_equals(), manufactured::SPhii{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-        m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
+        m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,time);
 #endif //DG_MANUFACTURED
     //m_UE2 now contains u_E^2; also update derivatives
     dg::blas2::symv( m_dx_P, m_phi[1], m_dP[1][0]);
@@ -746,16 +746,20 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_apar(
                               0., m_temp0);
     //----------Invert Induction Eq----------------------------//
     m_old_apar.extrapolate( time, m_apar);
-#ifdef DG_MANUFACTURED
-    dg::blas1::evaluate( m_temp0, dg::plus_equals(), manufactured::SA{
-        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-        m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,time);
-#endif //DG_MANUFACTURED
     std::vector<unsigned> number = m_multigrid.direct_solve(
         m_multi_induction, m_apar, m_temp0, m_p.eps_pol[0]);
     m_old_apar.update( time, m_apar);
     if(  number[0] == m_multigrid.max_iter())
         throw dg::Fail( m_p.eps_pol[0]);
+#ifdef DG_MANUFACTURED
+    //dg::blas1::evaluate( m_temp0, dg::plus_equals(), manufactured::SA{
+    //    m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
+    //    m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,time);
+    //here we cheat (a bit)
+    dg::blas1::evaluate( m_apar, dg::equals(), manufactured::A{
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
+        m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,time);
+#endif //DG_MANUFACTURED
     //----------Compute Derivatives----------------------------//
     dg::blas2::symv( m_dx_U, m_apar, m_dA[0]);
     dg::blas2::symv( m_dy_U, m_apar, m_dA[1]);
@@ -989,25 +993,25 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
                 dg::blas2::symv( -m_p.nu_perp, m_lapperpU,
                     m_fields[1][i],  1., yp[1][i]);
         }
-#endif
         //------------------Add Resistivity--------------------------//
         dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
             m_fields[0][0], m_fields[0][1],
             m_fields[1][0], m_fields[1][1], yp[1][0], yp[1][1]);
+#endif
     }
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( yp[0][0], dg::plus_equals(), manufactured::SNe{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-        m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
+        m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
     dg::blas1::evaluate( yp[0][1], dg::plus_equals(), manufactured::SNi{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-        m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
+        m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
     dg::blas1::evaluate( yp[1][0], dg::plus_equals(), manufactured::SUe{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-        m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
+        m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
     dg::blas1::evaluate( yp[1][1], dg::plus_equals(), manufactured::SUi{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-        m_p.beta,m_p.nu_perp,m_p.nu_parallel},m_R,m_Z,m_P,t);
+        m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
 #endif //DG_MANUFACTURED
     timer.toc();
     accu += timer.diff();
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index a2fbc7aec..08f665661 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -763,9 +763,9 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
     + \mu NU_\parallel\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
     =& -\tau \left(\bhat + {\vec b}_\perp\right)\cn N
     -N \left( \left(\bhat+{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right)
-    - \eta n_e^2(U_{\parallel,i}-u_{\parallel,e}) + \mu \nu_\parallel \Delta_\parallel U
+    - \eta n_e^2(U_{\parallel,i}-u_{\parallel,e}) 
     \nonumber\\
-    &+ \mu N\left(\Lambda_U + S_U\right) + \mu U_\parallel \left(\Lambda_N + S_N\right)
+    &+ \mu \nu_\parallel \Delta_\parallel U+ \mu N\left(\Lambda_U + S_U\right) + \mu U_\parallel \left(\Lambda_N + S_N\right)
 \label{}
 \end{align}
 with
@@ -1033,9 +1033,9 @@ two functions for which we have no boundary conditions
         -\tau U_\parallel\nc\vec{ \mathcal K_{\vn\times\bhat}}\nonumber\\&
         - \left(2\tau + {\mu}U_\parallel^2\right) \mathcal K_{\vn\times\bhat} (U_\parallel)
         -2\tau U_\parallel\mathcal K_{\vn\times\bhat}(\ln N)
-        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_{\parallel,i} - u_{\parallel,e})+ \frac{\nu_\parallel}{N} \Delta_\parallel U_\parallel
+        - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_{\parallel,i} - u_{\parallel,e})
  \nonumber\\&
-        - \nu_\perp\Delta_\perp^2 U_\parallel
+        + \frac{\nu_\parallel}{N} \Delta_\parallel U_\parallel - \nu_\perp\Delta_\perp^2 U_\parallel
                 + S_U,
         \label{eq:EgyrofluidU} \\
         W_\parallel&:= \left( U_\parallel + \frac{A_\parallel}{\mu}\right)
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index a556e6488..769b65305 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -90,6 +90,11 @@ struct ImplicitVelocity
     void construct( const Geometry& g, feltor::Parameters p,
             dg::geo::TokamakMagneticField mag)
     {
+#ifdef DG_MANUFACTURED
+        m_R= dg::pullback( dg::cooX3d, g);
+        m_Z= dg::pullback( dg::cooY3d, g);
+        m_P= dg::pullback( dg::cooZ3d, g);
+#endif //DG_MANUFACTURED
         m_p=p;
         m_lapM_perpU.construct( g, p.bcxU,p.bcyU,dg::PER,
             dg::normed, dg::centered);
@@ -158,6 +163,12 @@ struct ImplicitVelocity
         dg::blas1::copy( w, m_fields[1]);
         if( m_p.beta != 0){
             //let us solve for apar
+#ifdef DG_MANUFACTURED
+            //here we cheat (a bit)
+            dg::blas1::evaluate( m_apar, dg::equals(), manufactured::A{
+                m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
+                m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
+#else
             dg::blas1::pointwiseDot(  m_p.beta, m_fields[0][1], m_fields[1][1],
                                      -m_p.beta, m_fields[0][0], m_fields[1][0],
                                       0., m_temp);
@@ -170,6 +181,7 @@ struct ImplicitVelocity
             //m_old_apar.update( m_apar); //don't update here: makes the solver potentially unstable
             if(  number[0] == m_multigrid.max_iter())
                 throw dg::Fail( m_p.eps_pol[0]);
+#endif //DG_MANUFACTURED
 
             //compute u_e and U_i from w_e, W_i and apar
             dg::blas1::axpby( 1., m_fields[1][0], -1./m_p.mu[0],
@@ -191,13 +203,13 @@ struct ImplicitVelocity
                 dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU,
                     m_fields[1][i],  0., wp[i]);
         }
-#else
-        dg::blas1::copy( 0, wp);
-#endif
         //------------------Add Resistivity--------------------------//
         dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
             m_fields[0][0], m_fields[0][1],
             m_fields[1][0], m_fields[1][1], wp[0], wp[1]);
+#else
+        dg::blas1::copy( 0, wp);
+#endif
     }
 
     const Container& weights() const{
@@ -221,6 +233,9 @@ struct ImplicitVelocity
     std::vector<Container> m_multi_chi;
     std::array<std::array<Container,2>,2> m_fields;
     dg::Elliptic3d<Geometry, Matrix, Container> m_lapM_perpU;
+#ifdef DG_MANUFACTURED
+    Container m_R, m_Z, m_P; //coordinates
+#endif //DG_MANUFACTURED
 };
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
diff --git a/src/feltor/input/manufactured.json b/src/feltor/input/manufactured.json
index 47de7611d..809324237 100644
--- a/src/feltor/input/manufactured.json
+++ b/src/feltor/input/manufactured.json
@@ -2,7 +2,7 @@
     "n"  : 3,
     "Nx" : 64,
     "Ny" : 64,
-    "Nz" : 32,
+    "Nz" : 7,
     "dt" : 1e-4,
     "compression" : [2,2],
     "FCI":
@@ -22,12 +22,11 @@
     "rtol"       : 1e-4,
     "mu"          : -0.000272121,
     "tau"         : 0,
-    "beta"        : 10,
+    "beta"        : 0,
     "nu_perp"     : 0.1,
     "perp_diff"   : "viscous",
-    "nu_parallel" : 0,
-    "resistivity" : 0,
-    "curvmode"   : "true",
+    "resistivity" : 0.1,
+    "curvmode"   : "toroidal",
     "symmetric"  : false,
     "bc" :
     {
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 75ecf4fff..333f78e5e 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -46,19 +46,19 @@ int main( int argc, char* argv[])
     feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
 
     feltor::manufactured::Ne ne{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
-                                 p.beta,p.nu_perp,p.nu_parallel};
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
     feltor::manufactured::Ni ni{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
-                                 p.beta,p.nu_perp,p.nu_parallel};
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
     feltor::manufactured::Ue ue{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
-                                p.beta,p.nu_perp,p.nu_parallel};
+                                p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
     feltor::manufactured::Ui ui{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
-                                 p.beta,p.nu_perp,p.nu_parallel};
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
     feltor::manufactured::Phie phie{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
-                                     p.beta,p.nu_perp,p.nu_parallel};
+                                     p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
     feltor::manufactured::Phii phii{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
-                                     p.beta,p.nu_perp,p.nu_parallel};
+                                     p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
     feltor::manufactured::A aa{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
-                                p.beta,p.nu_perp,p.nu_parallel};
+                                p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
 
     dg::DVec R = dg::pullback( dg::cooX3d, grid);
     dg::DVec Z = dg::pullback( dg::cooY3d, grid);
diff --git a/src/feltor/manufactured.h b/src/feltor/manufactured.h
index 0a35a398f..dc31da535 100644
--- a/src/feltor/manufactured.h
+++ b/src/feltor/manufactured.h
@@ -14,886 +14,292 @@ DG_DEVICE double Sin(double x){ return sin(x);}
 DG_DEVICE double Cos(double x){ return cos(x);}
 static const double Pi = M_PI;
 struct Ne{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return 1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)
 ; }};
 struct Ni{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return 1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)
 ; }};
 struct Ue{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue))
 ; }};
 struct Ui{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3.
 ; }};
 struct Phie{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return (Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.
 ; }};
 struct Phii{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return (Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.
 ; }};
 struct GammaPhie{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return (Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.
 ; }};
 struct GammaNi{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return 1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)
 ; }};
 struct A{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return (beta*Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/4.
 ; }};
 struct SNe{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return 1.5707963267948966*Cos(Pi*t)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*Z) + 
-  FELTORPARALLEL*(-((nuparallel*
-          (0. + (1.*Z*((-1.5707963267948966*(-10 + R)*Cos(Pi*Z)*Sin(P)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (1.5707963267948966*Z*Cos(Pi*(-10 + R))*Sin(P)*
-                    Sin(Pi*t)*Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (10.*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                  (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-             Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-            (1.*(-10 + R)*R*((4.934802200544679*Z*Cos(Pi*(-10 + R))*
-                    Cos(Pi*Z)*Sin(P)*Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (31.41592653589793*Cos(P)*Cos(Pi*Z)*Sin(Pi*(-10 + R))*
-                    Sin(Pi*t))/
-                  (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-                 (1.5707963267948966*(-10 + R)*Z*Cos(Pi*Z)*Sin(P)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-                 (1.5707963267948966*Power(Z,2)*Cos(Pi*(-10 + R))*
-                    Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
-                 (1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*
-                    Sin(Pi*t)*Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-                 (10.*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                  (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
-                     1.5)) + (4.934802200544679*(-10 + R)*Sin(P)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
-             Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-            (20*((-1.5707963267948966*(-10 + R)*Cos(P)*Cos(Pi*Z)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (1.5707963267948966*Z*Cos(P)*Cos(Pi*(-10 + R))*
-                    Sin(Pi*t)*Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-                 (10.*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                  (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-             Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-            (1.*R*Z*((-4.934802200544679*(-10 + R)*Cos(Pi*(-10 + R))*
-                    Cos(Pi*Z)*Sin(P)*Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (1.5707963267948966*Power(-10 + R,2)*Cos(Pi*Z)*Sin(P)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-                 (1.5707963267948966*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*
-                    Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (31.41592653589793*Cos(P)*Cos(Pi*(-10 + R))*Sin(Pi*t)*
-                    Sin(Pi*Z))/
-                  (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-                 (1.5707963267948966*(-10 + R)*Z*Cos(Pi*(-10 + R))*
-                    Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-                 (10.*(-10 + R)*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                    Sin(Pi*Z))/
-                  (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
-                     1.5)) - (10.*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                    Sin(Pi*Z))/
-                  (Power(R,2)*
-                    Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-                 (4.934802200544679*Z*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                    Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
-             Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R) + 
-     (0. - (2.0943951023931953*(-10 + R)*R*Cos(2*Pi*Z)*Sin(2*P)*
-           Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
-         (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-        (0.5235987755982988*(-10 + R)*R*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-           Sin(Pi*(-10 + R))*Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*
-           Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (0.5235987755982988*R*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
-           Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z)\
-)/(Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (3.3333333333333335*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
-           Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z)\
-)/(Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (2.0943951023931953*R*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
-           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-           Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-           Sin(2*Pi*Z))/
-         (3.*Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (0.3333333333333333*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-           Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R) + 
-  FELTORPERP*(-((nuperp*((1.5707963267948966*(-10 + R)*Z*Cos(Pi*Z)*Sin(P)*
-               Sin(Pi*(-10 + R))*Sin(Pi*t))/
-             (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-            1.5707963267948966*
-             (1 - (1.*Power(Z,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-             Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
-            (10.*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-             (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-            R*((4.934802200544679*(-10 + R)*Z*Cos(Pi*(-10 + R))*
-                  Cos(Pi*Z)*Sin(P)*Sin(Pi*t))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (31.41592653589793*(-10 + R)*Cos(P)*Cos(Pi*Z)*
-                  Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-               (3.141592653589793*Power(-10 + R,2)*Z*Cos(Pi*Z)*Sin(P)*
-                  Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-               (3.141592653589793*(-10 + R)*Power(Z,2)*Cos(Pi*(-10 + R))*
-                  Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-               (1.5707963267948966*(-10 + R)*Cos(Pi*(-10 + R))*Sin(P)*
-                  Sin(Pi*t)*Sin(Pi*Z))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (20.*(-10 + R)*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                  Sin(Pi*Z))/
-                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) - 
-               4.934802200544679*
-                (1 - (1.*Power(-10 + R,2))/
-                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(P)*
-                Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)) + 
-            R*((31.41592653589793*(-10 + R)*Cos(P)*Cos(Pi*Z)*
-                  Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               (31.41592653589793*Z*Cos(P)*Cos(Pi*(-10 + R))*Sin(Pi*t)*
-                  Sin(Pi*Z))/
-                (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               0.5*(Power(R,-2) - 
-                  400/
-                   (Power(R,2)*
-                     (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*Sin(P)*
-                Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)) + 
-            R*((4.934802200544679*(-10 + R)*Z*Cos(Pi*(-10 + R))*Cos(Pi*Z)*
-                  Sin(P)*Sin(Pi*t))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (3.141592653589793*Power(-10 + R,2)*Z*Cos(Pi*Z)*Sin(P)*
-                  Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-               (1.5707963267948966*Z*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*
-                  Sin(Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (31.41592653589793*Z*Cos(P)*Cos(Pi*(-10 + R))*Sin(Pi*t)*
-                  Sin(Pi*Z))/
-                (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-               (3.141592653589793*(-10 + R)*Power(Z,2)*Cos(Pi*(-10 + R))*
-                  Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-               (20.*(-10 + R)*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                  Sin(Pi*Z))/
-                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-               (10.*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               4.934802200544679*
-                (1 - (1.*Power(Z,2))/
-                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(P)*
-                Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))))/R) + 
-     (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-         ((-4.*taue*Z*(-1000. + 20.*R - 2.*Power(Z,2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
-           (4.*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (3*Pi*Cos(3*Pi*Z)*(3.7699111843077517*R*Cos(3*Pi*R)*
-                 Sin(3*P) - 0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (27.925268031909273*(-50. + 1.*R - 0.1*Power(Z,2))*Cos(2*Pi*Z)*
-              Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
+  (FELTORPARALLEL*(0. - (2.0943951023931953*(-10 + R)*R*Cos(2*Pi*Z)*
+          Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+       (0.5235987755982988*(-10 + R)*R*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+          Sin(Pi*(-10 + R))*Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*
+          Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+       (0.5235987755982988*R*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
+          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+       (3.3333333333333335*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
+          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+       (2.0943951023931953*R*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
+          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+          Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+       (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+          Sin(2*Pi*Z))/
+        (3.*Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+       (0.3333333333333333*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+          Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/R + 
+  FELTORPERP*(-((nuperp*(0. + 1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*
+             Sin(Pi*t)*Sin(Pi*Z) - 
+            9.869604401089358*R*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)\
+))/R) + (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+         ((0.15000000000000002*R*(-20. + 2.*R)*taue*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
+           (0.1*taue*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (3*Pi*R*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
+              Sin(3*Pi*t))/
+            (100.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) \
+- (3*Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (2*beta*Power(Pi,2)*R*Cos(4*Pi*R)*Cos(4*Pi*Z)*Sin(2*P)*
+              Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
               Sin(2*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (4*beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(-3141.592653589793 + 125.66370614359172*R - 
-                   6.283185307179586*Power(R,2) - 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2)) - 
-           (80.*Z*(-50. + 1.*R - 0.1*Power(Z,2))*
-              (1.*taue - 0.1111111111111111*Power(Sin(2*P),2)*
-                 Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
-           (4.*Z*(1.*taue - 0.1111111111111111*Power(Sin(2*P),2)*
-                 Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (0.06*Cos(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (2.*Z*(3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
-                0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (2*beta*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(-3141.592653589793 + 125.66370614359172*R - 
-                   6.283185307179586*Power(R,2) - 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2)) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (-12.566370614359172*R*Z*Cos(4*Pi*R)*Sin(4*P) + 
-                (0.2*Power(Z,2)*Cos(4*P) + 
-                   (50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) - 1.*Z*Sin(4*P))*Sin(4*Pi*R))*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2)) - 
-           (1.3333333333333333*beta*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(-3141.592653589793 + 125.66370614359172*R - 
-                   6.283185307179586*Power(R,2) - 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              Sin(4*Pi*Z))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               3))) + 1.5707963267948966*R*Cos(Pi*Z)*Sin(P)*
-         Sin(Pi*(-10 + R))*Sin(Pi*t)*
-         ((taue*(-1000. + 20.*R - 2.*Power(Z,2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (20.*(-50. + 1.*R - 0.1*Power(Z,2))*
-              (1.*taue - 0.1111111111111111*Power(Sin(2*P),2)*
-                 Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           ((3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
-                0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(-3141.592653589793 + 125.66370614359172*R - 
-                   6.283185307179586*Power(R,2) - 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              Sin(4*Pi*Z))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2))) + 
-        R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-         ((4.*R*(-20. + 2.*R)*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
-           (2.*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (2.792526803190927*R*Z*Cos(2*Pi*R)*Power(Sin(2*P),2)*
-              Sin(2*Pi*R)*Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (2*R*(-20. + 2.*R)*Z*
-              (-2.*taue + 0.2222222222222222*Power(Sin(2*P),2)*
-                 Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) + 
-           (Z*(-2.*taue + 0.2222222222222222*Power(Sin(2*P),2)*
-                 Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (Sin(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*Cos(3*Pi*Z)*Sin(3*P) - 
-                0.06*Cos(3*P)*Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (3*Pi*Cos(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           ((-20. + 2.*R)*Sin(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(-125.66370614359172 + 12.566370614359172*R)*
-                 Cos(4*Pi*Z)*Sin(4*P) + 
-                (3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((70. - 6.*R + 0.30000000000000004*Power(R,2) + 
-                      0.1*Power(Z,2))*Cos(4*P) - 0.5*Z*Sin(4*P))*
-                 Sin(4*Pi*Z)))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2)) + 
-           (4*beta*Pi*Cos(4*Pi*R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2)) + 
-           (2*beta*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(4*Pi*R)*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2)) - 
-           (2*beta*(-20. + 2.*R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),3))) + 
+            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))) + 
+           (beta*Power(Pi,2)*R*Cos(2*Pi*(-10 + R))*Cos(4*Pi*Z)*Sin(2*P)*
+              Sin(4*P)*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))) - 
+           (beta*Pi*R*(-20. + 2.*R)*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*
+              Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*
+              Sin(2*Pi*Z))/
+            (60.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),1.5)) + 
+           (beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2)))) + 
         1.5707963267948966*R*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z)*
-         (0. - (2.*R*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (R*Z*(-2.*taue + 0.2222222222222222*Power(Sin(2*P),2)*
-                 Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (Sin(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2))) + 
+         ((-0.1*R*taue*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2)))) + 
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-         (0. - (2.*R*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (R*Z*(-2.*taue + 0.2222222222222222*Power(Sin(2*P),2)*
-                 Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (Sin(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2))) + 
+         ((-0.1*R*taue*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2)))) + 
         R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-         ((0.044444444444444446*
-              (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(2*P)*
-              Sin(2*P)*Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-              Power(Sin(2*Pi*Z),2))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (3*Cos(3*P)*Sin(3*Pi*t)*
-              (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
-                (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
-                 Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (4*beta*Cos(4*P)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                   0.3141592653589793*Power(R,2) - 
-                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) \
-+ ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                      0.3141592653589793*Power(R,3) + 
-                      3.141592653589793*Power(Z,2) + 
-                      R*(-219.9114857512855 - 
-                        0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
-                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2)) + 
-           (2*beta*Cos(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                   0.3141592653589793*Power(R,2) - 
-                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) \
-+ ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                      0.3141592653589793*Power(R,3) + 
-                      3.141592653589793*Power(Z,2) + 
-                      R*(-219.9114857512855 - 
-                         0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
-                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2))) + 
-        0.5*R*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)*
-         ((taue*(250000.00000000006 - 80.*Power(R,3) + 1.*Power(R,4) + 
-                1000.0000000000001*Power(Z,2) + 1.*Power(Z,4) + 
-                R*(-40000. - 80.*Power(Z,2)) + 
-                Power(R,2)*(2200. + 2.*Power(Z,2))))/
-            (R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3)) - 
-           (0.1*(900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              (1.*taue - 0.1111111111111111*Power(Sin(2*P),2)*
-                 Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (Sin(3*P)*Sin(3*Pi*t)*
-              (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
-                (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
-                 Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                   0.3141592653589793*Power(R,2) - 
-                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-                ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                      0.3141592653589793*Power(R,3) + 
-                      3.141592653589793*Power(Z,2) + 
-                      R*(-219.9114857512855 - 
-                         0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
-                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-            (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2))))/R)
+         ((-3.*taue*Z*(-50. + 1.*R - 0.1*Power(Z,2)))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
+           (0.2*taue*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (2*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (45.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+              Sin(2*Pi*Z))/
+            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))) - 
+           (0.011111111111111112*Z*
+              (-9*taue + Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (0.18849555921538758*R*Z*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*
+              Sin(3*Pi*Z))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (beta*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+              Sin(4*Pi*Z))/
+            (60.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))) + 
+           (0.008333333333333333*beta*Z*Sin(2*P)*Sin(4*P)*
+              Sin(2*Pi*(-10 + R))*(4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*
+              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
+            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
+               1.5))) + 1.5707963267948966*R*Cos(Pi*Z)*Sin(P)*
+         Sin(Pi*(-10 + R))*Sin(Pi*t)*
+         ((taue*(-50. + 1.*R - 0.1*Power(Z,2)))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (-9*taue + Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+               Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+            (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+              Sin(2*Pi*Z)*Sin(4*Pi*Z))/
+            (120.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2)))))/R)
 ; }};
 struct SNi{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return 1.5707963267948966*Cos(Pi*t)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*Z) + 
-  FELTORPARALLEL*(-((nuparallel*
-          (0. + (1.*Z*((-1.5707963267948966*(-10 + R)*Cos(Pi*Z)*Sin(P)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (1.5707963267948966*Z*Cos(Pi*(-10 + R))*Sin(P)*
-                    Sin(Pi*t)*Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (10.*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                  (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-             Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-            (1.*(-10 + R)*R*((4.934802200544679*Z*Cos(Pi*(-10 + R))*
-                    Cos(Pi*Z)*Sin(P)*Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (31.41592653589793*Cos(P)*Cos(Pi*Z)*Sin(Pi*(-10 + R))*
-                    Sin(Pi*t))/
-                  (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-                 (1.5707963267948966*(-10 + R)*Z*Cos(Pi*Z)*Sin(P)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-                 (1.5707963267948966*Power(Z,2)*Cos(Pi*(-10 + R))*
-                    Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
-                 (1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*
-                    Sin(Pi*t)*Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-                 (10.*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                  (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
-                     1.5)) + (4.934802200544679*(-10 + R)*Sin(P)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
-             Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-            (20*((-1.5707963267948966*(-10 + R)*Cos(P)*Cos(Pi*Z)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (1.5707963267948966*Z*Cos(P)*Cos(Pi*(-10 + R))*
-                    Sin(Pi*t)*Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-                 (10.*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                  (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-             Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-            (1.*R*Z*((-4.934802200544679*(-10 + R)*Cos(Pi*(-10 + R))*
-                    Cos(Pi*Z)*Sin(P)*Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (1.5707963267948966*Power(-10 + R,2)*Cos(Pi*Z)*Sin(P)*
-                    Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-                 (1.5707963267948966*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*
-                    Sin(Pi*t))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-                 (31.41592653589793*Cos(P)*Cos(Pi*(-10 + R))*Sin(Pi*t)*
-                    Sin(Pi*Z))/
-                  (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-                 (1.5707963267948966*(-10 + R)*Z*Cos(Pi*(-10 + R))*
-                    Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-                 (10.*(-10 + R)*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                    Sin(Pi*Z))/
-                  (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
-                     1.5)) - (10.*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                    Sin(Pi*Z))/
-                  (Power(R,2)*
-                    Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-                 (4.934802200544679*Z*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                    Sin(Pi*Z))/
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
-             Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R) + 
-     (0. - (2.0943951023931953*(-10 + R)*R*Cos(2*Pi*Z)*Sin(2*P)*
-           Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
-         Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-        (0.5235987755982988*(-10 + R)*R*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-           Sin(Pi*(-10 + R))*Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*
-           Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-        (0.5235987755982988*R*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
-           Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z)\
-)/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-        (3.3333333333333335*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
-           Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z)\
-)/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-        (2.0943951023931953*R*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
-           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-           Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-        (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-           Sin(2*Pi*Z))/
-         (3.*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (0.3333333333333333*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-           Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))/R) + 
-  FELTORPERP*(-((nuperp*((1.5707963267948966*(-10 + R)*Z*Cos(Pi*Z)*Sin(P)*
-               Sin(Pi*(-10 + R))*Sin(Pi*t))/
-             (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-            1.5707963267948966*
-             (1 - (1.*Power(Z,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-             Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
-            (10.*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-             (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-            R*((4.934802200544679*(-10 + R)*Z*Cos(Pi*(-10 + R))*
-                  Cos(Pi*Z)*Sin(P)*Sin(Pi*t))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (31.41592653589793*(-10 + R)*Cos(P)*Cos(Pi*Z)*
-                  Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-               (3.141592653589793*Power(-10 + R,2)*Z*Cos(Pi*Z)*Sin(P)*
-                  Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-               (3.141592653589793*(-10 + R)*Power(Z,2)*Cos(Pi*(-10 + R))*
-                  Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-               (1.5707963267948966*(-10 + R)*Cos(Pi*(-10 + R))*Sin(P)*
-                  Sin(Pi*t)*Sin(Pi*Z))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (20.*(-10 + R)*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                  Sin(Pi*Z))/
-                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) - 
-               4.934802200544679*
-                (1 - (1.*Power(-10 + R,2))/
-                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(P)*
-                Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)) + 
-            R*((31.41592653589793*(-10 + R)*Cos(P)*Cos(Pi*Z)*
-                  Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               (31.41592653589793*Z*Cos(P)*Cos(Pi*(-10 + R))*Sin(Pi*t)*
-                  Sin(Pi*Z))/
-                (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               0.5*(Power(R,-2) - 
-                  400/
-                   (Power(R,2)*
-                     (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*Sin(P)*
-                Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)) + 
-            R*((4.934802200544679*(-10 + R)*Z*Cos(Pi*(-10 + R))*Cos(Pi*Z)*
-                  Sin(P)*Sin(Pi*t))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (3.141592653589793*Power(-10 + R,2)*Z*Cos(Pi*Z)*Sin(P)*
-                  Sin(Pi*(-10 + R))*Sin(Pi*t))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-               (1.5707963267948966*Z*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*
-                  Sin(Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (31.41592653589793*Z*Cos(P)*Cos(Pi*(-10 + R))*Sin(Pi*t)*
-                  Sin(Pi*Z))/
-                (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-               (3.141592653589793*(-10 + R)*Power(Z,2)*Cos(Pi*(-10 + R))*
-                  Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-               (20.*(-10 + R)*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-                  Sin(Pi*Z))/
-                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-               (10.*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-                (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               4.934802200544679*
-                (1 - (1.*Power(Z,2))/
-                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(P)*
-                Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))))/R) + 
-     (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-         ((-4.*taui*Z*(-1000. + 20.*R - 2.*Power(Z,2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
-           (4.*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (3*Pi*Cos(3*Pi*Z)*(3.7699111843077517*R*Cos(3*Pi*R)*
-                 Sin(3*P) - 0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (27.925268031909273*mui*(-50. + 1.*R - 0.1*Power(Z,2))*
-              Cos(2*Pi*Z)*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (4*beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(-3141.592653589793 + 125.66370614359172*R - 
-                   6.283185307179586*Power(R,2) - 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) - 
-           (8.88888888888889*Z*(-50. + 1.*R - 0.1*Power(Z,2))*
-              (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
-           (0.4444444444444445*Z*
-              (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (0.06*Cos(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (2.*Z*(3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
-                0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (2*beta*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(-3141.592653589793 + 125.66370614359172*R - 
-                   6.283185307179586*Power(R,2) - 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (-12.566370614359172*R*Z*Cos(4*Pi*R)*Sin(4*P) + 
-                (0.2*Power(Z,2)*Cos(4*P) + 
-                   (50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) - 1.*Z*Sin(4*P))*Sin(4*Pi*R))*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) - 
-           (1.3333333333333333*beta*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(-3141.592653589793 + 125.66370614359172*R - 
-                   6.283185307179586*Power(R,2) - 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              Sin(4*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3)) + 
-        1.5707963267948966*R*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-         ((taui*(-1000. + 20.*R - 2.*Power(Z,2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (2.2222222222222223*(-50. + 1.*R - 0.1*Power(Z,2))*
-              (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           ((3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
-                0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              (R*(-3141.592653589793 + 125.66370614359172*R - 
-                   6.283185307179586*Power(R,2) - 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-                (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                    Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                 Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              Sin(4*Pi*Z))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))) + 
-        R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-         ((4.*R*(-20. + 2.*R)*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
-           (2.*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (2.792526803190927*mui*R*Z*Cos(2*Pi*R)*Power(Sin(2*P),2)*
-              Sin(2*Pi*R)*Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (0.4444444444444444*R*(-20. + 2.*R)*Z*
-              (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
-           (0.2222222222222222*Z*
-              (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (Sin(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*Cos(3*Pi*Z)*Sin(3*P) - 
-                0.06*Cos(3*P)*Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (3*Pi*Cos(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           ((-20. + 2.*R)*Sin(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(-125.66370614359172 + 12.566370614359172*R)*
-                 Cos(4*Pi*Z)*Sin(4*P) + 
-                (3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((70. - 6.*R + 0.30000000000000004*Power(R,2) + 
-                      0.1*Power(Z,2))*Cos(4*P) - 0.5*Z*Sin(4*P))*
-                 Sin(4*Pi*Z)))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
-           (4*beta*Pi*Cos(4*Pi*R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
-           (2*beta*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(4*Pi*R)*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) - 
-           (2*beta*(-20. + 2.*R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3))) + 
+  (FELTORPARALLEL*(0. - (2.0943951023931953*(-10 + R)*R*Cos(2*Pi*Z)*
+          Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
+        Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+       (0.5235987755982988*(-10 + R)*R*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+          Sin(Pi*(-10 + R))*Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*
+          Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+       (0.5235987755982988*R*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
+          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+        Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+       (3.3333333333333335*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
+          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+        Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+       (2.0943951023931953*R*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
+          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+          Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+       (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+          Sin(2*Pi*Z))/(3.*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) \
++ (0.3333333333333333*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+          Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R + 
+  FELTORPERP*(-((nuperp*(0. + 1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*
+             Sin(Pi*t)*Sin(Pi*Z) - 
+            9.869604401089358*R*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)\
+))/R) + (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+         ((0.15000000000000002*R*(-20. + 2.*R)*taui*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
+           (0.1*taui*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (3*Pi*R*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
+              Sin(3*Pi*t))/
+            (100.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) \
+- (3*Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (2*beta*Power(Pi,2)*R*Cos(4*Pi*R)*Cos(4*Pi*Z)*Sin(2*P)*
+              Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+              Sin(2*Pi*Z))/
+            (15.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (beta*Power(Pi,2)*R*Cos(2*Pi*(-10 + R))*Cos(4*Pi*Z)*Sin(2*P)*
+              Sin(4*P)*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+            (15.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (beta*Pi*R*(-20. + 2.*R)*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*
+              Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*
+              Sin(2*Pi*Z))/
+            (60.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) + 
+           (beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+            (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
         1.5707963267948966*R*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z)*
-         (0. - (2.*R*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (0.2222222222222222*R*Z*
-              (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (Sin(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))) + 
+         ((-0.1*R*taui*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+            (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-         (0. - (2.*R*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-           (0.2222222222222222*R*Z*
-              (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (Sin(3*Pi*R)*Sin(3*Pi*t)*
-              (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-                (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (R*(3141.592653589793 - 125.66370614359172*R + 
-                   6.283185307179586*Power(R,2) + 
-                   6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-                ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                      1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                   0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))) + 
+         ((-0.1*R*taui*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+            (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
         R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-         ((-0.044444444444444446*mui*
-              (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(2*P)*
-              Sin(2*P)*Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-              Power(Sin(2*Pi*Z),2))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (3*Cos(3*P)*Sin(3*Pi*t)*
-              (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
-                (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
-                 Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (4*beta*Cos(4*P)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                   0.3141592653589793*Power(R,2) - 
-                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) \
-+ ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                      0.3141592653589793*Power(R,3) + 
-                      3.141592653589793*Power(Z,2) + 
-                      R*(-219.9114857512855 - 
-                        0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
-                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
-           (2*beta*Cos(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                   0.3141592653589793*Power(R,2) - 
-                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) \
-+ ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                      0.3141592653589793*Power(R,3) + 
-                      3.141592653589793*Power(Z,2) + 
-                      R*(-219.9114857512855 - 
-                         0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
-                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))) + 
-        0.5*R*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)*
-         ((taui*(250000.00000000006 - 80.*Power(R,3) + 1.*Power(R,4) + 
-                1000.0000000000001*Power(Z,2) + 1.*Power(Z,4) + 
-                R*(-40000. - 80.*Power(Z,2)) + 
-                Power(R,2)*(2200. + 2.*Power(Z,2))))/
-            (R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3)) - 
-           (0.011111111111111112*
-              (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+         ((-3.*taui*Z*(-50. + 1.*R - 0.1*Power(Z,2)))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
+           (0.2*taui*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (2*mui*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (45.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+              Sin(2*Pi*Z))/
+            (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (0.011111111111111112*Z*
+              (-9*taui - mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
                  Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-           (Sin(3*P)*Sin(3*Pi*t)*
-              (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
-                (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
-                 Sin(3*Pi*Z)))/
-            (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-           (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-              Sin(4*Pi*t)*Sin(2*Pi*Z)*
-              (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                   0.3141592653589793*Power(R,2) - 
-                   0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-                ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                      0.3141592653589793*Power(R,3) + 
-                      3.141592653589793*Power(Z,2) + 
-                      R*(-219.9114857512855 - 
-                         0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
-                   (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                    Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-            (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))))/R)
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (0.18849555921538758*R*Z*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*
+              Sin(3*Pi*Z))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (beta*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+              Sin(4*Pi*Z))/
+            (60.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (0.008333333333333333*beta*Z*Sin(2*P)*Sin(4*P)*
+              Sin(2*Pi*(-10 + R))*(4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*
+              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) + 
+        1.5707963267948966*R*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
+         ((taui*(-50. + 1.*R - 0.1*Power(Z,2)))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (-9*taui - mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+               Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+            (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+              Sin(2*Pi*Z)*Sin(4*Pi*Z))/
+            (120.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/R)
 ; }};
 struct SUe{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return (2*Pi*Cos(2*Pi*t)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
   FELTORPARALLEL*((taue*Sin(Pi*t)*
@@ -911,8 +317,9 @@ struct SUe{
      (40*Cos(2*P)*Sin(2*P)*Power(Sin(2*Pi*(-10 + R)),2)*
         Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
       (9.*mue*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-     (nuparallel*(0. + (1.*Z*((-2.0943951023931953*(-10 + R)*
-                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+     (nuparallele*(0. + (1.*Z*
+             ((-2.0943951023931953*(-10 + R)*Cos(2*Pi*Z)*Sin(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
                 (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
                     1.*Power(Z,2))) + 
                (2.0943951023931953*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*
@@ -995,263 +402,73 @@ struct SUe{
                   Sin(2*Pi*t)*Sin(2*Pi*Z))/
                 (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
                     1.*Power(Z,2)))))/
-           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R + 
+           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
+      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
      (Sin(3*Pi*t)*((18.84955592153876 - 1.8849555921538759*R)*R*
            Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R) + 
           (1.8849555921538759*R*Z*Cos(3*Pi*R)*Sin(3*P) + 
              12.*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*Z)))/
       (mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
   (beta*Pi*Cos(4*Pi*t)*Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*Z))/mue + 
-  FELTORPERP*(0. + (eta*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-           Sin(Pi*Z))*((Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-             Sin(2*Pi*Z))/3. - 
+  FELTORPERP*(0. - (0.03333333333333333*taue*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+        Sin(2*Pi*t)*Sin(2*Pi*Z))/
+      (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) - 
+     (Pi*Cos(3*Pi*Z)*Sin(2*P)*Sin(3*P)*Sin(2*Pi*(-10 + R))*Sin(3*Pi*R)*
+        Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z))/
+      (50.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+     (eta*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+        ((Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3. - 
           (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (3.*Sqrt(-mue))))/mue + 
-     ((41.8879020478639*R*taue*(-50. + 1.*R - 0.1*Power(Z,2))*Cos(2*Pi*Z)*
-           Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
-         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) \
-+ (20.94395102393195*R*taue*(-50. + 1.*R - 0.1*Power(Z,2))*Cos(Pi*Z)*
-           Sin(P)*Sin(2*P)*Sin(Pi*R)*Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*
-           Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) \
-- (2.0943951023931953*Power(R,2)*taue*Z*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-           Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) \
-- (0.03333333333333333*R*taue*
-           (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(P)*Sin(2*P)*
-           Sin(Pi*R)*Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*
-           Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) \
-- (4.1887902047863905*Power(R,2)*taue*Z*Cos(2*Pi*R)*Sin(2*P)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) \
-- (0.06666666666666667*R*taue*
-           (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(2*P)*
-           Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) \
-+ (1.3333333333333333*Power(R,2)*(-20. + 2.*R)*taue*Z*Sin(2*P)*
-           Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3)) \
-- (26.666666666666664*R*taue*Z*(-50. + 1.*R - 0.1*Power(Z,2))*Sin(2*P)*
-           Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3)) \
-- (2.6666666666666665*R*taue*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))/
-      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) - 
-     (nuperp*((2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*Z)*Sin(2*P)*
-             Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-           (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-          (2*Pi*(1 - (1.*Power(Z,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-             Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (3.*Sqrt(-mue)) - (13.333333333333332*Z*Cos(2*P)*
-             Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-          R*((13.15947253478581*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
-                Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
-              (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (83.7758040957278*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (4.1887902047863905*Power(-10 + R,2)*Z*Cos(2*Pi*Z)*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
-                 2)) - (4.1887902047863905*(-10 + R)*Power(Z,2)*
-                Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
-                 2)) + (2.0943951023931953*(-10 + R)*Cos(2*Pi*(-10 + R))*
-                Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (26.666666666666664*(-10 + R)*Z*Cos(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*Power(400 + 1.*Power(-10 + R,2) + 
-                  1.*Power(Z,2),2)) - 
-             (4*Power(Pi,2)*(1 - 
-                  (1.*Power(-10 + R,2))/
-                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (3.*Sqrt(-mue))) + 
-          R*((83.7758040957278*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (83.7758040957278*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*
-                Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (4*(Power(R,-2) - 
-                  400/
-                   (Power(R,2)*
-                     (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
-                Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (3.*Sqrt(-mue))) + 
-          R*((13.15947253478581*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
-                Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
-              (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (4.1887902047863905*Power(-10 + R,2)*Z*Cos(2*Pi*Z)*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
-                 2)) + (2.0943951023931953*Z*Cos(2*Pi*Z)*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              (Sqrt(-mue)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (83.7758040957278*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (4.1887902047863905*(-10 + R)*Power(Z,2)*Cos(2*Pi*(-10 + R))*
-                Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),
-                 2)) + (26.666666666666664*(-10 + R)*Z*Cos(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*Power(400 + 1.*Power(-10 + R,2) + 
-                  1.*Power(Z,2),2)) + 
-             (13.333333333333332*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
-                Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*Power(R,2)*
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (4*Power(Pi,2)*(1 - 
-                  (1.*Power(Z,2))/
-                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue))\
-)))/R + (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z)*
-        ((-1884.9555921538758 + 37.69911184307752*R - 
-             3.7699111843077526*Power(Z,2))*Cos(3*Pi*Z)*Sin(3*P)*
-           Sin(3*Pi*R) + (-3.7699111843077517*R*Z*Cos(3*Pi*R)*Sin(3*P) + 
-             (-54. + 1.2000000000000002*R - 
-                0.06000000000000001*Power(R,2) - 
-                0.06000000000000001*Power(Z,2))*Cos(3*P)*Sin(3*Pi*R))*
-           Sin(3*Pi*Z)))/
-      (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
-     (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-        ((taue*(-1000. + 20.*R - 2.*Power(Z,2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-          (20.*(-50. + 1.*R - 0.1*Power(Z,2))*
-             (1.*taue - 0.1111111111111111*Power(Sin(2*P),2)*
-                Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                Power(Sin(2*Pi*Z),2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-          ((3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
-               0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-          (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-             (R*(-3141.592653589793 + 125.66370614359172*R - 
-                  6.283185307179586*Power(R,2) - 
-                  6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-               (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                   Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             Sin(4*Pi*Z))/
-           (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-               1.*Power(Z,2),2))))/(3.*Sqrt(-mue)) + 
+           (3.*Sqrt(-mue))))/mue - 
+     (nuperp*((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (3.*Sqrt(-mue)) - (8*Power(Pi,2)*R*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+             Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue))))/R + 
      (2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        (0. - (2.*R*taue*Z)/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-          (R*Z*(-2.*taue + 0.2222222222222222*Power(Sin(2*P),2)*
-                Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                Power(Sin(2*Pi*Z),2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-          (Sin(3*Pi*R)*Sin(3*Pi*t)*
-             (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-               (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-          (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
-             Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             (R*(3141.592653589793 - 125.66370614359172*R + 
-                  6.283185307179586*Power(R,2) + 
-                  6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-               ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                     1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                  0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-           (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-               1.*Power(Z,2),2))))/(3.*Sqrt(-mue)) + 
-     (beta*taue*Sin(Pi*t)*Sin(4*Pi*t)*
-        (1.5707963267948966*Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*
-           (R*(-3141.592653589793 + 125.66370614359172*R - 
-                6.283185307179586*Power(R,2) - 
-                6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-             (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                 Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-              Sin(4*Pi*R))*Sin(4*Pi*Z) + 
-          1.5707963267948966*Cos(Pi*R)*Sin(P)*Sin(4*Pi*R)*Sin(Pi*Z)*
-           (R*(3141.592653589793 - 125.66370614359172*R + 
-                6.283185307179586*Power(R,2) + 
-                6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-             ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                   1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)) + 
-          0.5*Cos(P)*Sin(4*P)*Sin(Pi*R)*Sin(Pi*Z)*
-           (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                0.3141592653589793*Power(R,2) - 
-                0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-             ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                   0.3141592653589793*Power(R,3) + 
-                   3.141592653589793*Power(Z,2) + 
-                   R*(-219.9114857512855 - 0.3141592653589793*Power(Z,2))\
-)*Cos(4*Pi*R) + (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                 Sin(4*Pi*R))*Sin(4*Pi*Z))))/
-      (mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)*
+        ((-0.1*R*taue*Z)/
+           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+          (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+             Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+           (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+               1.*Power(Z,2)))))/(3.*Sqrt(-mue)) + 
+     ((-0.20943951023931953*R*taue*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*R)*
+           Sin(2*Pi*t)*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
+         (Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+        (0.10471975511965977*R*taue*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+           Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+         (Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+        (0.03333333333333333*R*taue*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
+           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+         (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
+            1.5)))/(R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) \
++ (beta*taue*Sin(P)*Sin(4*P)*Sin(Pi*t)*Sin(4*Pi*t)*
+        (0.4934802200544679*R*Cos(Pi*R)*Cos(4*Pi*Z)*Sin(4*Pi*R)*Sin(Pi*Z) + 
+          Cos(Pi*Z)*Sin(Pi*R)*(-0.4934802200544679*R*Cos(4*Pi*R) - 
+             0.039269908169872414*Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+      (mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        ((taue*(250000.00000000006 - 80.*Power(R,3) + 1.*Power(R,4) + 
-               1000.0000000000001*Power(Z,2) + 1.*Power(Z,4) + 
-               R*(-40000. - 80.*Power(Z,2)) + 
-               Power(R,2)*(2200. + 2.*Power(Z,2))))/
-           (R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3)) - 
-          (0.1*(900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-             (1.*taue - 0.1111111111111111*Power(Sin(2*P),2)*
-                Power(Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                Power(Sin(2*Pi*Z),2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-          (Sin(3*P)*Sin(3*Pi*t)*
-             (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
-               (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
-                Sin(3*Pi*Z)))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-          (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-             Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                  0.3141592653589793*Power(R,2) - 
-                  0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-               ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                     0.3141592653589793*Power(R,3) + 
-                     3.141592653589793*Power(Z,2) + 
-                     R*(-219.9114857512855 - 
-                        0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
-                  (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                   Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-           (3.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-               1.*Power(Z,2),2))))/(3.*Sqrt(-mue)) + 
-     (3*beta*Sin(3*Pi*t)*Sin(4*Pi*t)*
-        (Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
-           (R*(-3141.592653589793 + 125.66370614359172*R - 
-                6.283185307179586*Power(R,2) - 
-                6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-             (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*Cos(4*P) + 
-                (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R))*
-           Sin(4*Pi*Z) + Pi*Cos(3*Pi*R)*Sin(3*P)*Sin(4*Pi*R)*Sin(3*Pi*Z)*
-           (R*(3141.592653589793 - 125.66370614359172*R + 
-                6.283185307179586*Power(R,2) + 
-                6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-             ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                   1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)) + 
-          Cos(3*P)*Sin(4*P)*Sin(3*Pi*R)*Sin(3*Pi*Z)*
-           (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                0.3141592653589793*Power(R,2) - 
-                0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-             ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                   0.3141592653589793*Power(R,3) + 
-                   3.141592653589793*Power(Z,2) + 
-                   R*(-219.9114857512855 - 0.3141592653589793*Power(Z,2)))*
-                 Cos(4*Pi*R) + 
-                (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                 Sin(4*Pi*R))*Sin(4*Pi*Z))))/
-      (5.*mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))
+     (3*beta*Pi*Sin(3*P)*Sin(4*P)*Sin(3*Pi*t)*Sin(4*Pi*t)*
+        (4*Pi*R*Cos(3*Pi*R)*Cos(4*Pi*Z)*Sin(4*Pi*R)*Sin(3*Pi*Z) - 
+          Cos(3*Pi*Z)*Sin(3*Pi*R)*(4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*
+           Sin(4*Pi*Z)))/
+      (200.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+     (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+        ((taue*(-50. + 1.*R - 0.1*Power(Z,2)))/
+           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+          (-9*taue + Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+              Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+           (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+          (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+             (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+             Sin(2*Pi*Z)*Sin(4*Pi*Z))/
+           (120.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+               1.*Power(Z,2)))))/(3.*Sqrt(-mue)))
 ; }};
 struct SUi{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return (2*Pi*Cos(2*Pi*t)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*Z))/3. + 
   FELTORPARALLEL*((taui*Sin(Pi*t)*
@@ -1269,8 +486,9 @@ struct SUi{
      (40*Cos(2*P)*Sin(2*P)*Power(Sin(2*Pi*(-10 + R)),2)*
         Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
       (9.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-     (nuparallel*(0. + (1.*Z*((-2.0943951023931953*(-10 + R)*
-                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+     (nuparalleli*(0. + (1.*Z*
+             ((-2.0943951023931953*(-10 + R)*Cos(2*Pi*Z)*Sin(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
                (2.0943951023931953*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*
                   Sin(2*Pi*t)*Sin(2*Pi*Z))/
@@ -1333,584 +551,124 @@ struct SUi{
                (13.15947253478581*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
                   Sin(2*Pi*t)*Sin(2*Pi*Z))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
-           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R + 
+           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
+      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
      (Sin(3*Pi*t)*((18.84955592153876 - 1.8849555921538759*R)*R*
            Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R) + 
           (1.8849555921538759*R*Z*Cos(3*Pi*R)*Sin(3*P) + 
              12.*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*Z)))/
       (mui*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
   (beta*Pi*Cos(4*Pi*t)*Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*Z))/mui + 
-  FELTORPERP*(0. + (eta*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-           Sin(Pi*Z))*((Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-             Sin(2*Pi*Z))/3. - 
+  FELTORPERP*(0. - (0.03333333333333333*taui*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+        Sin(2*Pi*t)*Sin(2*Pi*Z))/
+      Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+     (Pi*Cos(3*Pi*Z)*Sin(2*P)*Sin(3*P)*Sin(2*Pi*(-10 + R))*Sin(3*Pi*R)*
+        Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z))/
+      (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+     (eta*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+        ((Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3. - 
           (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (3.*Sqrt(-mue))))/mui + 
-     ((41.8879020478639*R*taui*(-50. + 1.*R - 0.1*Power(Z,2))*Cos(2*Pi*Z)*
-           Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-        (20.94395102393195*R*taui*(-50. + 1.*R - 0.1*Power(Z,2))*
-           Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*Sin(2*Pi*R)*Sin(Pi*t)*
-           Sin(2*Pi*t)*Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-        (2.0943951023931953*Power(R,2)*taui*Z*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-           Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-        (0.03333333333333333*R*taui*
-           (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(P)*Sin(2*P)*
-           Sin(Pi*R)*Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*
-           Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-        (4.1887902047863905*Power(R,2)*taui*Z*Cos(2*Pi*R)*Sin(2*P)*
-           Sin(2*Pi*t)*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
-           Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-        (0.06666666666666667*R*taui*
-           (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(2*P)*
-           Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-        (1.3333333333333333*Power(R,2)*(-20. + 2.*R)*taui*Z*Sin(2*P)*
-           Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
-        (26.666666666666664*R*taui*Z*(-50. + 1.*R - 0.1*Power(Z,2))*
-           Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3) - 
-        (2.6666666666666665*R*taui*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))/
-      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) - 
-     (nuperp*((2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*Z)*Sin(2*P)*
-             Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (2*Pi*(1 - (1.*Power(Z,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-             Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/3. - 
-          (13.333333333333332*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-             Sin(2*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-          R*((13.15947253478581*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
-                Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-             (83.7758040957278*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (4.1887902047863905*Power(-10 + R,2)*Z*Cos(2*Pi*Z)*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-             (4.1887902047863905*(-10 + R)*Power(Z,2)*
-                Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-             (2.0943951023931953*(-10 + R)*Cos(2*Pi*(-10 + R))*Sin(2*P)*
-                Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-             (26.666666666666664*(-10 + R)*Z*Cos(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) - 
-             (4*Power(Pi,2)*(1 - 
-                  (1.*Power(-10 + R,2))/
-                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3.) + 
-          R*((83.7758040957278*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (83.7758040957278*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*
-                Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (4*(Power(R,-2) - 
-                  400/
-                   (Power(R,2)*
-                     (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
-                Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3.) + 
-          R*((13.15947253478581*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
-                Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-             (4.1887902047863905*Power(-10 + R,2)*Z*Cos(2*Pi*Z)*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-             (2.0943951023931953*Z*Cos(2*Pi*Z)*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-             (83.7758040957278*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                Sin(2*Pi*Z))/
-              (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-             (4.1887902047863905*(-10 + R)*Power(Z,2)*Cos(2*Pi*(-10 + R))*
-                Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-             (26.666666666666664*(-10 + R)*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
-                Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-             (13.333333333333332*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
-                Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-             (4*Power(Pi,2)*(1 - 
-                  (1.*Power(Z,2))/
-                   (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(2*P)*
-                Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3.)))/R + 
-     (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z)*
-        ((-1884.9555921538758 + 37.69911184307752*R - 
-             3.7699111843077526*Power(Z,2))*Cos(3*Pi*Z)*Sin(3*P)*
-           Sin(3*Pi*R) + (-3.7699111843077517*R*Z*Cos(3*Pi*R)*Sin(3*P) + 
-             (-54. + 1.2000000000000002*R - 
-                0.06000000000000001*Power(R,2) - 
-                0.06000000000000001*Power(Z,2))*Cos(3*P)*Sin(3*Pi*R))*
-           Sin(3*Pi*Z)))/
-      (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)) + 
-     (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-        ((taui*(-1000. + 20.*R - 2.*Power(Z,2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-          (2.2222222222222223*(-50. + 1.*R - 0.1*Power(Z,2))*
-             (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-          ((3.7699111843077517*R*Cos(3*Pi*R)*Sin(3*P) - 
-               0.06*Z*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-          (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-             (R*(-3141.592653589793 + 125.66370614359172*R - 
-                  6.283185307179586*Power(R,2) - 
-                  6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-               (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                   Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-                Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             Sin(4*Pi*Z))/
-           (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))))/3. + 
+           (3.*Sqrt(-mue))))/mui - 
+     (nuperp*((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           3. - (8*Power(Pi,2)*R*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+             Sin(2*Pi*Z))/3.))/R + 
      (2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        (0. - (2.*R*taui*Z)/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) - 
-          (0.2222222222222222*R*Z*
-             (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-          (Sin(3*Pi*R)*Sin(3*Pi*t)*
-             (-3.7699111843077517*R*Cos(3*Pi*Z)*Sin(3*P) + 
-               (0.6 - 0.06*R)*Cos(3*P)*Sin(3*Pi*Z)))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-          (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*
-             Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             (R*(3141.592653589793 - 125.66370614359172*R + 
-                  6.283185307179586*Power(R,2) + 
-                  6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-               ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                     1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                  0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)))/
-           (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))))/3. + 
-     (beta*taui*Sin(Pi*t)*Sin(4*Pi*t)*
-        (1.5707963267948966*Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*
-           (R*(-3141.592653589793 + 125.66370614359172*R - 
-                6.283185307179586*Power(R,2) - 
-                6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-             (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*
-                 Cos(4*P) + (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*
-              Sin(4*Pi*R))*Sin(4*Pi*Z) + 
-          1.5707963267948966*Cos(Pi*R)*Sin(P)*Sin(4*Pi*R)*Sin(Pi*Z)*
-           (R*(3141.592653589793 - 125.66370614359172*R + 
-                6.283185307179586*Power(R,2) + 
-                6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-             ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                   1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)) + 
-          0.5*Cos(P)*Sin(4*P)*Sin(Pi*R)*Sin(Pi*Z)*
-           (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                0.3141592653589793*Power(R,2) - 
-                0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-             ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                   0.3141592653589793*Power(R,3) + 
-                   3.141592653589793*Power(Z,2) + 
-                   R*(-219.9114857512855 - 0.3141592653589793*Power(Z,2))\
-)*Cos(4*Pi*R) + (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                 Sin(4*Pi*R))*Sin(4*Pi*Z))))/
-      (mui*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)*
+        ((-0.1*R*taui*Z)/
+           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+          (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+             Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
+           (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/3. + 
+     ((-0.20943951023931953*R*taui*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*R)*
+           Sin(2*Pi*t)*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
+         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
+        (0.10471975511965977*R*taui*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+           Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+        (0.03333333333333333*R*taui*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
+           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5))/
+      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (beta*taui*Sin(P)*Sin(4*P)*Sin(Pi*t)*Sin(4*Pi*t)*
+        (0.4934802200544679*R*Cos(Pi*R)*Cos(4*Pi*Z)*Sin(4*Pi*R)*Sin(Pi*Z) + 
+          Cos(Pi*Z)*Sin(Pi*R)*(-0.4934802200544679*R*Cos(4*Pi*R) - 
+             0.039269908169872414*Sin(4*Pi*R))*Sin(4*Pi*Z)))/
+      (mui*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (2*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        ((taui*(250000.00000000006 - 80.*Power(R,3) + 1.*Power(R,4) + 
-               1000.0000000000001*Power(Z,2) + 1.*Power(Z,4) + 
-               R*(-40000. - 80.*Power(Z,2)) + 
-               Power(R,2)*(2200. + 2.*Power(Z,2))))/
-           (R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),3)) - 
-          (0.011111111111111112*
-             (900. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-             (9.*taui + mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2) + 
-          (Sin(3*P)*Sin(3*Pi*t)*
-             (0.1884955592153876*Z*Cos(3*Pi*Z)*Sin(3*Pi*R) + 
-               (-1.884955592153876 + 0.1884955592153876*R)*Cos(3*Pi*R)*
-                Sin(3*Pi*Z)))/
-           (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-          (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-             Sin(4*Pi*t)*Sin(2*Pi*Z)*
-             (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                  0.3141592653589793*Power(R,2) - 
-                  0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-               ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                     0.3141592653589793*Power(R,3) + 
-                     3.141592653589793*Power(Z,2) + 
-                     R*(-219.9114857512855 - 
-                        0.3141592653589793*Power(Z,2)))*Cos(4*Pi*R) + 
-                  (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                   Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-           (3.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))))/3. + 
-     (3*beta*Sin(3*Pi*t)*Sin(4*Pi*t)*
-        (Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
-           (R*(-3141.592653589793 + 125.66370614359172*R - 
-                6.283185307179586*Power(R,2) - 
-                6.283185307179586*Power(Z,2))*Cos(4*Pi*R)*Sin(4*P) + 
-             (Z*(50. - 2.*R + 0.1*Power(R,2) + 0.1*Power(Z,2))*Cos(4*P) + 
-                (-250. + 5.*R - 0.5*Power(Z,2))*Sin(4*P))*Sin(4*Pi*R))*
-           Sin(4*Pi*Z) + Pi*Cos(3*Pi*R)*Sin(3*P)*Sin(4*Pi*R)*Sin(3*Pi*Z)*
-           (R*(3141.592653589793 - 125.66370614359172*R + 
-                6.283185307179586*Power(R,2) + 
-                6.283185307179586*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*P) + 
-             ((-500. + 70.*R - 3.*Power(R,2) + 0.1*Power(R,3) - 
-                   1.*Power(Z,2) + 0.1*R*Power(Z,2))*Cos(4*P) - 
-                0.5*R*Z*Sin(4*P))*Sin(4*Pi*Z)) + 
-          Cos(3*P)*Sin(4*P)*Sin(3*Pi*R)*Sin(3*Pi*Z)*
-           (Z*(-157.07963267948966 + 6.283185307179586*R - 
-                0.3141592653589793*Power(R,2) - 
-                0.3141592653589793*Power(Z,2))*Cos(4*Pi*Z)*Sin(4*Pi*R) + 
-             ((1570.7963267948965 + 9.42477796076938*Power(R,2) - 
-                   0.3141592653589793*Power(R,3) + 
-                   3.141592653589793*Power(Z,2) + 
-                   R*(-219.9114857512855 - 0.3141592653589793*Power(Z,2)))*
-                 Cos(4*Pi*R) + 
-                (-22.5 + 0.5*R - 0.025*Power(R,2) - 0.025*Power(Z,2))*
-                 Sin(4*Pi*R))*Sin(4*Pi*Z))))/
-      (5.*mui*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)))
+     (3*beta*Pi*Sin(3*P)*Sin(4*P)*Sin(3*Pi*t)*Sin(4*Pi*t)*
+        (4*Pi*R*Cos(3*Pi*R)*Cos(4*Pi*Z)*Sin(4*Pi*R)*Sin(3*Pi*Z) - 
+          Cos(3*Pi*Z)*Sin(3*Pi*R)*(4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*
+           Sin(4*Pi*Z)))/
+      (200.*mui*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+     (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+        ((taui*(-50. + 1.*R - 0.1*Power(Z,2)))/
+           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+          (-9*taui - mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+              Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+           (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+          (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
+             (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+             Sin(2*Pi*Z)*Sin(4*Pi*Z))/
+           (120.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/3.)
 ; }};
 struct SPhie{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return 0. - ((0.015707963267948967*Power(R,3)*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*
-        Sin(Pi*t)*((3*Pi*(1 - 
-               (1.*Power(-10 + R,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Cos(3*Pi*Z)*
-             Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/5. + 
-          (1.8849555921538759*(-10 + R)*Z*Cos(3*Pi*(-10 + R))*Sin(3*P)*
-             Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (12.*(-10 + R)*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-             Sin(3*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-      (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-     (0.02*Power(R,3)*Z*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-           Sin(Pi*Z))*((3*Pi*(1 - 
-               (1.*Power(-10 + R,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Cos(3*Pi*Z)*
-             Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/5. + 
-          (1.8849555921538759*(-10 + R)*Z*Cos(3*Pi*(-10 + R))*Sin(3*P)*
-             Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (12.*(-10 + R)*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-             Sin(3*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
+    return 0. - ((0.029608813203268074*Power(R,3)*Cos(Pi*Z)*Cos(3*Pi*Z)*Sin(P)*
+        Sin(3*P)*Sin(Pi*(-10 + R))*Sin(3*Pi*(-10 + R))*Sin(Pi*t)*
+        Sin(3*Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+     (0.03769911184307752*Power(R,3)*Z*Cos(3*Pi*Z)*Sin(3*P)*
+        Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
+        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
       Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-     (0.015707963267948967*Power(R,3)*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*
-        Sin(Pi*Z)*((1.8849555921538759*(-10 + R)*Z*Cos(3*Pi*Z)*Sin(3*P)*
-             Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (3*Pi*(1 - (1.*Power(Z,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-             Cos(3*Pi*(-10 + R))*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/5. - 
-          (12.*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-      (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-     (0.02*(-10 + R)*Power(R,3)*
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-        ((1.8849555921538759*(-10 + R)*Z*Cos(3*Pi*Z)*Sin(3*P)*
-             Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (3*Pi*(1 - (1.*Power(Z,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-             Cos(3*Pi*(-10 + R))*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/5. - 
-          (12.*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
+     (0.029608813203268074*Power(R,3)*Cos(Pi*(-10 + R))*
+        Cos(3*Pi*(-10 + R))*Sin(P)*Sin(3*P)*Sin(Pi*t)*Sin(3*Pi*t)*
+        Sin(Pi*Z)*Sin(3*Pi*Z))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+     (0.03769911184307752*(-10 + R)*Power(R,3)*Cos(3*Pi*(-10 + R))*
+        Sin(3*P)*Sin(3*Pi*t)*(1 + 
+          0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(3*Pi*Z))/
       Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-     (3*Power(R,2)*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-        ((1.8849555921538759*(-10 + R)*Z*Cos(3*Pi*Z)*Sin(3*P)*
-             Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (3*Pi*(1 - (1.*Power(Z,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-             Cos(3*Pi*(-10 + R))*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/5. - 
-          (12.*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-      (100.*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-     (0.005*Power(R,3)*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)*
-        ((37.69911184307752*(-10 + R)*Cos(3*Pi*Z)*Sin(3*P)*
-             Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-           (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-          (37.69911184307752*Z*Cos(3*Pi*(-10 + R))*Sin(3*P)*Sin(3*Pi*t)*
-             Sin(3*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) \
-+ (3*(Power(R,-2) - 400/
-                (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
-             Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.))/
-      (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-     (Power(R,3)*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-        ((17.765287921960844*(-10 + R)*Z*Cos(3*Pi*(-10 + R))*Cos(3*Pi*Z)*
-             Sin(3*P)*Sin(3*Pi*t))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (113.09733552923255*(-10 + R)*Cos(3*P)*Cos(3*Pi*Z)*
-             Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-           (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-          (3.7699111843077517*Power(-10 + R,2)*Z*Cos(3*Pi*Z)*Sin(3*P)*
-             Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-           Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-          (3.7699111843077517*(-10 + R)*Power(Z,2)*Cos(3*Pi*(-10 + R))*
-             Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-          (1.8849555921538759*(-10 + R)*Cos(3*Pi*(-10 + R))*Sin(3*P)*
-             Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (24.*(-10 + R)*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-             Sin(3*Pi*Z))/
-           (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) - 
-          (9*Power(Pi,2)*(1 - 
-               (1.*Power(-10 + R,2))/
-                (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(3*P)*
-             Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.))/
-      (100.*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-     (Power(R,3)*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-        ((113.09733552923255*(-10 + R)*Cos(3*P)*Cos(3*Pi*Z)*
-             Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-           (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-          (113.09733552923255*Z*Cos(3*P)*Cos(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-             Sin(3*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) \
-- (9*(Power(R,-2) - 400/
-                (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
-             Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.))/
-      (100.*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-     (Power(R,3)*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-        ((17.765287921960844*(-10 + R)*Z*Cos(3*Pi*(-10 + R))*Cos(3*Pi*Z)*
-             Sin(3*P)*Sin(3*Pi*t))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (3.7699111843077517*Power(-10 + R,2)*Z*Cos(3*Pi*Z)*Sin(3*P)*
-             Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-           Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-          (1.8849555921538759*Z*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*(-10 + R))*
-             Sin(3*Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (113.09733552923255*Z*Cos(3*P)*Cos(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-             Sin(3*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-          (3.7699111843077517*(-10 + R)*Power(Z,2)*Cos(3*Pi*(-10 + R))*
-             Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-          (24.*(-10 + R)*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-             Sin(3*Pi*Z))/
-           (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-          (12.*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-          (9*Power(Pi,2)*(1 - 
-               (1.*Power(Z,2))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))\
-)*Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.))/
-      (100.*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R
+     (9*Pi*Power(R,2)*Cos(3*Pi*(-10 + R))*Sin(3*P)*Sin(3*Pi*t)*
+        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(3*Pi*Z))/
+      (500.*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+     (9*Power(Pi,2)*Power(R,3)*Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
+        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(3*Pi*Z))/
+      (250.*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R
 ; }};
 struct SPhii{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (0.017765287921960846*mui*Power(Sin(3*Pi*t),2)*
-    (Power(R,2)*(400.00000000000006 + 1.0000000000000002*Power(Z,2))*
-       Power(Cos(3*Pi*Z),2)*Power(Sin(3*P),2)*Power(Sin(3*Pi*R),2) + 
-      Power(R,2)*(500. - 20.000000000000004*R + 1.*Power(R,2))*
-       Power(Cos(3*Pi*R),2)*Power(Sin(3*P),2)*Power(Sin(3*Pi*Z),2) + 
-      10.13211836423378*Power(Cos(3*P),2)*Power(Sin(3*Pi*R),2)*
-       Power(Sin(3*Pi*Z),2) - 2.026423672846756*R*Power(Cos(3*P),2)*
-       Power(Sin(3*Pi*R),2)*Power(Sin(3*Pi*Z),2) + 
-      0.1013211836423378*Power(R,2)*Power(Cos(3*P),2)*Power(Sin(3*Pi*R),2)*
-       Power(Sin(3*Pi*Z),2) + 0.1013211836423378*Power(Z,2)*
-       Power(Cos(3*P),2)*Power(Sin(3*Pi*R),2)*Power(Sin(3*Pi*Z),2) - 
-      3.1830988618379075*R*Z*Sin(6*P)*Sin(6*Pi*R)*Power(Sin(3*Pi*Z),2) - 
-      31.830988618379074*R*Sin(6*P)*Power(Sin(3*Pi*R),2)*Sin(6*Pi*Z) + 
-      3.1830988618379075*Power(R,2)*Sin(6*P)*Power(Sin(3*Pi*R),2)*
-       Sin(6*Pi*Z) - 5.*Power(R,2)*Z*Power(Sin(3*P),2)*Sin(6*Pi*R)*
-       Sin(6*Pi*Z) + 0.5000000000000001*Power(R,3)*Z*Power(Sin(3*P),2)*
-       Sin(6*Pi*R)*Sin(6*Pi*Z)))/
-  Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)
+    return (mui*Power(R,2)*Power(Sin(3*P),2)*Power(Sin(3*Pi*t),2)*
+    (0.017765287921960846*Power(Cos(3*Pi*Z),2)*Power(Sin(3*Pi*R),2) + 
+      0.017765287921960846*Power(Cos(3*Pi*R),2)*Power(Sin(3*Pi*Z),2)))/
+  (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))
 ; }};
 struct SGammaPhie{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (-0.5*mui*taui*((1.8849555921538759*(-10 + R)*Z*Cos(3*Pi*Z)*Sin(3*P)*
-         Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-       (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-      (3*Pi*(1 - (1.*Power(Z,2))/
-            (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-         Cos(3*Pi*(-10 + R))*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/5. - 
-      (12.*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-       (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-      R*((17.765287921960844*(-10 + R)*Z*Cos(3*Pi*(-10 + R))*Cos(3*Pi*Z)*
-            Sin(3*P)*Sin(3*Pi*t))/
-          (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-         (113.09733552923255*(-10 + R)*Cos(3*P)*Cos(3*Pi*Z)*
-            Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-          (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-         (3.7699111843077517*Power(-10 + R,2)*Z*Cos(3*Pi*Z)*Sin(3*P)*
-            Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-          Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-         (3.7699111843077517*(-10 + R)*Power(Z,2)*Cos(3*Pi*(-10 + R))*
-            Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-          Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-         (1.8849555921538759*(-10 + R)*Cos(3*Pi*(-10 + R))*Sin(3*P)*
-            Sin(3*Pi*t)*Sin(3*Pi*Z))/
-          (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-         (24.*(-10 + R)*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-            Sin(3*Pi*Z))/
-          (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) - 
-         (9*Power(Pi,2)*(1 - (1.*Power(-10 + R,2))/
-               (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(3*P)*
-            Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.) + 
-      R*((113.09733552923255*(-10 + R)*Cos(3*P)*Cos(3*Pi*Z)*
-            Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-          (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-         (113.09733552923255*Z*Cos(3*P)*Cos(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-            Sin(3*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-         (9*(Power(R,-2) - 400/
-               (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
-            Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.) + 
-      R*((17.765287921960844*(-10 + R)*Z*Cos(3*Pi*(-10 + R))*Cos(3*Pi*Z)*
-            Sin(3*P)*Sin(3*Pi*t))/
-          (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-         (3.7699111843077517*Power(-10 + R,2)*Z*Cos(3*Pi*Z)*Sin(3*P)*
-            Sin(3*Pi*(-10 + R))*Sin(3*Pi*t))/
-          Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-         (1.8849555921538759*Z*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*(-10 + R))*
-            Sin(3*Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-         (113.09733552923255*Z*Cos(3*P)*Cos(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-            Sin(3*Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-         (3.7699111843077517*(-10 + R)*Power(Z,2)*Cos(3*Pi*(-10 + R))*
-            Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-          Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-         (24.*(-10 + R)*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-            Sin(3*Pi*Z))/
-          (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-         (12.*Z*Cos(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-          (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-         (9*Power(Pi,2)*(1 - (1.*Power(Z,2))/
-               (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(3*P)*
-            Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.)))/R
+    return (-0.5*mui*taui*((3*Pi*Cos(3*Pi*(-10 + R))*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+       5. - (18*Power(Pi,2)*R*Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
+         Sin(3*Pi*Z))/5.))/R
 ; }};
 struct SGammaNi{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return 0. - (0.5*mui*taui*((1.5707963267948966*(-10 + R)*Z*Cos(Pi*Z)*Sin(P)*
-          Sin(Pi*(-10 + R))*Sin(Pi*t))/
-        (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-       1.5707963267948966*(1 - 
-          (1.*Power(Z,2))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-        Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
-       (10.*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-        (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-       R*((4.934802200544679*(-10 + R)*Z*Cos(Pi*(-10 + R))*Cos(Pi*Z)*
-             Sin(P)*Sin(Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) \
-+ (31.41592653589793*(-10 + R)*Cos(P)*Cos(Pi*Z)*Sin(Pi*(-10 + R))*
-             Sin(Pi*t))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-          (3.141592653589793*Power(-10 + R,2)*Z*Cos(Pi*Z)*Sin(P)*
-             Sin(Pi*(-10 + R))*Sin(Pi*t))/
-           Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-          (3.141592653589793*(-10 + R)*Power(Z,2)*Cos(Pi*(-10 + R))*
-             Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-           Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-          (1.5707963267948966*(-10 + R)*Cos(Pi*(-10 + R))*Sin(P)*
-             Sin(Pi*t)*Sin(Pi*Z))/
-           (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (20.*(-10 + R)*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-           (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) - 
-          4.934802200544679*(1 - 
-             (1.*Power(-10 + R,2))/
-              (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(P)*
-           Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)) + 
-       R*((31.41592653589793*(-10 + R)*Cos(P)*Cos(Pi*Z)*Sin(Pi*(-10 + R))*
-             Sin(Pi*t))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-          (31.41592653589793*Z*Cos(P)*Cos(Pi*(-10 + R))*Sin(Pi*t)*
-             Sin(Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-          0.5*(Power(R,-2) - 400/
-              (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
-           Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)) + 
-       R*((4.934802200544679*(-10 + R)*Z*Cos(Pi*(-10 + R))*Cos(Pi*Z)*
-             Sin(P)*Sin(Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (3.141592653589793*Power(-10 + R,2)*Z*Cos(Pi*Z)*Sin(P)*
-             Sin(Pi*(-10 + R))*Sin(Pi*t))/
-           Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-          (1.5707963267948966*Z*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*
-             Sin(Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (31.41592653589793*Z*Cos(P)*Cos(Pi*(-10 + R))*Sin(Pi*t)*
-             Sin(Pi*Z))/(R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-          (3.141592653589793*(-10 + R)*Power(Z,2)*Cos(Pi*(-10 + R))*Sin(P)*
-             Sin(Pi*t)*Sin(Pi*Z))/
-           Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-          (20.*(-10 + R)*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-           (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-          (10.*Z*Cos(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))/
-           (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-          4.934802200544679*(1 - 
-             (1.*Power(Z,2))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-           Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))))/R
+    return 0. - (0.5*mui*taui*(0. + 1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*
+        Sin(Pi*t)*Sin(Pi*Z) - 9.869604401089358*R*Sin(P)*Sin(Pi*(-10 + R))*
+        Sin(Pi*t)*Sin(Pi*Z)))/R
 ; }};
 struct SA{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallel;
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return -(beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
       (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/3. \
 + (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
      (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-   (3.*Sqrt(-mue)) - ((3.141592653589793*beta*(-10 + R)*Z*Cos(4*Pi*Z)*
-        Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
-      (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-     beta*Pi*(1 - (1.*Power(Z,2))/
-         (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Cos(4*Pi*(-10 + R))*
-      Sin(4*P)*Sin(4*Pi*t)*Sin(4*Pi*Z) - 
-     (20.*beta*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-      (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-     R*((39.47841760435743*beta*(-10 + R)*Z*Cos(4*Pi*(-10 + R))*
-           Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*t))/
-         (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-        (251.32741228718345*beta*(-10 + R)*Cos(4*P)*Cos(4*Pi*Z)*
-           Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
-         (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (6.283185307179586*beta*Power(-10 + R,2)*Z*Cos(4*Pi*Z)*Sin(4*P)*
-           Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
-         Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) - 
-        (6.283185307179586*beta*(-10 + R)*Power(Z,2)*Cos(4*Pi*(-10 + R))*
-           Sin(4*P)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-         Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-        (3.141592653589793*beta*(-10 + R)*Cos(4*Pi*(-10 + R))*Sin(4*P)*
-           Sin(4*Pi*t)*Sin(4*Pi*Z))/
-         (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-        (40.*beta*(-10 + R)*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-           Sin(4*Pi*Z))/
-         (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) - 
-        4*beta*Power(Pi,2)*(1 - 
-           (1.*Power(-10 + R,2))/
-            (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*Sin(4*P)*
-         Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z)) + 
-     R*((251.32741228718345*beta*(-10 + R)*Cos(4*P)*Cos(4*Pi*Z)*
-           Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
-         (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-        (251.32741228718345*beta*Z*Cos(4*P)*Cos(4*Pi*(-10 + R))*
-           Sin(4*Pi*t)*Sin(4*Pi*Z))/
-         (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-        4*beta*(Power(R,-2) - 400/
-            (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))*
-         Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z)) + 
-     R*((39.47841760435743*beta*(-10 + R)*Z*Cos(4*Pi*(-10 + R))*
-           Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*t))/
-         (400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-        (6.283185307179586*beta*Power(-10 + R,2)*Z*Cos(4*Pi*Z)*Sin(4*P)*
-           Sin(4*Pi*(-10 + R))*Sin(4*Pi*t))/
-         Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-        (3.141592653589793*beta*Z*Cos(4*Pi*Z)*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-           Sin(4*Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-        (251.32741228718345*beta*Z*Cos(4*P)*Cos(4*Pi*(-10 + R))*
-           Sin(4*Pi*t)*Sin(4*Pi*Z))/
-         (R*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-        (6.283185307179586*beta*(-10 + R)*Power(Z,2)*Cos(4*Pi*(-10 + R))*
-           Sin(4*P)*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-         Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-        (40.*beta*(-10 + R)*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*
-           Sin(4*Pi*Z))/
-         (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2)) + 
-        (20.*beta*Z*Cos(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/
-         (Power(R,2)*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-        4*beta*Power(Pi,2)*(1 - 
-           (1.*Power(Z,2))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))*
-         Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z)))/R
+   (3.*Sqrt(-mue)) - (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*
+      Sin(4*Pi*Z) - 8*beta*Power(Pi,2)*R*Sin(4*P)*Sin(4*Pi*(-10 + R))*
+      Sin(4*Pi*t)*Sin(4*Pi*Z))/R
 ; }};
 }}//namespace feltor namespace manufactured
-- 
GitLab


From 38abf804acb01d00b271a6f24ba87f91321f9699 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 8 Oct 2020 16:26:19 +0200
Subject: [PATCH 356/540] Redesign construction and init of Extrapolation

problem was that in the initial phase the behaviour did not make much
sense
---
 inc/dg/cg.h      | 119 +++++++++++++++++------------------------------
 inc/dg/cg2d_t.cu |  16 +++++--
 2 files changed, 54 insertions(+), 81 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 7d762557d..9c9ca0528 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -285,64 +285,30 @@ struct Extrapolation
 {
     using value_type = get_value_type<ContainerType>;
     using container_type = ContainerType;
-    /*! @brief Set extrapolation order without initializing values
-     * @param number number of vectors to use for extrapolation.
-         Choose between 1 (constant), 2 (linear) or 3 (parabola) extrapolation.
-     * @attention the update function must be used at least \c number times before the extrapolate function can be called
+    /*! @brief Leave values uninitialized
      */
-    Extrapolation( unsigned number = 2){
-        set_number(number);
-    }
-    /*! @brief Set extrapolation order and initialize values
-     * @param number number of vectors to use for extrapolation.
-         Choose between 1 (constant), 2 (linear) or 3 (parabola) extrapolation.
-     * @param t_init the times are initialized with the values <tt> t_init, t_init-1 , t_init-2 </tt>
-     * @param init the vectors are initialized with this value
-     */
-    Extrapolation( unsigned number, value_type t_init, const ContainerType& init) {
-        set_number(number, t_init, init);
-    }
-    /*! @brief Set extrapolation order and initialize values (equidistant)
-     * @param number number of vectors to use for extrapolation.
-         Choose between 1 (constant), 2 (linear) or 3 (parabola) extrapolation.
-     * @param init the vectors are initialized with this value
-     * @note the times are initialized with the values <tt> 0, -1 , -2 </tt>
+    Extrapolation( ){ m_counter = 0; }
+    /*! @brief Set maximum extrapolation order and allocate memory
+     * @param max maximum of vectors to use for extrapolation.
+         Choose between 0 (no extrapolation) 1 (constant), 2 (linear) or 3 (parabola) extrapolation.
+         Higher values currently default back to a linear extrapolation
+     * @param copyable the memory is allocated based on this vector
      */
-    Extrapolation( unsigned number, const ContainerType& init) {
-        set_number(number, 0, init);
-    }
-    ///@copydoc Extrapolation(unsigned)
-    void set_number( unsigned number)
-    {
-        assert( number <= 3 );
-        m_number = number;
-        m_t.resize( number);
-        m_x.resize( number);
-        for(unsigned i=0; i<m_t.size(); i++)
-            m_t[i] = -(value_type)i;
-    }
-    ///@copydoc Extrapolation(unsigned,value_type,const ContainerType&)
-    void set_number( unsigned number, value_type t_init, const ContainerType& init)
-    {
-        //init times 0, -1, -2
-        assert( number <= 3 );
-        m_x.assign( number, init);
-        m_t.assign( number, t_init);
-        m_number = number;
-        for(unsigned i=0; i<m_t.size(); i++)
-            m_t[i] = t_init - (value_type)i;
+    Extrapolation( unsigned max, const ContainerType& copyable) {
+        set_max(max, copyable);
     }
     ///@copydoc Extrapolation(unsigned,const ContainerType&)
-    void set_number( unsigned number, const ContainerType& init)
+    void set_max( unsigned max, const ContainerType& copyable)
     {
-        //init times 0, -1, -2
-        set_number( number, 1, init);
-        for(unsigned i=0; i<m_t.size(); i++)
-            m_t[i] = -(value_type)i;
+        m_counter = 0;
+        m_x.assign( max, copyable);
+        m_t.assign( max, 0);
+        m_max = max;
     }
-    ///return the current extrapolation number
-    unsigned get_number( ) const{
-        return m_number;
+    ///return the current extrapolation max
+    ///This may not coincide with the max set in the constructor if values have not been updated yet
+    unsigned get_max( ) const{
+        return m_counter;
     }
 
     /**
@@ -352,12 +318,13 @@ struct Extrapolation
     * @param t time to which to extrapolate (or at which interpolating polynomial is evaluated)
     * @param new_x (write only) contains extrapolated value on output ( may alias the tail)
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
+    * @attention If the update function has not been called enough times to fill all values the result depends: (i) never called => new_x is zero (ii) called at least once => the interpolating polynomial is constructed with all available values
     */
     template<class ContainerType0>
     void extrapolate( value_type t, ContainerType0& new_x) const{
-        switch(m_number)
+        switch(m_counter)
         {
-            case(0):
+            case(0): dg::blas1::copy( 0, new_x);
                      break;
             case(1): dg::blas1::copy( m_x[0], new_x);
                      break;
@@ -385,19 +352,23 @@ struct Extrapolation
     * @param t time at which derivative of interpolating polynomial is evaluated
     * @param dot_x (write only) contains derived value on output
     * @note If t is chosen as the latest time of update t0, then the result coincides
-    * with the backward difference formula of order  \c number
-    * @attention If number==1, the result is 0 (derivative of a constant)
+    * with the backward difference formula of order  \c max
+    * @attention If max==1, the result is 0 (derivative of a constant)
+    * @attention If the update function has not been called enough times to fill all values the result depends: (i) never called => dot_x is zero (ii) called at least once => the interpolating polynomial is constructed with all available values
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
     */
     template<class ContainerType0>
     void derive( value_type t, ContainerType0& dot_x) const{
-        switch(m_number)
+        switch(m_counter)
         {
-            case(0):
+            case(0): dg::blas1::copy( 0, dot_x);
+            std::cout << "NO EXTRA\n";
                      break;
             case(1): dg::blas1::copy( 0, dot_x);
+            std::cout << "LINEAR EXTRA\n";
                      break;
             case(3): {
+            std::cout << "POLY EXTRA\n";
                 value_type f0 =-(-2.*t+m_t[1]+m_t[2])/(m_t[0]-m_t[1])/(m_t[0]-m_t[2]);
                 value_type f1 = (-2.*t+m_t[0]+m_t[2])/(m_t[0]-m_t[1])/(m_t[1]-m_t[2]);
                 value_type f2 =-(-2.*t+m_t[0]+m_t[1])/(m_t[2]-m_t[0])/(m_t[2]-m_t[1]);
@@ -406,6 +377,7 @@ struct Extrapolation
                 break;
             }
             default: {
+            std::cout << "SECOND EXTRA\n";
                 value_type f0 = 1./(m_t[0]-m_t[1]);
                 value_type f1 = 1./(m_t[1]-m_t[0]);
                 dg::blas1::axpby( f0, m_x[0], f1, m_x[1], dot_x);
@@ -421,7 +393,7 @@ struct Extrapolation
     */
     template<class ContainerType0>
     void extrapolate( ContainerType0& new_x) const{
-        if ( 0 == m_number)
+        if ( 0 == m_max)
             return;
         value_type t = m_t[0] +1.;
         extrapolate( t, new_x);
@@ -434,12 +406,11 @@ struct Extrapolation
     */
     template<class ContainerType0>
     void derive( ContainerType0& dot_x) const{
-        if ( 0 == m_number)
+        if ( 0 == m_max)
             return;
         derive( m_t[0], dot_x);
     }
 
-
     /**
     * @brief insert a new entry, deleting the oldest entry or update existing entry
     * @param t_new the time for the new entry
@@ -448,9 +419,11 @@ struct Extrapolation
     */
     template<class ContainerType0>
     void update( value_type t_new, const ContainerType0& new_entry){
-        if( m_number == 0) return;
+        if( m_max == 0) return;
+        if( m_counter < m_max)
+            m_counter++;
         //check if entry is already there to avoid division by zero errors
-        for( unsigned i=0; i<m_number; i++)
+        for( unsigned i=0; i<m_max; i++)
             if( fabs(t_new - m_t[i]) <1e-14)
             {
                 blas1::copy( new_entry, m_x[i]);
@@ -476,22 +449,22 @@ struct Extrapolation
 
     /**
      * @brief return the current head (the one most recently inserted)
-     * @return current head (undefined if number==0)
+     * @return current head (undefined if max==0)
      */
     const ContainerType& head()const{
         return m_x[0];
     }
     ///write access to tail value ( the one that will be deleted in the next update
     ContainerType& tail(){
-        return m_x[m_number-1];
+        return m_x[m_max-1];
     }
     ///read access to tail value ( the one that will be deleted in the next update
     const ContainerType& tail()const{
-        return m_x[m_number-1];
+        return m_x[m_max-1];
     }
 
     private:
-    unsigned m_number;
+    unsigned m_max, m_counter;
     std::vector<value_type> m_t;
     std::vector<ContainerType> m_x;
 };
@@ -549,7 +522,7 @@ struct Invert
      */
     void construct( const ContainerType& copyable, unsigned max_iter, value_type eps, int extrapolationType = 2, bool multiplyWeights = true, value_type nrmb_correction = 1.)
     {
-        m_ex.set_number( extrapolationType);
+        m_ex.set_max( extrapolationType, copyable);
         set_size( copyable, max_iter);
         set_accuracy( eps, nrmb_correction);
         multiplyWeights_=multiplyWeights;
@@ -564,7 +537,7 @@ struct Invert
      */
     void set_size( const ContainerType& assignable, unsigned max_iterations) {
         cg.construct( assignable, max_iterations);
-        m_ex.set_number( m_ex.get_number(), assignable);
+        m_ex.set_max( m_ex.get_max(), assignable);
     }
 
     /**
@@ -578,14 +551,6 @@ struct Invert
         nrmb_correction_ = nrmb_correction;
     }
 
-    /**
-     * @brief Set the extrapolation Type for following inversions
-     *
-     * @param extrapolationType number of last values to use for next extrapolation of initial guess
-     */
-    void set_extrapolationType( int extrapolationType) {
-        m_ex.set_number( extrapolationType);
-    }
     /**
      * @brief Set the maximum number of iterations
      * @param new_max New maximum number
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index 120ebe99e..6a76a0dcb 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -159,17 +159,21 @@ int main()
         std::cout << "L2 Norm of Residuum is        " << res.d<<"\n\n";
     }
     // Test Extrapolation object
-    dg::Extrapolation<double> extra(3);
+    double value;
+    dg::Extrapolation<double> extra(3,-1);
     extra.update( 0, 0);
     extra.update( 1, 1);
+    extra.extrapolate( 5, value);
+    std::cout << "Linear Extrapolated value is "<<value<< " (5)\n";
+    extra.derive( 4, value);
+    std::cout << "Linear Derived value is "<<value<< " (1)\n";
     extra.update( 3, 9);
     extra.update( 2, 4);
-    double value;
     extra.extrapolate( 5, value);
     std::cout << "Extrapolated value is "<<value<< " (25)\n";
     extra.derive( 4, value);
     std::cout << "Derived value is "<<value<< " (8)\n";
-    extra.set_number(2);
+    extra.set_max(2,-1);
     extra.update( 0, 0);
     extra.update( 1, 1);
     extra.update( 3, 9);
@@ -178,7 +182,11 @@ int main()
     std::cout << "Linear Extrapolated value is "<<value<< " (19)\n";
     extra.derive( 4, value);
     std::cout << "Linear Derived value is "<<value<< " (5)\n";
-    extra.set_number(1);
+    extra.set_max(1,-1);
+    extra.extrapolate( 5, value);
+    std::cout << "Empty Extrapolated value is "<<value<< " (0)\n";
+    extra.derive( 4, value);
+    std::cout << "Empty Derived value is "<<value<< " (0)\n";
     extra.update( 0, 0);
     extra.update( 1, 1);
     extra.update( 3, 9);
-- 
GitLab


From 6747185500cb9af217b85ab8a3fc02d4c608b6e2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 8 Oct 2020 23:57:53 +0200
Subject: [PATCH 357/540] Add description of solution of implicit

we should manufacture a solution for this equation, implement it
and then test it
---
 src/feltor/feltor.tex | 36 ++++++++++++++++++++++++++++++++++--
 1 file changed, 34 insertions(+), 2 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 08f665661..21473a5d6 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1444,10 +1444,42 @@ There seems to be no benefit in using the grid refinement technique except when
 \\
 time & Multistep "Karniadakis" & \\
 \qquad explicit & Multistep "Karniadakis" & $3$rd order explicit\\
-\qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion and Resistive terms. In every iteration of the implicit inversion we need to solve for $A_\parallel$\\
+\qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion terms. \\
 \bottomrule
 \end{longtable}
-
+Note that the explicit resistive and damping terms lead to absolute restriction
+on the timestep according to $\partial_t f = -\lambda f \Rightarrow \Delta t < \lambda^{-1}$. This is a quite weak restriction unless the resistivity exceeds $\eta > 10^{-4}$ or the damping coefficient excceeds $1$.
+The explicit parallel electron viscosity term leads to a restriction of the
+form $\Delta t < (2\pi (R_0 - a))^2/(N_z^2 \nu_{\parallel,e})$.
+
+In every iteration of the implicit inversion we need to solve an equation of the form
+\begin{align}
+    n_e + a \hat L n_e = \hat n_e \\
+    N_i + a \hat L N_i = \hat N_i \\
+    w_e + a \hat L u_e = \hat w_e \\
+    W_i + a \hat L U_i = \hat W_i
+\end{align}
+with $W=U+A_\parallel/\mu$ and $-\Delta_\perp A_\parallel = \beta (N_i U_i -n_e u_e)$
+    and $\hat L = -\nu_\perp \Delta_\perp^2$.
+This makes 7 equations for 7 unkown quantities.
+We solve the system by first isolating and solving the two density equations for $n_e$ and $N_i$. These can then be inserted into the velocity equations as
+fixed solutions, which makes these equations linear. As a next step we insert  the definition of $W$ into the velocity equations to get
+\begin{align}
+    u_e + \frac{A_\parallel}{\mu_e} + a \hat L u_e = \hat w_e \\
+    U_i + \frac{A_\parallel}{\mu_i} + a \hat L U_i = \hat W_i
+\end{align}
+We now introduce the two helper variables $-\Delta_\perp A_{\parallel,i} = \beta N_i U_i$ and $-\Delta_\perp A_{\parallel,e} = -\beta n_e u_e$ to get
+\begin{align}
+A_{\parallel,e} + \frac{\mu_e \Delta_\perp A_{\parallel,e}}{\beta n_e}
++ a \mu_e \hat L \frac{\Delta_\perp
+        A_{\parallel,e}}{ \beta n_e} + A_{\parallel,i} = \mu_e \hat w_e \\
+        A_{\parallel,e} -\frac{\mu_i \Delta_\perp A_{\parallel,i}}{\beta N_i}
+         - a \mu_i \hat L \frac{\Delta_\perp
+        A_{\parallel,i}}{\beta N_i}+ A_{\parallel,i} = \mu_i \hat W_i
+\end{align}
+which is a symmetric equation for $A_{\parallel,e}$, $A_{\parallel,i}$.
+Its solution yields $A_\parallel = A_{\parallel,i}+A_{\parallel,e}$ and thus a solution for $U_\parallel$ and $W_\parallel$ through backward substitutions.
+The resistive term cannot fit into this scheme as it is linear but not symmetric.
 \section{Usage}
 
 Compilation:\\
-- 
GitLab


From 91f7bc0abaad2a1e708c8cb5c69a0a6546ab795c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 9 Oct 2020 14:46:46 +0200
Subject: [PATCH 358/540] BUGFIX: update function in Extrapolation

if a value is rejected the counter wrongfully increased
---
 inc/dg/cg.h      | 10 +++-------
 inc/dg/cg2d_t.cu |  2 ++
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 9c9ca0528..348140670 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -362,13 +362,10 @@ struct Extrapolation
         switch(m_counter)
         {
             case(0): dg::blas1::copy( 0, dot_x);
-            std::cout << "NO EXTRA\n";
                      break;
             case(1): dg::blas1::copy( 0, dot_x);
-            std::cout << "LINEAR EXTRA\n";
                      break;
             case(3): {
-            std::cout << "POLY EXTRA\n";
                 value_type f0 =-(-2.*t+m_t[1]+m_t[2])/(m_t[0]-m_t[1])/(m_t[0]-m_t[2]);
                 value_type f1 = (-2.*t+m_t[0]+m_t[2])/(m_t[0]-m_t[1])/(m_t[1]-m_t[2]);
                 value_type f2 =-(-2.*t+m_t[0]+m_t[1])/(m_t[2]-m_t[0])/(m_t[2]-m_t[1]);
@@ -377,7 +374,6 @@ struct Extrapolation
                 break;
             }
             default: {
-            std::cout << "SECOND EXTRA\n";
                 value_type f0 = 1./(m_t[0]-m_t[1]);
                 value_type f1 = 1./(m_t[1]-m_t[0]);
                 dg::blas1::axpby( f0, m_x[0], f1, m_x[1], dot_x);
@@ -420,15 +416,15 @@ struct Extrapolation
     template<class ContainerType0>
     void update( value_type t_new, const ContainerType0& new_entry){
         if( m_max == 0) return;
-        if( m_counter < m_max)
-            m_counter++;
         //check if entry is already there to avoid division by zero errors
-        for( unsigned i=0; i<m_max; i++)
+        for( unsigned i=0; i<m_counter; i++)
             if( fabs(t_new - m_t[i]) <1e-14)
             {
                 blas1::copy( new_entry, m_x[i]);
                 return;
             }
+        if( m_counter < m_max) //don't update counter if Time entry was rejected
+            m_counter++;
         //push out last value (keep track of what is oldest value
         std::rotate( m_x.rbegin(), m_x.rbegin()+1, m_x.rend());
         std::rotate( m_t.rbegin(), m_t.rbegin()+1, m_t.rend());
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index 6a76a0dcb..a8c29ad5a 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -165,6 +165,7 @@ int main()
     extra.update( 1, 1);
     extra.extrapolate( 5, value);
     std::cout << "Linear Extrapolated value is "<<value<< " (5)\n";
+    extra.update( 1, 1); //should not change anything
     extra.derive( 4, value);
     std::cout << "Linear Derived value is "<<value<< " (1)\n";
     extra.update( 3, 9);
@@ -191,6 +192,7 @@ int main()
     extra.update( 1, 1);
     extra.update( 3, 9);
     extra.update( 2, 4);
+    extra.update( 2, 4);
     extra.extrapolate( 5, value);
     std::cout << "Monomial Extrapolated value is "<<value<< " (4)\n";
     extra.derive( 4, value);
-- 
GitLab


From 7e8f7adabf549e8ca1511beabe513703f50dd006 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 9 Oct 2020 14:50:22 +0200
Subject: [PATCH 359/540] EXPERIMENTAL: make Karniadakis use ARKStep to init

If successful we should also use it in other Multistep methods
---
 inc/dg/multistep.h | 59 ++++++++++++++++++++++++++++++++++------------
 1 file changed, 44 insertions(+), 15 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 548315cdb..6f67a902e 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include "implicit.h"
+#include "runge_kutta.h"
 
 
 /*! @file
@@ -215,6 +216,7 @@ struct Karniadakis
     Karniadakis( SolverParams&& ...ps):m_solver( std::forward<SolverParams>(ps)...){
         m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
         set_order(3);
+        m_counter = 0;
     }
     /**
      * @brief Reserve memory for the integration
@@ -227,6 +229,7 @@ struct Karniadakis
         m_solver = Solver( std::forward<SolverParams>(ps)...);
         m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
         set_order(3);
+        m_counter = 0;
     }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
@@ -238,9 +241,9 @@ struct Karniadakis
     const SolverType& solver() const { return m_solver;}
 
     /**
-     * @brief Initialize by integrating two timesteps backward in time
+     * @brief Initialize timestepper
      *
-     * The backward integration uses the Lie operator splitting method, with explicit Euler substeps for both explicit and implicit part
+     * This initializes the timestepper and sets the timestep for later use
      * @copydoc hide_explicit_implicit
      * @param t0 The intital time corresponding to u0
      * @param u0 The initial value of the integration
@@ -256,7 +259,8 @@ struct Karniadakis
     * @copydoc hide_explicit_implicit
     * @param t (write-only), contains timestep corresponding to \c u on output
     * @param u (write-only), contains next step of time-integration on output
-     * @note the implementation is such that on output the last call to the explicit part \c ex is at the new \c (t,u). This might be interesting if the call to \c ex changes its state.
+    * @note the implementation is such that on output the last call to the explicit part \c ex is at the new \c (t,u). This might be interesting if the call to \c ex changes its state.
+    * @attention The first two steps after the call to the init function are performed with a semi-implicit Runge-Kutta method
     */
     template< class Explicit, class Implicit>
     void step( Explicit& ex, Implicit& im, value_type& t, ContainerType& u);
@@ -296,6 +300,7 @@ struct Karniadakis
     value_type m_t, m_dt;
     value_type a[3];
     value_type b[3], g0 = 6./11.;
+    unsigned m_counter; //counts how often step has been called after init
 };
 
 ///@cond
@@ -303,25 +308,49 @@ template< class ContainerType, class SolverType>
 template< class RHS, class Diffusion>
 void Karniadakis<ContainerType, SolverType>::init( RHS& f, Diffusion& diff, value_type t0, const ContainerType& u0, value_type dt)
 {
-    //operator splitting using explicit Euler for both explicit and implicit part
     m_t = t0, m_dt = dt;
-    blas1::copy(  u0, m_u[0]);
-    f( t0, u0, m_f[0]); //f may not destroy u0
-    blas1::axpby( 1., m_u[0], -dt, m_f[0], m_f[1]); //Euler step
-    detail::Implicit<Diffusion, ContainerType> implicit( -dt, t0, diff);
-    implicit( m_f[1], m_u[1]); //explicit Euler step backwards
-    f( t0-dt, m_u[1], m_f[1]);
-    blas1::axpby( 1.,m_u[1], -dt, m_f[1], m_f[2]);
-    implicit.time() = t0 - dt;
-    implicit( m_f[2], m_u[2]);
-    f( t0-2*dt, m_u[2], m_f[2]); //evaluate f at the latest step
-    f( t0, u0, m_f[0]); // and set state in f to (t0,u0)
+    blas1::copy(  u0, m_u[2]);
+    f( t0, u0, m_f[2]); //f may not destroy u0
+    m_counter = 0;
+    //m_t = t0, m_dt = dt;
+    //blas1::copy(  u0, m_u[0]);
+    //f( t0, u0, m_f[0]); //f may not destroy u0
+    //blas1::axpby( 1., m_u[0], -dt, m_f[0], m_f[1]); //Euler step
+    //detail::Implicit<Diffusion, ContainerType> implicit( -dt, t0, diff);
+    //implicit( m_f[1], m_u[1]); //explicit Euler step backwards
+    //f( t0-dt, m_u[1], m_f[1]);
+    //blas1::axpby( 1.,m_u[1], -dt, m_f[1], m_f[2]);
+    //implicit.time() = t0 - dt;
+    //implicit( m_f[2], m_u[2]);
+    //f( t0-2*dt, m_u[2], m_f[2]); //evaluate f at the latest step
+    //f( t0, u0, m_f[0]); // and set state in f to (t0,u0)
 }
 
 template<class ContainerType, class SolverType>
 template< class RHS, class Diffusion>
 void Karniadakis<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, value_type& t, ContainerType& u)
 {
+    if( m_counter < 2)
+    {
+        ARKStep<ContainerType, SolverType> ark( "ARK-4-2-3", m_solver);
+        ContainerType tmp ( u);
+        ark.step( f, diff, t, u, t, u, m_dt, tmp);
+        m_counter++;
+        m_t = t;
+        if( m_counter == 1)
+        {
+            blas1::copy(  u, m_u[1]);
+            f( m_t, m_u[1], m_f[1]);
+        }
+        else
+        {
+            blas1::copy(  u, m_u[0]);
+            f( m_t, m_u[0], m_f[0]);
+        }
+        m_solver = ark.solver();
+        return;
+    }
+
     blas1::axpbypgz( m_dt*b[0], m_f[0], m_dt*b[1], m_f[1], m_dt*b[2], m_f[2]);
     blas1::axpbypgz( a[0], m_u[0], a[1], m_u[1], a[2], m_u[2]);
     //permute m_f[2], m_u[2]  to be the new m_f[0], m_u[0]
-- 
GitLab


From bd7007e214532d84723ffe85f248ece161725959 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 12 Oct 2020 19:51:33 +0200
Subject: [PATCH 360/540] Add a test file for the implicit feltor part

---
 src/feltor/Makefile        |    3 +
 src/feltor/feltor.tex      |   13 +-
 src/feltor/implicit_t.cu   |   88 +
 src/feltor/manufactured.cu |    3 +-
 src/feltor/manufactured.h  | 4100 +++++++++++++++++++++++++++++++-----
 5 files changed, 3673 insertions(+), 534 deletions(-)
 create mode 100644 src/feltor/implicit_t.cu

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index 6d6d091fb..c62587d4e 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -13,6 +13,9 @@ all: feltor_hpc feltor feltor_mpi manufactured feltordiag
 manufactured: manufactured.cu manufactured.h feltor.h implicit.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
 
+implicit_t: implicit_t.cu implicit.h  feltor.h implicit.h
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
+
 feltordiag: feltordiag.cu feltordiag.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g
 interpolate_in_3d: interpolate_in_3d.cu feltordiag.h
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 21473a5d6..b1bccd7b7 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1468,17 +1468,18 @@ fixed solutions, which makes these equations linear. As a next step we insert  t
     u_e + \frac{A_\parallel}{\mu_e} + a \hat L u_e = \hat w_e \\
     U_i + \frac{A_\parallel}{\mu_i} + a \hat L U_i = \hat W_i
 \end{align}
-We now introduce the two helper variables $-\Delta_\perp A_{\parallel,i} = \beta N_i U_i$ and $-\Delta_\perp A_{\parallel,e} = -\beta n_e u_e$ to get
+We now introduce the two auxiliary variables $-\Delta_\perp A_{\parallel,i} = N_i U_i$ and $-\Delta_\perp A_{\parallel,e} =  -n_e u_e$ to get
 \begin{align}
-A_{\parallel,e} + \frac{\mu_e \Delta_\perp A_{\parallel,e}}{\beta n_e}
+\beta A_{\parallel,e} + \frac{\mu_e \Delta_\perp A_{\parallel,e}}{n_e}
 + a \mu_e \hat L \frac{\Delta_\perp
-        A_{\parallel,e}}{ \beta n_e} + A_{\parallel,i} = \mu_e \hat w_e \\
-        A_{\parallel,e} -\frac{\mu_i \Delta_\perp A_{\parallel,i}}{\beta N_i}
+        A_{\parallel,e}}{ n_e} + \beta A_{\parallel,i} =  \mu_e \hat w_e \\
+        \beta A_{\parallel,e} -\frac{\mu_i \Delta_\perp A_{\parallel,i}}{N_i}
          - a \mu_i \hat L \frac{\Delta_\perp
-        A_{\parallel,i}}{\beta N_i}+ A_{\parallel,i} = \mu_i \hat W_i
+        A_{\parallel,i}}{ N_i}+ \beta A_{\parallel,i} = \mu_i \hat W_i
 \end{align}
+%Update in Mathematica
 which is a symmetric equation for $A_{\parallel,e}$, $A_{\parallel,i}$.
-Its solution yields $A_\parallel = A_{\parallel,i}+A_{\parallel,e}$ and thus a solution for $U_\parallel$ and $W_\parallel$ through backward substitutions.
+Its solution yields $A_\parallel = \beta(A_{\parallel,i}+A_{\parallel,e})$ and thus a solution for $U_\parallel$ and $W_\parallel$ through backward substitutions.
 The resistive term cannot fit into this scheme as it is linear but not symmetric.
 \section{Usage}
 
diff --git a/src/feltor/implicit_t.cu b/src/feltor/implicit_t.cu
new file mode 100644
index 000000000..b8059a115
--- /dev/null
+++ b/src/feltor/implicit_t.cu
@@ -0,0 +1,88 @@
+#include <iostream>
+
+#include "implicit.h"
+#include "manufactured.h"
+
+
+int main( int argc, char* argv[])
+{
+    Json::Value js, gs;
+    if( argc == 1)
+        file::file2Json( "input.json", js, file::comments::are_forbidden);
+    else if( argc == 2)
+        file::file2Json( argv[1], js, file::comments::are_forbidden);
+    else
+    {
+        std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile]\n";
+        return -1;
+    }
+    const feltor::Parameters p( js, file::error::is_throw);
+    p.display( std::cout);
+    const double R_0 = 10;
+    const double I_0 = 20; //q factor at r=1 is I_0/R_0
+    const double a  = 1; //small radius
+    dg::CylindricalGrid3d grid( R_0-a, R_0+a, -a, a, 0, 2.*M_PI,
+        p.n, p.Nx, p.Ny, p.Nz, p.bcxN, p.bcyN, dg::PER);
+    dg::DVec w3d = dg::create::volume( grid);
+
+    //create RHS
+    std::cout << "Initialize explicit" << std::endl;
+    dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
+    std::cout << "Initialize implicit" << std::endl;
+    feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
+    feltor::FeltorSpecialSolver<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> solver( grid, p, mag);
+    double alpha = -p.dt, time = 1;
+    feltor::manufactured::Snehat snehat{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1], alpha};
+    feltor::manufactured::SNihat snihat{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1], alpha};
+    feltor::manufactured::SWehat swehat{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1], alpha};
+    feltor::manufactured::SWihat swihat{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1], alpha};
+    feltor::manufactured::A aa{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
+    feltor::manufactured::Ne ne{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
+    feltor::manufactured::Ni ni{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
+    feltor::manufactured::we we{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
+    feltor::manufactured::Wi wi{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
+    dg::DVec R = dg::pullback( dg::cooX3d, grid);
+    dg::DVec Z = dg::pullback( dg::cooY3d, grid);
+    dg::DVec P = dg::pullback( dg::cooZ3d, grid);
+    dg::DVec zer = dg::evaluate( dg::zero, grid);
+    std::array<dg::DVec,2> w{zer,zer}, sol_w{w};
+    // ... init y to 0, rhs = rhs function and solutions to solutions
+    std::array<std::array<dg::DVec,2>,2> y{w,w}, rhs( y), sol{y};
+    dg::DVec apar{R}, sol_apar{apar};
+
+    dg::blas1::evaluate( rhs[0][0], dg::equals(), snehat, R,Z,P,time);
+    dg::blas1::evaluate( rhs[0][1], dg::equals(), snihat, R,Z,P,time);
+    dg::blas1::evaluate( rhs[1][0], dg::equals(), swehat, R,Z,P,time);
+    dg::blas1::evaluate( rhs[1][1], dg::equals(), swihat, R,Z,P,time);
+    dg::blas1::evaluate( sol[0][0], dg::equals(), ne, R,Z,P,time);
+    dg::blas1::evaluate( sol[0][1], dg::equals(), ni, R,Z,P,time);
+    dg::blas1::evaluate( sol[1][0], dg::equals(), we, R,Z,P,time);
+    dg::blas1::evaluate( sol[1][1], dg::equals(), wi, R,Z,P,time);
+    dg::blas1::evaluate( apar, dg::equals(), aa, R,Z,P,time);
+
+    solver.solve( -p.dt, im, time, y, rhs);
+    double normne = sqrt(dg::blas2::dot( w3d, sol[0][0]));
+    double normni = sqrt(dg::blas2::dot( w3d, sol[0][1]));
+    double normwe = sqrt(dg::blas2::dot( w3d, sol[1][0]));
+    double normwi = sqrt(dg::blas2::dot( w3d, sol[1][1]));
+
+    dg::blas1::axpby( 1., sol, -1., y);
+    std::cout <<"           rel. Error\tNorm: \n"
+              <<"    ne:   "<<sqrt(dg::blas2::dot( w3d, y[0][0]))/normne<<"\t"<<normne<<"\n"
+              <<"    ni:   "<<sqrt(dg::blas2::dot( w3d, y[0][1]))/normni<<"\t"<<normni<<"\n"
+              <<"    we:   "<<sqrt(dg::blas2::dot( w3d, y[1][0]))/normwe<<"\t"<<normwe<<"\n"
+              <<"    wi:   "<<sqrt(dg::blas2::dot( w3d, y[1][1]))/normwi<<"\t"<<normwi<<"\n";
+
+
+
+    return 0;
+}
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 333f78e5e..27d7ac9fe 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -28,9 +28,8 @@ int main( int argc, char* argv[])
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile]\n";
         return -1;
     }
-    const feltor::Parameters p( js, file::error::is_throw);// p.display( std::cout);
+    const feltor::Parameters p( js, file::error::is_throw);
     p.display( std::cout);
-//std::cout << "# "<<p.n<<" x "<<p.Nx<<" x "<<p.Ny<<" x "<<p.Nz<<"\n";
     const double R_0 = 10;
     const double I_0 = 20; //q factor at r=1 is I_0/R_0
     const double a  = 1; //small radius
diff --git a/src/feltor/manufactured.h b/src/feltor/manufactured.h
index dc31da535..48c3e5cc9 100644
--- a/src/feltor/manufactured.h
+++ b/src/feltor/manufactured.h
@@ -21,27 +21,65 @@ struct Ne{
 struct Ni{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return 1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)
+    return 1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/R + 
+  (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)
 ; }};
 struct Ue{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue))
+    return (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 52.63789013914324*R*Sin(2*Pi*R))*
+    Sin(2*Pi*t)*Sin(2*Pi*Z))/
+  (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))
 ; }};
 struct Ui{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3.
+    return (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+      10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+  (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+    R*(0.4052847345693511 + (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))
+; }};
+struct we{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+    DG_DEVICE double operator()(double R, double Z, double P, double t)const{
+    return (Sin(2*P)*Sin(2*Pi*t)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/mue + 
+      (12.566370614359172*Cos(2*Pi*R) - 157.91367041742973*R*Sin(2*Pi*R))/
+       (R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/
+  (3.*Sqrt(-mue))
+; }};
+struct Wi{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+    DG_DEVICE double operator()(double R, double Z, double P, double t)const{
+    return (Sin(2*P)*Sin(2*Pi*t)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/
+       (Sqrt(-mue)*mui) + (-2.5464790894703255*Cos(2*Pi*R) + 
+         31.999999999999996*R*Sin(2*Pi*R))/
+       (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+          Sin(Pi*Z) + R*(0.4052847345693511 + 
+            (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+             Sin(Pi*Z))))*Sin(2*Pi*Z))/3.
 ; }};
 struct Phie{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.
+    return (Sin(3*P)*(-0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+      R*(0.20000000000000004 + 17.765287921960844*mui*taui)*Sin(3*Pi*R))*
+    Sin(3*Pi*t)*Sin(3*Pi*Z))/R
 ; }};
 struct Phii{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.
+    return (Sin(3*P)*Sin(3*Pi*t)*(40*Sin(3*Pi*R)*Sin(3*Pi*Z) - 
+      (mui*Sin(3*P)*Sin(3*Pi*t)*
+         (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+            Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+              R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
+               Sin(3*Pi*R),2) + 
+           28034.10888465587*Power((0.005628954646796544*mui*taui + 
+                 Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
+               Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+            Power(Sin(3*Pi*Z),2)))/
+       (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/200.
 ; }};
 struct GammaPhie{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
@@ -51,43 +89,75 @@ struct GammaPhie{
 struct GammaNi{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return 1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)
+    return 1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)
 ; }};
 struct A{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (beta*Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*t)*Sin(4*Pi*Z))/4.
+    return (beta*(1 + Sqrt(-mue))*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+  (3.*Sqrt(-mue))
 ; }};
 struct SNe{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return 1.5707963267948966*Cos(Pi*t)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*Z) + 
-  (FELTORPARALLEL*(0. - (2.0943951023931953*(-10 + R)*R*Cos(2*Pi*Z)*
-          Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+  (FELTORPARALLEL*(0. - (6.283185307179586*(-10 + R)*Cos(2*Pi*Z)*Sin(2*P)*
+          (4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-       (0.5235987755982988*(-10 + R)*R*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-          Sin(Pi*(-10 + R))*Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*
-          Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-       (0.5235987755982988*R*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
-          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-       (3.3333333333333335*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
-          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-       (2.0943951023931953*R*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+       (3.141592653589793*(-10 + R)*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+          (4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
           Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-       (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+       (3.141592653589793*Z*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+          (4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+          Sin(Pi*Z)*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
+             Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+       (20.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+          (4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+          Sin(Pi*Z)*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
+             Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+       (1.5707963267948966*(-10 + R)*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+          Sin(Pi*(-10 + R))*(4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+          Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+       (1.5707963267948966*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
+          (4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+          Sin(Pi*Z)*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+       (10.*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
+          (4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+          Sin(Pi*Z)*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+       (1.*Z*Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
+            78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
           Sin(2*Pi*Z))/
-        (3.*Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-       (0.3333333333333333*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+       (40*Cos(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
           Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/R + 
+        (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/R + 
   FELTORPERP*(-((nuperp*(0. + 1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*
              Sin(Pi*t)*Sin(Pi*Z) - 
             9.869604401089358*R*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)\
@@ -95,580 +165,3558 @@ struct SNe{
          ((0.15000000000000002*R*(-20. + 2.*R)*taue*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
            (0.1*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (3*Pi*R*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
-              Sin(3*Pi*t))/
-            (100.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) \
-- (3*Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (2*beta*Power(Pi,2)*R*Cos(4*Pi*R)*Cos(4*Pi*Z)*Sin(2*P)*
-              Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (0.8882643960980423*Cos(3*Pi*Z)*Sin(3*P)*
+              (3*Pi*R*(-0.2122065907891938 - 
+                   18.849555921538755*mui*taui)*Cos(3*Pi*R) - 
+                9.42477796076938*mui*taui*Sin(3*Pi*R) + 
+                (-0.2122065907891938 - 18.849555921538755*mui*taui)*
+                 Sin(3*Pi*R))*Sin(3*Pi*t))/
+            Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
+           (0.44413219804902115*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*
+              (1.*mui*taui*Cos(3*Pi*R) + 
+                R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*
+                 Sin(3*Pi*R))*Sin(3*Pi*t))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (0.6579736267392905*beta*(1 + Sqrt(-mue))*Cos(Pi*R)*
+              Cos(2*Pi*Z)*Sin(P)*Power(Sin(2*P),2)*Sin(2*Pi*R)*
+              (4.1887902047863905*Cos(2*Pi*R) - 
+                52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+              Power(Sin(2*Pi*t),2)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+            (mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+              Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+           (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              (-330.73361792319804*R*Cos(2*Pi*R) - 
+                78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*R)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+           (2*beta*(1 + Sqrt(-mue))*Power(Pi,2)*Cos(2*Pi*R)*Cos(2*Pi*Z)*
+              Power(Sin(2*P),2)*
+              (4.1887902047863905*Cos(2*Pi*R) - 
+                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))) + 
-           (beta*Power(Pi,2)*R*Cos(2*Pi*(-10 + R))*Cos(4*Pi*Z)*Sin(2*P)*
-              Sin(4*P)*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))) - 
-           (beta*Pi*R*(-20. + 2.*R)*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*
-              Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*
+            (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+           (beta*(1 + Sqrt(-mue))*Pi*(-20. + 2.*R)*Cos(2*Pi*Z)*
+              Power(Sin(2*P),2)*Sin(2*Pi*R)*
+              (4.1887902047863905*Cos(2*Pi*R) - 
+                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (60.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),1.5)) + 
-           (beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2)))) + 
+            (30.*mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
+               1.5)*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
         1.5707963267948966*R*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z)*
          ((-0.1*R*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2)))) + 
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (0.8882643960980423*Cos(3*Pi*Z)*Sin(3*P)*
+              (1.*mui*taui*Cos(3*Pi*R) + 
+                R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*
+                 Sin(3*Pi*R))*Sin(3*Pi*t))/
+            Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
+           (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*R)*(4.1887902047863905*Cos(2*Pi*R) - 
+                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*Z))/
+            (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
          ((-0.1*R*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2)))) + 
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (0.8882643960980423*Cos(3*Pi*Z)*Sin(3*P)*
+              (1.*mui*taui*Cos(3*Pi*R) + 
+                R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*
+                 Sin(3*Pi*R))*Sin(3*Pi*t))/
+            Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
+           (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*R)*(4.1887902047863905*Cos(2*Pi*R) - 
+                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*Z))/
+            (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
         R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
          ((-3.*taue*Z*(-50. + 1.*R - 0.1*Power(Z,2)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
            (0.2*taue*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (2*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-            (45.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-           (beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+           (3*Pi*Cos(3*Pi*Z)*Sin(3*P)*
+              ((0.09424777960769379*mui*taui + 
+                   Power(R,2)*
+                    (0.1884955592153876 + 16.7433894073619*mui*taui))*
+                 Cos(3*Pi*R) + 0.8882643960980426*mui*R*taui*Sin(3*Pi*R))*
+              Sin(3*Pi*t))/
+            (R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (2*beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              (4.1887902047863905*Cos(2*Pi*R) - 
+                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))) - 
-           (0.011111111111111112*Z*
-              (-9*taue + Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (0.18849555921538758*R*Z*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*
-              Sin(3*Pi*Z))/
+            (15.*mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+           (0.10471975511965977*beta*(1 + Sqrt(-mue))*Cos(Pi*Z)*Sin(P)*
+              Power(Sin(2*P),2)*Sin(Pi*R)*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              (4.1887902047863905*Cos(2*Pi*R) - 
+                52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+              Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+            (mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+              Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+           (0.03333333333333333*beta*(1 + Sqrt(-mue))*Z*Power(Sin(2*P),2)*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              (4.1887902047863905*Cos(2*Pi*R) - 
+                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Power(Sin(2*Pi*Z),2))/
+            (mue*R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)*
+              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+           ((-34818.239691125345*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+                 Power(0.07957747154594767*Cos(2*Pi*R) - 
+                   1.*R*Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+               (Power(R,2)*Power(2. + 
+                   1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+              (17409.119845562673*Cos(Pi*Z)*Sin(P)*Power(Sin(2*P),2)*
+                 Sin(Pi*R)*Power(0.07957747154594767*Cos(2*Pi*R) - 
+                   1.*R*Sin(2*Pi*R),2)*Sin(Pi*t)*Power(Sin(2*Pi*t),2)*
+                 Power(Sin(2*Pi*Z),2))/
+               (Power(R,2)*Power(2. + 
+                   1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)))/
+            (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (0.1*Z*(taue - (2770.7474783005127*Power(Sin(2*P),2)*
+                   Power(0.07957747154594767*Cos(2*Pi*R) - 
+                     1.*R*Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
+                   Power(Sin(2*Pi*Z),2))/
+                 (Power(R,2)*Power(2. + 
+                     1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (beta*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
-              Sin(4*Pi*Z))/
-            (60.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))) + 
-           (0.008333333333333333*beta*Z*Sin(2*P)*Sin(4*P)*
-              Sin(2*Pi*(-10 + R))*(4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               1.5))) + 1.5707963267948966*R*Cos(Pi*Z)*Sin(P)*
-         Sin(Pi*(-10 + R))*Sin(Pi*t)*
+           (1.*Z*Sin(3*P)*((0.09424777960769379*mui*taui + 
+                   Power(R,2)*
+                    (0.1884955592153876 + 16.7433894073619*mui*taui))*
+                 Cos(3*Pi*R) + 0.8882643960980426*mui*R*taui*Sin(3*Pi*R))*
+              Sin(3*Pi*t)*Sin(3*Pi*Z))/
+            (R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5))) + 
+        1.5707963267948966*R*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
          ((taue*(-50. + 1.*R - 0.1*Power(Z,2)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (-9*taue + Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-               Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-            (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-           (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
-              Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-            (120.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2)))))/R)
+           (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              (4.1887902047863905*Cos(2*Pi*R) - 
+                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Power(Sin(2*Pi*Z),2))/
+            (30.*mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+           (taue - (2770.7474783005127*Power(Sin(2*P),2)*
+                 Power(0.07957747154594767*Cos(2*Pi*R) - 
+                   1.*R*Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
+                 Power(Sin(2*Pi*Z),2))/
+               (Power(R,2)*Power(2. + 
+                   1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)))/
+            (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (Sin(3*P)*((0.09424777960769379*mui*taui + 
+                   Power(R,2)*
+                    (0.1884955592153876 + 16.7433894073619*mui*taui))*
+                 Cos(3*Pi*R) + 0.8882643960980426*mui*R*taui*Sin(3*Pi*R))*
+              Sin(3*Pi*t)*Sin(3*Pi*Z))/
+            (R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/R)
 ; }};
 struct SNi{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return 1.5707963267948966*Cos(Pi*t)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*Z) + 
-  (FELTORPARALLEL*(0. - (2.0943951023931953*(-10 + R)*R*Cos(2*Pi*Z)*
-          Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
-        Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-       (0.5235987755982988*(-10 + R)*R*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-          Sin(Pi*(-10 + R))*Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*
-          Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-       (0.5235987755982988*R*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
-          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-        Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-       (3.3333333333333335*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
-          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-        Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-       (2.0943951023931953*R*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
-          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-          Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-       (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-          Sin(2*Pi*Z))/(3.*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) \
-+ (0.3333333333333333*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-          Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R + 
-  FELTORPERP*(-((nuperp*(0. + 1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*
-             Sin(Pi*t)*Sin(Pi*Z) - 
-            9.869604401089358*R*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)\
-))/R) + (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+    return (-1.2337005501361697*mui*taui*Cos(Pi*R)*Cos(Pi*t)*Sin(P)*Sin(Pi*Z))/R + 
+  Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*t)*Sin(P)*Sin(Pi*R)*
+   Sin(Pi*Z) + (FELTORPARALLEL*
+     (0. - (6.283185307179586*(-10 + R)*R*Cos(2*Pi*Z)*Sin(2*P)*
+          (-0.8488263631567752*Cos(2*Pi*R) + 
+            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z))/R + 
+            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
+             Sin(Pi*t)*Sin(Pi*Z)))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)))) + 
+       (1.*(-10 + R)*R*Sin(2*P)*
+          (-0.8488263631567752*Cos(2*Pi*R) + 
+            10.666666666666666*R*Sin(2*Pi*R))*
+          (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+            Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
+             Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t)*
+          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z))/R + 
+            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
+             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)),2)) - 
+       (20*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+          (-0.15915494309189535*mui*taui*Cos(P)*Cos(Pi*R)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+             Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
+          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z))/R + 
+            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
+             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)),2)) - 
+       (1.*R*Z*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+          (0.4052847345693511 + 
+            Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*Sin(P)*
+             Sin(Pi*t)*Sin(Pi*Z) + 
+            0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+            (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+             Sin(Pi*t)*Sin(Pi*Z))*
+          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z))/R + 
+            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
+             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)),2)) - 
+       (1.*(-10 + R)*R*Sin(2*P)*
+          (-0.8488263631567752*Cos(2*Pi*R) + 
+            10.666666666666666*R*Sin(2*Pi*R))*
+          ((-1.2337005501361697*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+               Sin(Pi*t))/R + 
+            Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*Z)*Sin(P)*
+             Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)))) + 
+       (20*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+          ((-0.39269908169872414*mui*taui*Cos(P)*Cos(Pi*R)*Sin(Pi*t)*
+               Sin(Pi*Z))/R + 
+            (0.25 + 2.4674011002723395*mui*taui)*Cos(P)*Sin(Pi*R)*
+             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)))) + 
+       (1.*R*Z*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+          ((0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+               Sin(Pi*Z))/Power(R,2) + 
+            Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*R)*Sin(P)*
+             Sin(Pi*t)*Sin(Pi*Z) + 
+            (1.2337005501361697*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+               Sin(Pi*Z))/R)*Sin(2*Pi*Z))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)))) + 
+       (1.*R*Z*Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 
+            16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
+          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z))/R + 
+            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
+             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)))) + 
+       (40*Cos(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z))/R + 
+            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
+             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)))) + 
+       (1.*Z*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z))/R + 
+            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
+             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z))))))/R + 
+  FELTORPERP*(-((nuperp*((0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z))/Power(R,2) + 
+            Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*R)*Sin(P)*
+             Sin(Pi*t)*Sin(Pi*Z) + 
+            (1.2337005501361697*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+               Sin(Pi*Z))/R + R*
+             ((3.875784585037477*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                  Sin(Pi*Z))/R - 
+               Power(Pi,2)*(0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+                Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)) + 
+            R*((-0.7853981633974483*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                  Sin(Pi*Z))/Power(R,3) + 
+               (3.875784585037477*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                  Sin(Pi*Z))/R - 
+               (2.4674011002723395*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                  Sin(Pi*Z))/Power(R,2) - 
+               Power(Pi,2)*(0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+                Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))/R) + 
+     (R*((0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z))/Power(R,2) + 
+           Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*R)*Sin(P)*
+            Sin(Pi*t)*Sin(Pi*Z) + 
+           (1.2337005501361697*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+              Sin(Pi*Z))/R)*((-0.1*R*taui*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*Z))/
+            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))*
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) - 
+           (R*Sin(3*P)*Sin(3*Pi*t)*
+              (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
+                (mui*Cos(3*Pi*Z)*Sin(3*P)*
+                   (528430.5031318273*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
+                     54*Power(Pi,3)*Power(R,2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
+                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) + 
+                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2))))/
+            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
+        (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+            Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
+         ((-0.1*R*taui*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*Z))/
+            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))*
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) - 
+           (R*Sin(3*P)*Sin(3*Pi*t)*
+              (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
+                (mui*Cos(3*Pi*Z)*Sin(3*P)*
+                   (528430.5031318273*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
+                     54*Power(Pi,3)*Power(R,2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
+                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) + 
+                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2))))/
+            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
+        R*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+            Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
          ((0.15000000000000002*R*(-20. + 2.*R)*taui*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
            (0.1*taui*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (3*Pi*R*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
-              Sin(3*Pi*t))/
-            (100.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) \
-- (3*Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (2*beta*Power(Pi,2)*R*Cos(4*Pi*R)*Cos(4*Pi*Z)*Sin(2*P)*
-              Sin(4*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
+           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              (0.4052847345693511 + 
+                Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
+                 Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))*
+              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                 Sin(Pi*t)*Sin(Pi*Z) + 
+                R*(0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*R)*(67.02064327658225*R*Cos(2*Pi*R) + 
+                16.*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))*
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) + 
+           (2*beta*(1 + Sqrt(-mue))*Power(Pi,2)*R*Cos(2*Pi*R)*Cos(2*Pi*Z)*
+              Power(Sin(2*P),2)*
+              (-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (15.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (beta*Power(Pi,2)*R*Cos(2*Pi*(-10 + R))*Cos(4*Pi*Z)*Sin(2*P)*
-              Sin(4*P)*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (15.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-           (beta*Pi*R*(-20. + 2.*R)*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*
-              Sin(2*Pi*(-10 + R))*Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*
+            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))*
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) - 
+           (beta*(1 + Sqrt(-mue))*Pi*R*(-20. + 2.*R)*Cos(2*Pi*Z)*
+              Power(Sin(2*P),2)*Sin(2*Pi*R)*
+              (-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (60.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) + 
-           (beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-        1.5707963267948966*R*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z)*
-         ((-0.1*R*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-         ((-0.1*R*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-            (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-        R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+            (30.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),1.5)*
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) + 
+           (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*Z))/
+            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))*
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) + 
+           (R*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
+              (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
+                (mui*Cos(3*Pi*Z)*Sin(3*P)*
+                   (528430.5031318273*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                       Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) \
+- 54*Power(Pi,3)*Power(R,2)*Power(0.942477796076938*mui*taui*
+                        Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
+                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) + 
+                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2))))/
+            (4000.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) \
+- (Sin(3*P)*Sin(3*Pi*t)*(120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
+                (mui*Cos(3*Pi*Z)*Sin(3*P)*
+                   (528430.5031318273*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                       Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) \
+- 54*Power(Pi,3)*Power(R,2)*Power(0.942477796076938*mui*taui*
+                        Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
+                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) + 
+                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2))))/
+            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (R*Sin(3*P)*Sin(3*Pi*t)*
+              (360*Power(Pi,2)*Cos(3*Pi*R)*Cos(3*Pi*Z) + 
+                (mui*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*
+                   (528430.5031318273*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
+                     54*Power(Pi,3)*Power(R,2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2)) + 
+                (2*mui*Cos(3*Pi*Z)*Sin(3*P)*
+                   (528430.5031318273*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
+                     54*Power(Pi,3)*Power(R,2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
+                 (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) - 
+                (mui*Cos(3*Pi*Z)*Sin(3*P)*
+                   (-108*Power(Pi,3)*Power(R,2)*
+                      (3*Pi*R*
+                        (-0.20000000000000004 - 
+                       17.765287921960844*mui*taui)*Cos(3*Pi*R) - 
+                        8.882643960980424*mui*taui*Sin(3*Pi*R) + 
+                        (-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R))*
+                      (0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R)) - 
+                     108*Power(Pi,3)*R*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     1.0568610062636547e6*
+                      ((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
+                      (0.5000000000000001*mui*R*taui*Cos(3*Pi*R) + 
+                        2*R*(0.011257909293593089 + 1.*mui*taui)*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*taui*Sin(3*Pi*R) - 
+                        3*Pi*
+                         (0.005628954646796544*mui*taui + 
+                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
+)*Sin(3*Pi*R)))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) - 
+                (4.*mui*(-20. + 2.*R)*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),3)) - 
+                (4.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,3)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2)) + 
+                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (18*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      (3*Pi*R*
+                        (-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Cos(3*Pi*R) - 
+                        8.882643960980424*mui*taui*Sin(3*Pi*R) + 
+                        (-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R))*
+                      (0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                         17.765287921960844*mui*taui)*Sin(3*Pi*R)) + 
+                     18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     56068.21776931174*
+                      ((0.005628954646796544*mui*taui + 
+                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
+)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
+                      (0.5000000000000001*mui*R*taui*Cos(3*Pi*R) + 
+                        2*R*(0.011257909293593089 + 1.*mui*taui)*
+                         Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*taui*Sin(3*Pi*R) - 
+                        3*Pi*(0.005628954646796544*mui*taui + 
+                         Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
+)*Sin(3*Pi*R))*Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2))))/
+            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
+        R*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+            Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
          ((-3.*taui*Z*(-50. + 1.*R - 0.1*Power(Z,2)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
            (0.2*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-           (2*mui*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-            (45.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-           (beta*Pi*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
-              Sin(2*Pi*Z))/
-            (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-           (0.011111111111111112*Z*
-              (-9*taui - mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (0.18849555921538758*R*Z*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*
-              Sin(3*Pi*Z))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (beta*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
-              Sin(4*Pi*Z))/
-            (60.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (0.008333333333333333*beta*Z*Sin(2*P)*Sin(4*P)*
-              Sin(2*Pi*(-10 + R))*(4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*
-              Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z)*Sin(4*Pi*Z))/
+           (2*beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              (-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*Z))/
+            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))*
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) + 
+           (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              (-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*
+              (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+                Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*
+                 Sin(P)*Sin(Pi*R)*Sin(Pi*t))*Power(Sin(2*Pi*t),2)*
+              Power(Sin(2*Pi*Z),2))/
+            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))*
+              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                 Sin(Pi*t)*Sin(Pi*Z) + 
+                R*(0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+           (0.03333333333333333*beta*(1 + Sqrt(-mue))*Z*Power(Sin(2*P),2)*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              (-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Power(Sin(2*Pi*Z),2))/
+            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),1.5)*
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) - 
+           ((4*mui*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+                 Power(0.8488263631567752*Cos(2*Pi*R) - 
+                   10.666666666666666*R*Sin(2*Pi*R),2)*
+                 Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+               Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                  Sin(Pi*t)*Sin(Pi*Z) + 
+                 R*(0.4052847345693511 + 
+                    (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                     Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+              (2*mui*Power(Sin(2*P),2)*
+                 Power(0.8488263631567752*Cos(2*Pi*R) - 
+                   10.666666666666666*R*Sin(2*Pi*R),2)*
+                 (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+                   Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*
+                    Sin(P)*Sin(Pi*R)*Sin(Pi*t))*Power(Sin(2*Pi*t),2)*
+                 Power(Sin(2*Pi*Z),2))/
+               Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                  Sin(Pi*t)*Sin(Pi*Z) + 
+                 R*(0.4052847345693511 + 
+                    (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                     Sin(Pi*t)*Sin(Pi*Z)),3))/
+            (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (0.1*Z*(taui + (mui*Power(Sin(2*P),2)*
+                   Power(0.8488263631567752*Cos(2*Pi*R) - 
+                     10.666666666666666*R*Sin(2*Pi*R),2)*
+                   Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+                 Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                    Sin(Pi*t)*Sin(Pi*Z) + 
+                   R*(0.4052847345693511 + 
+                      (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                       Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (R*Sin(3*P)*Sin(3*Pi*t)*
+              (360*Power(Pi,2)*Cos(3*Pi*R)*Cos(3*Pi*Z) + 
+                (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
+                   (528430.5031318273*Cos(3*Pi*Z)*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                       Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Sin(3*Pi*Z) - 
+                     54*Power(Pi,3)*Power(R,2)*Cos(3*Pi*Z)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2)*
+                      Sin(3*Pi*Z)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2)) + 
+                (2*mui*Sin(3*P)*Sin(3*Pi*t)*
+                   (528430.5031318273*Cos(3*Pi*Z)*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                       Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Sin(3*Pi*Z) - 
+                     54*Power(Pi,3)*Power(R,2)*Cos(3*Pi*Z)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2)*
+                      Sin(3*Pi*Z)))/
+                 (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) - 
+                (mui*Sin(3*P)*Sin(3*Pi*t)*
+                   (-108*Power(Pi,3)*R*Cos(3*Pi*Z)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                       17.765287921960844*mui*taui)*Sin(3*Pi*R),2)*
+                      Sin(3*Pi*Z) - 
+                     9.960680319430241e6*Power(R,2)*Cos(3*Pi*Z)*
+                      (R*(0.011257909293593089 + 1.*mui*taui)*
+                        Cos(3*Pi*R) + 
+                        (0.0011945012753036852 + 
+                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
+                      (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
+                        R*(0.011257909293593089 + 1.*mui*taui)*
+                        Sin(3*Pi*R))*Sin(3*Pi*Z) - 
+                     3.1705830187909645e6*Cos(3*Pi*Z)*
+                      ((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
+                      (R*(-0.007505272862395392 - 
+                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
+                        (-6.938893903907228e-18*mui*taui + 
+                        Power(R,2)*
+                        (0.03536776513153231 + 
+                        3.141592653589793*mui*taui))*Sin(3*Pi*R))*
+                      Sin(3*Pi*Z)))/
+                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) - 
+                (4.*mui*(-20. + 2.*R)*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),3)) - 
+                (4.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,3)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2)) + 
+                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                   (18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      (R*(0.011257909293593089 + 1.*mui*taui)*
+                        Cos(3*Pi*R) + 
+                        (0.0011945012753036852 + 
+                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
+                      (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
+                        R*(0.011257909293593089 + 1.*mui*taui)*
+                         Sin(3*Pi*R)) - 
+                     168204.65330793522*
+                      ((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
+                      (R*(-0.007505272862395392 - 
+                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
+                        (-6.938893903907228e-18*mui*taui + 
+                        Power(R,2)*
+                        (0.03536776513153231 + 
+                        3.141592653589793*mui*taui))*Sin(3*Pi*R))*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2))))/
+            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (0.0005*R*Z*Sin(3*P)*Sin(3*Pi*t)*
+              (120*Pi*Cos(3*Pi*R)*Sin(3*Pi*Z) + 
+                (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2)) + 
+                (2*mui*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) - 
+                (mui*Sin(3*P)*Sin(3*Pi*t)*
+                   (18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      (R*(0.011257909293593089 + 1.*mui*taui)*
+                        Cos(3*Pi*R) + 
+                        (0.0011945012753036852 + 
+                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
+                      (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
+                        R*(0.011257909293593089 + 1.*mui*taui)*
+                         Sin(3*Pi*R)) - 
+                     168204.65330793522*
+                      ((0.005628954646796544*mui*taui + 
+                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
+)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
+                      (R*(-0.007505272862395392 - 
+                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
+                        (-6.938893903907228e-18*mui*taui + 
+                         Power(R,2)*
+                         (0.03536776513153231 + 
+                         3.141592653589793*mui*taui))*Sin(3*Pi*R))*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2)))))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) + 
-        1.5707963267948966*R*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
+        R*((-1.2337005501361697*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+              Sin(Pi*t))/R + Pi*(0.25 + 2.4674011002723395*mui*taui)*
+            Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
          ((taui*(-50. + 1.*R - 0.1*Power(Z,2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (-9*taui - mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-               Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-            (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-           (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-              (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
-              Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-            (120.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/R)
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              (-0.8488263631567752*Cos(2*Pi*R) + 
+                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Power(Sin(2*Pi*Z),2))/
+            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))*
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) - 
+           (taui + (mui*Power(Sin(2*P),2)*
+                 Power(0.8488263631567752*Cos(2*Pi*R) - 
+                   10.666666666666666*R*Sin(2*Pi*R),2)*
+                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+               Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                  Sin(Pi*t)*Sin(Pi*Z) + 
+                 R*(0.4052847345693511 + 
+                    (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                     Sin(Pi*t)*Sin(Pi*Z)),2))/
+            (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (R*Sin(3*P)*Sin(3*Pi*t)*
+              (120*Pi*Cos(3*Pi*R)*Sin(3*Pi*Z) + 
+                (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2),2)) + 
+                (2*mui*Sin(3*P)*Sin(3*Pi*t)*
+                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     28034.10888465587*
+                      Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 
+                     1.*Power(Z,2))) - 
+                (mui*Sin(3*P)*Sin(3*Pi*t)*
+                   (18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
+                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                        R*(-0.20000000000000004 - 
+                         17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                     528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                      (R*(0.011257909293593089 + 1.*mui*taui)*
+                         Cos(3*Pi*R) + 
+                        (0.0011945012753036852 + 
+                         0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
+                      (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
+                        R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R)\
+) - 168204.65330793522*((0.005628954646796544*mui*taui + 
+                         Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
+)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
+                      (R*(-0.007505272862395392 - 
+                         0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
+                        (-6.938893903907228e-18*mui*taui + 
+                         Power(R,2)*
+                         (0.03536776513153231 + 
+                         3.141592653589793*mui*taui))*Sin(3*Pi*R))*
+                      Power(Sin(3*Pi*Z),2)))/
+                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))\
+))/(2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/R)
 ; }};
-struct SUe{
+struct SWe{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (2*Pi*Cos(2*Pi*t)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
-  FELTORPARALLEL*((taue*Sin(Pi*t)*
-        ((15.707963267948966 - 1.5707963267948966*R)*R*Cos(Pi*Z)*Sin(P)*
-           Sin(Pi*R) + (1.5707963267948966*R*Z*Cos(Pi*R)*Sin(P) + 
-             10.*Cos(P)*Sin(Pi*R))*Sin(Pi*Z)))/
+    return (-1.0471975511965976*Cos(Pi*t)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+     (12.566370614359172*Cos(2*Pi*R) - 157.91367041742973*R*Sin(2*Pi*R))*
+     Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+   (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+  (2*Pi*Cos(2*Pi*t)*Sin(2*P)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/mue + 
+       (12.566370614359172*Cos(2*Pi*R) - 157.91367041742973*R*Sin(2*Pi*R))/
+        (R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/
+   (3.*Sqrt(-mue)) + FELTORPARALLEL*
+   ((taue*Sin(Pi*t)*((15.707963267948966 - 1.5707963267948966*R)*R*
+           Cos(Pi*Z)*Sin(P)*Sin(Pi*R) + 
+          (1.5707963267948966*R*Z*Cos(Pi*R)*Sin(P) + 10.*Cos(P)*Sin(Pi*R))*
+           Sin(Pi*Z)))/
       (mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (0.6981317007977318*(-10 + R)*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-        Power(Sin(2*Pi*(-10 + R)),2)*Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-      (mue*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-     (0.6981317007977318*Z*Cos(2*Pi*(-10 + R))*Power(Sin(2*P),2)*
-        Sin(2*Pi*(-10 + R))*Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-      (mue*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-     (40*Cos(2*P)*Sin(2*P)*Power(Sin(2*Pi*(-10 + R)),2)*
-        Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-      (9.*mue*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) - 
+     (1.*(-10 + R)*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+          52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
+        ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+          (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+             Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*Power(2. + 
+               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+      (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+        (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (20*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+          52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
+        ((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+             Sin(Pi*Z)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*Power(2. + 
+               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+          (2*Cos(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+      (Sqrt(-mue)*Power(R,2)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+          1.*Power(Z,2))*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (1.*Z*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+          52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
+        ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+             Sin(Pi*Z)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*Power(2. + 
+               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+          (Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
+               78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+          (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*Power(R,2)*
+             (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+      (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+        (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
      (nuparallele*(0. + (1.*Z*
-             ((-2.0943951023931953*(-10 + R)*Cos(2*Pi*Z)*Sin(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2))) + 
-               (2.0943951023931953*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2))) + 
-               (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (3.*Sqrt(-mue)*R*
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
+             ((-1.*(-10 + R)*
+                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (20*((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (2*Cos(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
+)/(R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+               (1.*Z*((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (Sin(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
+           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+          (20*((-1.*(-10 + R)*
+                  ((-6.283185307179586*Cos(P)*Cos(2*Pi*Z)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
++ (6.283185307179586*Cos(P)*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+                       Power(Sin(Pi*R),2)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*
+                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(Pi*Z)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
+                    (6.283185307179586*Cos(2*P)*Cos(Pi*Z)*Sin(P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (3.141592653589793*Cos(P)*Cos(Pi*Z)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (1.*Z*((6.283185307179586*Cos(P)*Cos(Pi*R)*Sin(P)*
+                       Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*
+                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*
+                       Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
+                    (1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                       78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (6.283185307179586*Cos(2*P)*Cos(Pi*R)*Sin(P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (3.141592653589793*Cos(P)*Cos(Pi*R)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (2*Cos(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                       78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (2*Cos(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
+)/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (20*((2.*Power(Cos(P),2)*Sin(2*P)*Power(Sin(Pi*R),2)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*
+                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*
+                       Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
+                    (4.*Cos(P)*Cos(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (1.*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (4*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (1.*(-10 + R)*R*((13.15947253478581*Z*Cos(2*Pi*(-10 + R))*
-                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2))) + 
-               (80*Pi*Cos(2*P)*Cos(2*Pi*Z)*Sin(2*Pi*(-10 + R))*
-                  Sin(2*Pi*t))/
-                (3.*Sqrt(-mue)*R*
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-               (2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*Z)*Sin(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-                (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2),1.5)) - 
-               (2.0943951023931953*Power(Z,2)*Cos(2*Pi*(-10 + R))*
-                  Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2),1.5)) + 
-               (2.0943951023931953*Cos(2*Pi*(-10 + R))*Sin(2*P)*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2))) - 
-               (13.333333333333334*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (Sqrt(-mue)*R*
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) \
-+ (13.15947253478581*(-10 + R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                  Sin(2*Pi*Z))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2)))))/
+          (1.*(-10 + R)*R*((20*
+                  ((-6.283185307179586*Cos(P)*Cos(2*Pi*Z)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
++ (6.283185307179586*Cos(P)*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+                       Power(Sin(Pi*R),2)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*
+                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(Pi*Z)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
+                    (6.283185307179586*Cos(2*P)*Cos(Pi*Z)*Sin(P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (3.141592653589793*Cos(P)*Cos(Pi*Z)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+               (1.*Z*((-19.739208802178716*Cos(Pi*R)*Cos(2*Pi*Z)*
+                       Sin(P)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                       78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
++ (19.739208802178716*Cos(Pi*R)*Cos(Pi*Z)*Power(Sin(P),2)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*
+                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(Pi*Z)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
+                    (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                       78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (9.869604401089358*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                       Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (1.*(-10 + R)*Z*
+                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
+               (20.*Z*((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (2*Cos(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
+)/(R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) - 
+               (1.*Power(Z,2)*
+                  ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (Sin(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                       78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
+)/Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
+               (1.*((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (Sin(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                       78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
+)/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+               (1.*(-10 + R)*
+                  ((-39.47841760435743*Cos(Pi*Z)*Cos(2*Pi*Z)*Sin(P)*
+                       Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (19.739208802178716*Power(Cos(Pi*Z),2)*
+                       Power(Sin(P),2)*Sin(2*P)*Power(Sin(Pi*R),2)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*
+                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) + 
+                    (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (4*Power(Pi,2)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (20*((-4.1887902047863905*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2))) + 
-               (4.1887902047863905*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+          (1.*R*Z*((-1.*(-10 + R)*
+                  ((-19.739208802178716*Cos(Pi*R)*Cos(2*Pi*Z)*Sin(P)*
+                       Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
++ (19.739208802178716*Cos(Pi*R)*Cos(Pi*Z)*Power(Sin(P),2)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*
+                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(Pi*Z)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
+                    (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                        78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (9.869604401089358*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                       Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+                       Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (1.*Power(-10 + R,2)*
+                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
+               (1.*((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (20*((6.283185307179586*Cos(P)*Cos(Pi*R)*Sin(P)*
+                       Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*
+                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*
+                       Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
+                    (1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                        78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (6.283185307179586*Cos(2*P)*Cos(Pi*R)*Sin(P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (3.141592653589793*Cos(P)*Cos(Pi*R)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (2*Cos(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (2*Cos(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+               (20.*(-10 + R)*
+                  ((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (2*Cos(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) \
+- (20*((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (2*Cos(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                (Power(R,2)*Sqrt(400 + 1.*Power(-10 + R,2) + 
                     1.*Power(Z,2))) - 
-               (80*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (3.*Sqrt(-mue)*R*
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (1.*R*Z*((-13.15947253478581*(-10 + R)*Cos(2*Pi*(-10 + R))*
-                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2))) + 
-               (2.0943951023931953*Power(-10 + R,2)*Cos(2*Pi*Z)*
-                  Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-                (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2),1.5)) - 
-               (2.0943951023931953*Cos(2*Pi*Z)*Sin(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2))) + 
-               (80*Pi*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                  Sin(2*Pi*Z))/
-                (3.*Sqrt(-mue)*R*
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               (2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
-                  Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2),1.5)) - 
-               (13.333333333333334*(-10 + R)*Cos(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (Sqrt(-mue)*R*
-                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) - 
-               (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (3.*Sqrt(-mue)*Power(R,2)*
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               (13.15947253478581*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2)))))/
+               (1.*(-10 + R)*Z*
+                  ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (Sin(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
+- (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
+               (1.*Z*((19.739208802178716*Power(Cos(Pi*R),2)*
+                       Power(Sin(P),2)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*
+                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*
+                       Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
+                    (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                        78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+                    (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
+                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       Power(2. + 
+                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+                    (2*Sin(2*P)*
+                       (-330.73361792319804*R*Cos(2*Pi*R) - 
+                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,2)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+                    (2*Sin(2*P)*
+                       (4.1887902047863905*Cos(2*Pi*R) - 
+                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*Power(R,3)*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+                    (Sin(2*P)*
+                       (-826.834044807995*Cos(2*Pi*R) + 
+                        2078.060608725385*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (Sqrt(-mue)*R*
+                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
       (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (Sin(3*Pi*t)*((18.84955592153876 - 1.8849555921538759*R)*R*
-           Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R) + 
-          (1.8849555921538759*R*Z*Cos(3*Pi*R)*Sin(3*P) + 
-             12.*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*Z)))/
-      (mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-  (beta*Pi*Cos(4*Pi*t)*Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*Z))/mue + 
-  FELTORPERP*(0. - (0.03333333333333333*taue*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+     (Sin(3*Pi*t)*(R*Sin(3*Pi*R)*
+           (R*(18.849555921538762 + 1674.3389407361901*mui*taui + 
+                R*(-1.884955592153876 - 167.433894073619*mui*taui))*
+              Cos(3*Pi*Z)*Sin(3*P) + 
+             ((12. + 1065.9172753176506*mui*taui)*Cos(3*P) + 
+                8.882643960980428*mui*taui*Z*Sin(3*P))*Sin(3*Pi*Z)) + 
+          Cos(3*Pi*R)*(mui*R*(-88.82643960980423 + 8.882643960980422*R)*
+              taui*Cos(3*Pi*Z)*Sin(3*P) + 
+             (-56.54866776461628*mui*taui*Cos(3*P) + 
+                (0.9424777960769379*mui*taui + 
+                   Power(R,2)*
+                    (1.884955592153876 + 167.433894073619*mui*taui))*Z*
+                 Sin(3*P))*Sin(3*Pi*Z))))/
+      (mue*Power(R,2)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
+  FELTORPERP*(0. - (0.1*taue*Z*Sin(2*P)*
+        (4.1887902047863905*Cos(2*Pi*R) - 52.63789013914324*R*Sin(2*Pi*R))*
         Sin(2*Pi*t)*Sin(2*Pi*Z))/
-      (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) - 
-     (Pi*Cos(3*Pi*Z)*Sin(2*P)*Sin(3*P)*Sin(2*Pi*(-10 + R))*Sin(3*Pi*R)*
-        Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z))/
-      (50.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-     (eta*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-        ((Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3. - 
-          (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (3.*Sqrt(-mue))))/mue - 
-     (nuperp*((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (3.*Sqrt(-mue)) - (8*Power(Pi,2)*R*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-             Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue))))/R + 
-     (2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        ((-0.1*R*taue*Z)/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-          (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
-           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-          (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-             Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-           (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-               1.*Power(Z,2)))))/(3.*Sqrt(-mue)) + 
-     ((-0.20943951023931953*R*taue*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*R)*
-           Sin(2*Pi*t)*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
+      (Sqrt(-mue)*R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)*
+        (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (0.8882643960980423*Cos(3*Pi*Z)*Sin(2*P)*Sin(3*P)*
+        (4.1887902047863905*Cos(2*Pi*R) - 52.63789013914324*R*Sin(2*Pi*R))*
+        (1.*mui*taui*Cos(3*Pi*R) + 
+          R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*Sin(3*Pi*R)\
+)*Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z))/
+      (Sqrt(-mue)*Power(R,2)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+          1.*Power(Z,2))*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (0.32898681336964525*beta*(1. + Sqrt(-mue))*taue*Sin(P)*Sin(2*P)*
+        Sin(Pi*t)*Sin(2*Pi*t)*(1.*R*Cos(Pi*R)*Cos(2*Pi*Z)*Sin(2*Pi*R)*
+           Sin(Pi*Z) + Cos(Pi*Z)*Sin(Pi*R)*
+           (-1.*R*Cos(2*Pi*R) - 0.15915494309189535*Sin(2*Pi*R))*Sin(2*Pi*Z)\
+))/(Sqrt(-mue)*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
+     ((33.0733617923198*taue*Cos(2*Pi*Z)*Sin(2*P)*
+           (-0.07957747154594767*Cos(2*Pi*R) + 1.*R*Sin(2*Pi*R))*
+           Sin(2*Pi*t))/
          (Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-        (0.10471975511965977*R*taue*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-           Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-        (0.03333333333333333*R*taue*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+        (5.263789013914324*taue*Z*Sin(2*P)*
+           (-0.07957747154594767*Cos(2*Pi*R) + 1.*R*Sin(2*Pi*R))*
+           Sin(2*Pi*t)*Sin(2*Pi*Z))/
          (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
             1.5)))/(R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) \
-+ (beta*taue*Sin(P)*Sin(4*P)*Sin(Pi*t)*Sin(4*Pi*t)*
-        (0.4934802200544679*R*Cos(Pi*R)*Cos(4*Pi*Z)*Sin(4*Pi*R)*Sin(Pi*Z) + 
-          Cos(Pi*Z)*Sin(Pi*R)*(-0.4934802200544679*R*Cos(4*Pi*R) - 
-             0.039269908169872414*Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-      (mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (3*beta*Pi*Sin(3*P)*Sin(4*P)*Sin(3*Pi*t)*Sin(4*Pi*t)*
-        (4*Pi*R*Cos(3*Pi*R)*Cos(4*Pi*Z)*Sin(4*Pi*R)*Sin(3*Pi*Z) - 
-          Cos(3*Pi*Z)*Sin(3*Pi*R)*(4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*
-           Sin(4*Pi*Z)))/
-      (200.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-     (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-        ((taue*(-50. + 1.*R - 0.1*Power(Z,2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-          (-9*taue + Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
++ ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+           (4.1887902047863905*Cos(2*Pi*R) - 
+             52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+           Sin(Pi*Z)*Sin(2*Pi*Z))/
+         (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
+            2)) + (Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
+             78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+         (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+        (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+             52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+         (Sqrt(-mue)*Power(R,2)*
+           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))*
+      ((-0.1*R*taue*Z)/
+         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+        (0.8882643960980423*Cos(3*Pi*Z)*Sin(3*P)*
+           (1.*mui*taui*Cos(3*Pi*R) + 
+             R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*
+              Sin(3*Pi*R))*Sin(3*Pi*t))/
+         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
+        (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*Sin(2*Pi*R)*
+           (4.1887902047863905*Cos(2*Pi*R) - 
+             52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+           Sin(2*Pi*Z))/
+         (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
+     (eta*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+        (-((Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                 52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+             (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) \
++ (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)))))/mue - 
+     (nuperp*((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+             Sin(Pi*Z)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*Power(2. + 
+               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+          (Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
+               78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+          (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*Power(R,2)*
+             (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+          R*((-39.47841760435743*Cos(Pi*Z)*Cos(2*Pi*Z)*Sin(P)*Sin(2*P)*
+                Sin(Pi*R)*(4.1887902047863905*Cos(2*Pi*R) - 
+                  52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t))/
+              (Sqrt(-mue)*R*Power(2. + 
+                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+             (19.739208802178716*Power(Cos(Pi*Z),2)*Power(Sin(P),2)*
+                Sin(2*P)*Power(Sin(Pi*R),2)*
+                (4.1887902047863905*Cos(2*Pi*R) - 
+                  52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(Pi*t),2)*
+                Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*Power(2. + 
+                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) + 
+             (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+                (4.1887902047863905*Cos(2*Pi*R) - 
+                  52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+                Sin(Pi*Z)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*Power(2. + 
+                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+             (4*Power(Pi,2)*Sin(2*P)*
+                (4.1887902047863905*Cos(2*Pi*R) - 
+                  52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)\
+)/(Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
+          R*((19.739208802178716*Power(Cos(Pi*R),2)*Power(Sin(P),2)*
+                Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                  52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(Pi*t),2)*
+                Sin(2*Pi*t)*Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*Power(2. + 
+                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
+             (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+                (-330.73361792319804*R*Cos(2*Pi*R) - 
+                  78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+                Sin(Pi*Z)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*Power(2. + 
+                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+             (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+                (4.1887902047863905*Cos(2*Pi*R) - 
+                  52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+                Sin(Pi*Z)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*Power(R,2)*
+                Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+             (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+                (4.1887902047863905*Cos(2*Pi*R) - 
+                  52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+                Sin(Pi*Z)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*Power(2. + 
+                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+             (2*Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
+                  78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*Power(R,2)*
+                (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+             (2*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                  52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)\
+)/(Sqrt(-mue)*Power(R,3)*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+             (Sin(2*P)*(-826.834044807995*Cos(2*Pi*R) + 
+                  2078.060608725385*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
+))/R + ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+           (4.1887902047863905*Cos(2*Pi*R) - 
+             52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+         (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+        (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+           (4.1887902047863905*Cos(2*Pi*R) - 
+             52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+           Sin(2*Pi*Z))/
+         (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
+            2)))*((taue*(-50. + 1.*R - 0.1*Power(Z,2)))/
+         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+        (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
+           (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+           (4.1887902047863905*Cos(2*Pi*R) - 
+             52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+           Power(Sin(2*Pi*Z),2))/
+         (30.*mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+        (taue - (2770.7474783005127*Power(Sin(2*P),2)*
+              Power(0.07957747154594767*Cos(2*Pi*R) - 1.*R*Sin(2*Pi*R),2)*
               Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-           (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-          (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-          (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-             (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
-             Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-           (120.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-               1.*Power(Z,2)))))/(3.*Sqrt(-mue)))
+            (Power(R,2)*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
+               2)))/(10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) \
++ (Sin(3*P)*((0.09424777960769379*mui*taui + 
+                Power(R,2)*(0.1884955592153876 + 
+                   16.7433894073619*mui*taui))*Cos(3*Pi*R) + 
+             0.8882643960980426*mui*R*taui*Sin(3*Pi*R))*Sin(3*Pi*t)*
+           Sin(3*Pi*Z))/
+         (R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) - 
+     (35.06727277224087*beta*(1. + Sqrt(-mue))*Sin(2*P)*Sin(3*P)*Sin(2*Pi*t)*
+        Sin(3*Pi*t)*(R*Cos(2*Pi*R)*Cos(3*Pi*Z)*
+           (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
+             R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R))*Sin(2*Pi*Z) \
++ Sin(2*Pi*R)*(R*Sin(3*Pi*R)*((0.0017917519129555279 + 
+                   0.15915494309189535*mui*taui)*Cos(3*Pi*Z)*Sin(2*Pi*Z) - 
+                0.053051647697298476*mui*taui*Cos(2*Pi*Z)*Sin(3*Pi*Z)) + 
+             Cos(3*Pi*R)*(-0.008443431970194816*mui*taui*Cos(3*Pi*Z)*
+                 Sin(2*Pi*Z) + 
+                (-0.005628954646796544*mui*taui + 
+                   Power(R,2)*(-0.011257909293593089 - 1.*mui*taui))*
+                 Cos(2*Pi*Z)*Sin(3*Pi*Z)))))/
+      (Sqrt(-mue)*mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))))
 ; }};
-struct SUi{
+struct SWi{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (2*Pi*Cos(2*Pi*t)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*Z))/3. + 
+    return -(Sin(2*P)*(-2.5464790894703255*Cos(2*Pi*R) + 
+        31.999999999999996*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+      (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*t)*Sin(P)*Sin(Pi*Z) + 
+        Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*t)*Sin(P)*
+         Sin(Pi*R)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+   (3.*Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+        Sin(Pi*Z) + R*(0.4052847345693511 + 
+          (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+           Sin(Pi*Z)),2)) + (2*Pi*Cos(2*Pi*t)*Sin(2*P)*
+     ((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/(Sqrt(-mue)*mui) + 
+       (-2.5464790894703255*Cos(2*Pi*R) + 
+          31.999999999999996*R*Sin(2*Pi*R))/
+        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+           Sin(Pi*Z) + R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+              Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/3. + 
+  FELTORPERP*(0. + (1.623484850566707*beta*(1. + Sqrt(-mue))*taui*Sin(P)*
+        Sin(2*P)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*
+        (0.3183098861837907*mui*R*taui*Power(Cos(Pi*R),3)*
+           Power(Cos(Pi*Z),2) + 
+          R*(-0.06450306886639899 - 0.6366197723675813*mui*taui)*
+           Cos(Pi*R)*Power(Cos(Pi*Z),2)*Power(Sin(Pi*R),2) + 
+          Power(Cos(Pi*R),2)*Sin(Pi*R)*
+           (0.20264236728467555*mui*taui*Power(Cos(Pi*Z),2) + 
+             (-0.10132118364233778*mui*taui + 
+                Power(R,2)*(-0.20264236728467555 - 2.*mui*taui))*
+              Power(Sin(Pi*Z),2)) + 
+          R*Sin(Pi*R)*(R*(0.20264236728467555 + 2.*mui*taui)*
+              Power(Cos(Pi*Z),2)*Power(Sin(Pi*R),2) - 
+             0.15915494309189535*mui*taui*Sin(2*Pi*R)*Power(Sin(Pi*Z),2))))/
+      (Sqrt(-mue)*mui*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+     (0.1*taui*Z*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+      (Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)*
+        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+           Sin(Pi*Z) + R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+              Sin(Pi*Z)))) + ((-33.0733617923198*taui*Cos(2*Pi*Z)*
+           Sin(2*P)*(-0.07957747154594767*Cos(2*Pi*R) + 
+             1.*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+        (5.263789013914324*taui*Z*Sin(2*P)*
+           (-0.07957747154594767*Cos(2*Pi*R) + 1.*R*Sin(2*Pi*R))*
+           Sin(2*Pi*t)*Sin(2*Pi*Z))/
+         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5))/
+      (R*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (eta*Power(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z),2)*
+        (-((Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+                 52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)\
+)/(Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
+          (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)))))/
+      (mui*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+     (nuperp*(-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                 10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+               (0.4052847345693511 + 
+                 Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
+                  Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                 0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                 (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                  Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+             Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                Sin(Pi*t)*Sin(Pi*Z) + 
+               R*(0.4052847345693511 + 
+                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                   Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+          (Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
+             Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z))) + 
+          R*((2*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                  10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                Power(0.4052847345693511 + 
+                  Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
+                   Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                  0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                   Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                 Sin(Pi*t)*Sin(Pi*Z) + 
+                R*(0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)),3) - 
+             (2*Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 
+                  16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                (0.4052847345693511 + 
+                  Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
+                   Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                  0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                   Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                 Sin(Pi*t)*Sin(Pi*Z) + 
+                R*(0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)),2) - 
+             (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                  10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*
+                   Sin(Pi*t)*Sin(Pi*Z) + 
+                  2*Pi*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
+                   Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
+                  Power(Pi,2)*R*(0.10132118364233778 + 1.*mui*taui)*
+                   Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                 Sin(Pi*t)*Sin(Pi*Z) + 
+                R*(0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)),2) + 
+             (Sin(2*P)*(167.5516081914556*Cos(2*Pi*R) - 
+                  421.1031211131459*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                Sin(2*Pi*Z))/
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)))) + 
+          R*((-4*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                (-0.8488263631567752*Cos(2*Pi*R) + 
+                  10.666666666666666*R*Sin(2*Pi*R))*
+                (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+                  Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*
+                   Sin(P)*Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t))/
+              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                 Sin(Pi*t)*Sin(Pi*Z) + 
+                R*(0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)),2) + 
+             (2*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                  10.666666666666666*R*Sin(2*Pi*R))*
+                Power(-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                   Sin(Pi*t) + 
+                  Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*
+                   Sin(P)*Sin(Pi*R)*Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                 Sin(Pi*t)*Sin(Pi*Z) + 
+                R*(0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)),3) - 
+             (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                  10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*
+                   Sin(Pi*t)*Sin(Pi*Z) - 
+                  Power(Pi,2)*R*(0.10132118364233778 + 1.*mui*taui)*
+                   Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                 Sin(Pi*t)*Sin(Pi*Z) + 
+                R*(0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z)),2) - 
+             (4*Power(Pi,2)*Sin(2*P)*
+                (-0.8488263631567752*Cos(2*Pi*R) + 
+                  10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                Sin(2*Pi*Z))/
+              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                 Sin(Pi*Z) + R*
+                 (0.4052847345693511 + 
+                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                    Sin(Pi*t)*Sin(Pi*Z))))))/R - 
+     (Sin(2*P)*Sin(3*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(3*Pi*t)*
+        Sin(2*Pi*Z)*(120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
+          (mui*Cos(3*Pi*Z)*Sin(3*P)*
+             (528430.5031318273*
+                Power((0.005628954646796544*mui*taui + 
+                     Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
+)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
+               54*Power(Pi,3)*Power(R,2)*
+                Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                  R*(-0.20000000000000004 - 
+                     17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+             Sin(3*Pi*t)*Sin(3*Pi*Z))/
+           (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+             (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                  R*(-0.20000000000000004 - 
+                     17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+               28034.10888465587*
+                Power((0.005628954646796544*mui*taui + 
+                     Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
+                   Cos(3*Pi*R) + 
+                  0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                Power(Sin(3*Pi*Z),2)))/
+           (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
+              2))))/
+      (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+           Sin(Pi*Z) + R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+              Sin(Pi*Z)))) + (-((Sin(2*P)*
+             (-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+             (0.4052847345693511 + 
+               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
+                Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+               0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+              Sin(Pi*t)*Sin(Pi*Z) + 
+             R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+        (Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
+           Sin(2*Pi*t)*Sin(2*Pi*Z))/
+         (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+            Sin(Pi*Z) + R*(0.4052847345693511 + 
+              (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+               Sin(Pi*t)*Sin(Pi*Z))))*
+      ((-0.1*R*taui*Z)/
+         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+        (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+           Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
+             10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+           Sin(2*Pi*Z))/
+         (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+             1.*Power(Z,2))*(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+              Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+             R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)))) - 
+        (R*Sin(3*P)*Sin(3*Pi*t)*
+           (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
+             (mui*Cos(3*Pi*Z)*Sin(3*P)*
+                (528430.5031318273*
+                   Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                      Cos(3*Pi*R) + 
+                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
+                  54*Power(Pi,3)*Power(R,2)*
+                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                     R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+                Sin(3*Pi*t)*Sin(3*Pi*Z))/
+              (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) \
++ (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                     R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                  28034.10888465587*
+                   Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                      Cos(3*Pi*R) + 
+                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                   Power(Sin(3*Pi*Z),2)))/
+              (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                  1.*Power(Z,2),2))))/
+         (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
+     (beta*(1 + Sqrt(-mue))*Sin(2*P)*Sin(3*P)*Sin(2*Pi*t)*Sin(3*Pi*t)*
+        (-((2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Sin(2*Pi*Z)*
+             (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
+               (mui*Cos(3*Pi*Z)*Sin(3*P)*
+                  (528430.5031318273*
+                     Power((0.005628954646796544*mui*taui + 
+                       Power(R,2)*
+                       (0.011257909293593089 + 1.*mui*taui))*
+                       Cos(3*Pi*R) + 
+                       0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
+                    54*Power(Pi,3)*Power(R,2)*
+                     Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                       R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+                  Sin(3*Pi*t)*Sin(3*Pi*Z))/
+                (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
+                    1.*Power(Z,2))) + 
+               (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+                  (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                     Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                       R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                    28034.10888465587*
+                     Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                        Cos(3*Pi*R) + 
+                       0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                     Power(Sin(3*Pi*Z),2)))/
+                (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                    1.*Power(Z,2),2)))) + 
+          2*Pi*R*Cos(2*Pi*Z)*Sin(2*Pi*R)*
+           (120*Pi*Cos(3*Pi*R)*Sin(3*Pi*Z) + 
+             (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
+                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                     R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                  28034.10888465587*
+                   Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                      Cos(3*Pi*R) + 
+                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                   Power(Sin(3*Pi*Z),2)))/
+              (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                  1.*Power(Z,2),2)) + 
+             (2*mui*Sin(3*P)*Sin(3*Pi*t)*
+                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                     R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                  28034.10888465587*
+                   Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                      Cos(3*Pi*R) + 
+                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                   Power(Sin(3*Pi*Z),2)))/
+              (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) \
+- (mui*Sin(3*P)*Sin(3*Pi*t)*(18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
+                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                     R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                  528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                   (R*(0.011257909293593089 + 1.*mui*taui)*
+                      Cos(3*Pi*R) + 
+                     (0.0011945012753036852 + 
+                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
+                   (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
+                     R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R)) \
+- 168204.65330793522*((0.005628954646796544*mui*taui + 
+                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
+)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
+                   (R*(-0.007505272862395392 - 
+                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
+                     (-6.938893903907228e-18*mui*taui + 
+                        Power(R,2)*
+                         (0.03536776513153231 + 
+                         3.141592653589793*mui*taui))*Sin(3*Pi*R))*
+                   Power(Sin(3*Pi*Z),2)))/
+              (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))\
+)/(6000.*Sqrt(-mue)*mui*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) \
++ ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+           (-0.8488263631567752*Cos(2*Pi*R) + 
+             10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+         (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+            Sin(Pi*Z) + R*(0.4052847345693511 + 
+              (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+               Sin(Pi*t)*Sin(Pi*Z))) - 
+        (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+             10.666666666666666*R*Sin(2*Pi*R))*
+           (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+             Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
+              Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+         Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+            Sin(Pi*Z) + R*(0.4052847345693511 + 
+              (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+               Sin(Pi*t)*Sin(Pi*Z)),2))*
+      ((taui*(-50. + 1.*R - 0.1*Power(Z,2)))/
+         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+        (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
+           (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+           (-0.8488263631567752*Cos(2*Pi*R) + 
+             10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+           Power(Sin(2*Pi*Z),2))/
+         (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)))) - 
+        (taui + (mui*Power(Sin(2*P),2)*
+              Power(0.8488263631567752*Cos(2*Pi*R) - 
+                10.666666666666666*R*Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
+              Power(Sin(2*Pi*Z),2))/
+            Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z) + 
+              R*(0.4052847345693511 + 
+                 (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                  Sin(Pi*t)*Sin(Pi*Z)),2))/
+         (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+        (R*Sin(3*P)*Sin(3*Pi*t)*
+           (120*Pi*Cos(3*Pi*R)*Sin(3*Pi*Z) + 
+             (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
+                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                     R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                  28034.10888465587*
+                   Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                      Cos(3*Pi*R) + 
+                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                   Power(Sin(3*Pi*Z),2)))/
+              (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                  1.*Power(Z,2),2)) + 
+             (2*mui*Sin(3*P)*Sin(3*Pi*t)*
+                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                     R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                  28034.10888465587*
+                   Power((0.005628954646796544*mui*taui + 
+                        Power(R,2)*
+                        (0.011257909293593089 + 1.*mui*taui))*
+                      Cos(3*Pi*R) + 
+                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                   Power(Sin(3*Pi*Z),2)))/
+              (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+             (mui*Sin(3*P)*Sin(3*Pi*t)*
+                (18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
+                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                     R*(-0.20000000000000004 - 
+                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
+                  528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                   (R*(0.011257909293593089 + 1.*mui*taui)*Cos(3*Pi*R) + 
+                     (0.0011945012753036852 + 
+                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
+                   (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
+                     R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R)) - 
+                  168204.65330793522*
+                   ((0.005628954646796544*mui*taui + 
+                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
+                      Cos(3*Pi*R) + 
+                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
+                   (R*(-0.007505272862395392 - 
+                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
+                     (-6.938893903907228e-18*mui*taui + 
+                        Power(R,2)*
+                         (0.03536776513153231 + 
+                         3.141592653589793*mui*taui))*Sin(3*Pi*R))*
+                   Power(Sin(3*Pi*Z),2)))/
+              (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/
+         (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))))) + 
   FELTORPARALLEL*((taui*Sin(Pi*t)*
-        ((15.707963267948966 - 1.5707963267948966*R)*R*Cos(Pi*Z)*Sin(P)*
-           Sin(Pi*R) + (1.5707963267948966*R*Z*Cos(Pi*R)*Sin(P) + 
-             10.*Cos(P)*Sin(Pi*R))*Sin(Pi*Z)))/
-      (mui*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) - 
-     (0.6981317007977318*(-10 + R)*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-        Power(Sin(2*Pi*(-10 + R)),2)*Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-      Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-     (0.6981317007977318*Z*Cos(2*Pi*(-10 + R))*Power(Sin(2*P),2)*
-        Sin(2*Pi*(-10 + R))*Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-      Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-     (40*Cos(2*P)*Sin(2*P)*Power(Sin(2*Pi*(-10 + R)),2)*
-        Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-      (9.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-     (nuparalleli*(0. + (1.*Z*
-             ((-2.0943951023931953*(-10 + R)*Cos(2*Pi*Z)*Sin(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+        (R*Sin(Pi*R)*(R*(7.853981633974482 + 
+                77.51569170074954*mui*taui + 
+                R*(-0.7853981633974483 - 7.751569170074954*mui*taui))*
+              Cos(Pi*Z)*Sin(P) + 
+             ((5. + 49.34802200544679*mui*taui)*Cos(P) + 
+                1.2337005501361697*mui*taui*Z*Sin(P))*Sin(Pi*Z)) + 
+          Cos(Pi*R)*(mui*R*(-12.337005501361698 + 1.2337005501361697*R)*
+              taui*Cos(Pi*Z)*Sin(P) + 
+             (-7.853981633974484*mui*taui*Cos(P) + 
+                (0.39269908169872414*mui*taui + 
+                   Power(R,2)*
+                    (0.7853981633974483 + 7.751569170074954*mui*taui))*Z*
+                 Sin(P))*Sin(Pi*Z))))/
+      (mui*Power(R,2)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+     (1.*(-10 + R)*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
+        ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+             (-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z))) - 
+          (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*
+             (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
+                Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)),2)))/
+      (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+           Sin(Pi*Z) + R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+              Sin(Pi*Z)))) + (1.*Z*Sin(2*P)*
+        (-0.8488263631567752*Cos(2*Pi*R) + 
+          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
+        (-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                 10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+               (0.4052847345693511 + 
+                 Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
+                  Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                 0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                 (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                  Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+             Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                Sin(Pi*t)*Sin(Pi*Z) + 
+               R*(0.4052847345693511 + 
+                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                   Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+          (Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
+             Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)))))/
+      (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+           Sin(Pi*Z) + R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+              Sin(Pi*Z)))) + (20*Sin(2*P)*
+        (-0.8488263631567752*Cos(2*Pi*R) + 
+          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
+        (-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                 10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+               (-0.15915494309189535*mui*taui*Cos(P)*Cos(Pi*R)*
+                  Sin(Pi*t)*Sin(Pi*Z) + 
+                 R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*Sin(Pi*R)*
+                  Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+             Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                Sin(Pi*t)*Sin(Pi*Z) + 
+               R*(0.4052847345693511 + 
+                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                   Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+          (2*Cos(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)))))/
+      (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
+        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+           Sin(Pi*Z) + R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+              Sin(Pi*Z)))) - (nuparalleli*
+        (0. + (1.*R*Z*((-1.*(-10 + R)*
+                  ((-2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                        Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
+                    (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+                    (2*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                       Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*
+                       (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (Power(Pi,2)*R*
+                       (0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+                        1.5707963267948966*mui*taui*Cos(Pi*Z)*
+                       Sin(P)*Sin(Pi*R)*Sin(Pi*t) + 
+                        Pi*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (2.0943951023931953*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (3.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (1.*(-10 + R)*R*((13.15947253478581*Z*Cos(2*Pi*(-10 + R))*
-                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (80*Pi*Cos(2*P)*Cos(2*Pi*Z)*Sin(2*Pi*(-10 + R))*
-                  Sin(2*Pi*t))/
-                (3.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-               (2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*Z)*Sin(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+               (1.*Power(-10 + R,2)*
+                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
                 Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-               (2.0943951023931953*Power(Z,2)*Cos(2*Pi*(-10 + R))*
-                  Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+               (1.*((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (20*((2*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (-0.15915494309189535*mui*taui*Cos(P)*
+                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
+                       (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(P)*Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Cos(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (-0.15915494309189535*mui*taui*Cos(P)*
+                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (2*Cos(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
+                    (2*Cos(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+               (1.*(-10 + R)*Z*
+                  (-((Sin(2*P)*
+                        (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                        (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+                    (Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
                 Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
-               (2.0943951023931953*Cos(2*Pi*(-10 + R))*Sin(2*P)*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+               (1.*Z*((2*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Power(0.4052847345693511 + 
+                       Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                       (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                       Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
+                    (2*Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (1.5707963267948966*mui*taui*Cos(Pi*R)*
+                       Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        2*Pi*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
+                        Power(Pi,2)*R*
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
+                    (Sin(2*P)*
+                       (167.5516081914556*Cos(2*Pi*R) - 
+                        421.1031211131459*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (13.333333333333334*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+               (20.*(-10 + R)*
+                  (-((Sin(2*P)*
+                        (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                        (-0.15915494309189535*mui*taui*Cos(P)*
+                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+                    (2*Cos(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
                 (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) \
-+ (13.15947253478581*(-10 + R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                  Sin(2*Pi*Z))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
+- (20*(-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                        (-0.15915494309189535*mui*taui*Cos(P)*
+                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+                    (2*Cos(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                (Power(R,2)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2)))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (20*((-4.1887902047863905*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+          (1.*Z*((-1.*(-10 + R)*
+                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (4.1887902047863905*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (80*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (3.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
+               (1.*Z*(-((Sin(2*P)*
+                        (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                        (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+                    (Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (20*(-((Sin(2*P)*
+                        (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                        (-0.15915494309189535*mui*taui*Cos(P)*
+                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+                    (2*Cos(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (1.*R*Z*((-13.15947253478581*(-10 + R)*Cos(2*Pi*(-10 + R))*
-                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
+          (20*((-1.*(-10 + R)*
+                  ((-2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (-0.15915494309189535*mui*taui*Cos(P)*
+                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
+                    (4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+                    (2*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                       Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*
+                       (-0.15915494309189535*mui*taui*Cos(P)*
+                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(P)*Cos(Pi*R)*Cos(Pi*Z)*
+                       Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(P)*Cos(Pi*Z)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (2*Cos(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (2.0943951023931953*Power(-10 + R,2)*Cos(2*Pi*Z)*
-                  Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-               (2.0943951023931953*Cos(2*Pi*Z)*Sin(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+               (1.*Z*((2*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (-0.15915494309189535*mui*taui*Cos(P)*
+                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
+                       (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(P)*Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Cos(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (-0.15915494309189535*mui*taui*Cos(P)*
+                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (2*Cos(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                       Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
+                    (2*Cos(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (20*((2*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Power(-0.15915494309189535*mui*taui*Cos(P)*
+                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
+                    (4*Cos(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (-0.15915494309189535*mui*taui*Cos(P)*
+                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (4*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
+           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+          (1.*(-10 + R)*R*((1.*Z*
+                  ((-2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                        Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
+                    (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+                    (2*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*
+                       (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                        Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (Power(Pi,2)*R*
+                        (0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+                        1.5707963267948966*mui*taui*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t) + 
+                        Pi*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (80*Pi*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-                  Sin(2*Pi*Z))/
-                (3.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               (2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
-                  Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+               (20*((-2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (-0.15915494309189535*mui*taui*Cos(P)*
+                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
+                    (4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+                    (2*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*
+                       (-0.15915494309189535*mui*taui*Cos(P)*
+                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(P)*Cos(Pi*R)*Cos(Pi*Z)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(P)*Cos(Pi*Z)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (2*Cos(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
+                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+               (1.*(-10 + R)*Z*
+                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
                 Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-               (13.333333333333334*(-10 + R)*Cos(2*P)*
-                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) \
-- (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                (3.*Power(R,2)*
-                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               (13.15947253478581*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+               (1.*Power(Z,2)*
+                  (-((Sin(2*P)*
+                        (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                        (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                        Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+                    (Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
+               (1.*(-((Sin(2*P)*
+                        (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                        (0.4052847345693511 + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+                        Sin(Pi*Z) + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+                    (Sin(2*P)*
+                       (67.02064327658225*R*Cos(2*Pi*R) + 
+                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+               (20.*Z*(-((Sin(2*P)*
+                        (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                        (-0.15915494309189535*mui*taui*Cos(P)*
+                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                        R*(0.4052847345693511 + 
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+                    (2*Cos(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) - 
+               (1.*(-10 + R)*((-4*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
+                        Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+                       Sin(2*Pi*t))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
+                    (2*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*
+                       Power(-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*
+                        Sin(P)*Sin(Pi*t) + 
+                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
+                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t),2)*
+                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
+                    (Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) - 
+                        Power(Pi,2)*R*
+                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
+                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
+                    (4*Power(Pi,2)*Sin(2*P)*
+                       (-0.8488263631567752*Cos(2*Pi*R) + 
+                         10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+                       Sin(2*Pi*Z))/
+                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+                        Sin(Pi*t)*Sin(Pi*Z) + 
+                       R*(0.4052847345693511 + 
+                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
-      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (Sin(3*Pi*t)*((18.84955592153876 - 1.8849555921538759*R)*R*
-           Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R) + 
-          (1.8849555921538759*R*Z*Cos(3*Pi*R)*Sin(3*P) + 
-             12.*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*Z)))/
-      (mui*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-  (beta*Pi*Cos(4*Pi*t)*Sin(4*P)*Sin(4*Pi*(-10 + R))*Sin(4*Pi*Z))/mui + 
-  FELTORPERP*(0. - (0.03333333333333333*taui*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
-        Sin(2*Pi*t)*Sin(2*Pi*Z))/
-      Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-     (Pi*Cos(3*Pi*Z)*Sin(2*P)*Sin(3*P)*Sin(2*Pi*(-10 + R))*Sin(3*Pi*R)*
-        Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z))/
-      (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-     (eta*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-        ((Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3. - 
-          (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (3.*Sqrt(-mue))))/mui - 
-     (nuperp*((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           3. - (8*Power(Pi,2)*R*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-             Sin(2*Pi*Z))/3.))/R + 
-     (2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        ((-0.1*R*taui*Z)/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-          (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
-           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-          (beta*Pi*R*Cos(4*Pi*Z)*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-             Sin(4*Pi*R)*Sin(2*Pi*t)*Sin(4*Pi*t)*Sin(2*Pi*Z))/
-           (30.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/3. + 
-     ((-0.20943951023931953*R*taui*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*R)*
-           Sin(2*Pi*t)*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
-         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-        (0.10471975511965977*R*taui*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-           Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-        (0.03333333333333333*R*taui*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5))/
-      (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (beta*taui*Sin(P)*Sin(4*P)*Sin(Pi*t)*Sin(4*Pi*t)*
-        (0.4934802200544679*R*Cos(Pi*R)*Cos(4*Pi*Z)*Sin(4*Pi*R)*Sin(Pi*Z) + 
-          Cos(Pi*Z)*Sin(Pi*R)*(-0.4934802200544679*R*Cos(4*Pi*R) - 
-             0.039269908169872414*Sin(4*Pi*R))*Sin(4*Pi*Z)))/
-      (mui*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (3*beta*Pi*Sin(3*P)*Sin(4*P)*Sin(3*Pi*t)*Sin(4*Pi*t)*
-        (4*Pi*R*Cos(3*Pi*R)*Cos(4*Pi*Z)*Sin(4*Pi*R)*Sin(3*Pi*Z) - 
-          Cos(3*Pi*Z)*Sin(3*Pi*R)*(4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*
-           Sin(4*Pi*Z)))/
-      (200.*mui*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-     (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-        ((taui*(-50. + 1.*R - 0.1*Power(Z,2)))/
-           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-          (-9*taui - mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
-              Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-           (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-          (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-          (beta*Sin(2*P)*Sin(4*P)*Sin(2*Pi*(-10 + R))*
-             (4*Pi*R*Cos(4*Pi*R) + Sin(4*Pi*R))*Sin(2*Pi*t)*Sin(4*Pi*t)*
-             Sin(2*Pi*Z)*Sin(4*Pi*Z))/
-           (120.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/3.)
+      (R*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
+           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (Sin(3*Pi*t)*(-0.3*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+           Cos(3*P)*(-40.*Power(R,2)*
+              (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Sin(3*Pi*R)*
+              Sin(3*Pi*Z) + 2*mui*Sin(3*P)*Sin(3*Pi*t)*
+              (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
+                    Sin(3*Pi*R),2) + 
+                28034.10888465587*
+                 Power((0.005628954646796544*mui*taui + 
+                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
+                    Cos(3*Pi*R) + 
+                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                 Power(Sin(3*Pi*Z),2))) + 
+          0.6*(10 - R)*R*Sin(3*P)*
+           (Pi*Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),2)*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
+             0.008333333333333333*mui*
+              (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(3*Pi*Z)*
+              Sin(3*P)*(528430.5031318273*
+                 Power((0.005628954646796544*mui*taui + 
+                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
+                    Cos(3*Pi*R) + 
+                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
+                54*Power(Pi,3)*Power(R,2)*
+                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                   R*(-0.20000000000000004 - 
+                      17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
+              Sin(3*Pi*t)*Sin(3*Pi*Z) + 
+             0.016666666666666666*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
+              (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
+                    Sin(3*Pi*R),2) + 
+                28034.10888465587*
+                 Power((0.005628954646796544*mui*taui + 
+                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
+                    Cos(3*Pi*R) + 
+                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                 Power(Sin(3*Pi*Z),2))) + 
+          0.005*Z*Sin(3*P)*(376.99111843077515*Power(R,3)*
+              Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)*
+              Cos(3*Pi*R)*Sin(3*Pi*Z) - 
+             mui*(20. - 2.*R)*R*Sin(3*P)*Sin(3*Pi*t)*
+              (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
+                    Sin(3*Pi*R),2) + 
+                28034.10888465587*
+                 Power((0.005628954646796544*mui*taui + 
+                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
+                    Cos(3*Pi*R) + 
+                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                 Power(Sin(3*Pi*Z),2)) + 
+             2*mui*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Sin(3*P)*
+              Sin(3*Pi*t)*(9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
+                    Sin(3*Pi*R),2) + 
+                28034.10888465587*
+                 Power((0.005628954646796544*mui*taui + 
+                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
+                    Cos(3*Pi*R) + 
+                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
+                 Power(Sin(3*Pi*Z),2)) - 
+             mui*R*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Sin(3*P)*
+              Sin(3*Pi*t)*(18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
+                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
+                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
+                    Sin(3*Pi*R),2) + 
+                528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
+                 (R*(0.011257909293593089 + 1.*mui*taui)*Cos(3*Pi*R) + 
+                   (0.0011945012753036852 + 0.15915494309189537*mui*taui)*
+                    Sin(3*Pi*R))*
+                 (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
+                   R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R)) - 
+                168204.65330793522*
+                 ((0.005628954646796544*mui*taui + 
+                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
+                    Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R)\
+)*(R*(-0.007505272862395392 - 0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
+                   (-6.938893903907228e-18*mui*taui + 
+                      Power(R,2)*
+                       (0.03536776513153231 + 3.141592653589793*mui*taui))*
+                    Sin(3*Pi*R))*Power(Sin(3*Pi*Z),2)))))/
+      (mui*Power(R,3)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5))\
+)
 ; }};
 struct SPhie{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return 0. - ((0.029608813203268074*Power(R,3)*Cos(Pi*Z)*Cos(3*Pi*Z)*Sin(P)*
-        Sin(3*P)*Sin(Pi*(-10 + R))*Sin(3*Pi*(-10 + R))*Sin(Pi*t)*
-        Sin(3*Pi*t))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-     (0.03769911184307752*Power(R,3)*Z*Cos(3*Pi*Z)*Sin(3*P)*
-        Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
-      Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-     (0.029608813203268074*Power(R,3)*Cos(Pi*(-10 + R))*
-        Cos(3*Pi*(-10 + R))*Sin(P)*Sin(3*P)*Sin(Pi*t)*Sin(3*Pi*t)*
-        Sin(Pi*Z)*Sin(3*Pi*Z))/(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-     (0.03769911184307752*(-10 + R)*Power(R,3)*Cos(3*Pi*(-10 + R))*
-        Sin(3*P)*Sin(3*Pi*t)*(1 + 
-          0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(3*Pi*Z))/
-      Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),2) + 
-     (9*Pi*Power(R,2)*Cos(3*Pi*(-10 + R))*Sin(3*P)*Sin(3*Pi*t)*
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(3*Pi*Z))/
-      (500.*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-     (9*Power(Pi,2)*Power(R,3)*Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(3*Pi*Z))/
-      (250.*(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R
+    return 0.25*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+  (Sin(3*P)*Sin(3*Pi*t)*(0.2191704548265055*R*
+        (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(Pi*Z)*
+        Cos(3*Pi*Z)*Sin(P)*(-0.5*mui*taui*Cos(Pi*R) + 
+          R*(0.3183098861837907 + 3.141592653589793*mui*taui)*Sin(Pi*R))*
+        (1.*mui*taui*Cos(3*Pi*R) + 
+          R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*Sin(3*Pi*R)\
+)*Sin(Pi*t) - 0.438340909653011*R*Z*Cos(3*Pi*Z)*
+        (1.*mui*taui*Cos(3*Pi*R) + 
+          R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*Sin(3*Pi*R)\
+)*(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+          R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+              Sin(Pi*Z))) - 4.131265744601299*
+        (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        ((0.005628954646796544*mui*taui + 
+             Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*Cos(3*Pi*R) \
++ 0.053051647697298476*mui*R*taui*Sin(3*Pi*R))*
+        (0.4052847345693511 + R*
+           (0.3183098861837907 + 3.141592653589793*mui*taui)*Cos(Pi*R)*
+           Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+          (0.10132118364233778 + 1.5*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+           Sin(Pi*Z))*Sin(3*Pi*Z) + 
+       4.131265744601299*(-20. + 2.*R)*
+        ((0.005628954646796544*mui*taui + 
+             Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*Cos(3*Pi*R) \
++ 0.053051647697298476*mui*R*taui*Sin(3*Pi*R))*
+        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+           Sin(Pi*Z) + R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+              Sin(Pi*t)*Sin(Pi*Z)))*Sin(3*Pi*Z) - 
+       2.06563287230065*R*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        (1.*mui*taui*Cos(3*Pi*R) + 
+          R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*Sin(3*Pi*R)\
+)*(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+          R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+              Sin(Pi*t)*Sin(Pi*Z)))*Sin(3*Pi*Z) - 
+       4.131265744601299*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        (R*(0.022515818587186178 + 2.5*mui*taui)*Cos(3*Pi*R) + 
+          (3.122502256758253e-17*mui*taui + 
+             Power(R,2)*(-0.10610329539459692 - 9.42477796076938*mui*taui))*
+           Sin(3*Pi*R))*(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+           Sin(Pi*t)*Sin(Pi*Z) + 
+          R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+              Sin(Pi*Z)))*Sin(3*Pi*Z)))/
+   (R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))
 ; }};
-struct SPhii{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+struct Snehat{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli,alpha;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (mui*Power(R,2)*Power(Sin(3*P),2)*Power(Sin(3*Pi*t),2)*
-    (0.017765287921960846*Power(Cos(3*Pi*Z),2)*Power(Sin(3*Pi*R),2) + 
-      0.017765287921960846*Power(Cos(3*Pi*R),2)*Power(Sin(3*Pi*Z),2)))/
-  (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))
+    return 1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z) + 
+  (alpha*nuperp*(0. + 1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*
+        Sin(Pi*Z) - 9.869604401089358*R*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
+        Sin(Pi*Z)))/R
 ; }};
-struct SGammaPhie{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+struct SNihat{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli,alpha;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (-0.5*mui*taui*((3*Pi*Cos(3*Pi*(-10 + R))*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-       5. - (18*Power(Pi,2)*R*Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*
-         Sin(3*Pi*Z))/5.))/R
+    return 1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/R + 
+  (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+  (alpha*nuperp*((0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+          Sin(Pi*Z))/Power(R,2) + 
+       Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+        Sin(Pi*Z) + (1.2337005501361697*mui*taui*Sin(P)*Sin(Pi*R)*
+          Sin(Pi*t)*Sin(Pi*Z))/R + 
+       R*((3.875784585037477*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z))/R - Power(Pi,2)*
+           (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+           Sin(Pi*Z)) + R*((-0.7853981633974483*mui*taui*Cos(Pi*R)*Sin(P)*
+             Sin(Pi*t)*Sin(Pi*Z))/Power(R,3) + 
+          (3.875784585037477*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
+           R - (2.4674011002723395*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+             Sin(Pi*Z))/Power(R,2) - 
+          Power(Pi,2)*(0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
+           Sin(Pi*t)*Sin(Pi*Z))))/R
 ; }};
-struct SGammaNi{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+struct SWehat{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli,alpha;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return 0. - (0.5*mui*taui*(0. + 1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*
-        Sin(Pi*t)*Sin(Pi*Z) - 9.869604401089358*R*Sin(P)*Sin(Pi*(-10 + R))*
-        Sin(Pi*t)*Sin(Pi*Z)))/R
+    return (Sin(2*P)*Sin(2*Pi*t)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/mue + 
+       (12.566370614359172*Cos(2*Pi*R) - 157.91367041742973*R*Sin(2*Pi*R))/
+        (R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/
+   (3.*Sqrt(-mue)) + (alpha*nuperp*
+     ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+          (4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+          Sin(Pi*Z)*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) \
++ (Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
+            78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+       (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+            52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Power(R,2)*
+          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+       R*((-39.47841760435743*Cos(Pi*Z)*Cos(2*Pi*Z)*Sin(P)*Sin(2*P)*
+             Sin(Pi*R)*(4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t))/
+           (Sqrt(-mue)*R*Power(2. + 
+               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+          (19.739208802178716*Power(Cos(Pi*Z),2)*Power(Sin(P),2)*Sin(2*P)*
+             Power(Sin(Pi*R),2)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(Pi*t),2)*
+             Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*Power(2. + 
+               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) + 
+          (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+             Sin(Pi*Z)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*Power(2. + 
+               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
+          (4*Power(Pi,2)*Sin(2*P)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
+       R*((19.739208802178716*Power(Cos(Pi*R),2)*Power(Sin(P),2)*Sin(2*P)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(Pi*t),2)*
+             Sin(2*Pi*t)*Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
+              3)) - (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+             (-330.73361792319804*R*Cos(2*Pi*R) - 
+               78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+             Sin(Pi*Z)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
+              2)) + (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+             Sin(Pi*Z)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*Power(R,2)*
+             Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
+          (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+             (4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+             Sin(Pi*Z)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
+              2)) - (2*Sin(2*P)*
+             (-330.73361792319804*R*Cos(2*Pi*R) - 
+               78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*Power(R,2)*
+             (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+          (2*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
+               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*Power(R,3)*
+             (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+          (Sin(2*P)*(-826.834044807995*Cos(2*Pi*R) + 
+               2078.060608725385*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))))/R
 ; }};
-struct SA{
-    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+struct SWihat{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli,alpha;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return -(beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-      (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/3. \
-+ (beta*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
-     (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-   (3.*Sqrt(-mue)) - (beta*Pi*Cos(4*Pi*(-10 + R))*Sin(4*P)*Sin(4*Pi*t)*
-      Sin(4*Pi*Z) - 8*beta*Power(Pi,2)*R*Sin(4*P)*Sin(4*Pi*(-10 + R))*
-      Sin(4*Pi*t)*Sin(4*Pi*Z))/R
+    return (Sin(2*P)*Sin(2*Pi*t)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/
+        (Sqrt(-mue)*mui) + (-2.5464790894703255*Cos(2*Pi*R) + 
+          31.999999999999996*R*Sin(2*Pi*R))/
+        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+           Sin(Pi*Z) + R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+              Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/3. + 
+  (alpha*nuperp*(-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+              10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+            (0.4052847345693511 + 
+              Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*Sin(P)*
+               Sin(Pi*t)*Sin(Pi*Z) + 
+              0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+              (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+               Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+          Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+             Sin(Pi*Z) + R*(0.4052847345693511 + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z)),2)) + 
+       (Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
+          Sin(2*Pi*t)*Sin(2*Pi*Z))/
+        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+           Sin(Pi*Z) + R*(0.4052847345693511 + 
+             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
+              Sin(Pi*Z))) + R*((2*Sin(2*P)*
+             (-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+             Power(0.4052847345693511 + 
+               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
+                Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+               0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z),2)*Sin(2*Pi*Z))/
+           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+              Sin(Pi*t)*Sin(Pi*Z) + 
+             R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)),3) - 
+          (2*Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
+             Sin(2*Pi*t)*(0.4052847345693511 + 
+               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
+                Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
+               0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
+               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+              Sin(Pi*t)*Sin(Pi*Z) + 
+             R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)),2) - 
+          (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+             (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                Sin(Pi*Z) + 2*Pi*(0.10132118364233778 + 1.*mui*taui)*
+                Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
+               Power(Pi,2)*R*(0.10132118364233778 + 1.*mui*taui)*Sin(P)*
+                Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
+              Sin(Pi*t)*Sin(Pi*Z) + 
+             R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)),2) + 
+          (Sin(2*P)*(167.5516081914556*Cos(2*Pi*R) - 
+               421.1031211131459*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)))) + 
+       R*((-4*Pi*Cos(2*Pi*Z)*Sin(2*P)*
+             (-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*
+             (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
+                Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t))/
+           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)),2) + 
+          (2*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*
+             Power(-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
+               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
+                Sin(Pi*R)*Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)),3) - 
+          (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+             (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+                Sin(Pi*Z) - Power(Pi,2)*R*
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z)),2) - 
+          (4*Power(Pi,2)*Sin(2*P)*
+             (-0.8488263631567752*Cos(2*Pi*R) + 
+               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
+              Sin(Pi*Z) + R*(0.4052847345693511 + 
+                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
+                 Sin(Pi*t)*Sin(Pi*Z))))))/R
 ; }};
 }}//namespace feltor namespace manufactured
-- 
GitLab


From 82fd489eba5537ac6f51a65eaa49eb920b7b69fa Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 12 Oct 2020 19:52:56 +0200
Subject: [PATCH 361/540] Put Resistivity back into explicit part

it is linear, but it is not symmetric
---
 src/feltor/feltor.h   | 8 ++++----
 src/feltor/implicit.h | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 7480a589c..31688a24f 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -818,6 +818,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
             );
         }
     }
+    //------------------Add Resistivity--------------------------//
+    dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+        m_fields[0][0], m_fields[0][1],
+        m_fields[1][0], m_fields[1][1], yp[1][0], yp[1][1]);
 }
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
@@ -993,10 +997,6 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
                 dg::blas2::symv( -m_p.nu_perp, m_lapperpU,
                     m_fields[1][i],  1., yp[1][i]);
         }
-        //------------------Add Resistivity--------------------------//
-        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
-            m_fields[0][0], m_fields[0][1],
-            m_fields[1][0], m_fields[1][1], yp[1][0], yp[1][1]);
 #endif
     }
 #ifdef DG_MANUFACTURED
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 769b65305..e6dffb5b2 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -203,10 +203,10 @@ struct ImplicitVelocity
                 dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU,
                     m_fields[1][i],  0., wp[i]);
         }
-        //------------------Add Resistivity--------------------------//
-        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
-            m_fields[0][0], m_fields[0][1],
-            m_fields[1][0], m_fields[1][1], wp[0], wp[1]);
+        ////------------------Add Resistivity--------------------------//
+        //dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+        //    m_fields[0][0], m_fields[0][1],
+        //    m_fields[1][0], m_fields[1][1], wp[0], wp[1]);
 #else
         dg::blas1::copy( 0, wp);
 #endif
-- 
GitLab


From 3230dfcab643564e316fec2be0c19d6d1c00e0bd Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 13 Oct 2020 13:20:52 +0200
Subject: [PATCH 362/540] Give Elliptic a set_norm function

and fix bug in Invert and Extrapolate
---
 inc/dg/cg.h       | 23 +++++++++--------------
 inc/dg/elliptic.h | 17 +++++++++++++++++
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/inc/dg/cg.h b/inc/dg/cg.h
index 348140670..1633ca736 100644
--- a/inc/dg/cg.h
+++ b/inc/dg/cg.h
@@ -383,14 +383,12 @@ struct Extrapolation
 
     /**
     * @brief Extrapolate value (equidistant version)
-    * @param new_x (write only) contains extrapolated value on output ( may alias the tail)
+    * @param new_x (write only) contains extrapolated value on output ( may alias the tail if it exists)
     * @note Assumes that extrapolation time equals last inserted time+1
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
     */
     template<class ContainerType0>
     void extrapolate( ContainerType0& new_x) const{
-        if ( 0 == m_max)
-            return;
         value_type t = m_t[0] +1.;
         extrapolate( t, new_x);
     }
@@ -402,15 +400,13 @@ struct Extrapolation
     */
     template<class ContainerType0>
     void derive( ContainerType0& dot_x) const{
-        if ( 0 == m_max)
-            return;
         derive( m_t[0], dot_x);
     }
 
     /**
     * @brief insert a new entry, deleting the oldest entry or update existing entry
     * @param t_new the time for the new entry
-    * @param new_entry the new entry ( may alias the tail), replaces value of existing entry if \c t_new already exists
+    * @param new_entry the new entry ( replaces value of existing entry if \c t_new already exists
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
     */
     template<class ContainerType0>
@@ -433,7 +429,7 @@ struct Extrapolation
     }
     /**
     * @brief insert a new entry
-    * @param new_entry the new entry ( may alias the tail)
+    * @param new_entry the new entry ( may alias the tail if it exists)
     * @note Assumes new time equals last inserted time+1
     * @tparam ContainerType0 must be usable with \c ContainerType in \ref dispatch
     */
@@ -450,11 +446,11 @@ struct Extrapolation
     const ContainerType& head()const{
         return m_x[0];
     }
-    ///write access to tail value ( the one that will be deleted in the next update
+    ///write access to tail value ( the one that will be deleted in the next update, undefined if max==0)
     ContainerType& tail(){
         return m_x[m_max-1];
     }
-    ///read access to tail value ( the one that will be deleted in the next update
+    ///read access to tail value ( the one that will be deleted in the next update, undefined if max==0)
     const ContainerType& tail()const{
         return m_x[m_max-1];
     }
@@ -519,6 +515,7 @@ struct Invert
     void construct( const ContainerType& copyable, unsigned max_iter, value_type eps, int extrapolationType = 2, bool multiplyWeights = true, value_type nrmb_correction = 1.)
     {
         m_ex.set_max( extrapolationType, copyable);
+        m_rhs = copyable;
         set_size( copyable, max_iter);
         set_accuracy( eps, nrmb_correction);
         multiplyWeights_=multiplyWeights;
@@ -558,9 +555,6 @@ struct Invert
      */
     unsigned get_max() const {return cg.get_max();}
 
-    /// @brief Return last solution
-    const ContainerType& get_last() const { return m_ex.head();}
-
     /**
      * @brief Solve linear problem
      *
@@ -617,8 +611,8 @@ struct Invert
         unsigned number;
         if( multiplyWeights_ )
         {
-            dg::blas2::symv( weights, rho, m_ex.tail());
-            number = cg( op, phi, m_ex.tail(), p, inv_weights, eps_, nrmb_correction_);
+            dg::blas2::symv( weights, rho, m_rhs);
+            number = cg( op, phi, m_rhs, p, inv_weights, eps_, nrmb_correction_);
         }
         else
             number = cg( op, phi, rho, p, inv_weights, eps_, nrmb_correction_);
@@ -643,6 +637,7 @@ struct Invert
     value_type eps_, nrmb_correction_;
     dg::CG< ContainerType > cg;
     Extrapolation<ContainerType> m_ex;
+    ContainerType m_rhs;
     bool multiplyWeights_;
 };
 
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 4a90a35bf..7f697746d 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -293,6 +293,14 @@ class Elliptic
         if( m_no == not_normed)//multiply weights without volume
             dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
     }
+    /**
+     * @brief Determine if weights are multiplied to make operator symmetric or not
+     *
+     * @param new_norm new setting
+     */
+    void set_norm( dg::norm new_norm) {
+        m_no = new_norm;
+    }
     private:
     Matrix m_leftx, m_lefty, m_rightx, m_righty, m_jumpX, m_jumpY;
     Container m_weights, m_inv_weights, m_precond, m_weights_wo_vol;
@@ -562,6 +570,15 @@ class Elliptic3d
             dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
     }
 
+    /**
+     * @brief Determine if weights are multiplied to make operator symmetric or not
+     *
+     * @param new_norm new setting
+     */
+    void set_norm( dg::norm new_norm) {
+        m_no = new_norm;
+    }
+
     private:
     Matrix m_leftx, m_lefty, m_leftz, m_rightx, m_righty, m_rightz, m_jumpX, m_jumpY;
     Container m_weights, m_inv_weights, m_precond, m_weights_wo_vol;
-- 
GitLab


From 41b8f2401fdad8dc1eab9971fc1b2eba68d95c93 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 13 Oct 2020 13:22:36 +0200
Subject: [PATCH 363/540] Fix bug in manufactured part of feltor

and fix Makefile
---
 src/feltor/Makefile | 2 +-
 src/feltor/feltor.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index c62587d4e..79878dbd1 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -14,7 +14,7 @@ manufactured: manufactured.cu manufactured.h feltor.h implicit.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
 
 implicit_t: implicit_t.cu implicit.h  feltor.h implicit.h
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB) -g -DDG_BENCHMARK
 
 feltordiag: feltordiag.cu feltordiag.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 31688a24f..d6134d51e 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -1006,10 +1006,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     dg::blas1::evaluate( yp[0][1], dg::plus_equals(), manufactured::SNi{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
-    dg::blas1::evaluate( yp[1][0], dg::plus_equals(), manufactured::SUe{
+    dg::blas1::evaluate( yp[1][0], dg::plus_equals(), manufactured::SWe{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
-    dg::blas1::evaluate( yp[1][1], dg::plus_equals(), manufactured::SUi{
+    dg::blas1::evaluate( yp[1][1], dg::plus_equals(), manufactured::SWi{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
         m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
 #endif //DG_MANUFACTURED
-- 
GitLab


From 10bf4c57f87df71682fdeb77d076ac83fd2c0132 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 13 Oct 2020 13:23:06 +0200
Subject: [PATCH 364/540] Finally our implicit test converges

---
 src/feltor/implicit.h     |   72 +-
 src/feltor/implicit_t.cu  |   27 +-
 src/feltor/manufactured.h | 4063 +++++--------------------------------
 3 files changed, 606 insertions(+), 3556 deletions(-)

diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index e6dffb5b2..f87713618 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -90,11 +90,6 @@ struct ImplicitVelocity
     void construct( const Geometry& g, feltor::Parameters p,
             dg::geo::TokamakMagneticField mag)
     {
-#ifdef DG_MANUFACTURED
-        m_R= dg::pullback( dg::cooX3d, g);
-        m_Z= dg::pullback( dg::cooY3d, g);
-        m_P= dg::pullback( dg::cooZ3d, g);
-#endif //DG_MANUFACTURED
         m_p=p;
         m_lapM_perpU.construct( g, p.bcxU,p.bcyU,dg::PER,
             dg::normed, dg::centered);
@@ -118,7 +113,9 @@ struct ImplicitVelocity
         //m_induction.construct(  g,
         //    p.bcxU, p.bcyU, dg::PER, -1., dg::centered);
         //m_induction.elliptic().set_chi( hh);
-        //m_invert.construct( m_temp, g.size(), p.eps_pol[0],1 );
+#ifdef DG_MANUFACTURED
+        m_invert.construct( m_temp, g.size(), p.eps_pol[0],1 );
+#endif // DG_MANUFACTURED
         //Multigrid setup
         m_multi_induction.resize(p.stages);
         m_multigrid.construct( g, p.stages);
@@ -163,12 +160,6 @@ struct ImplicitVelocity
         dg::blas1::copy( w, m_fields[1]);
         if( m_p.beta != 0){
             //let us solve for apar
-#ifdef DG_MANUFACTURED
-            //here we cheat (a bit)
-            dg::blas1::evaluate( m_apar, dg::equals(), manufactured::A{
-                m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-                m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
-#else
             dg::blas1::pointwiseDot(  m_p.beta, m_fields[0][1], m_fields[1][1],
                                      -m_p.beta, m_fields[0][0], m_fields[1][0],
                                       0., m_temp);
@@ -181,7 +172,6 @@ struct ImplicitVelocity
             //m_old_apar.update( m_apar); //don't update here: makes the solver potentially unstable
             if(  number[0] == m_multigrid.max_iter())
                 throw dg::Fail( m_p.eps_pol[0]);
-#endif //DG_MANUFACTURED
 
             //compute u_e and U_i from w_e, W_i and apar
             dg::blas1::axpby( 1., m_fields[1][0], -1./m_p.mu[0],
@@ -222,10 +212,37 @@ struct ImplicitVelocity
         return m_lapM_perpU.precond();
     }
 
+
+#ifdef DG_MANUFACTURED
+    // -Delta_perp inv_SA = SA
+    void invert_SA( const  Container& SA, Container& inv_SA)
+    {
+        //m_lapM_perpU.set_norm( dg::not_normed);
+        //m_invert( m_lapM_perpU, inv_SA, SA);
+        //m_lapM_perpU.set_norm( dg::normed);
+        std::vector<unsigned> number = m_multigrid.direct_solve(
+            m_multi_induction, inv_SA, SA, m_p.eps_pol[0]); //eps_pol[0] on all grids
+        if(  number[0] == m_multigrid.max_iter())
+            throw dg::Fail( m_p.eps_pol[0]);
+        if( m_p.perp_diff == "hyperviscous")
+        {
+            dg::blas2::symv( m_lapM_perpU, inv_SA,      m_temp);
+            dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, m_temp, 0., inv_SA);
+        }
+        else // m_p.perp_diff == "viscous"
+        {
+            dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU,
+                inv_SA,  0., m_temp);
+            dg::blas1::copy( m_temp, inv_SA);
+        }
+    }
+#endif //DG_MANUFACTURED
   private:
     feltor::Parameters m_p;
     Container m_temp, m_apar;
-    //dg::Invert<Container> m_invert;
+#ifdef DG_MANUFACTURED
+    dg::Invert<Container> m_invert;
+#endif //DG_MANUFACTURED
     //dg::Helmholtz3d<Geometry, Matrix, Container> m_induction;
     dg::MultigridCG2d<Geometry, Matrix, Container> m_multigrid;
     std::vector<dg::Helmholtz3d<Geometry, Matrix, Container>> m_multi_induction;
@@ -233,9 +250,6 @@ struct ImplicitVelocity
     std::vector<Container> m_multi_chi;
     std::array<std::array<Container,2>,2> m_fields;
     dg::Elliptic3d<Geometry, Matrix, Container> m_lapM_perpU;
-#ifdef DG_MANUFACTURED
-    Container m_R, m_Z, m_P; //coordinates
-#endif //DG_MANUFACTURED
 };
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
@@ -272,6 +286,14 @@ struct FeltorSpecialSolver
     FeltorSpecialSolver( const Geometry& grid, Parameters p,
         dg::geo::TokamakMagneticField mag)
     {
+#ifdef DG_MANUFACTURED
+        m_p = p;
+        m_R= dg::pullback( dg::cooX3d, grid);
+        m_Z= dg::pullback( dg::cooY3d, grid);
+        m_P= dg::pullback( dg::cooZ3d, grid);
+        m_inv_sa = m_sa = m_R;
+        m_rhs = std::array<Container,2>{m_R,m_R};
+#endif //DG_MANUFACTURED
         std::array<Container,2> temp = dg::construct<std::array<Container,2>>( dg::evaluate( dg::zero, grid));
         m_eps = p.eps_time;
         m_solver = dg::DefaultSolver<std::array<Container,2>>(
@@ -296,7 +318,17 @@ struct FeltorSpecialSolver
 
         m_solver.solve( alpha, m_imdens, t, y[0], rhs[0]);
         m_imvelo.set_density( y[0]);
+#ifdef DG_MANUFACTURED
+        dg::blas1::evaluate( m_sa, dg::equals(), manufactured::SA{
+            m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
+            m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
+        m_imvelo.invert_SA( m_sa, m_inv_sa);
+        dg::blas1::axpby( +alpha/m_p.mu[0], m_inv_sa, 1., rhs[1][0], m_rhs[0]);
+        dg::blas1::axpby( +alpha/m_p.mu[1], m_inv_sa, 1., rhs[1][1], m_rhs[1]);
+        m_solver.solve( alpha, m_imvelo, t, y[1], m_rhs);
+#else
         m_solver.solve( alpha, m_imvelo, t, y[1], rhs[1]);
+#endif //DG_MANUFACTURED
         m_imvelo.update();
     }
     private:
@@ -304,6 +336,12 @@ struct FeltorSpecialSolver
     ImplicitDensity<Geometry,IMatrix, Matrix,Container> m_imdens;
     ImplicitVelocity<Geometry,IMatrix, Matrix,Container> m_imvelo;
     value_type m_eps;
+#ifdef DG_MANUFACTURED
+    feltor::Parameters m_p;
+    std::array<Container, 2> m_rhs;
+    Container m_sa, m_inv_sa;
+    Container m_R, m_Z, m_P; //coordinates
+#endif // DG_MANUFACTURED
 };
 
 }//namespace feltor
diff --git a/src/feltor/implicit_t.cu b/src/feltor/implicit_t.cu
index b8059a115..348628e28 100644
--- a/src/feltor/implicit_t.cu
+++ b/src/feltor/implicit_t.cu
@@ -1,7 +1,15 @@
 #include <iostream>
 
-#include "implicit.h"
+#include "dg/algorithm.h"
+#include "dg/geometries/geometries.h"
+#include "dg/file/json_utilities.h"
+#include "parameters.h"
+
+#define DG_MANUFACTURED
+#define FELTORPARALLEL 1
+#define FELTORPERP 1
 #include "manufactured.h"
+#include "implicit.h"
 
 
 int main( int argc, char* argv[])
@@ -31,7 +39,7 @@ int main( int argc, char* argv[])
     std::cout << "Initialize implicit" << std::endl;
     feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
     feltor::FeltorSpecialSolver<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> solver( grid, p, mag);
-    double alpha = -p.dt, time = 1;
+    double alpha = -p.dt, time = 0.237; //Sin(3*Pi*t)
     feltor::manufactured::Snehat snehat{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                  p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1], alpha};
     feltor::manufactured::SNihat snihat{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
@@ -46,10 +54,14 @@ int main( int argc, char* argv[])
                                  p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
     feltor::manufactured::Ni ni{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                  p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
-    feltor::manufactured::we we{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+    feltor::manufactured::We we{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
     feltor::manufactured::Wi wi{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                  p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
+    feltor::manufactured::Ue ue{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
+    feltor::manufactured::Ui ui{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
+                                 p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
     dg::DVec R = dg::pullback( dg::cooX3d, grid);
     dg::DVec Z = dg::pullback( dg::cooY3d, grid);
     dg::DVec P = dg::pullback( dg::cooZ3d, grid);
@@ -61,15 +73,18 @@ int main( int argc, char* argv[])
 
     dg::blas1::evaluate( rhs[0][0], dg::equals(), snehat, R,Z,P,time);
     dg::blas1::evaluate( rhs[0][1], dg::equals(), snihat, R,Z,P,time);
+    dg::blas1::plus( rhs[0], -1);
     dg::blas1::evaluate( rhs[1][0], dg::equals(), swehat, R,Z,P,time);
     dg::blas1::evaluate( rhs[1][1], dg::equals(), swihat, R,Z,P,time);
+    dg::blas1::evaluate( apar, dg::equals(), aa, R,Z,P,time);
+
+    solver.solve( alpha, im, time, y, rhs);
+
+    dg::blas1::plus( y[0], +1); //we solve for n-1
     dg::blas1::evaluate( sol[0][0], dg::equals(), ne, R,Z,P,time);
     dg::blas1::evaluate( sol[0][1], dg::equals(), ni, R,Z,P,time);
     dg::blas1::evaluate( sol[1][0], dg::equals(), we, R,Z,P,time);
     dg::blas1::evaluate( sol[1][1], dg::equals(), wi, R,Z,P,time);
-    dg::blas1::evaluate( apar, dg::equals(), aa, R,Z,P,time);
-
-    solver.solve( -p.dt, im, time, y, rhs);
     double normne = sqrt(dg::blas2::dot( w3d, sol[0][0]));
     double normni = sqrt(dg::blas2::dot( w3d, sol[0][1]));
     double normwe = sqrt(dg::blas2::dot( w3d, sol[1][0]));
diff --git a/src/feltor/manufactured.h b/src/feltor/manufactured.h
index 48c3e5cc9..f5d8bb37e 100644
--- a/src/feltor/manufactured.h
+++ b/src/feltor/manufactured.h
@@ -21,65 +21,39 @@ struct Ne{
 struct Ni{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return 1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/R + 
-  (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)
+    return 1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)
 ; }};
 struct Ue{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 52.63789013914324*R*Sin(2*Pi*R))*
-    Sin(2*Pi*t)*Sin(2*Pi*Z))/
-  (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))
+    return -(Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue))
 ; }};
 struct Ui{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-      10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-  (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-    R*(0.4052847345693511 + (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))
+    return (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3.
 ; }};
-struct we{
+struct We{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(2*P)*Sin(2*Pi*t)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/mue + 
-      (12.566370614359172*Cos(2*Pi*R) - 157.91367041742973*R*Sin(2*Pi*R))/
-       (R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/
-  (3.*Sqrt(-mue))
+    return -((beta + beta*Sqrt(-mue) - mue)*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
+     Sin(2*Pi*Z))/(3.*Power(-mue,1.5))
 ; }};
 struct Wi{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(2*P)*Sin(2*Pi*t)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/
-       (Sqrt(-mue)*mui) + (-2.5464790894703255*Cos(2*Pi*R) + 
-         31.999999999999996*R*Sin(2*Pi*R))/
-       (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-          Sin(Pi*Z) + R*(0.4052847345693511 + 
-            (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-             Sin(Pi*Z))))*Sin(2*Pi*Z))/3.
+    return ((beta + beta*Sqrt(-mue) + Sqrt(-mue)*mui)*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
+    Sin(2*Pi*Z))/(3.*Sqrt(-mue)*mui)
 ; }};
 struct Phie{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(3*P)*(-0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-      R*(0.20000000000000004 + 17.765287921960844*mui*taui)*Sin(3*Pi*R))*
-    Sin(3*Pi*t)*Sin(3*Pi*Z))/R
+    return (Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.
 ; }};
 struct Phii{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(3*P)*Sin(3*Pi*t)*(40*Sin(3*Pi*R)*Sin(3*Pi*Z) - 
-      (mui*Sin(3*P)*Sin(3*Pi*t)*
-         (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-            Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-              R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
-               Sin(3*Pi*R),2) + 
-           28034.10888465587*Power((0.005628954646796544*mui*taui + 
-                 Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
-               Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-            Power(Sin(3*Pi*Z),2)))/
-       (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/200.
+    return (Sin(3*P)*Sin(3*Pi*(-10 + R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/5.
 ; }};
 struct GammaPhie{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
@@ -101,63 +75,32 @@ struct SNe{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return 1.5707963267948966*Cos(Pi*t)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*Z) + 
-  (FELTORPARALLEL*(0. - (6.283185307179586*(-10 + R)*Cos(2*Pi*Z)*Sin(2*P)*
-          (4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+  (FELTORPARALLEL*(0. + (2.0943951023931953*(-10 + R)*R*Cos(2*Pi*Z)*
+          Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-       (3.141592653589793*(-10 + R)*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-          (4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+       (0.5235987755982988*(-10 + R)*R*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+          Sin(Pi*(-10 + R))*Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*
           Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-       (3.141592653589793*Z*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-          (4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-          Sin(Pi*Z)*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-             Sin(Pi*Z))*Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-       (20.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-          (4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-          Sin(Pi*Z)*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
-             Sin(Pi*Z))*Sin(2*Pi*Z))/
-        (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-       (1.5707963267948966*(-10 + R)*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-          Sin(Pi*(-10 + R))*(4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+       (0.5235987755982988*R*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
+          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+       (3.3333333333333335*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
+          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+       (2.0943951023931953*R*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
+          (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
           Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-       (1.5707963267948966*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
-          (4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-          Sin(Pi*Z)*Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-       (10.*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
-          (4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-          Sin(Pi*Z)*Sin(2*Pi*Z))/
-        (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-       (1.*Z*Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
-            78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+       (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
           Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-       (40*Cos(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
+        (3.*Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+       (0.3333333333333333*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
           (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
           Sin(2*Pi*Z))/
-        (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/R + 
+        (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/R + 
   FELTORPERP*(-((nuperp*(0. + 1.5707963267948966*Cos(Pi*(-10 + R))*Sin(P)*
              Sin(Pi*t)*Sin(Pi*Z) - 
             9.869604401089358*R*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)\
@@ -165,3347 +108,578 @@ struct SNe{
          ((0.15000000000000002*R*(-20. + 2.*R)*taue*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
            (0.1*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (0.8882643960980423*Cos(3*Pi*Z)*Sin(3*P)*
-              (3*Pi*R*(-0.2122065907891938 - 
-                   18.849555921538755*mui*taui)*Cos(3*Pi*R) - 
-                9.42477796076938*mui*taui*Sin(3*Pi*R) + 
-                (-0.2122065907891938 - 18.849555921538755*mui*taui)*
-                 Sin(3*Pi*R))*Sin(3*Pi*t))/
-            Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (0.44413219804902115*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*
-              (1.*mui*taui*Cos(3*Pi*R) + 
-                R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*
-                 Sin(3*Pi*R))*Sin(3*Pi*t))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (0.6579736267392905*beta*(1 + Sqrt(-mue))*Cos(Pi*R)*
-              Cos(2*Pi*Z)*Sin(P)*Power(Sin(2*P),2)*Sin(2*Pi*R)*
-              (4.1887902047863905*Cos(2*Pi*R) - 
-                52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-              Power(Sin(2*Pi*t),2)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-            (mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-           (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              (-330.73361792319804*R*Cos(2*Pi*R) - 
-                78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*R)*
-              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-            (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-           (2*beta*(1 + Sqrt(-mue))*Power(Pi,2)*Cos(2*Pi*R)*Cos(2*Pi*Z)*
-              Power(Sin(2*P),2)*
-              (4.1887902047863905*Cos(2*Pi*R) - 
-                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-              Sin(2*Pi*Z))/
-            (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-           (beta*(1 + Sqrt(-mue))*Pi*(-20. + 2.*R)*Cos(2*Pi*Z)*
-              Power(Sin(2*P),2)*Sin(2*Pi*R)*
-              (4.1887902047863905*Cos(2*Pi*R) - 
-                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (3*Pi*R*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
+              Sin(3*Pi*t))/
+            (100.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) \
+- (3*Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (2*beta*(1 + Sqrt(-mue))*Power(Pi,2)*R*Cos(2*Pi*R)*Cos(2*Pi*Z)*
+              Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (30.*mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-               1.5)*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
+            (45.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (2*beta*(1 + Sqrt(-mue))*Power(Pi,2)*R*Cos(2*Pi*(-10 + R))*
+              Cos(2*Pi*Z)*Power(Sin(2*P),2)*Sin(2*Pi*R)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (45.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (beta*(1 + Sqrt(-mue))*Pi*R*(-20. + 2.*R)*Cos(2*Pi*Z)*
+              Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (90.*mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
+               1.5)) + (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*
+              Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (45.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
         1.5707963267948966*R*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z)*
          ((-0.1*R*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (0.8882643960980423*Cos(3*Pi*Z)*Sin(3*P)*
-              (1.*mui*taui*Cos(3*Pi*R) + 
-                R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*
-                 Sin(3*Pi*R))*Sin(3*Pi*t))/
-            Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              Sin(2*Pi*R)*(4.1887902047863905*Cos(2*Pi*R) - 
-                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
+            (45.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
          ((-0.1*R*taue*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (0.8882643960980423*Cos(3*Pi*Z)*Sin(3*P)*
-              (1.*mui*taui*Cos(3*Pi*R) + 
-                R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*
-                 Sin(3*Pi*R))*Sin(3*Pi*t))/
-            Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-           (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              Sin(2*Pi*R)*(4.1887902047863905*Cos(2*Pi*R) - 
-                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
+            (45.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
         R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
          ((-3.*taue*Z*(-50. + 1.*R - 0.1*Power(Z,2)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
            (0.2*taue*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (3*Pi*Cos(3*Pi*Z)*Sin(3*P)*
-              ((0.09424777960769379*mui*taui + 
-                   Power(R,2)*
-                    (0.1884955592153876 + 16.7433894073619*mui*taui))*
-                 Cos(3*Pi*R) + 0.8882643960980426*mui*R*taui*Sin(3*Pi*R))*
-              Sin(3*Pi*t))/
-            (R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (2*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (45.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
            (2*beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-              (4.1887902047863905*Cos(2*Pi*R) - 
-                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-              Sin(2*Pi*Z))/
-            (15.*mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-           (0.10471975511965977*beta*(1 + Sqrt(-mue))*Cos(Pi*Z)*Sin(P)*
-              Power(Sin(2*P),2)*Sin(Pi*R)*
-              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-              (4.1887902047863905*Cos(2*Pi*R) - 
-                52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-              Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-            (mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-           (0.03333333333333333*beta*(1 + Sqrt(-mue))*Z*Power(Sin(2*P),2)*
-              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-              (4.1887902047863905*Cos(2*Pi*R) - 
-                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*(-10 + R))*(2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (45.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (0.011111111111111112*beta*(1 + Sqrt(-mue))*Z*
+              Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
               Power(Sin(2*Pi*Z),2))/
-            (mue*R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)*
-              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-           ((-34818.239691125345*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-                 Power(0.07957747154594767*Cos(2*Pi*R) - 
-                   1.*R*Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-               (Power(R,2)*Power(2. + 
-                   1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-              (17409.119845562673*Cos(Pi*Z)*Sin(P)*Power(Sin(2*P),2)*
-                 Sin(Pi*R)*Power(0.07957747154594767*Cos(2*Pi*R) - 
-                   1.*R*Sin(2*Pi*R),2)*Sin(Pi*t)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2))/
-               (Power(R,2)*Power(2. + 
-                   1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)))/
-            (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (0.1*Z*(taue - (2770.7474783005127*Power(Sin(2*P),2)*
-                   Power(0.07957747154594767*Cos(2*Pi*R) - 
-                     1.*R*Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                   Power(Sin(2*Pi*Z),2))/
-                 (Power(R,2)*Power(2. + 
-                     1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
+            (mue*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) - 
+           (0.011111111111111112*Z*
+              (-9*taue + Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (1.*Z*Sin(3*P)*((0.09424777960769379*mui*taui + 
-                   Power(R,2)*
-                    (0.1884955592153876 + 16.7433894073619*mui*taui))*
-                 Cos(3*Pi*R) + 0.8882643960980426*mui*R*taui*Sin(3*Pi*R))*
-              Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5))) + 
+           (0.18849555921538758*R*Z*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*
+              Sin(3*Pi*Z))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) + 
         1.5707963267948966*R*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
          ((taue*(-50. + 1.*R - 0.1*Power(Z,2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
-              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-              (4.1887902047863905*Cos(2*Pi*R) - 
-                52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
               Power(Sin(2*Pi*Z),2))/
-            (30.*mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-              (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-           (taue - (2770.7474783005127*Power(Sin(2*P),2)*
-                 Power(0.07957747154594767*Cos(2*Pi*R) - 
-                   1.*R*Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2))/
-               (Power(R,2)*Power(2. + 
-                   1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)))/
-            (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (Sin(3*P)*((0.09424777960769379*mui*taui + 
-                   Power(R,2)*
-                    (0.1884955592153876 + 16.7433894073619*mui*taui))*
-                 Cos(3*Pi*R) + 0.8882643960980426*mui*R*taui*Sin(3*Pi*R))*
-              Sin(3*Pi*t)*Sin(3*Pi*Z))/
-            (R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/R)
+            (90.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (-9*taue + Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+               Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+            (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/R)
 ; }};
 struct SNi{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (-1.2337005501361697*mui*taui*Cos(Pi*R)*Cos(Pi*t)*Sin(P)*Sin(Pi*Z))/R + 
-  Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*t)*Sin(P)*Sin(Pi*R)*
-   Sin(Pi*Z) + (FELTORPARALLEL*
-     (0. - (6.283185307179586*(-10 + R)*R*Cos(2*Pi*Z)*Sin(2*P)*
-          (-0.8488263631567752*Cos(2*Pi*R) + 
-            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z))/R + 
-            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
-             Sin(Pi*t)*Sin(Pi*Z)))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)))) + 
-       (1.*(-10 + R)*R*Sin(2*P)*
-          (-0.8488263631567752*Cos(2*Pi*R) + 
-            10.666666666666666*R*Sin(2*Pi*R))*
-          (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-            Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
-             Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t)*
-          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z))/R + 
-            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
-             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)),2)) - 
-       (20*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-          (-0.15915494309189535*mui*taui*Cos(P)*Cos(Pi*R)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-             Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
-          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z))/R + 
-            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
-             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)),2)) - 
-       (1.*R*Z*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-          (0.4052847345693511 + 
-            Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*Sin(P)*
-             Sin(Pi*t)*Sin(Pi*Z) + 
-            0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-            (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-             Sin(Pi*t)*Sin(Pi*Z))*
-          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z))/R + 
-            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
-             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)),2)) - 
-       (1.*(-10 + R)*R*Sin(2*P)*
-          (-0.8488263631567752*Cos(2*Pi*R) + 
-            10.666666666666666*R*Sin(2*Pi*R))*
-          ((-1.2337005501361697*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-               Sin(Pi*t))/R + 
-            Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*Z)*Sin(P)*
-             Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)))) + 
-       (20*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-          ((-0.39269908169872414*mui*taui*Cos(P)*Cos(Pi*R)*Sin(Pi*t)*
-               Sin(Pi*Z))/R + 
-            (0.25 + 2.4674011002723395*mui*taui)*Cos(P)*Sin(Pi*R)*
-             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)))) + 
-       (1.*R*Z*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-          ((0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-               Sin(Pi*Z))/Power(R,2) + 
-            Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*R)*Sin(P)*
-             Sin(Pi*t)*Sin(Pi*Z) + 
-            (1.2337005501361697*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-               Sin(Pi*Z))/R)*Sin(2*Pi*Z))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)))) + 
-       (1.*R*Z*Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 
-            16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
-          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z))/R + 
-            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
-             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)))) + 
-       (40*Cos(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z))/R + 
-            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
-             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)))) + 
-       (1.*Z*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-            10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-          (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z))/R + 
-            (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
-             Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-        (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-          (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z))))))/R + 
-  FELTORPERP*(-((nuperp*((0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z))/Power(R,2) + 
-            Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*R)*Sin(P)*
-             Sin(Pi*t)*Sin(Pi*Z) + 
-            (1.2337005501361697*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-               Sin(Pi*Z))/R + R*
-             ((3.875784585037477*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                  Sin(Pi*Z))/R - 
-               Power(Pi,2)*(0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-                Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)) + 
-            R*((-0.7853981633974483*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                  Sin(Pi*Z))/Power(R,3) + 
-               (3.875784585037477*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                  Sin(Pi*Z))/R - 
-               (2.4674011002723395*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                  Sin(Pi*Z))/Power(R,2) - 
-               Power(Pi,2)*(0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-                Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))/R) + 
-     (R*((0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z))/Power(R,2) + 
-           Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*R)*Sin(P)*
-            Sin(Pi*t)*Sin(Pi*Z) + 
-           (1.2337005501361697*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-              Sin(Pi*Z))/R)*((-0.1*R*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-              Sin(2*Pi*Z))/
-            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) - 
-           (R*Sin(3*P)*Sin(3*Pi*t)*
-              (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
-                (mui*Cos(3*Pi*Z)*Sin(3*P)*
-                   (528430.5031318273*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
-                     54*Power(Pi,3)*Power(R,2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
-                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) + 
-                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2))))/
-            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-        (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-            Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
-         ((-0.1*R*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-              Sin(2*Pi*Z))/
-            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) - 
-           (R*Sin(3*P)*Sin(3*Pi*t)*
-              (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
-                (mui*Cos(3*Pi*Z)*Sin(3*P)*
-                   (528430.5031318273*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
-                     54*Power(Pi,3)*Power(R,2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
-                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) + 
-                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2))))/
-            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-        R*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-            Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
+    return 0.7853981633974483*Cos(Pi*t)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*Z) + 
+  (FELTORPARALLEL*(0. - (2.0943951023931953*(-10 + R)*R*Cos(2*Pi*Z)*
+          Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          (1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)))/
+        Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+       (0.2617993877991494*(-10 + R)*R*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
+          Sin(Pi*(-10 + R))*Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*
+          Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+       (0.2617993877991494*R*Z*Cos(Pi*(-10 + R))*Sin(P)*Sin(2*P)*
+          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+        Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+       (1.6666666666666667*Cos(P)*Sin(2*P)*Sin(Pi*(-10 + R))*
+          Sin(2*Pi*(-10 + R))*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
+        Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+       (2.0943951023931953*R*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
+          (1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+          Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+       (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          (1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+          Sin(2*Pi*Z))/(3.*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) \
++ (0.3333333333333333*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          (1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+          Sin(2*Pi*Z))/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/R + 
+  FELTORPERP*(-((nuperp*(0. + 0.7853981633974483*Cos(Pi*(-10 + R))*Sin(P)*
+             Sin(Pi*t)*Sin(Pi*Z) - 
+            4.934802200544679*R*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z)\
+))/R) + (R*(1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
          ((0.15000000000000002*R*(-20. + 2.*R)*taui*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
            (0.1*taui*Z)/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-              (0.4052847345693511 + 
-                Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
-                 Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*
-              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                 Sin(Pi*t)*Sin(Pi*Z) + 
-                R*(0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              Sin(2*Pi*R)*(67.02064327658225*R*Cos(2*Pi*R) + 
-                16.*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) + 
+           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (3*Pi*R*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*
+              Sin(3*Pi*t))/
+            (100.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) \
+- (3*Pi*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
            (2*beta*(1 + Sqrt(-mue))*Power(Pi,2)*R*Cos(2*Pi*R)*Cos(2*Pi*Z)*
-              Power(Sin(2*P),2)*
-              (-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) - 
+            (45.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))) + 
+           (2*beta*(1 + Sqrt(-mue))*Power(Pi,2)*R*Cos(2*Pi*(-10 + R))*
+              Cos(2*Pi*Z)*Power(Sin(2*P),2)*Sin(2*Pi*R)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (45.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))) - 
            (beta*(1 + Sqrt(-mue))*Pi*R*(-20. + 2.*R)*Cos(2*Pi*Z)*
-              Power(Sin(2*P),2)*Sin(2*Pi*R)*
-              (-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-              Sin(2*Pi*Z))/
-            (30.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),1.5)*
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) + 
+              Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (90.*Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2),1.5)) + 
            (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*Z))/
+            (45.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2)))) + 
+        0.7853981633974483*R*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*Sin(Pi*Z)*
+         ((-0.1*R*taui*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*Z))/
+            (45.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2)))) + 
+        (1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+         ((-0.1*R*taui*Z)/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+              Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*Power(Sin(2*Pi*t),2)*
               Sin(2*Pi*Z))/
-            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) + 
-           (R*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
-              (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
-                (mui*Cos(3*Pi*Z)*Sin(3*P)*
-                   (528430.5031318273*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                       Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) \
-- 54*Power(Pi,3)*Power(R,2)*Power(0.942477796076938*mui*taui*
-                        Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
-                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) + 
-                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2))))/
-            (4000.*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) \
-- (Sin(3*P)*Sin(3*Pi*t)*(120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
-                (mui*Cos(3*Pi*Z)*Sin(3*P)*
-                   (528430.5031318273*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                       Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) \
-- 54*Power(Pi,3)*Power(R,2)*Power(0.942477796076938*mui*taui*
-                        Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
-                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) + 
-                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2))))/
-            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-           (R*Sin(3*P)*Sin(3*Pi*t)*
-              (360*Power(Pi,2)*Cos(3*Pi*R)*Cos(3*Pi*Z) + 
-                (mui*(-20. + 2.*R)*Cos(3*Pi*Z)*Sin(3*P)*
-                   (528430.5031318273*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
-                     54*Power(Pi,3)*Power(R,2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2)) + 
-                (2*mui*Cos(3*Pi*Z)*Sin(3*P)*
-                   (528430.5031318273*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
-                     54*Power(Pi,3)*Power(R,2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-                   Sin(3*Pi*t)*Sin(3*Pi*Z))/
-                 (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) - 
-                (mui*Cos(3*Pi*Z)*Sin(3*P)*
-                   (-108*Power(Pi,3)*Power(R,2)*
-                      (3*Pi*R*
-                        (-0.20000000000000004 - 
-                       17.765287921960844*mui*taui)*Cos(3*Pi*R) - 
-                        8.882643960980424*mui*taui*Sin(3*Pi*R) + 
-                        (-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R))*
-                      (0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R)) - 
-                     108*Power(Pi,3)*R*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     1.0568610062636547e6*
-                      ((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
-                      (0.5000000000000001*mui*R*taui*Cos(3*Pi*R) + 
-                        2*R*(0.011257909293593089 + 1.*mui*taui)*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*taui*Sin(3*Pi*R) - 
-                        3*Pi*
-                         (0.005628954646796544*mui*taui + 
-                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
-)*Sin(3*Pi*R)))*Sin(3*Pi*t)*Sin(3*Pi*Z))/
-                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) - 
-                (4.*mui*(-20. + 2.*R)*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),3)) - 
-                (4.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,3)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2)) + 
-                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (18*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      (3*Pi*R*
-                        (-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Cos(3*Pi*R) - 
-                        8.882643960980424*mui*taui*Sin(3*Pi*R) + 
-                        (-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R))*
-                      (0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                         17.765287921960844*mui*taui)*Sin(3*Pi*R)) + 
-                     18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     56068.21776931174*
-                      ((0.005628954646796544*mui*taui + 
-                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
-)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
-                      (0.5000000000000001*mui*R*taui*Cos(3*Pi*R) + 
-                        2*R*(0.011257909293593089 + 1.*mui*taui)*
-                         Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*taui*Sin(3*Pi*R) - 
-                        3*Pi*(0.005628954646796544*mui*taui + 
-                         Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
-)*Sin(3*Pi*R))*Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2))))/
-            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-        R*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-            Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
+            (45.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2)))) + 
+        R*(1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
          ((-3.*taui*Z*(-50. + 1.*R - 0.1*Power(Z,2)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5) - 
            (0.2*taui*Z)/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
+           (9*Power(Pi,2)*R*Cos(3*Pi*R)*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*t))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
+           (2*mui*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (45.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
            (2*beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-              (-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-              Sin(2*Pi*Z))/
-            (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) + 
-           (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
-              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-              (-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*
-              (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-                Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*
-                 Sin(P)*Sin(Pi*R)*Sin(Pi*t))*Power(Sin(2*Pi*t),2)*
-              Power(Sin(2*Pi*Z),2))/
-            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*
-              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                 Sin(Pi*t)*Sin(Pi*Z) + 
-                R*(0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-           (0.03333333333333333*beta*(1 + Sqrt(-mue))*Z*Power(Sin(2*P),2)*
-              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-              (-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+              Sin(2*Pi*(-10 + R))*(2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
+              Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+            (45.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))) + 
+           (0.011111111111111112*beta*(1 + Sqrt(-mue))*Z*
+              Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
               Power(Sin(2*Pi*Z),2))/
-            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),1.5)*
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) - 
-           ((4*mui*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-                 Power(0.8488263631567752*Cos(2*Pi*R) - 
-                   10.666666666666666*R*Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
-               Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                  Sin(Pi*t)*Sin(Pi*Z) + 
-                 R*(0.4052847345693511 + 
-                    (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                     Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-              (2*mui*Power(Sin(2*P),2)*
-                 Power(0.8488263631567752*Cos(2*Pi*R) - 
-                   10.666666666666666*R*Sin(2*Pi*R),2)*
-                 (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-                   Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*
-                    Sin(P)*Sin(Pi*R)*Sin(Pi*t))*Power(Sin(2*Pi*t),2)*
-                 Power(Sin(2*Pi*Z),2))/
-               Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                  Sin(Pi*t)*Sin(Pi*Z) + 
-                 R*(0.4052847345693511 + 
-                    (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                     Sin(Pi*t)*Sin(Pi*Z)),3))/
-            (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (0.1*Z*(taui + (mui*Power(Sin(2*P),2)*
-                   Power(0.8488263631567752*Cos(2*Pi*R) - 
-                     10.666666666666666*R*Sin(2*Pi*R),2)*
-                   Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-                 Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                    Sin(Pi*t)*Sin(Pi*Z) + 
-                   R*(0.4052847345693511 + 
-                      (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                       Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
-            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-           (R*Sin(3*P)*Sin(3*Pi*t)*
-              (360*Power(Pi,2)*Cos(3*Pi*R)*Cos(3*Pi*Z) + 
-                (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
-                   (528430.5031318273*Cos(3*Pi*Z)*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                       Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Sin(3*Pi*Z) - 
-                     54*Power(Pi,3)*Power(R,2)*Cos(3*Pi*Z)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2)*
-                      Sin(3*Pi*Z)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2)) + 
-                (2*mui*Sin(3*P)*Sin(3*Pi*t)*
-                   (528430.5031318273*Cos(3*Pi*Z)*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                       Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Sin(3*Pi*Z) - 
-                     54*Power(Pi,3)*Power(R,2)*Cos(3*Pi*Z)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2)*
-                      Sin(3*Pi*Z)))/
-                 (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) - 
-                (mui*Sin(3*P)*Sin(3*Pi*t)*
-                   (-108*Power(Pi,3)*R*Cos(3*Pi*Z)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                       17.765287921960844*mui*taui)*Sin(3*Pi*R),2)*
-                      Sin(3*Pi*Z) - 
-                     9.960680319430241e6*Power(R,2)*Cos(3*Pi*Z)*
-                      (R*(0.011257909293593089 + 1.*mui*taui)*
-                        Cos(3*Pi*R) + 
-                        (0.0011945012753036852 + 
-                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
-                      (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
-                        R*(0.011257909293593089 + 1.*mui*taui)*
-                        Sin(3*Pi*R))*Sin(3*Pi*Z) - 
-                     3.1705830187909645e6*Cos(3*Pi*Z)*
-                      ((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
-                      (R*(-0.007505272862395392 - 
-                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
-                        (-6.938893903907228e-18*mui*taui + 
-                        Power(R,2)*
-                        (0.03536776513153231 + 
-                        3.141592653589793*mui*taui))*Sin(3*Pi*R))*
-                      Sin(3*Pi*Z)))/
-                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) - 
-                (4.*mui*(-20. + 2.*R)*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),3)) - 
-                (4.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,3)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2)) + 
-                (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                   (18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      (R*(0.011257909293593089 + 1.*mui*taui)*
-                        Cos(3*Pi*R) + 
-                        (0.0011945012753036852 + 
-                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
-                      (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
-                        R*(0.011257909293593089 + 1.*mui*taui)*
-                         Sin(3*Pi*R)) - 
-                     168204.65330793522*
-                      ((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
-                      (R*(-0.007505272862395392 - 
-                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
-                        (-6.938893903907228e-18*mui*taui + 
-                        Power(R,2)*
-                        (0.03536776513153231 + 
-                        3.141592653589793*mui*taui))*Sin(3*Pi*R))*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2))))/
-            (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-           (0.0005*R*Z*Sin(3*P)*Sin(3*Pi*t)*
-              (120*Pi*Cos(3*Pi*R)*Sin(3*Pi*Z) + 
-                (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2)) + 
-                (2*mui*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) - 
-                (mui*Sin(3*P)*Sin(3*Pi*t)*
-                   (18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      (R*(0.011257909293593089 + 1.*mui*taui)*
-                        Cos(3*Pi*R) + 
-                        (0.0011945012753036852 + 
-                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
-                      (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
-                        R*(0.011257909293593089 + 1.*mui*taui)*
-                         Sin(3*Pi*R)) - 
-                     168204.65330793522*
-                      ((0.005628954646796544*mui*taui + 
-                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
-)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
-                      (R*(-0.007505272862395392 - 
-                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
-                        (-6.938893903907228e-18*mui*taui + 
-                         Power(R,2)*
-                         (0.03536776513153231 + 
-                         3.141592653589793*mui*taui))*Sin(3*Pi*R))*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2)))))/
+            (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
+               1.5)) - (0.011111111111111112*Z*
+              (-9*taui - mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2)))/
+            Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+           (0.18849555921538758*R*Z*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*
+              Sin(3*Pi*Z))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) + 
-        R*((-1.2337005501361697*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-              Sin(Pi*t))/R + Pi*(0.25 + 2.4674011002723395*mui*taui)*
-            Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
+        0.7853981633974483*R*Cos(Pi*Z)*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
          ((taui*(-50. + 1.*R - 0.1*Power(Z,2)))/
             Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-           (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
-              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-              (-0.8488263631567752*Cos(2*Pi*R) + 
-                10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+           (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*
+              (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
               Power(Sin(2*Pi*Z),2))/
-            (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2))*
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) - 
-           (taui + (mui*Power(Sin(2*P),2)*
-                 Power(0.8488263631567752*Cos(2*Pi*R) - 
-                   10.666666666666666*R*Sin(2*Pi*R),2)*
-                 Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-               Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                  Sin(Pi*t)*Sin(Pi*Z) + 
-                 R*(0.4052847345693511 + 
-                    (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                     Sin(Pi*t)*Sin(Pi*Z)),2))/
-            (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-           (R*Sin(3*P)*Sin(3*Pi*t)*
-              (120*Pi*Cos(3*Pi*R)*Sin(3*Pi*Z) + 
-                (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2),2)) + 
-                (2*mui*Sin(3*P)*Sin(3*Pi*t)*
-                   (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     28034.10888465587*
-                      Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                        0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 
-                     1.*Power(Z,2))) - 
-                (mui*Sin(3*P)*Sin(3*Pi*t)*
-                   (18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
-                      Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                        R*(-0.20000000000000004 - 
-                         17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                     528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                      (R*(0.011257909293593089 + 1.*mui*taui)*
-                         Cos(3*Pi*R) + 
-                        (0.0011945012753036852 + 
-                         0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
-                      (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
-                        R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R)\
-) - 168204.65330793522*((0.005628954646796544*mui*taui + 
-                         Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
-)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
-                      (R*(-0.007505272862395392 - 
-                         0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
-                        (-6.938893903907228e-18*mui*taui + 
-                         Power(R,2)*
-                         (0.03536776513153231 + 
-                         3.141592653589793*mui*taui))*Sin(3*Pi*R))*
-                      Power(Sin(3*Pi*Z),2)))/
-                 (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))\
-))/(2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/R)
+            (90.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+                1.*Power(Z,2))) + 
+           (-9*taui - mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+               Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+            (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+           (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+            (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/R)
 ; }};
 struct SWe{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (-1.0471975511965976*Cos(Pi*t)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-     (12.566370614359172*Cos(2*Pi*R) - 157.91367041742973*R*Sin(2*Pi*R))*
-     Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-   (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-  (2*Pi*Cos(2*Pi*t)*Sin(2*P)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/mue + 
-       (12.566370614359172*Cos(2*Pi*R) - 157.91367041742973*R*Sin(2*Pi*R))/
-        (R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/
-   (3.*Sqrt(-mue)) + FELTORPARALLEL*
-   ((taue*Sin(Pi*t)*((15.707963267948966 - 1.5707963267948966*R)*R*
-           Cos(Pi*Z)*Sin(P)*Sin(Pi*R) + 
-          (1.5707963267948966*R*Z*Cos(Pi*R)*Sin(P) + 10.*Cos(P)*Sin(Pi*R))*
-           Sin(Pi*Z)))/
+    return (-2*(beta + beta*Sqrt(-mue) - mue)*Pi*Cos(2*Pi*t)*Sin(2*P)*Sin(2*Pi*R)*
+     Sin(2*Pi*Z))/(3.*Power(-mue,1.5)) + 
+  FELTORPARALLEL*((taue*Sin(Pi*t)*
+        ((15.707963267948966 - 1.5707963267948966*R)*R*Cos(Pi*Z)*Sin(P)*
+           Sin(Pi*R) + (1.5707963267948966*R*Z*Cos(Pi*R)*Sin(P) + 
+             10.*Cos(P)*Sin(Pi*R))*Sin(Pi*Z)))/
       (mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) - 
-     (1.*(-10 + R)*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-          52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-          (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-             Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*Power(2. + 
-               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
-      (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-        (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (20*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-          52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        ((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-             Sin(Pi*Z)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*Power(2. + 
-               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-          (2*Cos(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-      (Sqrt(-mue)*Power(R,2)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-          1.*Power(Z,2))*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (1.*Z*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-          52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-             Sin(Pi*Z)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*Power(2. + 
-               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-          (Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
-               78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-          (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*Power(R,2)*
-             (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-      (Sqrt(-mue)*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-        (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
+        (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (0.6981317007977318*(-10 + R)*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+        Power(Sin(2*Pi*(-10 + R)),2)*Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+      (mue*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+     (0.6981317007977318*Z*Cos(2*Pi*(-10 + R))*Power(Sin(2*P),2)*
+        Sin(2*Pi*(-10 + R))*Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+      (mue*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+     (40*Cos(2*P)*Sin(2*P)*Power(Sin(2*Pi*(-10 + R)),2)*
+        Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+      (9.*mue*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
      (nuparallele*(0. + (1.*Z*
-             ((-1.*(-10 + R)*
-                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (20*((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (2*Cos(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
-)/(R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-               (1.*Z*((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (Sin(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
-           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (20*((-1.*(-10 + R)*
-                  ((-6.283185307179586*Cos(P)*Cos(2*Pi*Z)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-+ (6.283185307179586*Cos(P)*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-                       Power(Sin(Pi*R),2)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*
-                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(Pi*Z)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
-                    (6.283185307179586*Cos(2*P)*Cos(Pi*Z)*Sin(P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (3.141592653589793*Cos(P)*Cos(Pi*Z)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (1.*Z*((6.283185307179586*Cos(P)*Cos(Pi*R)*Sin(P)*
-                       Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*
-                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*
-                       Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
-                    (1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                       78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (6.283185307179586*Cos(2*P)*Cos(Pi*R)*Sin(P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (3.141592653589793*Cos(P)*Cos(Pi*R)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (2*Cos(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                       78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (2*Cos(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
-)/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (20*((2.*Power(Cos(P),2)*Sin(2*P)*Power(Sin(Pi*R),2)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*
-                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*
-                       Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
-                    (4.*Cos(P)*Cos(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (1.*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (4*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
+             ((2.0943951023931953*(-10 + R)*Cos(2*Pi*Z)*Sin(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2))) - 
+               (2.0943951023931953*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2))) - 
+               (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (3.*Sqrt(-mue)*R*
+                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (1.*(-10 + R)*R*((20*
-                  ((-6.283185307179586*Cos(P)*Cos(2*Pi*Z)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-+ (6.283185307179586*Cos(P)*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-                       Power(Sin(Pi*R),2)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*
-                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(Pi*Z)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
-                    (6.283185307179586*Cos(2*P)*Cos(Pi*Z)*Sin(P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (3.141592653589793*Cos(P)*Cos(Pi*Z)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
-                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-               (1.*Z*((-19.739208802178716*Cos(Pi*R)*Cos(2*Pi*Z)*
-                       Sin(P)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                       78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-+ (19.739208802178716*Cos(Pi*R)*Cos(Pi*Z)*Power(Sin(P),2)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*
-                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(Pi*Z)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
-                    (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                       78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (9.869604401089358*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                       Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (1.*(-10 + R)*Z*
-                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-               (20.*Z*((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (2*Cos(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
-)/(R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) - 
-               (1.*Power(Z,2)*
-                  ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (Sin(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                       78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
-)/Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
-               (1.*((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                       52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (Sin(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                       78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
-)/Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (1.*(-10 + R)*
-                  ((-39.47841760435743*Cos(Pi*Z)*Cos(2*Pi*Z)*Sin(P)*
-                       Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (19.739208802178716*Power(Cos(Pi*Z),2)*
-                       Power(Sin(P),2)*Sin(2*P)*Power(Sin(Pi*R),2)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*
-                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) + 
-                    (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (4*Power(Pi,2)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
+          (1.*(-10 + R)*R*((-13.15947253478581*Z*Cos(2*Pi*(-10 + R))*
+                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2))) - 
+               (80*Pi*Cos(2*P)*Cos(2*Pi*Z)*Sin(2*Pi*(-10 + R))*
+                  Sin(2*Pi*t))/
+                (3.*Sqrt(-mue)*R*
+                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+               (2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*Z)*Sin(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+                (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2),1.5)) + 
+               (2.0943951023931953*Power(Z,2)*Cos(2*Pi*(-10 + R))*
+                  Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2),1.5)) - 
+               (2.0943951023931953*Cos(2*Pi*(-10 + R))*Sin(2*P)*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2))) + 
+               (13.333333333333334*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (Sqrt(-mue)*R*
+                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) \
+- (13.15947253478581*(-10 + R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+                  Sin(2*Pi*Z))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2)))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (1.*R*Z*((-1.*(-10 + R)*
-                  ((-19.739208802178716*Cos(Pi*R)*Cos(2*Pi*Z)*Sin(P)*
-                       Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-+ (19.739208802178716*Cos(Pi*R)*Cos(Pi*Z)*Power(Sin(P),2)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*
-                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(Pi*Z)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
-                    (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                        78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (9.869604401089358*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                       Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*
-                       Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (1.*Power(-10 + R,2)*
-                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-               (1.*((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2))))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (20*((6.283185307179586*Cos(P)*Cos(Pi*R)*Sin(P)*
-                       Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*
-                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*
-                       Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
-                    (1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                        78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (6.283185307179586*Cos(2*P)*Cos(Pi*R)*Sin(P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (3.141592653589793*Cos(P)*Cos(Pi*R)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (2*Cos(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (2*Cos(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               (20.*(-10 + R)*
-                  ((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (2*Cos(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) \
-- (20*((-1.*Cos(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (2*Cos(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                (Power(R,2)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+          (20*((4.1887902047863905*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
                     1.*Power(Z,2))) - 
-               (1.*(-10 + R)*Z*
-                  ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (Sin(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) \
-- (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
-               (1.*Z*((19.739208802178716*Power(Cos(Pi*R),2)*
-                       Power(Sin(P),2)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*
-                       Power(Sin(Pi*t),2)*Sin(2*Pi*t)*
-                       Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
-                    (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                        78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-                    (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*
-                       Sin(2*Pi*t)*Sin(Pi*Z)*Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       Power(2. + 
-                        1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-                    (2*Sin(2*P)*
-                       (-330.73361792319804*R*Cos(2*Pi*R) - 
-                        78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,2)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-                    (2*Sin(2*P)*
-                       (4.1887902047863905*Cos(2*Pi*R) - 
-                        52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*Power(R,3)*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-                    (Sin(2*P)*
-                       (-826.834044807995*Cos(2*Pi*R) + 
-                        2078.060608725385*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (Sqrt(-mue)*R*
-                       (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
+               (4.1887902047863905*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2))) + 
+               (80*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (3.*Sqrt(-mue)*R*
+                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
+           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+          (1.*R*Z*((13.15947253478581*(-10 + R)*Cos(2*Pi*(-10 + R))*
+                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2))) - 
+               (2.0943951023931953*Power(-10 + R,2)*Cos(2*Pi*Z)*
+                  Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+                (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2),1.5)) + 
+               (2.0943951023931953*Cos(2*Pi*Z)*Sin(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2))) - 
+               (80*Pi*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+                  Sin(2*Pi*Z))/
+                (3.*Sqrt(-mue)*R*
+                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+               (2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
+                  Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (Sqrt(-mue)*Power(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2),1.5)) + 
+               (13.333333333333334*(-10 + R)*Cos(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (Sqrt(-mue)*R*
+                  Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) + 
+               (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (3.*Sqrt(-mue)*Power(R,2)*
+                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+               (13.15947253478581*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (Sqrt(-mue)*Sqrt(400 + 1.*Power(-10 + R,2) + 
+                    1.*Power(Z,2)))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
       (R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (Sin(3*Pi*t)*(R*Sin(3*Pi*R)*
-           (R*(18.849555921538762 + 1674.3389407361901*mui*taui + 
-                R*(-1.884955592153876 - 167.433894073619*mui*taui))*
-              Cos(3*Pi*Z)*Sin(3*P) + 
-             ((12. + 1065.9172753176506*mui*taui)*Cos(3*P) + 
-                8.882643960980428*mui*taui*Z*Sin(3*P))*Sin(3*Pi*Z)) + 
-          Cos(3*Pi*R)*(mui*R*(-88.82643960980423 + 8.882643960980422*R)*
-              taui*Cos(3*Pi*Z)*Sin(3*P) + 
-             (-56.54866776461628*mui*taui*Cos(3*P) + 
-                (0.9424777960769379*mui*taui + 
-                   Power(R,2)*
-                    (1.884955592153876 + 167.433894073619*mui*taui))*Z*
-                 Sin(3*P))*Sin(3*Pi*Z))))/
-      (mue*Power(R,2)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-  FELTORPERP*(0. - (0.1*taue*Z*Sin(2*P)*
-        (4.1887902047863905*Cos(2*Pi*R) - 52.63789013914324*R*Sin(2*Pi*R))*
+     (Sin(3*Pi*t)*((18.84955592153876 - 1.8849555921538759*R)*R*
+           Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R) + 
+          (1.8849555921538759*R*Z*Cos(3*Pi*R)*Sin(3*P) + 
+             12.*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*Z)))/
+      (mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
+  FELTORPERP*(0. + (0.03333333333333333*taue*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
         Sin(2*Pi*t)*Sin(2*Pi*Z))/
-      (Sqrt(-mue)*R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)*
-        (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (0.8882643960980423*Cos(3*Pi*Z)*Sin(2*P)*Sin(3*P)*
-        (4.1887902047863905*Cos(2*Pi*R) - 52.63789013914324*R*Sin(2*Pi*R))*
-        (1.*mui*taui*Cos(3*Pi*R) + 
-          R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*Sin(3*Pi*R)\
-)*Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z))/
-      (Sqrt(-mue)*Power(R,2)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-          1.*Power(Z,2))*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
+      (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)) + 
+     (Pi*Cos(3*Pi*Z)*Sin(2*P)*Sin(3*P)*Sin(2*Pi*(-10 + R))*Sin(3*Pi*R)*
+        Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z))/
+      (50.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
      (0.32898681336964525*beta*(1. + Sqrt(-mue))*taue*Sin(P)*Sin(2*P)*
         Sin(Pi*t)*Sin(2*Pi*t)*(1.*R*Cos(Pi*R)*Cos(2*Pi*Z)*Sin(2*Pi*R)*
            Sin(Pi*Z) + Cos(Pi*Z)*Sin(Pi*R)*
            (-1.*R*Cos(2*Pi*R) - 0.15915494309189535*Sin(2*Pi*R))*Sin(2*Pi*Z)\
 ))/(Sqrt(-mue)*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
         (1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
-     ((33.0733617923198*taue*Cos(2*Pi*Z)*Sin(2*P)*
-           (-0.07957747154594767*Cos(2*Pi*R) + 1.*R*Sin(2*Pi*R))*
-           Sin(2*Pi*t))/
+     (eta*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
+        ((Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3. + 
+          (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (3.*Sqrt(-mue))))/mue - 
+     (nuperp*((-2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (3.*Sqrt(-mue)) + (8*Power(Pi,2)*R*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+             Sin(2*Pi*t)*Sin(2*Pi*Z))/(3.*Sqrt(-mue))))/R - 
+     (2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z)*
+        ((-0.1*R*taue*Z)/
+           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+          (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+             Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*Power(Sin(2*Pi*t),2)*
+             Sin(2*Pi*Z))/
+           (45.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/
+      (3.*Sqrt(-mue)) + ((0.20943951023931953*R*taue*Cos(2*Pi*Z)*Sin(2*P)*
+           Sin(2*Pi*R)*Sin(2*Pi*t)*
+           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
+         (Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+        (0.10471975511965977*R*taue*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+           Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
          (Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-        (5.263789013914324*taue*Z*Sin(2*P)*
-           (-0.07957747154594767*Cos(2*Pi*R) + 1.*R*Sin(2*Pi*R))*
-           Sin(2*Pi*t)*Sin(2*Pi*Z))/
+        (0.03333333333333333*R*taue*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
+           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
          (Sqrt(-mue)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
             1.5)))/(R*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) \
-+ ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-           (4.1887902047863905*Cos(2*Pi*R) - 
-             52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-           Sin(Pi*Z)*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
-            2)) + (Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
-             78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-        (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-             52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-         (Sqrt(-mue)*Power(R,2)*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))*
-      ((-0.1*R*taue*Z)/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-        (0.8882643960980423*Cos(3*Pi*Z)*Sin(3*P)*
-           (1.*mui*taui*Cos(3*Pi*R) + 
-             R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*
-              Sin(3*Pi*R))*Sin(3*Pi*t))/
-         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
-        (beta*(1 + Sqrt(-mue))*Pi*Cos(2*Pi*Z)*Power(Sin(2*P),2)*Sin(2*Pi*R)*
-           (4.1887902047863905*Cos(2*Pi*R) - 
-             52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-           Sin(2*Pi*Z))/
-         (15.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
-     (eta*(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))*
-        (-((Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                 52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-             (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) \
-+ (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)))))/mue - 
-     (nuperp*((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-             Sin(Pi*Z)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*Power(2. + 
-               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-          (Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
-               78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-          (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*Power(R,2)*
-             (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-          R*((-39.47841760435743*Cos(Pi*Z)*Cos(2*Pi*Z)*Sin(P)*Sin(2*P)*
-                Sin(Pi*R)*(4.1887902047863905*Cos(2*Pi*R) - 
-                  52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t))/
-              (Sqrt(-mue)*R*Power(2. + 
-                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-             (19.739208802178716*Power(Cos(Pi*Z),2)*Power(Sin(P),2)*
-                Sin(2*P)*Power(Sin(Pi*R),2)*
-                (4.1887902047863905*Cos(2*Pi*R) - 
-                  52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(Pi*t),2)*
-                Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*Power(2. + 
-                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) + 
-             (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-                (4.1887902047863905*Cos(2*Pi*R) - 
-                  52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-                Sin(Pi*Z)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*Power(2. + 
-                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-             (4*Power(Pi,2)*Sin(2*P)*
-                (4.1887902047863905*Cos(2*Pi*R) - 
-                  52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)\
-)/(Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
-          R*((19.739208802178716*Power(Cos(Pi*R),2)*Power(Sin(P),2)*
-                Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                  52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(Pi*t),2)*
-                Sin(2*Pi*t)*Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*Power(2. + 
-                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) - 
-             (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-                (-330.73361792319804*R*Cos(2*Pi*R) - 
-                  78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-                Sin(Pi*Z)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*Power(2. + 
-                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-             (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-                (4.1887902047863905*Cos(2*Pi*R) - 
-                  52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-                Sin(Pi*Z)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*Power(R,2)*
-                Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-             (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-                (4.1887902047863905*Cos(2*Pi*R) - 
-                  52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-                Sin(Pi*Z)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*Power(2. + 
-                  1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-             (2*Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
-                  78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*Power(R,2)*
-                (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-             (2*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                  52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)\
-)/(Sqrt(-mue)*Power(R,3)*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-             (Sin(2*P)*(-826.834044807995*Cos(2*Pi*R) + 
-                  2078.060608725385*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))\
-))/R + ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-           (4.1887902047863905*Cos(2*Pi*R) - 
-             52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-         (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-        (3.141592653589793*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-           (4.1887902047863905*Cos(2*Pi*R) - 
-             52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-           Sin(2*Pi*Z))/
-         (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
-            2)))*((taue*(-50. + 1.*R - 0.1*Power(Z,2)))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-        (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
-           (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-           (4.1887902047863905*Cos(2*Pi*R) - 
-             52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-           Power(Sin(2*Pi*Z),2))/
-         (30.*mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-           (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-        (taue - (2770.7474783005127*Power(Sin(2*P),2)*
-              Power(0.07957747154594767*Cos(2*Pi*R) - 1.*R*Sin(2*Pi*R),2)*
+- (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+        ((taue*(-50. + 1.*R - 0.1*Power(Z,2)))/
+           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+          (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*
+             (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+             Power(Sin(2*Pi*Z),2))/
+           (90.*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (-9*taue + Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
               Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
-            (Power(R,2)*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
-               2)))/(10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) \
-+ (Sin(3*P)*((0.09424777960769379*mui*taui + 
-                Power(R,2)*(0.1884955592153876 + 
-                   16.7433894073619*mui*taui))*Cos(3*Pi*R) + 
-             0.8882643960980426*mui*R*taui*Sin(3*Pi*R))*Sin(3*Pi*t)*
-           Sin(3*Pi*Z))/
-         (R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) - 
-     (35.06727277224087*beta*(1. + Sqrt(-mue))*Sin(2*P)*Sin(3*P)*Sin(2*Pi*t)*
-        Sin(3*Pi*t)*(R*Cos(2*Pi*R)*Cos(3*Pi*Z)*
-           (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
-             R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R))*Sin(2*Pi*Z) \
-+ Sin(2*Pi*R)*(R*Sin(3*Pi*R)*((0.0017917519129555279 + 
-                   0.15915494309189535*mui*taui)*Cos(3*Pi*Z)*Sin(2*Pi*Z) - 
-                0.053051647697298476*mui*taui*Cos(2*Pi*Z)*Sin(3*Pi*Z)) + 
-             Cos(3*Pi*R)*(-0.008443431970194816*mui*taui*Cos(3*Pi*Z)*
-                 Sin(2*Pi*Z) + 
-                (-0.005628954646796544*mui*taui + 
-                   Power(R,2)*(-0.011257909293593089 - 1.*mui*taui))*
-                 Cos(2*Pi*Z)*Sin(3*Pi*Z)))))/
-      (Sqrt(-mue)*mue*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))))
+           (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/
+      (3.*Sqrt(-mue)) + (beta*(1 + Sqrt(-mue))*Pi*Sin(2*P)*Sin(3*P)*
+        Sin(2*Pi*t)*Sin(3*Pi*t)*
+        (-(Cos(3*Pi*Z)*(2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Sin(3*Pi*R)*
+             Sin(2*Pi*Z)) + Pi*R*Cos(3*Pi*R)*Sin(2*Pi*R)*
+           (Sin(Pi*Z) + Sin(5*Pi*Z))))/
+      (50.*Sqrt(-mue)*mue*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))))
 ; }};
 struct SWi{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return -(Sin(2*P)*(-2.5464790894703255*Cos(2*Pi*R) + 
-        31.999999999999996*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-      (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*t)*Sin(P)*Sin(Pi*Z) + 
-        Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*t)*Sin(P)*
-         Sin(Pi*R)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-   (3.*Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-        Sin(Pi*Z) + R*(0.4052847345693511 + 
-          (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-           Sin(Pi*Z)),2)) + (2*Pi*Cos(2*Pi*t)*Sin(2*P)*
-     ((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/(Sqrt(-mue)*mui) + 
-       (-2.5464790894703255*Cos(2*Pi*R) + 
-          31.999999999999996*R*Sin(2*Pi*R))/
-        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-           Sin(Pi*Z) + R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-              Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/3. + 
-  FELTORPERP*(0. + (1.623484850566707*beta*(1. + Sqrt(-mue))*taui*Sin(P)*
-        Sin(2*P)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(Pi*Z)*
-        (0.3183098861837907*mui*R*taui*Power(Cos(Pi*R),3)*
-           Power(Cos(Pi*Z),2) + 
-          R*(-0.06450306886639899 - 0.6366197723675813*mui*taui)*
-           Cos(Pi*R)*Power(Cos(Pi*Z),2)*Power(Sin(Pi*R),2) + 
-          Power(Cos(Pi*R),2)*Sin(Pi*R)*
-           (0.20264236728467555*mui*taui*Power(Cos(Pi*Z),2) + 
-             (-0.10132118364233778*mui*taui + 
-                Power(R,2)*(-0.20264236728467555 - 2.*mui*taui))*
-              Power(Sin(Pi*Z),2)) + 
-          R*Sin(Pi*R)*(R*(0.20264236728467555 + 2.*mui*taui)*
-              Power(Cos(Pi*Z),2)*Power(Sin(Pi*R),2) - 
-             0.15915494309189535*mui*taui*Sin(2*Pi*R)*Power(Sin(Pi*Z),2))))/
-      (Sqrt(-mue)*mui*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-     (0.1*taui*Z*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-      (Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5)*
-        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-           Sin(Pi*Z) + R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-              Sin(Pi*Z)))) + ((-33.0733617923198*taui*Cos(2*Pi*Z)*
-           Sin(2*P)*(-0.07957747154594767*Cos(2*Pi*R) + 
-             1.*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
-        (5.263789013914324*taui*Z*Sin(2*P)*
-           (-0.07957747154594767*Cos(2*Pi*R) + 1.*R*Sin(2*Pi*R))*
-           Sin(2*Pi*t)*Sin(2*Pi*Z))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5))/
-      (R*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (eta*Power(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z),2)*
-        (-((Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-                 52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)\
-)/(Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
-          (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)))))/
-      (mui*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-     (nuperp*(-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                 10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-               (0.4052847345693511 + 
-                 Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
-                  Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                 0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                 (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                  Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-             Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                Sin(Pi*t)*Sin(Pi*Z) + 
-               R*(0.4052847345693511 + 
-                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                   Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-          (Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
-             Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z))) + 
-          R*((2*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                  10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                Power(0.4052847345693511 + 
-                  Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
-                   Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                  0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                   Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                 Sin(Pi*t)*Sin(Pi*Z) + 
-                R*(0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)),3) - 
-             (2*Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 
-                  16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                (0.4052847345693511 + 
-                  Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
-                   Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                  0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                   Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                 Sin(Pi*t)*Sin(Pi*Z) + 
-                R*(0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)),2) - 
-             (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                  10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*
-                   Sin(Pi*t)*Sin(Pi*Z) + 
-                  2*Pi*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
-                   Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
-                  Power(Pi,2)*R*(0.10132118364233778 + 1.*mui*taui)*
-                   Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                 Sin(Pi*t)*Sin(Pi*Z) + 
-                R*(0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)),2) + 
-             (Sin(2*P)*(167.5516081914556*Cos(2*Pi*R) - 
-                  421.1031211131459*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                Sin(2*Pi*Z))/
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)))) + 
-          R*((-4*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                (-0.8488263631567752*Cos(2*Pi*R) + 
-                  10.666666666666666*R*Sin(2*Pi*R))*
-                (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-                  Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*
-                   Sin(P)*Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t))/
-              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                 Sin(Pi*t)*Sin(Pi*Z) + 
-                R*(0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)),2) + 
-             (2*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                  10.666666666666666*R*Sin(2*Pi*R))*
-                Power(-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                   Sin(Pi*t) + 
-                  Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*
-                   Sin(P)*Sin(Pi*R)*Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                 Sin(Pi*t)*Sin(Pi*Z) + 
-                R*(0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)),3) - 
-             (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                  10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*
-                   Sin(Pi*t)*Sin(Pi*Z) - 
-                  Power(Pi,2)*R*(0.10132118364233778 + 1.*mui*taui)*
-                   Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-              Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                 Sin(Pi*t)*Sin(Pi*Z) + 
-                R*(0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z)),2) - 
-             (4*Power(Pi,2)*Sin(2*P)*
-                (-0.8488263631567752*Cos(2*Pi*R) + 
-                  10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                Sin(2*Pi*Z))/
-              (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                 Sin(Pi*Z) + R*
-                 (0.4052847345693511 + 
-                   (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                    Sin(Pi*t)*Sin(Pi*Z))))))/R - 
-     (Sin(2*P)*Sin(3*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(3*Pi*t)*
-        Sin(2*Pi*Z)*(120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
-          (mui*Cos(3*Pi*Z)*Sin(3*P)*
-             (528430.5031318273*
-                Power((0.005628954646796544*mui*taui + 
-                     Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
-)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
-               54*Power(Pi,3)*Power(R,2)*
-                Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                  R*(-0.20000000000000004 - 
-                     17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-             Sin(3*Pi*t)*Sin(3*Pi*Z))/
-           (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-          (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-             (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                  R*(-0.20000000000000004 - 
-                     17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-               28034.10888465587*
-                Power((0.005628954646796544*mui*taui + 
-                     Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
-                   Cos(3*Pi*R) + 
-                  0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                Power(Sin(3*Pi*Z),2)))/
-           (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),
-              2))))/
-      (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-           Sin(Pi*Z) + R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-              Sin(Pi*Z)))) + (-((Sin(2*P)*
-             (-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-             (0.4052847345693511 + 
-               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
-                Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-               0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-              Sin(Pi*t)*Sin(Pi*Z) + 
-             R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-        (Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
-           Sin(2*Pi*t)*Sin(2*Pi*Z))/
-         (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-            Sin(Pi*Z) + R*(0.4052847345693511 + 
-              (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-               Sin(Pi*t)*Sin(Pi*Z))))*
-      ((-0.1*R*taui*Z)/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) + 
-        (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
-           Sin(2*Pi*R)*(-0.8488263631567752*Cos(2*Pi*R) + 
-             10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-           Sin(2*Pi*Z))/
-         (15.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
-             1.*Power(Z,2))*(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-              Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-             R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)))) - 
-        (R*Sin(3*P)*Sin(3*Pi*t)*
-           (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
-             (mui*Cos(3*Pi*Z)*Sin(3*P)*
-                (528430.5031318273*
-                   Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                      Cos(3*Pi*R) + 
-                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
-                  54*Power(Pi,3)*Power(R,2)*
-                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                     R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-                Sin(3*Pi*t)*Sin(3*Pi*Z))/
-              (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) \
-+ (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                     R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                  28034.10888465587*
-                   Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                      Cos(3*Pi*R) + 
-                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                   Power(Sin(3*Pi*Z),2)))/
-              (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                  1.*Power(Z,2),2))))/
-         (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
-     (beta*(1 + Sqrt(-mue))*Sin(2*P)*Sin(3*P)*Sin(2*Pi*t)*Sin(3*Pi*t)*
-        (-((2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Sin(2*Pi*Z)*
-             (120*Pi*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
-               (mui*Cos(3*Pi*Z)*Sin(3*P)*
-                  (528430.5031318273*
-                     Power((0.005628954646796544*mui*taui + 
-                       Power(R,2)*
-                       (0.011257909293593089 + 1.*mui*taui))*
-                       Cos(3*Pi*R) + 
-                       0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
-                    54*Power(Pi,3)*Power(R,2)*
-                     Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                       R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-                  Sin(3*Pi*t)*Sin(3*Pi*Z))/
-                (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 
-                    1.*Power(Z,2))) + 
-               (2.*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-                  (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                     Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                       R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                    28034.10888465587*
-                     Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                        Cos(3*Pi*R) + 
-                       0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                     Power(Sin(3*Pi*Z),2)))/
-                (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                    1.*Power(Z,2),2)))) + 
-          2*Pi*R*Cos(2*Pi*Z)*Sin(2*Pi*R)*
-           (120*Pi*Cos(3*Pi*R)*Sin(3*Pi*Z) + 
-             (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
-                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                     R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                  28034.10888465587*
-                   Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                      Cos(3*Pi*R) + 
-                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                   Power(Sin(3*Pi*Z),2)))/
-              (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                  1.*Power(Z,2),2)) + 
-             (2*mui*Sin(3*P)*Sin(3*Pi*t)*
-                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                     R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                  28034.10888465587*
-                   Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                      Cos(3*Pi*R) + 
-                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                   Power(Sin(3*Pi*Z),2)))/
-              (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) \
-- (mui*Sin(3*P)*Sin(3*Pi*t)*(18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
-                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                     R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                  528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                   (R*(0.011257909293593089 + 1.*mui*taui)*
-                      Cos(3*Pi*R) + 
-                     (0.0011945012753036852 + 
-                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
-                   (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
-                     R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R)) \
-- 168204.65330793522*((0.005628954646796544*mui*taui + 
-                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui)\
-)*Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
-                   (R*(-0.007505272862395392 - 
-                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
-                     (-6.938893903907228e-18*mui*taui + 
-                        Power(R,2)*
-                         (0.03536776513153231 + 
-                         3.141592653589793*mui*taui))*Sin(3*Pi*R))*
-                   Power(Sin(3*Pi*Z),2)))/
-              (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))\
-)/(6000.*Sqrt(-mue)*mui*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) \
-+ ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-           (-0.8488263631567752*Cos(2*Pi*R) + 
-             10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-         (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-            Sin(Pi*Z) + R*(0.4052847345693511 + 
-              (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-               Sin(Pi*t)*Sin(Pi*Z))) - 
-        (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-             10.666666666666666*R*Sin(2*Pi*R))*
-           (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-             Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
-              Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-         Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-            Sin(Pi*Z) + R*(0.4052847345693511 + 
-              (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-               Sin(Pi*t)*Sin(Pi*Z)),2))*
-      ((taui*(-50. + 1.*R - 0.1*Power(Z,2)))/
-         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
-        (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*
-           (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*
-           (-0.8488263631567752*Cos(2*Pi*R) + 
-             10.666666666666666*R*Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
-           Power(Sin(2*Pi*Z),2))/
-         (30.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)))) - 
-        (taui + (mui*Power(Sin(2*P),2)*
-              Power(0.8488263631567752*Cos(2*Pi*R) - 
-                10.666666666666666*R*Sin(2*Pi*R),2)*Power(Sin(2*Pi*t),2)*
-              Power(Sin(2*Pi*Z),2))/
-            Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z) + 
-              R*(0.4052847345693511 + 
-                 (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                  Sin(Pi*t)*Sin(Pi*Z)),2))/
-         (10.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
-        (R*Sin(3*P)*Sin(3*Pi*t)*
-           (120*Pi*Cos(3*Pi*R)*Sin(3*Pi*Z) + 
-             (mui*(-20. + 2.*R)*Sin(3*P)*Sin(3*Pi*t)*
-                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                     R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                  28034.10888465587*
-                   Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                      Cos(3*Pi*R) + 
-                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                   Power(Sin(3*Pi*Z),2)))/
-              (Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                  1.*Power(Z,2),2)) + 
-             (2*mui*Sin(3*P)*Sin(3*Pi*t)*
-                (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                     R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                  28034.10888465587*
-                   Power((0.005628954646796544*mui*taui + 
-                        Power(R,2)*
-                        (0.011257909293593089 + 1.*mui*taui))*
-                      Cos(3*Pi*R) + 
-                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                   Power(Sin(3*Pi*Z),2)))/
-              (Power(R,3)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) - 
-             (mui*Sin(3*P)*Sin(3*Pi*t)*
-                (18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
-                   Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                     R*(-0.20000000000000004 - 
-                        17.765287921960844*mui*taui)*Sin(3*Pi*R),2) + 
-                  528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                   (R*(0.011257909293593089 + 1.*mui*taui)*Cos(3*Pi*R) + 
-                     (0.0011945012753036852 + 
-                        0.15915494309189537*mui*taui)*Sin(3*Pi*R))*
-                   (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
-                     R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R)) - 
-                  168204.65330793522*
-                   ((0.005628954646796544*mui*taui + 
-                        Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
-                      Cos(3*Pi*R) + 
-                     0.05305164769729846*mui*R*taui*Sin(3*Pi*R))*
-                   (R*(-0.007505272862395392 - 
-                        0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
-                     (-6.938893903907228e-18*mui*taui + 
-                        Power(R,2)*
-                         (0.03536776513153231 + 
-                         3.141592653589793*mui*taui))*Sin(3*Pi*R))*
-                   Power(Sin(3*Pi*Z),2)))/
-              (Power(R,2)*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/
-         (2000.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))))) + 
+    return (2*(beta + beta*Sqrt(-mue) + Sqrt(-mue)*mui)*Pi*Cos(2*Pi*t)*Sin(2*P)*
+     Sin(2*Pi*R)*Sin(2*Pi*Z))/(3.*Sqrt(-mue)*mui) + 
   FELTORPARALLEL*((taui*Sin(Pi*t)*
-        (R*Sin(Pi*R)*(R*(7.853981633974482 + 
-                77.51569170074954*mui*taui + 
-                R*(-0.7853981633974483 - 7.751569170074954*mui*taui))*
-              Cos(Pi*Z)*Sin(P) + 
-             ((5. + 49.34802200544679*mui*taui)*Cos(P) + 
-                1.2337005501361697*mui*taui*Z*Sin(P))*Sin(Pi*Z)) + 
-          Cos(Pi*R)*(mui*R*(-12.337005501361698 + 1.2337005501361697*R)*
-              taui*Cos(Pi*Z)*Sin(P) + 
-             (-7.853981633974484*mui*taui*Cos(P) + 
-                (0.39269908169872414*mui*taui + 
-                   Power(R,2)*
-                    (0.7853981633974483 + 7.751569170074954*mui*taui))*Z*
-                 Sin(P))*Sin(Pi*Z))))/
-      (mui*Power(R,2)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-     (1.*(-10 + R)*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-             (-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z))) - 
-          (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*
-             (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
-                Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)),2)))/
-      (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-           Sin(Pi*Z) + R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-              Sin(Pi*Z)))) + (1.*Z*Sin(2*P)*
-        (-0.8488263631567752*Cos(2*Pi*R) + 
-          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        (-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                 10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-               (0.4052847345693511 + 
-                 Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
-                  Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                 0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                 (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                  Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-             Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                Sin(Pi*t)*Sin(Pi*Z) + 
-               R*(0.4052847345693511 + 
-                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                   Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-          (Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
-             Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)))))/
-      (Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-           Sin(Pi*Z) + R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-              Sin(Pi*Z)))) + (20*Sin(2*P)*
-        (-0.8488263631567752*Cos(2*Pi*R) + 
-          10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z)*
-        (-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                 10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-               (-0.15915494309189535*mui*taui*Cos(P)*Cos(Pi*R)*
-                  Sin(Pi*t)*Sin(Pi*Z) + 
-                 R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*Sin(Pi*R)*
-                  Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-             Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                Sin(Pi*t)*Sin(Pi*Z) + 
-               R*(0.4052847345693511 + 
-                  (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                   Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-          (2*Cos(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)))))/
-      (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))*
-        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-           Sin(Pi*Z) + R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-              Sin(Pi*Z)))) - (nuparalleli*
-        (0. + (1.*R*Z*((-1.*(-10 + R)*
-                  ((-2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                        Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
-                    (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-                    (2*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                       Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*
-                       (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (Power(Pi,2)*R*
-                       (0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-                        1.5707963267948966*mui*taui*Cos(Pi*Z)*
-                       Sin(P)*Sin(Pi*R)*Sin(Pi*t) + 
-                        Pi*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
+        ((7.853981633974483 - 0.7853981633974483*R)*R*Cos(Pi*Z)*Sin(P)*
+           Sin(Pi*R) + (0.7853981633974483*R*Z*Cos(Pi*R)*Sin(P) + 
+             5.*Cos(P)*Sin(Pi*R))*Sin(Pi*Z)))/
+      (mui*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        (1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) - 
+     (0.6981317007977318*(-10 + R)*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+        Power(Sin(2*Pi*(-10 + R)),2)*Power(Sin(2*Pi*t),2)*Sin(2*Pi*Z))/
+      Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+     (0.6981317007977318*Z*Cos(2*Pi*(-10 + R))*Power(Sin(2*P),2)*
+        Sin(2*Pi*(-10 + R))*Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+      Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+     (40*Cos(2*P)*Sin(2*P)*Power(Sin(2*Pi*(-10 + R)),2)*
+        Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+      (9.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+     (nuparalleli*(0. + (1.*Z*
+             ((-2.0943951023931953*(-10 + R)*Cos(2*Pi*Z)*Sin(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (1.*Power(-10 + R,2)*
-                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-               (1.*((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
+               (2.0943951023931953*Z*Cos(2*Pi*(-10 + R))*Sin(2*P)*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
+               (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (3.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
+           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+          (1.*(-10 + R)*R*((13.15947253478581*Z*Cos(2*Pi*(-10 + R))*
+                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (20*((2*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (-0.15915494309189535*mui*taui*Cos(P)*
-                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
-                       (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(P)*Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Cos(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (-0.15915494309189535*mui*taui*Cos(P)*
-                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (2*Cos(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
-                    (2*Cos(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
-               (1.*(-10 + R)*Z*
-                  (-((Sin(2*P)*
-                        (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                        (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-                    (Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+               (80*Pi*Cos(2*P)*Cos(2*Pi*Z)*Sin(2*Pi*(-10 + R))*
+                  Sin(2*Pi*t))/
+                (3.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
+               (2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*Z)*Sin(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
+               (2.0943951023931953*Power(Z,2)*Cos(2*Pi*(-10 + R))*
+                  Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
                 Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
-               (1.*Z*((2*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Power(0.4052847345693511 + 
-                       Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                       (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                       Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
-                    (2*Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (1.5707963267948966*mui*taui*Cos(Pi*R)*
-                       Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        2*Pi*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
-                        Power(Pi,2)*R*
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
-                    (Sin(2*P)*
-                       (167.5516081914556*Cos(2*Pi*R) - 
-                        421.1031211131459*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+               (2.0943951023931953*Cos(2*Pi*(-10 + R))*Sin(2*P)*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (20.*(-10 + R)*
-                  (-((Sin(2*P)*
-                        (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                        (-0.15915494309189535*mui*taui*Cos(P)*
-                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-                    (2*Cos(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+               (13.333333333333334*Z*Cos(2*P)*Sin(2*Pi*(-10 + R))*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
                 (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) \
-- (20*(-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                        (-0.15915494309189535*mui*taui*Cos(P)*
-                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-                    (2*Cos(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                (Power(R,2)*Sqrt(400 + 1.*Power(-10 + R,2) + 
-                    1.*Power(Z,2)))))/
++ (13.15947253478581*(-10 + R)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+                  Sin(2*Pi*Z))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (1.*Z*((-1.*(-10 + R)*
-                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (1.*Z*(-((Sin(2*P)*
-                        (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                        (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-                    (Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+          (20*((-4.1887902047863905*(-10 + R)*Cos(2*P)*Cos(2*Pi*Z)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (20*(-((Sin(2*P)*
-                        (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                        (-0.15915494309189535*mui*taui*Cos(P)*
-                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-                    (2*Cos(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
+               (4.1887902047863905*Z*Cos(2*P)*Cos(2*Pi*(-10 + R))*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
+               (80*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (3.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-          (20*((-1.*(-10 + R)*
-                  ((-2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (-0.15915494309189535*mui*taui*Cos(P)*
-                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
-                    (4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-                    (2*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                       Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*
-                       (-0.15915494309189535*mui*taui*Cos(P)*
-                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(P)*Cos(Pi*R)*Cos(Pi*Z)*
-                       Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(P)*Cos(Pi*Z)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (2*Cos(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (1.*Z*((2*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (-0.15915494309189535*mui*taui*Cos(P)*
-                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
-                       (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(P)*Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Cos(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (-0.15915494309189535*mui*taui*Cos(P)*
-                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (2*Cos(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                       Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                       Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
-                    (2*Cos(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+          (1.*R*Z*((-13.15947253478581*(-10 + R)*Cos(2*Pi*(-10 + R))*
+                  Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*t))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (20*((2*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Power(-0.15915494309189535*mui*taui*Cos(P)*
-                       Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
-                    (4*Cos(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (-0.15915494309189535*mui*taui*Cos(P)*
-                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (4*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)))))/
-           Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-          (1.*(-10 + R)*R*((1.*Z*
-                  ((-2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                        Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
-                    (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-                    (2*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*
-                       (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                        Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (Power(Pi,2)*R*
-                        (0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-                        1.5707963267948966*mui*taui*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t) + 
-                        Pi*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
+               (2.0943951023931953*Power(-10 + R,2)*Cos(2*Pi*Z)*
+                  Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
+                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
+               (2.0943951023931953*Cos(2*Pi*Z)*Sin(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) + 
-               (20*((-2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (-0.15915494309189535*mui*taui*Cos(P)*
-                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
-                    (4*Pi*Cos(2*P)*Cos(2*Pi*Z)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-                    (2*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*
-                       (-0.15915494309189535*mui*taui*Cos(P)*
-                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(P)*Cos(Pi*R)*Cos(Pi*Z)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(P)*Cos(Pi*Z)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (2*Cos(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
-                (R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) + 
-               (1.*(-10 + R)*Z*
-                  ((2*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)))/
+               (80*Pi*Cos(2*P)*Cos(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+                  Sin(2*Pi*Z))/
+                (3.*R*Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+               (2.0943951023931953*(-10 + R)*Z*Cos(2*Pi*(-10 + R))*
+                  Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
                 Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) - 
-               (1.*Power(Z,2)*
-                  (-((Sin(2*P)*
-                        (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                        (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                        Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-                    (Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5) + 
-               (1.*(-((Sin(2*P)*
-                        (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                        (0.4052847345693511 + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-                        Sin(Pi*Z) + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-                    (Sin(2*P)*
-                       (67.02064327658225*R*Cos(2*Pi*R) + 
-                        16.*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2)) - 
-               (20.*Z*(-((Sin(2*P)*
-                        (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                        (-0.15915494309189535*mui*taui*Cos(P)*
-                        Cos(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.10132118364233778 + 1.*mui*taui)*Cos(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                       Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                        R*(0.4052847345693511 + 
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-                    (2*Cos(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
-                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) - 
-               (1.*(-10 + R)*((-4*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*
-                        Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t))*
-                       Sin(2*Pi*t))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) + 
-                    (2*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*
-                       Power(-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*
-                        Sin(P)*Sin(Pi*t) + 
-                        Pi*R*(0.10132118364233778 + 1.*mui*taui)*
-                        Cos(Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(Pi*t),2)*
-                       Sin(2*Pi*t)*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),3) - 
-                    (Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                        10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) - 
-                        Power(Pi,2)*R*
-                        (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                        Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-                     Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*
-                        Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)),2) - 
-                    (4*Power(Pi,2)*Sin(2*P)*
-                       (-0.8488263631567752*Cos(2*Pi*R) + 
-                         10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-                       Sin(2*Pi*Z))/
-                     (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-                        Sin(Pi*t)*Sin(Pi*Z) + 
-                       R*(0.4052847345693511 + 
-                         (0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                         Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))))/
+               (13.333333333333334*(-10 + R)*Cos(2*P)*
+                  Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (R*Power(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2),1.5)) \
+- (40*Cos(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+                (3.*Power(R,2)*
+                  Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))) - 
+               (13.15947253478581*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+                  Sin(2*Pi*t)*Sin(2*Pi*Z))/
                 Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
            Sqrt(400 + 1.*Power(-10 + R,2) + 1.*Power(Z,2))))/
-      (R*(1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z))/R + (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*
-           Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-     (Sin(3*Pi*t)*(-0.3*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-           Cos(3*P)*(-40.*Power(R,2)*
-              (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Sin(3*Pi*R)*
-              Sin(3*Pi*Z) + 2*mui*Sin(3*P)*Sin(3*Pi*t)*
-              (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
-                    Sin(3*Pi*R),2) + 
-                28034.10888465587*
-                 Power((0.005628954646796544*mui*taui + 
-                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
-                    Cos(3*Pi*R) + 
-                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                 Power(Sin(3*Pi*Z),2))) + 
-          0.6*(10 - R)*R*Sin(3*P)*
-           (Pi*Power(R,2)*Power(500. - 20.*R + 1.*Power(R,2) + 
-                1.*Power(Z,2),2)*Cos(3*Pi*Z)*Sin(3*Pi*R) - 
-             0.008333333333333333*mui*
-              (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(3*Pi*Z)*
-              Sin(3*P)*(528430.5031318273*
-                 Power((0.005628954646796544*mui*taui + 
-                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
-                    Cos(3*Pi*R) + 
-                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2) - 
-                54*Power(Pi,3)*Power(R,2)*
-                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                   R*(-0.20000000000000004 - 
-                      17.765287921960844*mui*taui)*Sin(3*Pi*R),2))*
-              Sin(3*Pi*t)*Sin(3*Pi*Z) + 
-             0.016666666666666666*mui*Z*Sin(3*P)*Sin(3*Pi*t)*
-              (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
-                    Sin(3*Pi*R),2) + 
-                28034.10888465587*
-                 Power((0.005628954646796544*mui*taui + 
-                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
-                    Cos(3*Pi*R) + 
-                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                 Power(Sin(3*Pi*Z),2))) + 
-          0.005*Z*Sin(3*P)*(376.99111843077515*Power(R,3)*
-              Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)*
-              Cos(3*Pi*R)*Sin(3*Pi*Z) - 
-             mui*(20. - 2.*R)*R*Sin(3*P)*Sin(3*Pi*t)*
-              (9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
-                    Sin(3*Pi*R),2) + 
-                28034.10888465587*
-                 Power((0.005628954646796544*mui*taui + 
-                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
-                    Cos(3*Pi*R) + 
-                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                 Power(Sin(3*Pi*Z),2)) + 
-             2*mui*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Sin(3*P)*
-              Sin(3*Pi*t)*(9*Power(Pi,2)*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
-                    Sin(3*Pi*R),2) + 
-                28034.10888465587*
-                 Power((0.005628954646796544*mui*taui + 
-                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
-                    Cos(3*Pi*R) + 
-                   0.05305164769729846*mui*R*taui*Sin(3*Pi*R),2)*
-                 Power(Sin(3*Pi*Z),2)) - 
-             mui*R*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Sin(3*P)*
-              Sin(3*Pi*t)*(18*Power(Pi,2)*R*Power(Cos(3*Pi*Z),2)*
-                 Power(0.942477796076938*mui*taui*Cos(3*Pi*R) + 
-                   R*(-0.20000000000000004 - 17.765287921960844*mui*taui)*
-                    Sin(3*Pi*R),2) + 
-                528430.5031318273*Power(R,2)*Power(Cos(3*Pi*Z),2)*
-                 (R*(0.011257909293593089 + 1.*mui*taui)*Cos(3*Pi*R) + 
-                   (0.0011945012753036852 + 0.15915494309189537*mui*taui)*
-                    Sin(3*Pi*R))*
-                 (-0.053051647697298455*mui*taui*Cos(3*Pi*R) + 
-                   R*(0.011257909293593089 + 1.*mui*taui)*Sin(3*Pi*R)) - 
-                168204.65330793522*
-                 ((0.005628954646796544*mui*taui + 
-                      Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*
-                    Cos(3*Pi*R) + 0.05305164769729846*mui*R*taui*Sin(3*Pi*R)\
-)*(R*(-0.007505272862395392 - 0.8333333333333334*mui*taui)*Cos(3*Pi*R) + 
-                   (-6.938893903907228e-18*mui*taui + 
-                      Power(R,2)*
-                       (0.03536776513153231 + 3.141592653589793*mui*taui))*
-                    Sin(3*Pi*R))*Power(Sin(3*Pi*Z),2)))))/
-      (mui*Power(R,3)*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2.5))\
-)
+      (R*(1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (Sin(3*Pi*t)*((18.84955592153876 - 1.8849555921538759*R)*R*
+           Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R) + 
+          (1.8849555921538759*R*Z*Cos(3*Pi*R)*Sin(3*P) + 
+             12.*Cos(3*P)*Sin(3*Pi*R))*Sin(3*Pi*Z)))/
+      (mui*R*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))) + 
+  FELTORPERP*(0. - (0.03333333333333333*taui*Z*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+        Sin(2*Pi*t)*Sin(2*Pi*Z))/
+      Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+     (Pi*Cos(3*Pi*Z)*Sin(2*P)*Sin(3*P)*Sin(2*Pi*(-10 + R))*Sin(3*Pi*R)*
+        Sin(2*Pi*t)*Sin(3*Pi*t)*Sin(2*Pi*Z))/
+      (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+     (0.16449340668482262*beta*(1. + Sqrt(-mue))*taui*Sin(P)*Sin(2*P)*
+        Sin(Pi*t)*Sin(2*Pi*t)*(1.*R*Cos(Pi*R)*Cos(2*Pi*Z)*Sin(2*Pi*R)*
+           Sin(Pi*Z) + Cos(Pi*Z)*Sin(Pi*R)*
+           (-1.*R*Cos(2*Pi*R) - 0.15915494309189535*Sin(2*Pi*R))*Sin(2*Pi*Z)\
+))/(Sqrt(-mue)*mui*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        (1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (eta*Power(1 + 0.5*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z),2)*
+        ((Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/3. + 
+          (Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           (3.*Sqrt(-mue))))/
+      (mui*(1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) - 
+     (nuperp*((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+           3. - (8*Power(Pi,2)*R*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+             Sin(2*Pi*Z))/3.))/R + 
+     (2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*Sin(2*Pi*Z)*
+        ((-0.1*R*taui*Z)/
+           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+          (3*Pi*R*Cos(3*Pi*Z)*Sin(3*P)*Sin(3*Pi*R)*Sin(3*Pi*t))/
+           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (beta*(1 + Sqrt(-mue))*Pi*R*Cos(2*Pi*Z)*Power(Sin(2*P),2)*
+             Sin(2*Pi*(-10 + R))*Sin(2*Pi*R)*Power(Sin(2*Pi*t),2)*
+             Sin(2*Pi*Z))/
+           (45.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+               1.*Power(Z,2)))))/3. + 
+     ((-0.10471975511965977*R*taui*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*R)*
+           Sin(2*Pi*t)*(4. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))/
+         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) - 
+        (0.05235987755982988*R*taui*Cos(Pi*Z)*Sin(P)*Sin(2*P)*Sin(Pi*R)*
+           Sin(2*Pi*R)*Sin(Pi*t)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
+         Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)) + 
+        (0.016666666666666666*R*taui*Z*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
+           (4. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
+         Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5))/
+      (R*(1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z))) + 
+     (2*Pi*Cos(2*Pi*Z)*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+        ((taui*(-50. + 1.*R - 0.1*Power(Z,2)))/
+           Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),1.5) - 
+          (beta*(1 + Sqrt(-mue))*Power(Sin(2*P),2)*Sin(2*Pi*(-10 + R))*
+             (2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Power(Sin(2*Pi*t),2)*
+             Power(Sin(2*Pi*Z),2))/
+           (90.*Sqrt(-mue)*Sqrt(500. - 20.*R + 1.*Power(R,2) + 
+               1.*Power(Z,2))) + 
+          (-9*taui - mui*Power(Sin(2*P),2)*Power(Sin(2*Pi*R),2)*
+              Power(Sin(2*Pi*t),2)*Power(Sin(2*Pi*Z),2))/
+           (90.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))) + 
+          (3*Pi*R*Cos(3*Pi*R)*Sin(3*P)*Sin(3*Pi*t)*Sin(3*Pi*Z))/
+           (50.*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2)))))/3. + 
+     (beta*(1 + Sqrt(-mue))*Pi*Sin(2*P)*Sin(3*P)*Sin(2*Pi*t)*Sin(3*Pi*t)*
+        (-(Cos(3*Pi*Z)*(2*Pi*R*Cos(2*Pi*R) + Sin(2*Pi*R))*Sin(3*Pi*R)*
+             Sin(2*Pi*Z)) + Pi*R*Cos(3*Pi*R)*Sin(2*Pi*R)*
+           (Sin(Pi*Z) + Sin(5*Pi*Z))))/
+      (50.*Sqrt(-mue)*mui*Sqrt(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))))
 ; }};
 struct SPhie{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
     return 0.25*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-  (Sin(3*P)*Sin(3*Pi*t)*(0.2191704548265055*R*
+  (R*Sin(3*P)*Sin(3*Pi*t)*(-0.014804406601634037*R*
         (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Cos(Pi*Z)*
-        Cos(3*Pi*Z)*Sin(P)*(-0.5*mui*taui*Cos(Pi*R) + 
-          R*(0.3183098861837907 + 3.141592653589793*mui*taui)*Sin(Pi*R))*
-        (1.*mui*taui*Cos(3*Pi*R) + 
-          R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*Sin(3*Pi*R)\
-)*Sin(Pi*t) - 0.438340909653011*R*Z*Cos(3*Pi*Z)*
-        (1.*mui*taui*Cos(3*Pi*R) + 
-          R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*Sin(3*Pi*R)\
-)*(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-          R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-              Sin(Pi*Z))) - 4.131265744601299*
-        (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        ((0.005628954646796544*mui*taui + 
-             Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*Cos(3*Pi*R) \
-+ 0.053051647697298476*mui*R*taui*Sin(3*Pi*R))*
-        (0.4052847345693511 + R*
-           (0.3183098861837907 + 3.141592653589793*mui*taui)*Cos(Pi*R)*
-           Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-          (0.10132118364233778 + 1.5*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-           Sin(Pi*Z))*Sin(3*Pi*Z) + 
-       4.131265744601299*(-20. + 2.*R)*
-        ((0.005628954646796544*mui*taui + 
-             Power(R,2)*(0.011257909293593089 + 1.*mui*taui))*Cos(3*Pi*R) \
-+ 0.053051647697298476*mui*R*taui*Sin(3*Pi*R))*
-        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-           Sin(Pi*Z) + R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-              Sin(Pi*t)*Sin(Pi*Z)))*Sin(3*Pi*Z) - 
-       2.06563287230065*R*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (1.*mui*taui*Cos(3*Pi*R) + 
-          R*(-0.2122065907891938 - 18.849555921538755*mui*taui)*Sin(3*Pi*R)\
-)*(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-          R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-              Sin(Pi*t)*Sin(Pi*Z)))*Sin(3*Pi*Z) - 
-       4.131265744601299*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
-        (R*(0.022515818587186178 + 2.5*mui*taui)*Cos(3*Pi*R) + 
-          (3.122502256758253e-17*mui*taui + 
-             Power(R,2)*(-0.10610329539459692 - 9.42477796076938*mui*taui))*
-           Sin(3*Pi*R))*(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-           Sin(Pi*t)*Sin(Pi*Z) + 
-          R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-              Sin(Pi*Z)))*Sin(3*Pi*Z)))/
-   (R*Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2))
+        Cos(3*Pi*Z)*Sin(P)*Sin(Pi*R)*Sin(3*Pi*R)*Sin(Pi*t) + 
+       0.00942477796076938*R*Z*Cos(3*Pi*Z)*Sin(3*Pi*R)*
+        (4. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)) - 
+       0.014804406601634037*R*(500. - 20.*R + 1.*Power(R,2) + 
+          1.*Power(Z,2))*Cos(Pi*R)*Cos(3*Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z)*
+        Sin(3*Pi*Z) + 0.00942477796076938*R*(-10. + 1.*R)*Cos(3*Pi*R)*
+        (4. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(3*Pi*Z) - 
+       0.01413716694115407*(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*
+        Cos(3*Pi*R)*(4. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*
+        Sin(3*Pi*Z) + 0.08882643960980423*R*
+        (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))*Sin(3*Pi*R)*
+        (4. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(3*Pi*Z)))/
+   Power(500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2),2)
+; }};
+struct SPhii{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+    DG_DEVICE double operator()(double R, double Z, double P, double t)const{
+    return (mui*Power(R,2)*Power(Sin(3*P),2)*Power(Sin(3*Pi*t),2)*
+    (0.017765287921960846*Power(Cos(3*Pi*Z),2)*Power(Sin(3*Pi*R),2) + 
+      0.017765287921960846*Power(Cos(3*Pi*R),2)*Power(Sin(3*Pi*Z),2)))/
+  (500. - 20.*R + 1.*Power(R,2) + 1.*Power(Z,2))
+; }};
+struct SGammaPhie{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+    DG_DEVICE double operator()(double R, double Z, double P, double t)const{
+    return (mui*taui*Sin(3*P)*(-0.9424777960769379*Cos(3*Pi*R) + 
+      17.765287921960844*R*Sin(3*Pi*R))*Sin(3*Pi*t)*Sin(3*Pi*Z))/R
+; }};
+struct SGammaNi{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+    DG_DEVICE double operator()(double R, double Z, double P, double t)const{
+    return (mui*taui*Sin(P)*(-0.39269908169872414*Cos(Pi*R) + 
+      2.4674011002723395*R*Sin(Pi*R))*Sin(Pi*t)*Sin(Pi*Z))/R
+; }};
+struct SA{
+    double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli;
+    DG_DEVICE double operator()(double R, double Z, double P, double t)const{
+    return (beta*Sin(2*P)*Sin(2*Pi*t)*((-2.0943951023931953 - 
+         2.0943951023931953*Sqrt(-mue))*Cos(2*Pi*R) + 
+      R*Sin(2*Pi*R)*(25.985611736238287 + 25.985611736238287*Sqrt(-mue) + 
+         (-0.16666666666666666 - 0.08333333333333333*Sqrt(-mue))*Sin(P)*
+          Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))*Sin(2*Pi*Z))/(Sqrt(-mue)*R)
 ; }};
 struct Snehat{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli,alpha;
@@ -3518,205 +692,28 @@ struct Snehat{
 struct SNihat{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli,alpha;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return 1. - (0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/R + 
-  (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-  (alpha*nuperp*((0.39269908169872414*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-          Sin(Pi*Z))/Power(R,2) + 
-       Pi*(0.25 + 2.4674011002723395*mui*taui)*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-        Sin(Pi*Z) + (1.2337005501361697*mui*taui*Sin(P)*Sin(Pi*R)*
-          Sin(Pi*t)*Sin(Pi*Z))/R + 
-       R*((3.875784585037477*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z))/R - Power(Pi,2)*
-           (0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-           Sin(Pi*Z)) + R*((-0.7853981633974483*mui*taui*Cos(Pi*R)*Sin(P)*
-             Sin(Pi*t)*Sin(Pi*Z))/Power(R,3) + 
-          (3.875784585037477*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z))/
-           R - (2.4674011002723395*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-             Sin(Pi*Z))/Power(R,2) - 
-          Power(Pi,2)*(0.25 + 2.4674011002723395*mui*taui)*Sin(P)*Sin(Pi*R)*
-           Sin(Pi*t)*Sin(Pi*Z))))/R
+    return 1 + 0.25*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*Sin(Pi*Z) + 
+  (alpha*nuperp*(0. + 0.7853981633974483*Cos(Pi*(-10 + R))*Sin(P)*Sin(Pi*t)*
+        Sin(Pi*Z) - 4.934802200544679*R*Sin(P)*Sin(Pi*(-10 + R))*Sin(Pi*t)*
+        Sin(Pi*Z)))/R
 ; }};
 struct SWehat{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli,alpha;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(2*P)*Sin(2*Pi*t)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/mue + 
-       (12.566370614359172*Cos(2*Pi*R) - 157.91367041742973*R*Sin(2*Pi*R))/
-        (R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/
-   (3.*Sqrt(-mue)) + (alpha*nuperp*
-     ((-3.141592653589793*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-          (4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-          Sin(Pi*Z)*Sin(2*Pi*Z))/
-        (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) \
-+ (Sin(2*P)*(-330.73361792319804*R*Cos(2*Pi*R) - 
-            78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-        (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) - 
-       (Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-            52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-        (Sqrt(-mue)*Power(R,2)*
-          (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-       R*((-39.47841760435743*Cos(Pi*Z)*Cos(2*Pi*Z)*Sin(P)*Sin(2*P)*
-             Sin(Pi*R)*(4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t))/
-           (Sqrt(-mue)*R*Power(2. + 
-               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-          (19.739208802178716*Power(Cos(Pi*Z),2)*Power(Sin(P),2)*Sin(2*P)*
-             Power(Sin(Pi*R),2)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(Pi*t),2)*
-             Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*Power(2. + 
-               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),3)) + 
-          (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-             Sin(Pi*Z)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*Power(2. + 
-               1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) - 
-          (4*Power(Pi,2)*Sin(2*P)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z)))) + 
-       R*((19.739208802178716*Power(Cos(Pi*R),2)*Power(Sin(P),2)*Sin(2*P)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Power(Sin(Pi*t),2)*
-             Sin(2*Pi*t)*Power(Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
-              3)) - (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-             (-330.73361792319804*R*Cos(2*Pi*R) - 
-               78.95683520871486*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-             Sin(Pi*Z)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
-              2)) + (6.283185307179586*Cos(Pi*R)*Sin(P)*Sin(2*P)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-             Sin(Pi*Z)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*Power(R,2)*
-             Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),2)) + 
-          (9.869604401089358*Sin(P)*Sin(2*P)*Sin(Pi*R)*
-             (4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(Pi*t)*Sin(2*Pi*t)*
-             Sin(Pi*Z)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*Power(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z),
-              2)) - (2*Sin(2*P)*
-             (-330.73361792319804*R*Cos(2*Pi*R) - 
-               78.95683520871486*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*Power(R,2)*
-             (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-          (2*Sin(2*P)*(4.1887902047863905*Cos(2*Pi*R) - 
-               52.63789013914324*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*Power(R,3)*
-             (2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))) + 
-          (Sin(2*P)*(-826.834044807995*Cos(2*Pi*R) + 
-               2078.060608725385*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (Sqrt(-mue)*R*(2. + 1.*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))))))/R
+    return -((beta + beta*Sqrt(-mue) - mue)*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
+      Sin(2*Pi*Z))/(3.*Power(-mue,1.5)) + 
+  (alpha*nuperp*((-2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
+          Sin(2*Pi*Z))/(3.*Sqrt(-mue)) + 
+       (8*Power(Pi,2)*R*Sin(2*P)*Sin(2*Pi*(-10 + R))*Sin(2*Pi*t)*
+          Sin(2*Pi*Z))/(3.*Sqrt(-mue))))/R
 ; }};
 struct SWihat{
     double mue,mui,taue,taui,eta,beta,nuperp,nuparallele,nuparalleli,alpha;
     DG_DEVICE double operator()(double R, double Z, double P, double t)const{
-    return (Sin(2*P)*Sin(2*Pi*t)*((beta*(1 + Sqrt(-mue))*Sin(2*Pi*R))/
-        (Sqrt(-mue)*mui) + (-2.5464790894703255*Cos(2*Pi*R) + 
-          31.999999999999996*R*Sin(2*Pi*R))/
-        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-           Sin(Pi*Z) + R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-              Sin(Pi*t)*Sin(Pi*Z))))*Sin(2*Pi*Z))/3. + 
-  (alpha*nuperp*(-((Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-              10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-            (0.4052847345693511 + 
-              Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*Sin(P)*
-               Sin(Pi*t)*Sin(Pi*Z) + 
-              0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-              (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-               Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-          Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-             Sin(Pi*Z) + R*(0.4052847345693511 + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z)),2)) + 
-       (Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
-          Sin(2*Pi*t)*Sin(2*Pi*Z))/
-        (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-           Sin(Pi*Z) + R*(0.4052847345693511 + 
-             (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*
-              Sin(Pi*Z))) + R*((2*Sin(2*P)*
-             (-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-             Power(0.4052847345693511 + 
-               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
-                Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-               0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z),2)*Sin(2*Pi*Z))/
-           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-              Sin(Pi*t)*Sin(Pi*Z) + 
-             R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)),3) - 
-          (2*Sin(2*P)*(67.02064327658225*R*Cos(2*Pi*R) + 16.*Sin(2*Pi*R))*
-             Sin(2*Pi*t)*(0.4052847345693511 + 
-               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*R)*
-                Sin(P)*Sin(Pi*t)*Sin(Pi*Z) + 
-               0.5*mui*taui*Sin(P)*Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z) + 
-               (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-              Sin(Pi*t)*Sin(Pi*Z) + 
-             R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)),2) - 
-          (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-             (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                Sin(Pi*Z) + 2*Pi*(0.10132118364233778 + 1.*mui*taui)*
-                Cos(Pi*R)*Sin(P)*Sin(Pi*t)*Sin(Pi*Z) - 
-               Power(Pi,2)*R*(0.10132118364233778 + 1.*mui*taui)*Sin(P)*
-                Sin(Pi*R)*Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*
-              Sin(Pi*t)*Sin(Pi*Z) + 
-             R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)),2) + 
-          (Sin(2*P)*(167.5516081914556*Cos(2*Pi*R) - 
-               421.1031211131459*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)))) + 
-       R*((-4*Pi*Cos(2*Pi*Z)*Sin(2*P)*
-             (-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*
-             (-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
-                Sin(Pi*R)*Sin(Pi*t))*Sin(2*Pi*t))/
-           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)),2) + 
-          (2*Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*
-             Power(-0.5*mui*taui*Cos(Pi*R)*Cos(Pi*Z)*Sin(P)*Sin(Pi*t) + 
-               Pi*R*(0.10132118364233778 + 1.*mui*taui)*Cos(Pi*Z)*Sin(P)*
-                Sin(Pi*R)*Sin(Pi*t),2)*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)),3) - 
-          (Sin(2*P)*(-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*
-             (1.5707963267948966*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-                Sin(Pi*Z) - Power(Pi,2)*R*
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                Sin(Pi*t)*Sin(Pi*Z))*Sin(2*Pi*Z))/
-           Power(-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z)),2) - 
-          (4*Power(Pi,2)*Sin(2*P)*
-             (-0.8488263631567752*Cos(2*Pi*R) + 
-               10.666666666666666*R*Sin(2*Pi*R))*Sin(2*Pi*t)*Sin(2*Pi*Z))/
-           (-0.15915494309189535*mui*taui*Cos(Pi*R)*Sin(P)*Sin(Pi*t)*
-              Sin(Pi*Z) + R*(0.4052847345693511 + 
-                (0.10132118364233778 + 1.*mui*taui)*Sin(P)*Sin(Pi*R)*
-                 Sin(Pi*t)*Sin(Pi*Z))))))/R
+    return ((beta + beta*Sqrt(-mue) + Sqrt(-mue)*mui)*Sin(2*P)*Sin(2*Pi*R)*Sin(2*Pi*t)*
+     Sin(2*Pi*Z))/(3.*Sqrt(-mue)*mui) + 
+  (alpha*nuperp*((2*Pi*Cos(2*Pi*(-10 + R))*Sin(2*P)*Sin(2*Pi*t)*
+          Sin(2*Pi*Z))/3. - (8*Power(Pi,2)*R*Sin(2*P)*Sin(2*Pi*(-10 + R))*
+          Sin(2*Pi*t)*Sin(2*Pi*Z))/3.))/R
 ; }};
 }}//namespace feltor namespace manufactured
-- 
GitLab


From 1ee72cd460804f61612ee92920b8fdb0ad8d6205 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 16 Oct 2020 18:28:05 +0200
Subject: [PATCH 365/540] Update compass input and geometry

---
 src/feltor/geometry/compass.json | 36 ++++++++++++++++----------------
 src/feltor/input/compass.json    |  6 +++---
 2 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/src/feltor/geometry/compass.json b/src/feltor/geometry/compass.json
index 9608a108e..79849bea8 100644
--- a/src/feltor/geometry/compass.json
+++ b/src/feltor/geometry/compass.json
@@ -1,26 +1,26 @@
 //Compass geometry file T_e = 200eV, B = 0.8T, Deuterium plasma
 {
     "A": 0,
-   	"R_0": 219.23,
+   	"R_0": 213.36,
    	"PP": 1,
    	"PI": 1,
-   	"c":[
-   	    0.10587834884347505,
-   	    0.18967653484360376,
-   	    -0.41197076060549486,
-   	    -0.15733959980607443,
-   	    0.29149170772913691,
-   	    -0.29079253027214297,
-   	    -0.012380284150980477,
-   	    0.092294585192278152,
-   	    0.58087399002810055,
-   	    -0.30962849734343311,
-   	    -0.075321444509503473,
-   	    0.010040461725044062
-   	],
-   	"elongation": 1.6594,
+	"c":[
+		0.072597888572520090,
+		-0.14926096478076946,
+		-0.031407992567660930,
+		-0.0023298277478519561,
+		0.058103163908191134,
+		-0.079234863442382291,
+		-0.0033337854432893672,
+		0.033813030145633505,
+		0.37706830466684815,
+		-0.17520562483628400,
+		-0.051326331829125358,
+		0.0061149094270187820
+	],
    	"equilibrium": "solovev",
     "description" : "standardX",
-   	"inverseaspectratio": 0.2857142857142857,
-   	"triangularity": 0.4
+	"inverseaspectratio": 0.3211009174311926,
+    "triangularity": 0.3,
+    "elongation": 1.44
 }
diff --git a/src/feltor/input/compass.json b/src/feltor/input/compass.json
index a3ab20f77..0f83cedf8 100644
--- a/src/feltor/input/compass.json
+++ b/src/feltor/input/compass.json
@@ -1,6 +1,6 @@
 {
     "n"  : 3,
-    "Nx" : 224,
+    "Nx" : 240,
     "Ny" : 384,
     "Nz" : 32,
     "dt" : 2e-3,
@@ -37,7 +37,7 @@
     "box" :
     {
         "scaleR" :  [1.3,1.25],
-        "scaleZ" :  [1.4,1.3]
+        "scaleZ" :  [1.45,1.4]
     },
     "initne"     : "turbulence",
     "initphi"    : "zero",
@@ -55,7 +55,7 @@
     "source" :
     {
         "rate": 2e-3,
-        "type": "influx",
+        "type": "profile_influx",
         "boundary": 0.55,
         "alpha" : 0.2
     },
-- 
GitLab


From 550852078968f8c62e3b9f0a205d1e654ff89bbb Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 16 Oct 2020 18:29:50 +0200
Subject: [PATCH 366/540] Add profile_influx as source profile

---
 src/feltor/feltor.tex | 30 ++++++++----------------------
 src/feltor/init.h     | 21 ++++++++++++++++++++-
 2 files changed, 28 insertions(+), 23 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index b1bccd7b7..3f06c7931 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1089,8 +1089,9 @@ has a Dirichlet boundary condition, then $N'$ satisfies a correspondingly scaled
 The standard helicity of the magnetic field would be a right handed screw,
 where the magnetic field and the toroidal plasma current point in the clockwise
 direction if the tokamak is seen from above.
-The helicity should however not have any influence on the plasma dynamics as
-the tokamak viewed in the mirror has the reversed helicity.
+The helicity should however not have any influence on the plasma dynamics. With
+a mirror transformation (view the tokamak in a mirror) we should be able
+to transform the solution to one with reversed helicity.
 
 \subsection{Conservation laws} \label{sec:conservation}
 \subsubsection{Mass conservation}
@@ -1447,9 +1448,9 @@ time & Multistep "Karniadakis" & \\
 \qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion terms. \\
 \bottomrule
 \end{longtable}
-Note that the explicit resistive and damping terms lead to absolute restriction
-on the timestep according to $\partial_t f = -\lambda f \Rightarrow \Delta t < \lambda^{-1}$. This is a quite weak restriction unless the resistivity exceeds $\eta > 10^{-4}$ or the damping coefficient excceeds $1$.
-The explicit parallel electron viscosity term leads to a restriction of the
+Note that the explicit resistive and damping terms lead to absolute (CFL) restriction
+on the timestep according to $\partial_t u_e \propto \frac{\eta}{\mu_e} u_e \Rightarrow \Delta t < \frac{-\mu_e}{\eta}$. This is a quite weak restriction unless the resistivity exceeds $\eta > 10^{-4}$ or the damping coefficient excceeds $1$.
+The explicit parallel electron viscosity term leads to a CFL condition of the
 form $\Delta t < (2\pi (R_0 - a))^2/(N_z^2 \nu_{\parallel,e})$.
 
 In every iteration of the implicit inversion we need to solve an equation of the form
@@ -1463,23 +1464,8 @@ with $W=U+A_\parallel/\mu$ and $-\Delta_\perp A_\parallel = \beta (N_i U_i -n_e
     and $\hat L = -\nu_\perp \Delta_\perp^2$.
 This makes 7 equations for 7 unkown quantities.
 We solve the system by first isolating and solving the two density equations for $n_e$ and $N_i$. These can then be inserted into the velocity equations as
-fixed solutions, which makes these equations linear. As a next step we insert  the definition of $W$ into the velocity equations to get
-\begin{align}
-    u_e + \frac{A_\parallel}{\mu_e} + a \hat L u_e = \hat w_e \\
-    U_i + \frac{A_\parallel}{\mu_i} + a \hat L U_i = \hat W_i
-\end{align}
-We now introduce the two auxiliary variables $-\Delta_\perp A_{\parallel,i} = N_i U_i$ and $-\Delta_\perp A_{\parallel,e} =  -n_e u_e$ to get
-\begin{align}
-\beta A_{\parallel,e} + \frac{\mu_e \Delta_\perp A_{\parallel,e}}{n_e}
-+ a \mu_e \hat L \frac{\Delta_\perp
-        A_{\parallel,e}}{ n_e} + \beta A_{\parallel,i} =  \mu_e \hat w_e \\
-        \beta A_{\parallel,e} -\frac{\mu_i \Delta_\perp A_{\parallel,i}}{N_i}
-         - a \mu_i \hat L \frac{\Delta_\perp
-        A_{\parallel,i}}{ N_i}+ \beta A_{\parallel,i} = \mu_i \hat W_i
-\end{align}
-%Update in Mathematica
-which is a symmetric equation for $A_{\parallel,e}$, $A_{\parallel,i}$.
-Its solution yields $A_\parallel = \beta(A_{\parallel,i}+A_{\parallel,e})$ and thus a solution for $U_\parallel$ and $W_\parallel$ through backward substitutions.
+fixed solutions, which makes these equations linear.
+
 The resistive term cannot fit into this scheme as it is linear but not symmetric.
 \section{Usage}
 
diff --git a/src/feltor/init.h b/src/feltor/init.h
index a9fdcca7b..fc987da7c 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -84,7 +84,7 @@ HVec profile(const Geometry& grid,
     double psipO = mag.psip()( RO, ZO);
     //First the profile and the source (on the host since we want to output those)
     HVec profile = dg::pullback( dg::compose(dg::LinearX(
-        p.nprofamp/psipO, 0.), mag.psip()), grid);
+        1./psipO, 0.), mag.psip()), grid);
     dg::blas1::pointwiseDot( profile_damping(grid,p,mag), profile, profile);
     return profile;
 }
@@ -143,6 +143,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
         {
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,mag));
+            dg::blas1::scal( y0, p.nprofamp );
             HVec ntilde = dg::evaluate(dg::zero,grid);
             if( p.sigma_z == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in straight blob initial condition\n");
@@ -175,6 +176,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
         {
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,mag));
+            dg::blas1::scal( y0, p.nprofamp );
             HVec ntilde = dg::evaluate(dg::zero,grid);
             if( p.sigma_z == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in straight blob initial condition\n");
@@ -207,6 +209,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
         {
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,mag));
+            dg::blas1::scal( y0, p.nprofamp );
             HVec ntilde = dg::evaluate(dg::zero,grid);
             if( p.sigma_z == 0)
                 throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in turbulence initial condition\n");
@@ -238,6 +241,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
         {
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,mag));
+            dg::blas1::scal( y0, p.nprofamp );
             HVec ntilde = dg::evaluate(dg::zero,grid);
             dg::SinX sinX( p.amp, 0., p.k_psi);
             ntilde = dg::pullback( dg::compose( sinX, mag.psip()), grid);
@@ -304,10 +308,23 @@ std::map<std::string, std::function< HVec(
         {
             fixed_profile = true;
             ne_profile = dg::construct<HVec>( detail::profile(grid, p,mag));
+            dg::blas1::scal( ne_profile, p.nprofamp );
             HVec source_profile = dg::construct<HVec> ( detail::source_damping( grid, p,mag));
             return source_profile;
         }
     },
+    {"profile_influx",
+        []( bool& fixed_profile, HVec& ne_profile,
+        Geometry& grid, const feltor::Parameters& p,
+        dg::geo::TokamakMagneticField& mag )
+        {
+            fixed_profile = false;
+            ne_profile = dg::construct<HVec>( detail::profile(grid, p,mag));
+            dg::blas1::scal( ne_profile, p.nprofamp );
+            HVec source_profile = dg::construct<HVec> ( detail::profile( grid, p,mag));
+            return source_profile;
+        }
+    },
     {"influx",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
@@ -315,6 +332,7 @@ std::map<std::string, std::function< HVec(
         {
             fixed_profile = false;
             ne_profile = dg::construct<HVec>( detail::profile(grid, p,mag));
+            dg::blas1::scal( ne_profile, p.nprofamp );
             HVec source_profile = dg::construct<HVec> ( detail::source_damping( grid, p,mag));
             return source_profile;
         }
@@ -348,6 +366,7 @@ std::map<std::string, std::function< HVec(
 
             fixed_profile = false;
             ne_profile = dg::construct<HVec>( detail::profile(grid, p,mag));
+            dg::blas1::scal( ne_profile, p.nprofamp );
             HVec source_profile = dg::pullback(
                 dg::compose( dg::GaussianX( psip0, sigma, 1.),  mag.psip() ), grid);
             dg::blas1::pointwiseDot( detail::xpoint_damping(grid,p,mag),
-- 
GitLab


From dffd480a49a3c7bc4d706aedb52035be2a873518 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 19 Oct 2020 18:55:16 +0200
Subject: [PATCH 367/540] Add a comment about A_parallel diffusion

---
 src/feltor/feltor.tex | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 3f06c7931..8ede90d28 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -820,6 +820,10 @@ The dynamic viscosity $\mu\nu_\parallel$ is larger for ions than for electrons.
  The approximate Spitzer current \(J_{\parallel,s}:= n_e \left(U_{\parallel,i} - u_{\parallel,e}\right)\)
  determines the parallel resistive terms to $R_\parallel:= n_e\eta J_{\parallel,s}$.
 
+ Note also that since $A_\parallel/\mu_e$ is (potentially) much larger than $u_e$
+ it is important that the diffusive operators act on $u_e$ rather than $w_e$.
+ The latter essentially would entail that electron diffusion acts on $A_\parallel$.
+
  For the perpendicular terms we use ad-hoc numerical diffusion terms for numerical
  stabilisation.
 \begin{align}
-- 
GitLab


From 674cf8dda1c9749a283442282bf51f70ead3d4d3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 19 Oct 2020 21:52:08 +0200
Subject: [PATCH 368/540] Add boundary formulas for field-aligned boundary

in parallel.tex
---
 doc/related_pages/parallel/parallel.tex | 146 ++++++++++++++++++------
 1 file changed, 113 insertions(+), 33 deletions(-)

diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index 1f8b750fd..4124233d5 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -60,13 +60,14 @@ Thus, it is better to reparameterize Eq.~\eqref{eq:fieldline_original} with $\va
 \end{align}
 \label{eq:fieldline}
 \end{subequations}
-We integrate Eqs.~\eqref{eq:fieldline} from $\varphi=0$ to $\varphi=\pm \Delta \varphi$
+We integrate Eqs.~\eqref{eq:fieldline} from $\varphi=\varphi_k$ to $\varphi=\varphi_k\pm \Delta \varphi$
 with initial condition
 \begin{align}
-    (R(0), Z(0), s(0) ) = (R, Z, 0).
+    (R(\varphi_k), Z(\varphi_k), s(\varphi_k) ) = (R, Z, s_k).
     \label{}
 \end{align}
-Let us characterize the solution $(R(\pm \Delta \varphi), Z(\pm \Delta \varphi))$ to Eqs.~\eqref{eq:fieldline} as the flow generated by $\vec v/v^\varphi$
+where $k$ is the numbering of the planes and $s_k$ is arbitrary.
+Let us characterize the solution $(R(\varphi_k\pm \Delta \varphi), Z(\varphi_k\pm \Delta \varphi))$ to Eqs.~\eqref{eq:fieldline} as the flow generated by $\vec v/v^\varphi$
 \begin{align}
     \Tpm\vec z \equiv \Tpm[R, Z, \varphi]:= ( R(\pm \Delta\varphi), Z( \pm \Delta\varphi), \varphi\pm\Delta \varphi),
     \label{}
@@ -74,12 +75,20 @@ Let us characterize the solution $(R(\pm \Delta \varphi), Z(\pm \Delta \varphi))
 Obviously we have $\Tm\circ\Tp = \Eins$, but $\Tpm$ is not necessarily unitary since $\vec v/v^\varphi$ is in general
 not divergence free.
 
-We now propose a centered discretization for the parallel derivative
+We now fit a second order polynomial through $(R,Z,\varphi)$, $\Tp [R,Z,\varphi]$ and $\Tm[R,Z,\varphi]$. We approximate the first and second derivative at $(R,Z,\varphi)$ by evaluating the derivative of the
+polynomial at that point.
 \begin{align} \label{eq:paralleldis}
     \nabla_\parallel f \equiv \frac{df}{ds}
-    \rightarrow \frac{f\left(T_{\Delta\varphi}^+\vec z_k\right)-f\left(T_{\Delta\varphi}^-\vec z_k\right)}{s_{k+1}-s_{k-1}},
-\end{align}
-where $\vec z_k = (R,Z,\varphi_k)$.
+    \rightarrow
+    \left(\frac{ 1}{s_{k+1}-s_{k-1}} - \frac{1}{s_k - s_{k-1}}\right) &f_{k-1}  \nonumber\\
+    +\left(\frac{ 1}{s_{k}-s_{k-1}} - \frac{1}{s_{k+1} - s_{k}}\right) &f_k\nonumber\\
+    +\left(\frac{ 1}{s_{k+1}-s_{k}} - \frac{1}{s_{k+1} - s_{k-1}}\right) &f_{k+1}
+    %\frac{f\left(T_{\Delta\varphi}^+\vec z_k\right)-f\left(T_{\Delta\varphi}^-\vec z_k\right)}{s_{k+1}-s_{k-1}},
+\end{align}
+where $\vec z_k = (R,Z,\varphi_k)$ and $f_k = f(\vec z_k)$
+and $f_{k\pm 1} := f(\Tpm \vec z_k)$.
+Note that Eq.~\eqref{eq:paralleldis} reduces to
+the familiar centered difference formula in case of equidistant spacings.
 Eq.~\eqref{eq:paralleldis} is slightly different from Reference~\cite{Hariri2014}, where the $\d f/\d\varphi v^\varphi$ is discretized to avoid integrating $s$.
 However, our discretization has the advantage that it can be used for higher
 order derivatives as well (e.g. $\d^2f/\d \varphi^2$), which is not possible otherwise due to the chain rule
@@ -107,10 +116,11 @@ A consistency check is the relation $I^+\circ I^- = \Eins$.
 
 The discretization~\eqref{eq:paralleldis} can now be written as a matrix vector product
 \begin{align}
-    \nabla_\parallel f \rightarrow S \circ \left[ \Eins^+\otimes I^+ - \Eins^- \otimes I^-  \right] \vec f, 
+\nabla_\parallel f \rightarrow  \left[S^+ \circ \Eins^+\otimes I^+ + S^0  + S^-\circ \Eins^- \otimes I^-  \right] \vec f,
     \label{}
 \end{align}
-where $S$ is the diagonal matrix that contains the entries $1/(s_{k+1} - s_{k-1}$.
+where $S^+$, $S^0$ and $S^-$ are the diagonal matrices that contain the prefactors
+in Eq.~\eqref{eq:paralleldis}.
 This discretization is not skew-symmetric since the
 field lines are not volume-preserving, or~$(I^+)^\mathrm{T} \neq I^-$.
 In fact, the adjoint of the parallel derivative is
@@ -135,7 +145,6 @@ discretized using
     -\frac{2f_{k}}{(s_{k+1}-s_k)(s_{k}-s_{k-1})} \nonumber\\
     +\frac{2f_{k-1}}{(s_{k}-s_{k-1})(s_{k+1}-s_{k-1})}
 \end{align}
-where $f_k = f(\vec z_k)$ with $\vec z_k = (R,Z,\varphi_k)$ and $f_{k\pm 1} := f(\Tpm \vec z_k)$.
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \subsection{Change of coordinates}
@@ -192,28 +201,12 @@ on the boundary of the domain (Dirichlet) or by fixing
 the derivative perpendicularly to the boundary (Neumann), or
 a combination of both ( Robin).
 
-One approach for Dirichlet boundary conditions
-is to find the exact place where the fieldline intersects the boundary.
-We have to find
-$\varphi_b$ such that the result of the integration of Eq.~\eqref{eq:fieldline} from
-$0$ to $\varphi_b$ lies on the boundary.
-The angle $\varphi_b$ can be found by a bisection algorithm knowing that $0<\varphi_b < \Delta\varphi$.
-This kind of procedure is known as a shooting method.
-Another possibility is to trick the fieldline integrator into finding the point for us.
-We do this by setting $\vec v \equiv 0$ on all points outside the simulation box, which
-makes the ODE integrator stop once it crosses the domain boundary.
-This works fairly well with the adaptive embedded Runge Kutta method that we use.
-However, the problem with this procedure in practice is the small
-distance between the starting point and the point where the corresponding
-fieldline intersects the boundary. This seriously deteriorates the
-CFL condition. (To ease the CFL condition
-was the reason to devise the algorithm in the first place)
-
-One problem with Neumann boundaries is that they usually prescribe
+
+\subsubsection{ Boundary conditions perpendicular to the wall}
+The issue with Neumann boundary conditions is that they usually prescribe
 derivatives perpendicular to the boundary
 while the fieldlines are in general not perpendicular to the boundary.
-
-The \textbf{approach we currently adopted} is to introduce ghostcells at the
+The \textbf{approach we currently adopt} is to introduce ghostcells at the
 places where fieldlines end. The value of the ghostcells are
 as if we Fourier transformed the fields on the simulation domain
 with the correct boundary conditions and thus have a periodic
@@ -237,10 +230,97 @@ evaluate and thus integrate the vector field $\vec v$ even outside the domain.
 This is unfortunately not true for transformed coordinates.
 For now we have to rely on the fieldlines being aligned to the
 boundary in these cases to avoid boundary conditions altogether.
-This is possible in \textsc{Feltor} at least for the closed field line region
-(read the geometries section).
 
-\subsection{Avoiding boundary conditions in non-aligned systems} \label{sec:avoid}
+The downside of this approach is that the mirrored point of a field-line
+can lie very far away from the wall if the resolution in $\varphi$ is low (which
+is the motivation to implement FCI in the first place).
+In principle we need to increase the $\varphi$ resolution until we resolve
+the perpendicular direction, in order to reliably converge with this method.
+In practice we can often run a simulation stably nevertheless, however for
+large parallel flows or strong poloidal fields there may be numerical instabilities.
+
+\subsubsection{ Boundary conditions parallel to the fieldline}
+The natural way to implement boundary conditions is to work them into the
+interpolating polynomial. The boundary condition then replaces a third
+point by setting
+either a predefined value (Dirichlet) or derivative (Neumann) on the boundary.
+Evaluating the interpolating polynomial at the middle point then yields
+(assuming that $\Tp \vec z$ lies outside the boundary and $s_k <s_b^+ < s_{k+1}$ is where the boundary lies)
+\begin{align} \label{eq:paralleldis_neup}
+    \frac{df}{ds}
+    \underset{\textsc{NEU}}{\rightarrow}
+    \left(\frac{ 1}{s_{k}-s_{k-1}} - \frac{1}{2(s_b^+-s_k) + (s_k-s_{k-1})} \right) &(f_k-f_{k-1})\nonumber\\
+    +\left(\frac{s_k - s_{k-1}}{2(s_b^+-s_k) + (s_k-s_{k-1})}\right) &f_{b+}'\\
+\label{eq:second_order_neup}
+    \frac{\d^2 f}{\d s^2}
+    \underset{\textsc{NEU}}{\rightarrow}
+    \frac{2}{2(s_b^+-s_k) + (s_k-s_{k-1})}\left(  f_{b+}' - \frac{1}{s_k - s_{k-1}}(f_k-f_{k-1})\right)
+\end{align}
+where $f_{b+}'$ is the value of the  derivative on the boundary (currently we only allow homogeneous boundary conditions, i.e. $f_{b+}' \equiv 0$).
+If the boundary lies at $s_{k-1}<s_b^-<s_k$ then we have
+\begin{align} \label{eq:paralleldis_neum}
+    \frac{df}{ds}
+    \underset{\textsc{NEU}}{\rightarrow}
+    \left(\frac{ 1}{s_{k+1}-s_k} - \frac{1}{2(s_k - s_b^-) + (s_{k+1}-s_k)} \right) &(f_{k+1}-f_k)\nonumber\\
+    +\left(\frac{s_{k+1} - s_k}{2(s_k - s_b^-) + (s_{k+1}-s_k)}\right) &f_{b-}'
+    \\
+\label{eq:second_order_neum}
+    \frac{\d^2 f}{\d s^2}
+    \underset{\textsc{NEU}}{\rightarrow}
+    \frac{2}{2(s_k -s_b^-) + (s_{k+1}-s_{k})}\left(  -f_{b-}' + \frac{1}{s_{k+1}-s_k} (f_{k+1} - f_k)\right)
+\end{align}
+while if the fieldline intersects the wall on both ends we have
+\begin{align} \label{eq:paralleldis_neupm}
+    \frac{df}{ds}
+    \underset{\textsc{NEU}}{\rightarrow}
+    \frac{ 1}{s_b^+ - s_b^-}\left[({ s_k - s_b^-})f_{b+}' + ({s_{b+}'-s_k}) f_{b-}'\right]    \\
+\label{eq:second_order_neupm}
+    \frac{\d^2 f}{\d s^2}
+    \underset{\textsc{NEU}}{\rightarrow}
+    \frac{ 1}{s_b^+ - s_b^-}( f_{b+}' - f_{b-}')
+\end{align}
+The formulas for Dirichlet boundary conditions are the same as the original
+formulas, with $s_{k+1}$, $f_{k+1}$ replaced by $s_b^+$, $f_b^+$ and/or $s_{k-1}$, $f_{k-1}$ replaced by $s_b^-$, $f_b^-$.
+%\begin{align} \label{eq:paralleldis_dir}
+%    \frac{df}{ds}
+%    \underset{\textsc{DIR}}{\rightarrow}
+%    \left(\frac{ 1}{s_{b}-s_k + s_k-s_{k-1}} - \frac{1}{s_k - s_{k-1}}\right) &f_{k-1}  \nonumber\\
+%    +\left(\frac{ 1}{s_{k}-s_{k-1}} - \frac{1}{s_{b} - s_{k}}\right) &f_k\nonumber\\
+%    +\left(\frac{ s_k - s_{k-1}}{(s_b -s_k + s_k - s_{k-1})(s_b-s_{k})} \right) &f_b
+%\end{align}
+%\begin{align}\label{eq:second_order_dir}
+%    \frac{\d^2 f}{\d s^2}
+%    \underset{\textsc{DIR}}{\rightarrow}
+%     \frac{2f_b}{(s_{b}-s_k)(s_{b}-s_k + s_k - s_{k-1})}
+%    -\frac{2f_{k}}{(s_{b}-s_k)(s_{k}-s_{k-1})} \nonumber\\
+%    +\frac{2f_{k-1}}{(s_{k}-s_{k-1})(s_{b}-s_k + s_k-s_{k-1})}
+%\end{align}
+
+There is no difference between those formulas and actually evaluating the
+interpolating polynomial at a ghost point and then using that point in the
+original formulas. The reason is that the interpolating polynomial is unique
+and thus is the value of its derivatives at $s_k$.
+
+For the above formulas to work we need to find the exact place where the fieldline intersects
+the boundary.
+We have to find
+$\varphi_b$ such that the result of the integration of Eq.~\eqref{eq:fieldline} from
+$\varphi$ to $\varphi_b$ lies on the boundary.
+The angle $\varphi_b$ can be found by a bisection algorithm knowing that $\varphi_k<\varphi_b < \varphi_k + \Delta\varphi$.
+This kind of procedure is known as a shooting method.
+Another possibility is to trick the fieldline integrator into finding the point for us.
+We do this by setting $\vec v \equiv 0$ on all points outside the simulation box, which
+makes the ODE integrator stop once it crosses the domain boundary.
+This works fairly well with the adaptive embedded Runge Kutta method that we use
+and in particular also works in transformed coordinates.
+
+%However, the problem with this procedure in practice is the small
+%distance between the starting point and the point where the corresponding
+%fieldline intersects the boundary. This seriously deteriorates the
+%CFL condition. (To ease the CFL condition
+%was the reason to devise the algorithm in the first place)
+
+\subsubsection{Avoiding boundary conditions in non-aligned systems} \label{sec:avoid}
 
 When computing in non-aligned coordinate systems
 one idea to avoid boundary conditions
-- 
GitLab


From 8c51f74b1b854a6d7639958eae018c8f50064f2d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 21 Oct 2020 15:20:02 +0200
Subject: [PATCH 369/540] Add some comments on parallel boundary condition

---
 .../parallel/ds_cylindrical_dirichlet10.tex   |  2 +-
 doc/related_pages/parallel/parallel.tex       | 37 ++++++++-----------
 2 files changed, 16 insertions(+), 23 deletions(-)

diff --git a/doc/related_pages/parallel/ds_cylindrical_dirichlet10.tex b/doc/related_pages/parallel/ds_cylindrical_dirichlet10.tex
index 75e7d241e..2966e322e 100644
--- a/doc/related_pages/parallel/ds_cylindrical_dirichlet10.tex
+++ b/doc/related_pages/parallel/ds_cylindrical_dirichlet10.tex
@@ -10,7 +10,7 @@
   &    &     &     error & order &             error & order &     error & order &     error & order &       error &  order &          error & order \\
 \textbf{m} & \textbf{N} & \textbf{Nz} &           &       &                   &       &           &       &           &       &             &        &                &       \\
 \midrule
-\textbf{1} & \textbf{6 } & \textbf{5  } &  3.56e-01 &   n/a &          3.56e-01 &   n/a &  4.64e-01 &   n/a &  4.64e-01 &   n/a &    4.48e-01 &    n/a &       7.56e-03 &   n/a \\
+\textbf{10} & \textbf{6 } & \textbf{5  } &  3.56e-01 &   n/a &          3.56e-01 &   n/a &  4.64e-01 &   n/a &  4.64e-01 &   n/a &    4.48e-01 &    n/a &       7.56e-03 &   n/a \\
   & \textbf{12} & \textbf{10 } &  1.34e-01 &  1.41 &          1.34e-01 &  1.41 &  1.64e-01 &  1.50 &  1.64e-01 &  1.50 &    6.82e-01 &  -0.61 &       6.32e-03 &  0.26 \\
   & \textbf{18} & \textbf{20 } &  3.76e-02 &  1.83 &          3.76e-02 &  1.83 &  4.48e-02 &  1.87 &  4.48e-02 &  1.87 &    1.42e+00 &  -1.05 &       5.61e-03 &  0.17 \\
   & \textbf{24} & \textbf{40 } &  9.68e-03 &  1.96 &          9.68e-03 &  1.96 &  1.15e-02 &  1.97 &  1.15e-02 &  1.97 &    3.03e+00 &  -1.10 &       4.39e-03 &  0.35 \\
diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index 4124233d5..f5fff05fa 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -233,11 +233,15 @@ boundary in these cases to avoid boundary conditions altogether.
 
 The downside of this approach is that the mirrored point of a field-line
 can lie very far away from the wall if the resolution in $\varphi$ is low (which
-is the motivation to implement FCI in the first place).
+is the motivation to implement FCI in the first place) and
+in addition also on a different flux surface.
 In principle we need to increase the $\varphi$ resolution until we resolve
 the perpendicular direction, in order to reliably converge with this method.
-In practice we can often run a simulation stably nevertheless, however for
-large parallel flows or strong poloidal fields there may be numerical instabilities.
+In practice we can often run a simulation stably with low resolution nevertheless.
+
+In order to avoid coupling different flux surfaces through the boundary condition, we have the possibility to
+mirror also the flux surfaces at the boundary.
+This approach converges well and works stably if the flux surface is (or is close to) perpendicular to the wall.
 
 \subsubsection{ Boundary conditions parallel to the fieldline}
 The natural way to implement boundary conditions is to work them into the
@@ -273,7 +277,7 @@ while if the fieldline intersects the wall on both ends we have
 \begin{align} \label{eq:paralleldis_neupm}
     \frac{df}{ds}
     \underset{\textsc{NEU}}{\rightarrow}
-    \frac{ 1}{s_b^+ - s_b^-}\left[({ s_k - s_b^-})f_{b+}' + ({s_{b+}'-s_k}) f_{b-}'\right]    \\
+    \frac{ 1}{s_b^+ - s_b^-}\left[({ s_k - s_b^-})f_{b+}' + ({s_{b}^+-s_k}) f_{b-}'\right]    \\
 \label{eq:second_order_neupm}
     \frac{\d^2 f}{\d s^2}
     \underset{\textsc{NEU}}{\rightarrow}
@@ -281,20 +285,6 @@ while if the fieldline intersects the wall on both ends we have
 \end{align}
 The formulas for Dirichlet boundary conditions are the same as the original
 formulas, with $s_{k+1}$, $f_{k+1}$ replaced by $s_b^+$, $f_b^+$ and/or $s_{k-1}$, $f_{k-1}$ replaced by $s_b^-$, $f_b^-$.
-%\begin{align} \label{eq:paralleldis_dir}
-%    \frac{df}{ds}
-%    \underset{\textsc{DIR}}{\rightarrow}
-%    \left(\frac{ 1}{s_{b}-s_k + s_k-s_{k-1}} - \frac{1}{s_k - s_{k-1}}\right) &f_{k-1}  \nonumber\\
-%    +\left(\frac{ 1}{s_{k}-s_{k-1}} - \frac{1}{s_{b} - s_{k}}\right) &f_k\nonumber\\
-%    +\left(\frac{ s_k - s_{k-1}}{(s_b -s_k + s_k - s_{k-1})(s_b-s_{k})} \right) &f_b
-%\end{align}
-%\begin{align}\label{eq:second_order_dir}
-%    \frac{\d^2 f}{\d s^2}
-%    \underset{\textsc{DIR}}{\rightarrow}
-%     \frac{2f_b}{(s_{b}-s_k)(s_{b}-s_k + s_k - s_{k-1})}
-%    -\frac{2f_{k}}{(s_{b}-s_k)(s_{k}-s_{k-1})} \nonumber\\
-%    +\frac{2f_{k-1}}{(s_{k}-s_{k-1})(s_{b}-s_k + s_k-s_{k-1})}
-%\end{align}
 
 There is no difference between those formulas and actually evaluating the
 interpolating polynomial at a ghost point and then using that point in the
@@ -314,6 +304,9 @@ makes the ODE integrator stop once it crosses the domain boundary.
 This works fairly well with the adaptive embedded Runge Kutta method that we use
 and in particular also works in transformed coordinates.
 
+The advantage of this method is that the parallel derivative
+does not couple different field lines through the boundary conditions. Thus
+the dynamics on each field-line can be completely independent.
 %However, the problem with this procedure in practice is the small
 %distance between the starting point and the point where the corresponding
 %fieldline intersects the boundary. This seriously deteriorates the
@@ -585,7 +578,7 @@ where we used $\psi_R \equiv \partial \psi_p /\partial R$ and
 $I_R \equiv \partial I(\psi_p) /\partial R$.
 
 \subsection{Cylindrical grid and boundary conditions}
-A very simple non-trivial choice for the poloidal flux is
+A simple but non-trivial choice for the poloidal flux is
 \begin{align}
   \psi_p = \frac{1}{2} \left( (R-R_0)^2 + Z^2 \right) \equiv \frac{1}{2} r^2
   \label{eq:circular}
@@ -596,11 +589,11 @@ $R\in[R_0-1, R_0+1]$,
 $Z\in[-1,1]$ and
 $\varphi \in [0,2\pi]$ and choose
 \begin{align}
-  f_N(R,Z,\varphi) = \sin(\pi (R-R_0)/2) \sin(\pi Z /2)\sin(\varphi)\\
-  f_D(R,Z,\varphi) = \cos(\pi (R-R_0)/2) \cos(\pi Z /2)\sin(\varphi)
+    f(R,Z,\varphi) = (\cos(\pi (R-R_0))+1)( \cos(\pi Z /2)+1)\sin(\varphi)
   \label{}
 \end{align}
-On the chosen domain these functions have Neumann respectively Dirichlet boundary conditions.
+On the chosen domain $f$ respects both Neumann and Dirichlet boundary conditions
+both along the field and perpendicular to the wall.
 In Tables~\ref{tab:ds_cylindrical_dirichlet1} and
 \ref{tab:ds_cylindrical_dirichlet10} we show the convergence of the solution
 for various operators and Dirichlet boundary conditions. Note that the same
-- 
GitLab


From 271357fc2ce09a652b7efa8c1f8f32e4797a7b3f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 21 Oct 2020 15:26:38 +0200
Subject: [PATCH 370/540] Change testfunction for ds_t

---
 inc/geometries/ds_curv_t.cu                |  2 +-
 inc/geometries/ds_mpit.cu                  | 18 ++----
 inc/geometries/ds_t.cu                     | 18 ++----
 inc/geometries/elliptic3d_t.cu             |  6 +-
 inc/geometries/geometry_params_Xpoint.json |  4 +-
 inc/geometries/testfunctors.h              | 64 +++++-----------------
 6 files changed, 31 insertions(+), 81 deletions(-)

diff --git a/inc/geometries/ds_curv_t.cu b/inc/geometries/ds_curv_t.cu
index 5598de3bf..267c7de25 100644
--- a/inc/geometries/ds_curv_t.cu
+++ b/inc/geometries/ds_curv_t.cu
@@ -16,7 +16,7 @@ int main(int argc, char * argv[])
     std::cout << "# Test DS on flux grid (No Boundary conditions)!\n";
     Json::Value js;
     if( argc==1) {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else {
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index 35f298a86..1f71aaa71 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -47,14 +47,14 @@ int main(int argc, char* argv[])
     dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-8, mx[0], mx[1]);
     dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( dsFA, dg::centered);
     ///##########################################################///
-    dg::MDVec fun = dg::evaluate( dg::geo::TestFunctionSin(mag), g3d);
+    dg::MDVec fun = dg::evaluate( dg::geo::TestFunctionDirNeu(mag), g3d);
     dg::MDVec derivative(fun);
     dg::MDVec divb = dg::evaluate( dg::geo::Divb(mag), g3d);
-    dg::MDVec sol0 = dg::evaluate( dg::geo::DsFunction<dg::geo::TestFunctionSin>(mag), g3d);
-    dg::MDVec sol1 = dg::evaluate( dg::geo::DssFunction<dg::geo::TestFunctionSin>(mag), g3d);
-    dg::MDVec sol2 = dg::evaluate( dg::geo::DsDivFunction<dg::geo::TestFunctionSin>(mag), g3d);
-    dg::MDVec sol3 = dg::evaluate( dg::geo::DsDivDsFunction<dg::geo::TestFunctionSin>(mag), g3d);
-    dg::MDVec sol4 =dg::evaluate( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionSin>(mag), g3d);
+    dg::MDVec sol0 = dg::evaluate( dg::geo::DsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    dg::MDVec sol1 = dg::evaluate( dg::geo::DssFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    dg::MDVec sol2 = dg::evaluate( dg::geo::DsDivFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    dg::MDVec sol3 = dg::evaluate( dg::geo::DsDivDsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    dg::MDVec sol4 =dg::evaluate( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
     std::vector<std::pair<std::string, std::array<const dg::MDVec*,2>>> names{
          {"forward",{&fun,&sol0}},          {"backward",{&fun,&sol0}},
          {"centered",{&fun,&sol0}},         {"dss",{&fun,&sol1}},
@@ -85,14 +85,8 @@ int main(int argc, char* argv[])
     }
     ///##########################################################///
     if(rank==0)std::cout << "# Reconstruct parallel derivative!\n";
-    fun = dg::evaluate( dg::geo::TestFunctionCos(mag), g3d);
     dsFA.construct( bhat, g3d, dg::DIR, dg::DIR, dg::geo::NoLimiter(), 1e-8, mx[0], mx[1]);
     ds.construct( dsFA, dg::centered);
-    sol0 = dg::evaluate( dg::geo::DsFunction<dg::geo::TestFunctionCos>(mag), g3d);
-    sol1 = dg::evaluate( dg::geo::DssFunction<dg::geo::TestFunctionCos>(mag), g3d);
-    sol2 = dg::evaluate( dg::geo::DsDivFunction<dg::geo::TestFunctionCos>(mag), g3d);
-    sol3 = dg::evaluate( dg::geo::DsDivDsFunction<dg::geo::TestFunctionCos>(mag), g3d);
-    sol4 = dg::evaluate( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionCos>(mag), g3d);
     if(rank==0)std::cout << "# TEST DIR Boundary conditions!\n";
     ///##########################################################///
     if(rank==0)std::cout << "Dirichlet: \n";
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index f6b67bcf8..c1110f285 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -40,14 +40,14 @@ int main(int argc, char * argv[])
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::centered);
     //![doxygen]
     ///##########################################################///
-    dg::DVec fun = dg::pullback( dg::geo::TestFunctionSin(mag), g3d);
+    dg::DVec fun = dg::pullback( dg::geo::TestFunctionDirNeu(mag), g3d);
     dg::DVec derivative(fun);
     dg::DVec divb = dg::pullback( dg::geo::Divb(mag), g3d);
-    dg::DVec sol0 = dg::pullback( dg::geo::DsFunction<dg::geo::TestFunctionSin>(mag), g3d);
-    dg::DVec sol1 = dg::pullback( dg::geo::DssFunction<dg::geo::TestFunctionSin>(mag), g3d);
-    dg::DVec sol2 = dg::pullback( dg::geo::DsDivFunction<dg::geo::TestFunctionSin>(mag), g3d);
-    dg::DVec sol3 = dg::pullback( dg::geo::DsDivDsFunction<dg::geo::TestFunctionSin>(mag), g3d);
-    dg::DVec sol4 = dg::pullback( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionSin>(mag), g3d);
+    dg::DVec sol0 = dg::pullback( dg::geo::DsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    dg::DVec sol1 = dg::pullback( dg::geo::DssFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    dg::DVec sol2 = dg::pullback( dg::geo::DsDivFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    dg::DVec sol3 = dg::pullback( dg::geo::DsDivDsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    dg::DVec sol4 = dg::pullback( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
     std::vector<std::pair<std::string, std::array<const dg::DVec*,2>>> names{
          {"forward",{&fun,&sol0}},          {"backward",{&fun,&sol0}},
          {"centered",{&fun,&sol0}},         {"dss",{&fun,&sol1}},
@@ -78,14 +78,8 @@ int main(int argc, char * argv[])
     }
     ///##########################################################///
     std::cout << "# Reconstruct parallel derivative!\n";
-    fun = dg::evaluate( dg::geo::TestFunctionCos(mag), g3d);
     dsFA.construct( bhat, g3d, dg::DIR, dg::DIR, dg::geo::NoLimiter(), 1e-8, mx, my);
     ds.construct( dsFA, dg::centered);
-    sol0 = dg::evaluate( dg::geo::DsFunction<dg::geo::TestFunctionCos>(mag), g3d);
-    sol1 = dg::evaluate( dg::geo::DssFunction<dg::geo::TestFunctionCos>(mag), g3d);
-    sol2 = dg::evaluate( dg::geo::DsDivFunction<dg::geo::TestFunctionCos>(mag), g3d);
-    sol3 = dg::evaluate( dg::geo::DsDivDsFunction<dg::geo::TestFunctionCos>(mag), g3d);
-    sol4 = dg::evaluate( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionCos>(mag), g3d);
     std::cout << "# TEST DIR Boundary conditions!\n";
     ///##########################################################///
     std::cout << "Dirichlet: \n";
diff --git a/inc/geometries/elliptic3d_t.cu b/inc/geometries/elliptic3d_t.cu
index f2501882d..b80269c71 100644
--- a/inc/geometries/elliptic3d_t.cu
+++ b/inc/geometries/elliptic3d_t.cu
@@ -37,8 +37,8 @@ int main()
     dg::geo::TestInvertDS<dg::Elliptic3d<dg::CylindricalGrid3d, dg::DMatrix, dg::DVec>, dg::DVec> test( elliptic, 1.);
     dg::Invert< dg::DVec > invert( w3d, n*n*Nx*Ny, eps);
 
-    const dg::DVec sol = dg::evaluate( dg::geo::TestFunctionCos(mag), g3d);
-    dg::DVec b = dg::evaluate( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionCos>(mag), g3d);
+    const dg::DVec sol = dg::evaluate( dg::geo::TestFunctionDirNeu(mag), g3d);
+    dg::DVec b = dg::evaluate( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
     dg::DVec x = b;
 
     std::cout << "# --------- Alignment Tensor:\n";
@@ -62,7 +62,7 @@ int main()
     dg::blas1::copy( 1., one);
     ellipticP.set_chi( one);
 
-    b = dg::evaluate( dg::geo::DPerpFunction<dg::geo::TestFunctionCos>(mag), g3d);
+    b = dg::evaluate( dg::geo::DPerpFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
 
     std::cout << "# --------- Projection Tensor:\n";
     t.tic();
diff --git a/inc/geometries/geometry_params_Xpoint.json b/inc/geometries/geometry_params_Xpoint.json
index dcf20ca3b..e944e759e 100644
--- a/inc/geometries/geometry_params_Xpoint.json
+++ b/inc/geometries/geometry_params_Xpoint.json
@@ -5,8 +5,8 @@
 {
     //----------------------Solovev coefficients---------------
     "A" : 0.0,
-    "PP": -1,
-    "PI": -1,
+    "PP": 1,
+    "PI": 1,
     "c" :[  0.07350114445500399706283007092406934834526,
            -0.08662417436317227513877947632069712210813,
            -0.1463931543401102620740934776490506239925,
diff --git a/inc/geometries/testfunctors.h b/inc/geometries/testfunctors.h
index ad85e72ff..6a5bb0cdc 100644
--- a/inc/geometries/testfunctors.h
+++ b/inc/geometries/testfunctors.h
@@ -94,78 +94,40 @@ struct TestFunctionPsi2
     TokamakMagneticField c_;
 };
 
-// sin(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*sin(phi);
-struct TestFunctionSin{
-    TestFunctionSin( const TokamakMagneticField& c){
+// (cos(M_PI*(R-R_0))+1)*(cos(M_PI*Z/2.)+1)*sin(phi);
+struct TestFunctionDirNeu{
+    TestFunctionDirNeu( const TokamakMagneticField& c){
         R_0 = c.R0();
     }
     double operator()(double R, double Z, double phi)const{
-        return sin(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*sin(phi);
+        return (cos(M_PI*(R-R_0))+1.)*(cos(M_PI*Z)+1.)*sin(phi);
     }
     double dR(double R, double Z, double phi)const{
-        return M_PI/2.*cos(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*sin(phi);
+        return -M_PI*sin(M_PI*(R-R_0))*(cos(M_PI*Z)+1.)*sin(phi);
     }
     double dZ(double R, double Z, double phi)const{
-        return M_PI/2.*sin(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*sin(phi);
+        return -M_PI*(cos(M_PI*(R-R_0))+1.)*sin(M_PI*Z)*sin(phi);
     }
     double dP(double R, double Z, double phi)const{
-        return sin(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*cos(phi);
+        return (cos(M_PI*(R-R_0))+1.)*(cos(M_PI*Z)+1.)*cos(phi);
     }
     double dRR(double R, double Z, double phi)const{
-        return -M_PI*M_PI/4.*sin(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*sin(phi);
+        return -M_PI*M_PI*cos(M_PI*(R-R_0))*(cos(M_PI*Z)+1.)*sin(phi);
     }
     double dZZ(double R, double Z, double phi)const{
-        return -M_PI*M_PI/4.*sin(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*sin(phi);
+        return -M_PI*M_PI*(cos(M_PI*(R-R_0))+1.)*cos(M_PI*Z)*sin(phi);
     }
     double dPP(double R, double Z, double phi)const{
-        return -sin(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*sin(phi);
+        return -(cos(M_PI*(R-R_0))+1.)*(cos(M_PI*Z)+1.)*sin(phi);
     }
     double dRZ(double R, double Z, double phi)const{
-        return M_PI*M_PI/4.*cos(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*sin(phi);
+        return M_PI*M_PI*sin(M_PI*(R-R_0))*sin(M_PI*Z)*sin(phi);
     }
     double dRP(double R, double Z, double phi)const{
-        return M_PI/2.*cos(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*cos(phi);
+        return -M_PI*sin(M_PI*(R-R_0))*(cos(M_PI*Z)+1.)*cos(phi);
     }
     double dZP(double R, double Z, double phi)const{
-        return M_PI/2.*sin(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*cos(phi);
-    }
-    private:
-    double R_0;
-};
-// cos(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*sin(phi);
-struct TestFunctionCos{
-    TestFunctionCos( const TokamakMagneticField& c){
-        R_0 = c.R0();
-    }
-    double operator()(double R, double Z, double phi)const{
-        return cos(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*sin(phi);
-    }
-    double dR(double R, double Z, double phi)const{
-        return -M_PI/2.*sin(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*sin(phi);
-    }
-    double dZ(double R, double Z, double phi)const{
-        return -M_PI/2.*cos(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*sin(phi);
-    }
-    double dP(double R, double Z, double phi)const{
-        return cos(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*cos(phi);
-    }
-    double dRR(double R, double Z, double phi)const{
-        return -M_PI*M_PI/4.*cos(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*sin(phi);
-    }
-    double dZZ(double R, double Z, double phi)const{
-        return -M_PI*M_PI/4.*cos(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*sin(phi);
-    }
-    double dPP(double R, double Z, double phi)const{
-        return -cos(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*sin(phi);
-    }
-    double dRZ(double R, double Z, double phi)const{
-        return M_PI*M_PI/4.*sin(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*sin(phi);
-    }
-    double dRP(double R, double Z, double phi)const{
-        return -M_PI/2.*sin(M_PI*(R-R_0)/2.)*cos(M_PI*Z/2.)*cos(phi);
-    }
-    double dZP(double R, double Z, double phi)const{
-        return -M_PI/2.*cos(M_PI*(R-R_0)/2.)*sin(M_PI*Z/2.)*cos(phi);
+        return -M_PI*(cos(M_PI*(R-R_0))+1.)*sin(M_PI*Z)*cos(phi);
     }
     private:
     double R_0;
-- 
GitLab


From 8d694106dff6cfe4b6bf9e40fb6fb2a64b34134d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 21 Oct 2020 16:09:47 +0200
Subject: [PATCH 371/540] Add boundary mode Along fieldline

and (ii) change the centered discretization formula to
a more exact one
---
 inc/geometries/ds.h               | 318 +++++++++++++++++++++++-------
 inc/geometries/ds_curv_mpit.cu    |   2 +-
 inc/geometries/ds_curv_t.cu       |   2 +-
 inc/geometries/ds_mpit.cu         |   4 +-
 inc/geometries/ds_t.cu            |   4 +-
 inc/geometries/fieldaligned.h     | 190 +++++++++++++-----
 inc/geometries/mpi_fieldaligned.h |  95 ++++++---
 src/feltor/feltor.h               |   6 +-
 src/heat/heat.cuh                 |   4 +-
 9 files changed, 463 insertions(+), 162 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 4225e276f..06159b2d7 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -17,19 +17,174 @@
 
 namespace dg{
 namespace geo{
+///@cond
+namespace detail{
+
+struct ComputeSymv{
+    DG_DEVICE
+    void operator()( double& fp, double fm, double hp, double hm,  double vol3d) const{
+        fp = ( fp-fm)/(hp+hm);
+        fp = vol3d*fp/(hp+hm);
+    }
+    DG_DEVICE
+    void operator()( double& fp, double& fm, double f0, double hp, double hm, double vol3d) const{
+        fp = ( fp-f0)/hp;
+        fp = 0.5*vol3d*fp/hp;
+        fm = ( f0-fm)/hm;
+        fm = 0.5*vol3d*fm/hm;
+    }
+};
+struct ComputeSymvEnd{
+    DG_DEVICE
+    void operator()( double& fm, double fp, double weights) const{
+        fm = ( fp-fm)/weights;
+    }
+    DG_DEVICE
+    void operator()( double& efm, double fm, double fp, double efp, double weights) const{
+        efm = ( efm- fm + fp -efp)/weights;
+    }
+};
+
+struct ComputeDSCentered{
+
+    ComputeDSCentered( double alpha, double beta):m_alpha(alpha), m_beta(beta){}
+    DG_DEVICE
+    void operator()( double& dsf, double fm, double fo, double fp,
+            double hm, double hp)
+    {
+        dsf = m_alpha*( fm*( 1./(hp+hm) - 1/hm)
+            + fo*( 1./hm - 1./hp)
+            + fp*( 1./hp - 1/(hp+hm))
+            ) + m_beta*dsf;
+    }
+    private:
+    double m_alpha, m_beta;
+};
+struct ComputeDSCenteredNEU{
+    ComputeDSCenteredNEU( double alpha, double beta):m_alpha(alpha), m_beta(beta), m_ds(1.,0.){}
+    DG_DEVICE
+    void operator()( double& dsf, double fm, double fo, double fp,
+            double hm, double hp, double hbm, double hbp,
+            double bpm, double bpo, double bpp)
+    {
+        double inner=0, plus=0, minus=0, both=0;
+        m_ds( inner, fm, fo, fp, hm, hp);
+        plus  = ( 1./hm - 1./( 2.*hbp + hm))*(fo-fm);
+        minus = ( 1./hp - 1./( 2.*hbm + hp))*(fp-fm);
+        both = 0.;
+        dsf = m_alpha*((1.-bpm-bpo-bpp)*inner + bpp*plus + bpm*minus + bpo*both)
+            + m_beta*dsf;
+    }
+    private:
+    double m_alpha, m_beta;
+    ComputeDSCentered m_ds;
+};
+struct ComputeDSCenteredDIR{
+    ComputeDSCenteredDIR( double alpha, double beta):m_alpha(alpha), m_beta(beta), m_ds(1.,0.){}
+    DG_DEVICE
+    void operator()( double& dsf, double fm, double fo, double fp,
+            double hm, double hp, double hbm, double hbp,
+            double bpm, double bpo, double bpp)
+    {
+        double inner=0, plus=0, minus=0, both=0;
+        m_ds( inner, fm, fo, fp, hm, hp);
+        m_ds( plus, fm, fo, 0., hm, hbp);
+        m_ds( minus, 0., fo, fp, hbm, hp);
+        m_ds( both, 0., fo, 0., hbm, hbp);
+        dsf = m_alpha*((1.-bpm-bpo-bpp)*inner + bpp*plus + bpm*minus + bpo*both)
+            + m_beta*dsf;
+    }
+    private:
+    double m_alpha, m_beta;
+    ComputeDSCentered m_ds;
+};
+struct ComputeDSS{
+    ComputeDSS( double alpha, double beta):m_alpha(alpha), m_beta(beta){}
+    DG_DEVICE
+    void operator()( double& dssf, double fm, double fo, double fp,
+            double hm, double hp)
+    {
+        dssf = m_alpha*(
+               2.*fm/(hp+hm)/hm
+             - 2.*fo/hp/hm
+             + 2.*fp/(hp+hm)/hp) + m_beta*dssf;
+    }
+    private:
+    double m_alpha, m_beta;
+};
+struct ComputeDSSNEU{
+    ComputeDSSNEU( double alpha, double beta):m_alpha(alpha), m_beta(beta), m_dss(1.,0.){}
+    DG_DEVICE
+    void operator()( double& dssf, double fm, double fo, double fp,
+            double hm, double hp, double hbm, double hbp,
+            double bpm, double bpo, double bpp)
+    {
+        double inner=0, plus=0, minus=0, both=0;
+        m_dss( inner, fm, fo, fp, hm, hp);
+        plus  = -2.*(fo-fm)/( 2.*hbp + hm)/hm;
+        minus =  2.*(fp-fo)/( 2.*hbm + hp)/hp;
+        both = 0.;
+        dssf = m_alpha*((1.-bpm-bpo-bpp)*inner + bpp*plus + bpm*minus + bpo*both)
+            + m_beta*dssf;
+    }
+    private:
+    double m_alpha, m_beta;
+    ComputeDSS m_dss;
+};
+struct ComputeDSSDIR{
+    ComputeDSSDIR( double alpha, double beta):m_alpha(alpha), m_beta(beta), m_dss(1.,0.){}
+    DG_DEVICE
+    void operator()( double& dssf, double fm, double fo, double fp,
+            double hm, double hp, double hbm, double hbp,
+            double bpm, double bpo, double bpp)
+    {
+        double inner=0, plus=0, minus=0, both=0;
+        m_dss( inner, fm, fo, fp, hm, hp);
+        m_dss( plus, fm, fo, 0., hm, hbp);
+        m_dss( minus, 0., fo, fp, hbm, hp);
+        m_dss( both, 0., fo, 0., hbm, hbp);
+        dssf = m_alpha*((1.-bpm-bpo-bpp)*inner + bpp*plus + bpm*minus + bpo*both)
+            +m_beta*dssf;
+    }
+    private:
+    double m_alpha, m_beta;
+    ComputeDSS m_dss;
+};
+
+}//namespace detail
+///@endcond
+
+///@brief How boundary conditions are implemented in DS
+enum class boundary
+{
+    perp, //!< boundary condition is implemented perpendicular to boundary
+    along_field //!< boundary condition is implemented along the fieldline
+};
+
+static const std::map < std::string, boundary> str2boundary {
+    { "perp", boundary::perp},
+    { "along_field", boundary::perp}
+};
+
+/*!@class hide_ds_parameters2
+* @param f The vector to derive
+* @param g contains result on output (write only)
+* @note the vector sizes need to equal the grid size in the constructor
+*/
+/*!@class hide_ds_parameters4
+* @param alpha Scalar
+* @param f The vector to derive
+* @param beta Scalar
+* @param g contains result on output (write only)
+* @note the vector sizes need to equal the grid size in the constructor
+*/
+/*!@class hide_ds_dir_and_mode
+ * @param dir indicate the direction in the bracket operator and in symv
+ * @param mode indicate how boundary conditions should be treated: if \c dg::geo::boundary::perp then the boundary conditions are implemented by mirroring points perpendicular to the boundary, which has some drawbacks as to the numerical stability and toroidal resolution. if \c dg::geo::boundary::along_field then the boundary condition is implemented along the field-line which is numerically more desirable because it decouples the parallel direction.
+ * @attention The \c along_field boundary modes only works for \c centered and \c dss members and only if both \c bcx and \c bcy are the same (either \c dg::NEU or \c dg::DIR but not \c dg::NEU_DIR or \c dg::DIR_NEU) In all other cases \c perp mode is used by default
+ *
+*/
 
-    /*!@class hide_ds_parameters2
-    * @param f The vector to derive
-    * @param g contains result on output (write only)
-    * @note the vector sizes need to equal the grid size in the constructor
-    */
-    /*!@class hide_ds_parameters4
-    * @param alpha Scalar
-    * @param f The vector to derive
-    * @param beta Scalar
-    * @param g contains result on output (write only)
-    * @note the vector sizes need to equal the grid size in the constructor
-    */
 /*!@class hide_ds_attention
 @attention The \c div and \c symv member functions reliably converge only if fieldlines
 do not(!) intersect the boundary and then only if the \c mx and \c my
@@ -71,7 +226,7 @@ struct DS
      * @brief Create the magnetic unit vector field and construct
 
      * @copydoc hide_fieldaligned_physics_parameters
-     * @param dir indicate the direction in the bracket operator and in symv
+     * @copydoc hide_ds_dir_and_mode
      * @copydoc hide_fieldaligned_numerics_parameters
      * @sa \c Fieldaligned
      */
@@ -81,17 +236,18 @@ struct DS
         dg::bc bcy = dg::NEU,
         Limiter limit = FullLimiter(),
         dg::direction dir = dg::centered,
+        boundary mode = boundary::perp,
         double eps = 1e-5,
         unsigned mx=10, unsigned my=10,
         double deltaPhi=-1):
-        DS( FA( vec, grid, bcx, bcy, limit, eps, mx, my, deltaPhi), dir)
+        DS( FA( vec, grid, bcx, bcy, limit, eps, mx, my, deltaPhi), dir, mode )
     {
     }
     /**
      * @brief Use the given vector field to construct
      *
      * @copydoc hide_fieldaligned_physics_parameters
-     * @param dir indicate the direction in the bracket operator and in symv
+     * @copydoc hide_ds_dir_and_mode
      * @copydoc hide_fieldaligned_numerics_parameters
      * @sa \c Fieldaligned
      */
@@ -101,19 +257,20 @@ struct DS
         dg::bc bcy = dg::NEU,
         Limiter limit = FullLimiter(),
         dg::direction dir = dg::centered,
+        boundary mode = boundary::perp,
         double eps = 1e-5,
         unsigned mx=10, unsigned my=10,
         double deltaPhi=-1):
-        DS( FA( vec, grid, bcx, bcy, limit, eps, mx, my, deltaPhi), dir)
+        DS( FA( vec, grid, bcx, bcy, limit, eps, mx, my, deltaPhi), dir, mode)
     {
     }
     /**
      * @brief Re-construct from a given \c Fieldaligned object
      *
      * @param fieldaligned this object will be used in all further member calls
-     * @param dir indicate the direction in the bracket operator and in symv
+     * @copydoc hide_ds_dir_and_mode
      */
-    DS( FA fieldaligned, dg::direction dir = dg::centered);
+    DS( FA fieldaligned, dg::direction dir = dg::centered, boundary mode = boundary::perp);
     /**
     * @brief Perfect forward parameters to one of the constructors
     * @tparam Params deduced by the compiler
@@ -149,7 +306,7 @@ struct DS
     /**
     * @brief forward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
     *
-    * forward derivative \f$ g_i = \alpha \frac{1}{h_z}(f_{i+1} - f_{i}) + \beta g_i\f$
+    * forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
     * @copydoc hide_ds_parameters4
     */
     void forward( double alpha, const container& f, double beta, container& g){
@@ -158,7 +315,7 @@ struct DS
     /**
     * @brief backward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
     *
-    * backward derivative \f$ g_i = \alpha \frac{1}{2h_z}(f_{i} - f_{i-1}) + \beta g_i \f$
+    * backward derivative \f$ g_i = \alpha \frac{1}{h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
     * @copydoc hide_ds_parameters4
     */
     void backward( double alpha, const container& f, double beta, container& g){
@@ -167,7 +324,7 @@ struct DS
     /**
     * @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
     *
-    * centered derivative \f$ g_i = \alpha \frac{1}{2h_z}(f_{i+1} - f_{i-1}) + \beta g_i\f$
+    * The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup
     * @copydoc hide_ds_parameters4
     */
     void centered( double alpha, const container& f, double beta, container& g){
@@ -176,7 +333,7 @@ struct DS
     /**
     * @brief backward derivative \f$ g = \vec v \cdot \nabla f \f$
     *
-    * backward derivative \f$ g_i = \frac{1}{2h_z}(f_{i} - f_{i-1}) \f$
+    * backward derivative \f$ g_i = \frac{1}{h_z^-}(f_{i} - f_{i-1}) \f$
     * @copydoc hide_ds_parameters2
     */
     void backward( const container& f, container& g){
@@ -185,7 +342,7 @@ struct DS
     /**
     * @brief forward derivative \f$ g = \vec v \cdot \nabla f \f$
     *
-    * forward derivative \f$ g_i = \frac{1}{h_z}(f_{i+1} - f_{i})\f$
+    * forward derivative \f$ g_i = \frac{1}{h_z^+}(f_{i+1} - f_{i})\f$
     * @copydoc hide_ds_parameters2
     */
     void forward( const container& f, container& g){
@@ -194,7 +351,7 @@ struct DS
     /**
     * @brief centered derivative \f$ g = \vec v \cdot \nabla f \f$
     *
-    * centered derivative \f$ g_i = \frac{1}{2h_z}(f_{i+1} - f_{i-1})\f$
+    * The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup
     * @copydoc hide_ds_parameters2
     */
     void centered( const container& f, container& g){
@@ -319,6 +476,7 @@ struct DS
      * @brief Discretizes \f$ g = (\vec v\cdot \nabla)^2 f \f$
      *
      * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
+     * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
      * @copydoc hide_ds_parameters2
      */
     void dss( const container& f, container& g){ do_dss( 1., f, 0., g);}
@@ -326,6 +484,7 @@ struct DS
      * @brief Discretizes \f$ g = \alpha (\vec v\cdot \nabla)^2 f + \beta g \f$
      *
      * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
+     * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
      * @copydoc hide_ds_parameters4
      */
     void dss( double alpha, const container& f, double beta, container& g){ do_dss( alpha, f, beta, g);}
@@ -341,9 +500,9 @@ struct DS
     }
 
     /**
-    * @brief access the underlying Fieldaligned object for evaluate
+    * @brief access the underlying Fieldaligned object
     *
-    * @return acces to fieldaligned object
+    * @return acces to Fieldaligned object
     */
     const FA& fieldaligned() const{return m_fa;}
     private:
@@ -363,13 +522,14 @@ struct DS
     container m_vol3d, m_inv3d, m_weights_wo_vol;
     dg::direction m_dir;
     Matrix m_jumpX, m_jumpY;
+    dg::geo::boundary m_bound_mode;
 };
 
 ///@cond
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
 template<class Geometry, class I, class M, class container>
-DS<Geometry, I, M,container>::DS( Fieldaligned<Geometry, I, container> fa, dg::direction dir): m_fa(fa)
+DS<Geometry, I, M,container>::DS( Fieldaligned<Geometry, I, container> fa, dg::direction dir, boundary mode): m_fa(fa)
 {
     dg::assign( dg::create::volume(     fa.grid()), m_vol3d);
     dg::assign( dg::create::weights(    fa.grid()), m_weights_wo_vol);
@@ -377,6 +537,7 @@ DS<Geometry, I, M,container>::DS( Fieldaligned<Geometry, I, container> fa, dg::d
     dg::blas2::transfer( dg::create::jumpX( fa.grid(), fa.bcx()), m_jumpX);
     dg::blas2::transfer( dg::create::jumpY( fa.grid(), fa.bcy()), m_jumpY);
     m_temp = m_vol3d, m_tempP = m_temp, m_temp0 = m_temp, m_tempM = m_temp;
+    m_bound_mode = mode;
 }
 
 template<class G, class I, class M, class container>
@@ -407,14 +568,18 @@ void DS<G,I,M,container>::do_forward( double alpha, const container& f, double b
 {
     //direct
     m_fa(einsPlus, f, m_tempP);
-    dg::blas1::pointwiseDot( alpha, m_tempP, m_fa.hp_inv(), -alpha, f, m_fa.hp_inv(), beta, dsf);
+    dg::blas1::pointwiseDivide( m_tempP, m_fa.hp(), m_tempP);
+    dg::blas1::pointwiseDivide(       f, m_fa.hp(), m_tempM);
+    dg::blas1::axpbypgz( alpha, m_tempP, -alpha, m_tempM, beta, dsf);
 }
 template<class G,class I, class M, class container>
 void DS<G,I,M,container>::do_backward( double alpha, const container& f, double beta, container& dsf)
 {
     //direct
     m_fa(einsMinus, f, m_tempM);
-    dg::blas1::pointwiseDot( alpha, f, m_fa.hm_inv(), -alpha, m_tempM, m_fa.hm_inv(), beta, dsf);
+    dg::blas1::pointwiseDivide( m_tempM, m_fa.hm(), m_tempM);
+    dg::blas1::pointwiseDivide(       f, m_fa.hm(), m_tempP);
+    dg::blas1::axpbypgz( alpha, m_tempP, -alpha, m_tempM, beta, dsf);
 }
 template<class G, class I, class M, class container>
 void DS<G, I,M,container>::do_centered( double alpha, const container& f, double beta, container& dsf)
@@ -422,13 +587,33 @@ void DS<G, I,M,container>::do_centered( double alpha, const container& f, double
     //direct discretisation
     m_fa(einsPlus, f, m_tempP);
     m_fa(einsMinus, f, m_tempM);
-    dg::blas1::pointwiseDot( alpha, m_tempP, m_fa.h0_inv(), -alpha, m_tempM, m_fa.h0_inv(), beta, dsf);
+    if( boundary::along_field == m_bound_mode
+            && m_fa.bcx() == dg::NEU && m_fa.bcy() == dg::NEU)
+    {
+        dg::blas1::subroutine( detail::ComputeDSCenteredNEU( alpha, beta),
+                dsf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
+                m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
+    }
+    else if( boundary::along_field == m_bound_mode
+            && m_fa.bcx() == dg::DIR && m_fa.bcy() == dg::DIR)
+    {
+        dg::blas1::subroutine( detail::ComputeDSCenteredDIR( alpha, beta),
+                dsf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
+                m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
+    }
+    else
+    {
+        dg::blas1::subroutine( detail::ComputeDSCentered( alpha, beta),
+                dsf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp());
+    }
 }
+
 template<class G, class I, class M, class container>
 void DS<G,I,M,container>::do_divBackward( double alpha, const container& f, double beta, container& dsf)
 {
     //adjoint discretisation
-    dg::blas1::pointwiseDot( 1., m_vol3d, f, m_fa.hp_inv(), 0., m_temp0);
+    dg::blas1::pointwiseDot(  m_vol3d, f, m_temp0);
+    dg::blas1::pointwiseDivide( m_temp0, m_fa.hp(), m_temp0);
     m_fa(einsPlusT, m_temp0, m_tempP);
     dg::blas1::pointwiseDot( alpha, m_temp0, m_inv3d, -alpha, m_tempP, m_inv3d, beta, dsf);
 }
@@ -436,7 +621,8 @@ template<class G,class I, class M, class container>
 void DS<G,I,M,container>::do_divForward( double alpha, const container& f, double beta, container& dsf)
 {
     //adjoint discretisation
-    dg::blas1::pointwiseDot( 1., m_vol3d, f, m_fa.hm_inv(),0., m_temp0);
+    dg::blas1::pointwiseDot(  m_vol3d, f, m_temp0);
+    dg::blas1::pointwiseDivide( m_temp0, m_fa.hm(), m_temp0);
     m_fa(einsMinusT, m_temp0, m_tempM);
     dg::blas1::pointwiseDot( alpha, m_tempM, m_inv3d, -alpha, m_temp0, m_inv3d, beta, dsf);
 }
@@ -444,51 +630,15 @@ template<class G, class I, class M, class container>
 void DS<G, I,M,container>::do_divCentered( double alpha, const container& f, double beta, container& dsf)
 {
     //adjoint discretisation
-    dg::blas1::pointwiseDot( 1., m_vol3d, f, m_fa.h0_inv(), 0.,m_temp0);
+    dg::blas1::pointwiseDot(  m_vol3d, f, m_temp0);
+    dg::blas1::axpby( 1., m_fa.hp(), 1., m_fa.hm(), m_tempP);
+    dg::blas1::pointwiseDivide( m_temp0, m_tempP, m_temp0);
     m_fa(einsPlusT,  m_temp0, m_tempP);
     m_fa(einsMinusT, m_temp0, m_tempM);
     dg::blas1::pointwiseDot( alpha, m_tempM, m_inv3d, -alpha, m_tempP, m_inv3d, beta, dsf);
 
 }
 
-namespace detail{
-struct ComputeDSS{
-    ComputeDSS( double alpha, double beta):m_alpha(alpha), m_beta(beta){}
-    DG_DEVICE
-    void operator()( double& dssf, double fp, double f, double fm, double hp_inv, double h0_inv, double hm_inv) const{
-        dssf = m_alpha*( 2.*fp*hp_inv*h0_inv - 2.*f*hp_inv*hm_inv + 2*fm*hm_inv*h0_inv) + m_beta*dssf;
-        //dssf = m_alpha*(fp - 2.*f + fm)*h_inv*h_inv + m_beta*dssf;
-    }
-    private:
-    double m_alpha, m_beta;
-};
-struct ComputeSymv{
-    DG_DEVICE
-    void operator()( double& fp, double fm, double h_inv, double vol3d) const{
-        fp = ( fp-fm)*h_inv;
-        fp = vol3d*fp*h_inv;
-    }
-    DG_DEVICE
-    void operator()( double& fp, double& fm, double f0, double hp_inv, double hm_inv, double vol3d) const{
-        fp = ( fp-f0)*hp_inv;
-        fp = 0.5*vol3d*fp*hp_inv;
-        fm = ( f0-fm)*hm_inv;
-        fm = 0.5*vol3d*fm*hm_inv;
-    }
-};
-struct ComputeSymvEnd{
-    DG_DEVICE
-    void operator()( double& fm, double fp, double weights) const{
-        fm = ( fp-fm)/weights;
-    }
-    DG_DEVICE
-    void operator()( double& efm, double fm, double fp, double efp, double weights) const{
-        efm = ( efm- fm + fp -efp)/weights;
-    }
-};
-
-}//namespace detail
-
 template<class G,class I, class M, class container>
 void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta, container& dsTdsf)
 {
@@ -497,7 +647,8 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
     {
         m_fa(einsPlus, f, m_tempP);
         m_fa(einsMinus, f, m_tempM);
-        dg::blas1::subroutine( detail::ComputeSymv(), m_tempP, m_tempM, m_fa.h0_inv(), m_vol3d);
+        dg::blas1::subroutine( detail::ComputeSymv(), m_tempP, m_tempM,
+                m_fa.hp(), m_fa.hm(), m_vol3d);
         m_fa(einsPlusT,  m_tempP, m_temp);
         m_fa(einsMinusT, m_tempP, m_tempM);
         dg::blas1::subroutine( detail::ComputeSymvEnd(), m_temp,
@@ -508,7 +659,7 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
         m_fa(einsPlus, f, m_tempP);
         m_fa(einsMinus, f, m_tempM);
         dg::blas1::subroutine( detail::ComputeSymv(), m_tempP, m_tempM, f,
-            m_fa.hp_inv(), m_fa.hm_inv(), m_vol3d);
+            m_fa.hp(), m_fa.hm(), m_vol3d);
         m_fa(einsPlusT, m_tempP, m_temp0);
         m_fa(einsMinusT, m_tempM, m_temp);
         dg::blas1::subroutine( detail::ComputeSymvEnd(), m_temp,
@@ -528,8 +679,25 @@ void DS<G,I,M,container>::do_dss( double alpha, const container& f, double beta,
 {
     m_fa(einsPlus,  f, m_tempP);
     m_fa(einsMinus, f, m_tempM);
-    dg::blas1::subroutine( detail::ComputeDSS( alpha, beta),
-            dssf, m_tempP, f, m_tempM, m_fa.hp_inv(), m_fa.h0_inv(), m_fa.hm_inv());
+    if( boundary::along_field == m_bound_mode
+            && m_fa.bcx() == dg::NEU && m_fa.bcy() == dg::NEU)
+    {
+        dg::blas1::subroutine( detail::ComputeDSSNEU( alpha, beta),
+                dssf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
+                m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
+    }
+    else if( boundary::along_field == m_bound_mode
+            && m_fa.bcx() == dg::DIR && m_fa.bcy() == dg::DIR)
+    {
+        dg::blas1::subroutine( detail::ComputeDSSDIR( alpha, beta),
+                dssf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
+                m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
+    }
+    else
+    {
+        dg::blas1::subroutine( detail::ComputeDSS( alpha, beta),
+                dssf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp());
+    }
 }
 ///@endcond
 
diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 9979409ac..53eea6075 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -57,7 +57,7 @@ int main(int argc, char * argv[])
     if(rank==0)std::cout << "# Constructing Grid..."<<std::endl;
     dg::geo::CurvilinearProductMPIGrid3d g3d(flux, n, Nx, Ny,Nz, dg::NEU, dg::PER, dg::PER, comm);
     if(rank==0)std::cout << "# Constructing Fieldlines..."<<std::endl;
-    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( mag, g3d, dg::NEU, dg::PER, dg::geo::FullLimiter(), dg::centered, 1e-8, mx[0], mx[1]);
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( mag, g3d, dg::NEU, dg::PER, dg::geo::FullLimiter(), dg::centered, dg::geo::boundary::along_field, 1e-8, mx[0], mx[1]);
 
     t.toc();
     if(rank==0)std::cout << "# Construction took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/ds_curv_t.cu b/inc/geometries/ds_curv_t.cu
index 267c7de25..1e4d2b797 100644
--- a/inc/geometries/ds_curv_t.cu
+++ b/inc/geometries/ds_curv_t.cu
@@ -46,7 +46,7 @@ int main(int argc, char * argv[])
     std::cout << "# Constructing Grid..."<<std::endl;
     dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::NEU);
     std::cout << "# Constructing Fieldlines..."<<std::endl;
-    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( mag, g3d, dg::NEU, dg::PER, dg::geo::FullLimiter(), dg::centered, 1e-8, mx, my);
+    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( mag, g3d, dg::NEU, dg::PER, dg::geo::FullLimiter(), dg::centered, dg::geo::boundary::along_field, 1e-8, mx, my);
 
     t.toc();
     std::cout << "# Construction took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index 1f71aaa71..d5f9759aa 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -45,7 +45,7 @@ int main(int argc, char* argv[])
     const dg::geo::CylindricalVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
     //create Fieldaligned object and construct DS from it
     dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-8, mx[0], mx[1]);
-    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( dsFA, dg::centered);
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( dsFA, dg::centered, dg::geo::boundary::along_field);
     ///##########################################################///
     dg::MDVec fun = dg::evaluate( dg::geo::TestFunctionDirNeu(mag), g3d);
     dg::MDVec derivative(fun);
@@ -86,7 +86,7 @@ int main(int argc, char* argv[])
     ///##########################################################///
     if(rank==0)std::cout << "# Reconstruct parallel derivative!\n";
     dsFA.construct( bhat, g3d, dg::DIR, dg::DIR, dg::geo::NoLimiter(), 1e-8, mx[0], mx[1]);
-    ds.construct( dsFA, dg::centered);
+    ds.construct( dsFA, dg::centered, dg::geo::boundary::along_field);
     if(rank==0)std::cout << "# TEST DIR Boundary conditions!\n";
     ///##########################################################///
     if(rank==0)std::cout << "Dirichlet: \n";
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index c1110f285..12cc7e127 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -37,7 +37,7 @@ int main(int argc, char * argv[])
     const dg::geo::CylindricalVectorLvl0 bhat = dg::geo::createBHat(mag);
     //create Fieldaligned object and construct DS from it
     dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-8, mx, my);
-    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::centered);
+    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::centered, dg::geo::boundary::along_field );
     //![doxygen]
     ///##########################################################///
     dg::DVec fun = dg::pullback( dg::geo::TestFunctionDirNeu(mag), g3d);
@@ -79,7 +79,7 @@ int main(int argc, char * argv[])
     ///##########################################################///
     std::cout << "# Reconstruct parallel derivative!\n";
     dsFA.construct( bhat, g3d, dg::DIR, dg::DIR, dg::geo::NoLimiter(), 1e-8, mx, my);
-    ds.construct( dsFA, dg::centered);
+    ds.construct( dsFA, dg::centered, dg::geo::boundary::along_field);
     std::cout << "# TEST DIR Boundary conditions!\n";
     ///##########################################################///
     std::cout << "Dirichlet: \n";
diff --git a/inc/geometries/fieldaligned.h b/inc/geometries/fieldaligned.h
index 1dae3afae..e3247f3e5 100644
--- a/inc/geometries/fieldaligned.h
+++ b/inc/geometries/fieldaligned.h
@@ -46,25 +46,32 @@ namespace detail{
 
 struct DSFieldCylindrical
 {
-    DSFieldCylindrical( const dg::geo::CylindricalVectorLvl0& v):v_(v){ }
+    DSFieldCylindrical( const dg::geo::CylindricalVectorLvl0& v,
+            const dg::aGeometry2d& g, bool stay_in_box):
+        m_v(v), m_g(g), m_in_box( stay_in_box){}
     void operator()( double t, const std::array<double,3>& y, std::array<double,3>& yp) const {
         double R = y[0], Z = y[1];
-        double vz = v_.z()(R, Z);
-        yp[0] = v_.x()(R, Z)/vz;
-        yp[1] = v_.y()(R, Z)/vz;
+        if( m_in_box && !m_g->contains( R,Z) )
+        {
+            yp[0] = yp[1] = yp[2] = 0.;
+            return;
+        }
+        double vz = m_v.z()(R, Z);
+        yp[0] = m_v.x()(R, Z)/vz;
+        yp[1] = m_v.y()(R, Z)/vz;
         yp[2] = 1./vz;
     }
 
     private:
-    dg::geo::CylindricalVectorLvl0 v_;
+    dg::geo::CylindricalVectorLvl0 m_v;
+    dg::ClonePtr<dg::aGeometry2d> m_g;
+    bool m_in_box;
 };
 
 struct DSField
 {
     //z component of v may not vanish
-    //Fields outside g are extended according to bc in g
-    //this extension is not(!) the same as extending the flux function with these bc
-    DSField( const dg::geo::CylindricalVectorLvl0& v, const dg::aGeometry2d& g): g_(g)
+    DSField( const dg::geo::CylindricalVectorLvl0& v, const dg::aGeometry2d& g, bool stay_in_box ): m_g(g), m_in_box( stay_in_box)
     {
         thrust::host_vector<double> v_zeta, v_eta;
         dg::pushForwardPerp( v.x(), v.y(), v_zeta, v_eta, g);
@@ -79,13 +86,20 @@ struct DSField
     //interpolate the vectors given in the constructor on the given point
     void operator()(double t, const std::array<double,3>& y, std::array<double,3>& yp) const
     {
-        yp[0] = interpolate(dg::lspace, dzetadphi_, y[0], y[1], *g_);
-        yp[1] = interpolate(dg::lspace, detadphi_,  y[0], y[1], *g_);
-        yp[2] = interpolate(dg::lspace, dsdphi_,    y[0], y[1], *g_);
+        if( m_in_box && !m_g->contains( y[0],y[1]) )
+        {
+            yp[0] = yp[1] = yp[2] = 0.;
+            return;
+        }
+        // else shift point into domain
+        yp[0] = interpolate(dg::lspace, dzetadphi_, y[0], y[1], *m_g);
+        yp[1] = interpolate(dg::lspace, detadphi_,  y[0], y[1], *m_g);
+        yp[2] = interpolate(dg::lspace, dsdphi_,    y[0], y[1], *m_g);
     }
     private:
     thrust::host_vector<double> dzetadphi_, detadphi_, dsdphi_;
-    dg::ClonePtr<dg::aGeometry2d> g_;
+    dg::ClonePtr<dg::aGeometry2d> m_g;
+    bool m_in_box;
 };
 
 template<class real_type>
@@ -100,6 +114,10 @@ void integrate_all_fieldlines2d( const dg::geo::CylindricalVectorLvl0& vec,
     const dg::aRealTopology2d<real_type>& grid_evaluate,
     std::array<thrust::host_vector<real_type>,3>& yp,
     std::array<thrust::host_vector<real_type>,3>& ym,
+    thrust::host_vector<real_type>& yp2b,
+    thrust::host_vector<real_type>& ym2b,
+    thrust::host_vector<bool>& in_boxp,
+    thrust::host_vector<bool>& in_boxm,
     real_type deltaPhi, real_type eps)
 {
     //grid_field contains the global geometry for the field and the boundaries
@@ -108,12 +126,13 @@ void integrate_all_fieldlines2d( const dg::geo::CylindricalVectorLvl0& vec,
     std::array<thrust::host_vector<real_type>,3> y{tmp,tmp,tmp};; //x
     y[1] = dg::evaluate( dg::cooY2d, grid_evaluate); //y
     y[2] = dg::evaluate( dg::zero, grid_evaluate); //s
-    yp.fill(tmp); ym.fill(tmp); //allocate memory for output
+    yp.fill(tmp); ym.fill(tmp); yp2b = ym2b = tmp; //allocate memory for output
+    in_boxp.resize( tmp.size()), in_boxm.resize( tmp.size() );
     //construct field on high polynomial grid, then integrate it
-    dg::geo::detail::DSField field( vec, grid_field);
+    dg::geo::detail::DSField field( vec, grid_field, false);
     //field in case of cartesian grid
-    dg::geo::detail::DSFieldCylindrical cyl_field(vec);
-    unsigned size = grid_evaluate.size();
+    dg::geo::detail::DSFieldCylindrical cyl_field(vec, grid_field, false);
+    const unsigned size = grid_evaluate.size();
     for( unsigned i=0; i<size; i++)
     {
         std::array<real_type,3> coords{y[0][i],y[1][i],y[2][i]}, coordsP, coordsM;
@@ -130,6 +149,35 @@ void integrate_all_fieldlines2d( const dg::geo::CylindricalVectorLvl0& vec,
             dg::integrateERK( "Dormand-Prince-7-4-5", field, 0., coords, phi1, coordsM, 0., dg::pid_control, ds_norm, eps,1e-10); //integration
         yp[0][i] = coordsP[0], yp[1][i] = coordsP[1], yp[2][i] = coordsP[2];
         ym[0][i] = coordsM[0], ym[1][i] = coordsM[1], ym[2][i] = coordsM[2];
+        in_boxp[i] = grid_field.contains( yp[0][i], yp[1][i]) ? true : false;
+        in_boxm[i] = grid_field.contains( ym[0][i], ym[1][i]) ? true : false;
+    }
+    yp2b = yp[2], ym2b = ym[2];
+    //Now integrate again but this time find the boundary distance
+    field = dg::geo::detail::DSField( vec, grid_field,true);
+    cyl_field = dg::geo::detail::DSFieldCylindrical(vec, grid_field, true);
+    for( unsigned i=0; i<size; i++)
+    {
+        std::array<real_type,3> coords{y[0][i],y[1][i],y[2][i]}, coordsP, coordsM;
+        if( false == in_boxp[i])
+        {
+            //x,y,s
+            real_type phi1 = deltaPhi;
+            if( dynamic_cast<const dg::CartesianGrid2d*>( &grid_field))
+                dg::integrateERK( "Dormand-Prince-7-4-5", cyl_field, 0., coords, phi1, coordsP, 0., dg::pid_control, ds_norm, eps,1e-10); //integration
+            else
+                dg::integrateERK( "Dormand-Prince-7-4-5", field, 0., coords, phi1, coordsP, 0., dg::pid_control, ds_norm, eps,1e-10); //integration
+            yp2b[i] = coordsP[2];
+        }
+        if( false == in_boxm[i])
+        {
+            real_type phi1 =  - deltaPhi;
+            if( dynamic_cast<const dg::CartesianGrid2d*>( &grid_field))
+                dg::integrateERK( "Dormand-Prince-7-4-5", cyl_field, 0., coords, phi1, coordsM, 0., dg::pid_control, ds_norm, eps,1e-10); //integration
+            else
+                dg::integrateERK( "Dormand-Prince-7-4-5", field, 0., coords, phi1, coordsM, 0., dg::pid_control, ds_norm, eps,1e-10); //integration
+            ym2b[i] = coordsM[2];
+        }
     }
 }
 
@@ -331,20 +379,40 @@ struct Fieldaligned
     */
     void operator()(enum whichMatrix which, const container& in, container& out);
 
-    ///@brief Inverse distance between the planes \f$ (s^{k}-s^{k-1})^{-1} \f$
+    ///@brief Distance between the planes \f$ (s_{k}-s_{k-1}) \f$
     ///@return three-dimensional vector
-    const container& hm_inv()const {
-        return m_hm_inv;
+    const container& hm()const {
+        return m_hm;
     }
-    ///@brief Inverse distance between the planes \f$ (s^{k+1}-s^{k})^{-1} \f$
+    ///@brief Distance between the planes \f$ (s_{k+1}-s_{k}) \f$
     ///@return three-dimensional vector
-    const container& hp_inv()const {
-        return m_hp_inv;
+    const container& hp()const {
+        return m_hp;
     }
-    ///@brief Inverse distance between the planes \f$ (s^{k+1}-s^{k-1})^{-1} \f$
+    ///@brief Distance between the planes and the boundary \f$ (s_{k}-s_{b}^-) \f$
     ///@return three-dimensional vector
-    const container& h0_inv()const {
-        return m_h0_inv;
+    const container& hbm()const {
+        return m_hbm;
+    }
+    ///@brief Distance between the planes \f$ (s_b^+-s_{k}) \f$
+    ///@return three-dimensional vector
+    const container& hbp()const {
+        return m_hbp;
+    }
+    ///@brief Mask minus, 1 if fieldline intersects wall in minus direction but not in plus direction, 0 else
+    ///@return three-dimensional vector
+    const container& bbm()const {
+        return m_bbm;
+    }
+    ///@brief Mask both, 1 if fieldline intersects wall in plus direction and in minus direction, 0 else
+    ///@return three-dimensional vector
+    const container& bbo()const {
+        return m_bbo;
+    }
+    ///@brief Mask plus, 1 if fieldline intersects wall in plus direction but not in minus direction, 0 else
+    ///@return three-dimensional vector
+    const container& bbp()const {
+        return m_bbp;
     }
     ///Grid used for construction
     const ProductGeometry& grid()const{return *m_g;}
@@ -375,8 +443,9 @@ struct Fieldaligned
     void ePlus( enum whichMatrix which, const container& in, container& out);
     void eMinus(enum whichMatrix which, const container& in, container& out);
     IMatrix m_plus, m_minus, m_plusT, m_minusT; //2d interpolation matrices
-    container m_h0_inv, m_hm_inv, m_hp_inv; //3d size
-    container m_hm, m_hp; //2d size
+    container m_hm, m_hp, m_hbm, m_hbp;         //3d size
+    container m_bbm, m_bbp, m_bbo;  //3d size masks
+    container m_hm2d, m_hp2d;       //2d size
     container m_left, m_right;      //perp_size
     container m_limiter;            //perp_size
     container m_ghostM, m_ghostP;   //perp_size
@@ -385,13 +454,20 @@ struct Fieldaligned
     std::vector<dg::View<const container>> m_f;
     std::vector<dg::View< container>> m_temp;
     dg::ClonePtr<ProductGeometry> m_g;
+    template<class Geometry>
+    void assign3dfrom2d( const thrust::host_vector<double>& in2d, container& out, const Geometry& grid)
+    {
+        dg::split( out, m_temp, grid); //3d vector
+        container tmp2d;
+        dg::assign( in2d, tmp2d);
+        for( unsigned i=0; i<m_Nz; i++)
+            dg::blas1::copy( tmp2d, m_temp[i]);
+    }
 };
 
 ///@cond
-////////////////////////////////////DEFINITIONS////////////////////////////////////////
-//
-
 
+////////////////////////////////////DEFINITIONS///////////////////////////////////////
 template<class Geometry, class IMatrix, class container>
 template <class Limiter>
 Fieldaligned<Geometry, IMatrix, container>::Fieldaligned(
@@ -430,7 +506,10 @@ Fieldaligned<Geometry, IMatrix, container>::Fieldaligned(
     std::cout << "# DS: High order grid gen  took: "<<t.diff()<<"\n";
     t.tic();
 #endif //DG_BENCHMARK
-    detail::integrate_all_fieldlines2d( vec, *grid_magnetic, *grid_coarse, yp_coarse, ym_coarse, deltaPhi, eps);
+    thrust::host_vector<bool> in_boxp, in_boxm;
+    thrust::host_vector<double> hbp, hbm;
+    detail::integrate_all_fieldlines2d( vec, *grid_magnetic, *grid_coarse,
+            yp_coarse, ym_coarse, hbp, hbm, in_boxp, in_boxm, deltaPhi, eps);
     dg::IHMatrix interpolate = dg::create::interpolation( grid_fine, *grid_coarse);  //INTERPOLATE TO FINE GRID
     yp.fill(dg::evaluate( dg::zero, grid_fine));
     ym = yp;
@@ -469,21 +548,36 @@ Fieldaligned<Geometry, IMatrix, container>::Fieldaligned(
     dg::blas2::transfer( minus, m_minus);
     dg::blas2::transfer( minusT, m_minusT);
     ///%%%%%%%%%%%%%%%%%%%%copy into h vectors %%%%%%%%%%%%%%%%%%%//
-    dg::assign( dg::evaluate( dg::zero, grid), m_h0_inv);
-    m_hp_inv = m_hm_inv = m_h0_inv;
-    dg::assign( yp_coarse[2], m_hp); //2d vector
-    m_temp = dg::split( m_hp_inv, grid); //3d vector
-    m_f = dg::split( (const container&)m_hp_inv, grid);
-    for( unsigned i=0; i<m_Nz; i++)
-        dg::blas1::copy( m_hp, m_temp[i]);
-    dg::assign( ym_coarse[2], m_hm); //2d vector
-    dg::split( m_hm_inv, m_temp, grid); //3d vector
-    for( unsigned i=0; i<m_Nz; i++)
-        dg::blas1::copy( m_hm, m_temp[i]);
-    dg::blas1::axpby( 1., m_hp_inv, -1., m_hm_inv, m_h0_inv); //hp-hm
-    dg::blas1::pointwiseDivide( -1., m_hm_inv, m_hm_inv); //0-hm
-    dg::blas1::pointwiseDivide(  1., m_hp_inv, m_hp_inv); //hp-0
-    dg::blas1::pointwiseDivide(  1., m_h0_inv, m_h0_inv);
+    dg::assign( dg::evaluate( dg::zero, grid), m_hm);
+    m_temp  = dg::split( m_hm, grid); //3d vector
+    m_f     = dg::split( (const container&)m_hm, grid);
+    m_hbp = m_hbm = m_hp = m_hm;
+    dg::assign( yp_coarse[2], m_hp2d); //2d vector
+    dg::assign( ym_coarse[2], m_hm2d); //2d vector
+    assign3dfrom2d( hbp, m_hbp, grid);
+    assign3dfrom2d( hbm, m_hbm, grid);
+    assign3dfrom2d( yp_coarse[2], m_hp, grid);
+    assign3dfrom2d( ym_coarse[2], m_hm, grid);
+    dg::blas1::scal( m_hm2d, -1.);
+    dg::blas1::scal( m_hbm, -1.);
+    dg::blas1::scal( m_hm, -1.);
+
+    ///%%%%%%%%%%%%%%%%%%%%create mask vectors %%%%%%%%%%%%%%%%%%%//
+    thrust::host_vector<double> bbm( in_boxp.size(),0.), bbo(bbm), bbp(bbm);
+    for( unsigned i=0; i<in_boxp.size(); i++)
+    {
+        if( !in_boxp[i] && !in_boxm[i])
+            bbo[i] = 1.;
+        else if( !in_boxp[i] && in_boxm[i])
+            bbp[i] = 1.;
+        else if( in_boxp[i] && !in_boxm[i])
+            bbm[i] = 1.;
+        // else all are 0
+    }
+    m_bbm = m_bbo = m_bbp = m_hm;
+    assign3dfrom2d( bbm, m_bbm, grid);
+    assign3dfrom2d( bbo, m_bbo, grid);
+    assign3dfrom2d( bbp, m_bbp, grid);
 }
 
 template<class G, class I, class container>
@@ -649,7 +743,7 @@ void Fieldaligned<G, I, container>::ePlus( enum whichMatrix which, const contain
             dg::blas1::axpby( 2, m_right, -1., m_f[i0], m_ghostP);
         if( m_bcz == dg::NEU || m_bcz == dg::DIR_NEU)
         {
-            dg::blas1::pointwiseDot( m_right, m_hp, m_ghostP);
+            dg::blas1::pointwiseDot( m_right, m_hp2d, m_ghostP);
             dg::blas1::axpby( 1., m_ghostP, 1., m_f[i0], m_ghostP);
         }
         //interlay ghostcells with periodic cells: L*g + (1-L)*fpe
@@ -678,7 +772,7 @@ void Fieldaligned<G, I, container>::eMinus( enum whichMatrix which, const contai
             dg::blas1::axpby( 2., m_left,  -1., m_f[i0], m_ghostM);
         if( m_bcz == dg::NEU || m_bcz == dg::NEU_DIR)
         {
-            dg::blas1::pointwiseDot( m_left, m_hm, m_ghostM);
+            dg::blas1::pointwiseDot( m_left, m_hm2d, m_ghostM);
             dg::blas1::axpby( -1., m_ghostM, 1., m_f[i0], m_ghostM);
         }
         //interlay ghostcells with periodic cells: L*g + (1-L)*fme
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 79b5de23e..581367063 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -131,22 +131,35 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
 
     void operator()(enum whichMatrix which, const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
 
-    const MPI_Vector<LocalContainer>& hm_inv()const {
-        return m_hm_inv;
+    const MPI_Vector<LocalContainer>& hm()const {
+        return m_hm;
     }
-    const MPI_Vector<LocalContainer>& hp_inv()const {
-        return m_hp_inv;
+    const MPI_Vector<LocalContainer>& hp()const {
+        return m_hp;
     }
-    const MPI_Vector<LocalContainer>& h0_inv()const {
-        return m_h0_inv;
+    const MPI_Vector<LocalContainer>& hbm()const {
+        return m_hbm;
+    }
+    const MPI_Vector<LocalContainer>& hbp()const {
+        return m_hbp;
+    }
+    const MPI_Vector<LocalContainer>& bbm()const {
+        return m_bbm;
+    }
+    const MPI_Vector<LocalContainer>& bbo()const {
+        return m_bbo;
+    }
+    const MPI_Vector<LocalContainer>& bbp()const {
+        return m_bbp;
     }
     const ProductMPIGeometry& grid() const{return *m_g;}
   private:
     void ePlus( enum whichMatrix which, const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
     void eMinus(enum whichMatrix which, const MPI_Vector<LocalContainer>& in, MPI_Vector<LocalContainer>& out);
     MPIDistMat<LocalIMatrix, CommunicatorXY> m_plus, m_minus, m_plusT, m_minusT; //2d interpolation matrices
-    MPI_Vector<LocalContainer> m_h0_inv, m_hm_inv, m_hp_inv; //3d size
-    MPI_Vector<LocalContainer> m_hm, m_hp; //2d size
+    MPI_Vector<LocalContainer> m_hm, m_hp, m_hbm, m_hbp; //3d size
+    MPI_Vector<LocalContainer> m_bbm, m_bbp, m_bbo; //3d size masks
+    MPI_Vector<LocalContainer> m_hm2d, m_hp2d; //2d size
     MPI_Vector<LocalContainer> m_left, m_right; //2d size
     MPI_Vector<LocalContainer> m_limiter; //2d size
     MPI_Vector<LocalContainer> m_ghostM, m_ghostP; //2d size
@@ -160,6 +173,15 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
     //we need to manually send data through the host
     thrust::host_vector<double> m_send_buffer, m_recv_buffer; //2d size
 #endif
+    template<class MPIGeometry>
+    void assign3dfrom2d( const thrust::host_vector<double>& in2d, MPI_Vector<LocalContainer>& out, const MPIGeometry& grid)
+    {
+        dg::split( out, m_temp, grid); //3d vector
+        LocalContainer tmp2d;
+        dg::assign( in2d, tmp2d);
+        for( unsigned i=0; i<m_Nz; i++)
+            dg::blas1::copy( tmp2d, m_temp[i].data());
+    }
 };
 //////////////////////////////////////DEFINITIONS/////////////////////////////////////
 template<class MPIGeometry, class LocalIMatrix, class CommunicatorXY, class LocalContainer>
@@ -209,7 +231,10 @@ Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     if(rank==0) std::cout << "# DS: High order grid gen   took: "<<t.diff()<<"\n";
     t.tic();
 #endif
-    detail::integrate_all_fieldlines2d( vec, *global_grid_magnetic, grid_coarse->local(), yp_coarse, ym_coarse, deltaPhi, eps);
+    thrust::host_vector<bool> in_boxp, in_boxm;
+    thrust::host_vector<double> hbp, hbm;
+    detail::integrate_all_fieldlines2d( vec, *global_grid_magnetic, grid_coarse->local(),
+            yp_coarse, ym_coarse, hbp, hbm, in_boxp, in_boxm, deltaPhi, eps);
     dg::IHMatrix interpolate = dg::create::interpolation( grid_fine.local(), grid_coarse->local());  //INTERPOLATE TO FINE GRID
     yp.fill(dg::evaluate( dg::zero, grid_fine.local())); ym = yp;
     for( int i=0; i<2; i++)
@@ -255,23 +280,37 @@ Fieldaligned<MPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY>, MPI_Vector<L
     if(rank==0) std::cout << "# DS: Conversion            took: "<<t.diff()<<"\n";
 #endif
     ///%%%%%%%%%%%%%%%%%%%%copy into h vectors %%%%%%%%%%%%%%%%%%%//
-    dg::assign( dg::evaluate( dg::zero, grid), m_h0_inv);
-    m_hp_inv = m_hm_inv = m_h0_inv;
-    dg::assign( dg::evaluate( dg::zero, *grid_coarse), m_hp);
-    m_hm = m_hp;
-    dg::assign( yp_coarse[2], m_hp.data()); //2d vector
-    m_temp = dg::split( m_hp_inv, grid); //3d vector
-    m_f = dg::split( (const MPI_Vector<LocalContainer>&)m_hp_inv, grid);
-    for( unsigned i=0; i<m_Nz; i++)
-        dg::blas1::copy( m_hp, m_temp[i]);
-    dg::assign( ym_coarse[2], m_hm.data()); //2d vector
-    dg::split( m_hm_inv, m_temp, grid); //3d vector
-    for( unsigned i=0; i<m_Nz; i++)
-        dg::blas1::copy( m_hm, m_temp[i]);
-    dg::blas1::axpby( 1., m_hp_inv, -1., m_hm_inv, m_h0_inv);//hm is negative
-    dg::blas1::pointwiseDivide( -1., m_hm_inv, m_hm_inv);
-    dg::blas1::pointwiseDivide(  1., m_hp_inv, m_hp_inv);
-    dg::blas1::pointwiseDivide(  1., m_h0_inv, m_h0_inv);
+    dg::assign( dg::evaluate( dg::zero, grid), m_hm);
+    m_temp = dg::split( m_hm, grid); //3d vector
+    m_f = dg::split( (const MPI_Vector<LocalContainer>&)m_hm, grid);
+    m_hbp = m_hbm = m_hp = m_hm;
+    dg::assign( dg::evaluate( dg::zero, *grid_coarse), m_hp2d);
+    dg::assign( yp_coarse[2], m_hp2d.data()); //2d vector
+    dg::assign( dg::evaluate( dg::zero, *grid_coarse), m_hm2d);
+    dg::assign( ym_coarse[2], m_hm2d.data()); //2d vector
+    assign3dfrom2d( hbp, m_hbp, grid);
+    assign3dfrom2d( hbm, m_hbm, grid);
+    assign3dfrom2d( yp_coarse[2], m_hp, grid);
+    assign3dfrom2d( ym_coarse[2], m_hm, grid);
+    dg::blas1::scal( m_hm2d, -1.);
+    dg::blas1::scal( m_hbm, -1.);
+    dg::blas1::scal( m_hm, -1.);
+    ///%%%%%%%%%%%%%%%%%%%%create mask vectors %%%%%%%%%%%%%%%%%%%//
+    thrust::host_vector<double> bbm( in_boxp.size(),0.), bbo(bbm), bbp(bbm);
+    for( unsigned i=0; i<in_boxp.size(); i++)
+    {
+        if( !in_boxp[i] && !in_boxm[i])
+            bbo[i] = 1.;
+        else if( !in_boxp[i] && in_boxm[i])
+            bbp[i] = 1.;
+        else if( in_boxp[i] && !in_boxm[i])
+            bbm[i] = 1.;
+        // else all are 0
+    }
+    m_bbm = m_bbo = m_bbp = m_hm;
+    assign3dfrom2d( bbm, m_bbm, grid);
+    assign3dfrom2d( bbo, m_bbo, grid);
+    assign3dfrom2d( bbp, m_bbp, grid);
 }
 
 template<class G, class M, class C, class container>
@@ -385,7 +424,7 @@ void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichM
             dg::blas1::axpby( 2, m_right, -1., m_f[i0], m_ghostP);
         if( m_bcz == dg::NEU || m_bcz == dg::DIR_NEU)
         {
-            dg::blas1::pointwiseDot( m_right, m_hp, m_ghostP);
+            dg::blas1::pointwiseDot( m_right, m_hp2d, m_ghostP);
             dg::blas1::axpby( 1., m_ghostP, 1., m_f[i0], m_ghostP);
         }
         //interlay ghostcells with periodic cells: L*g + (1-L)*fpe
@@ -431,7 +470,7 @@ void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::eMinus( enum which
             dg::blas1::axpby( 2., m_left,  -1., m_f[i0], m_ghostM);
         if( m_bcz == dg::NEU || m_bcz == dg::NEU_DIR)
         {
-            dg::blas1::pointwiseDot( m_left, m_hm, m_ghostM);
+            dg::blas1::pointwiseDot( m_left, m_hm2d, m_ghostM);
             dg::blas1::axpby( -1., m_ghostM, 1., m_f[i0], m_ghostM);
         }
         //interlay ghostcells with periodic cells: L*g + (1-L)*fme
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index d6134d51e..6869399ac 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -460,19 +460,19 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
     //in DS we take the true bhat
     auto bhat = dg::geo::createBHat( mag);
     m_ds_N.construct( bhat, g, p.bcxN, p.bcyN, dg::geo::NoLimiter(),
-        dg::forward, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz );
+        dg::forward, dg::geo::boundary::perp, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz );
     if( p.bcxU == p.bcxN && p.bcyU == p.bcyN)
         m_ds_U.construct( m_ds_N);
     else
         m_ds_U.construct( bhat, g, p.bcxU, p.bcyU, dg::geo::NoLimiter(),
-            dg::forward, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
+            dg::forward, dg::geo::boundary::perp, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
     if( p.bcxP == p.bcxN && p.bcyP == p.bcyN)
         m_ds_P.construct( m_ds_N);
     else if( p.bcxP == p.bcxU && p.bcyP == p.bcyU)
         m_ds_P.construct( m_ds_U);
     else
         m_ds_P.construct( bhat, g, p.bcxP, p.bcyP, dg::geo::NoLimiter(),
-            dg::forward, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
+            dg::forward, dg::geo::boundary::perp, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
 
     // in Poisson we take EPhi except for the true curvmode
     bhat = dg::geo::createEPhi(+1);
diff --git a/src/heat/heat.cuh b/src/heat/heat.cuh
index aee45e0be..e17989743 100644
--- a/src/heat/heat.cuh
+++ b/src/heat/heat.cuh
@@ -13,7 +13,7 @@ struct Implicit
     Implicit( const Geometry& g, Parameters p,
         dg::geo::TokamakMagneticField mag):
         m_p(p),
-        m_ds( mag, g, p.bcx, p.bcy, dg::geo::NoLimiter(), dg::forward,
+        m_ds( mag, g, p.bcx, p.bcy, dg::geo::NoLimiter(), dg::forward, dg::geo::boundary::perp,
               p.rk4eps, p.mx, p.my),
         m_ellipticForward( g, dg::normed, dg::forward),
         m_ellipticBackward( g, dg::normed, dg::backward),
@@ -107,7 +107,7 @@ Explicit<Geometry,IMatrix,Matrix,container>::Explicit( const Geometry& g,
     m_Z( dg::pullback( dg::cooY3d, g)),
     m_P( dg::pullback( dg::cooZ3d, g)),
 #endif //DG_MANUFACTURED
-    m_ds( mag, g, p.bcx, p.bcy, dg::geo::NoLimiter(), dg::forward,
+    m_ds( mag, g, p.bcx, p.bcy, dg::geo::NoLimiter(), dg::forward, dg::geo::boundary::perp,
           p.rk4eps, p.mx, p.my),
     m_p(p),
     m_ellipticForward( g, dg::normed, dg::forward),
-- 
GitLab


From 63271ee7a39b950bd96c73b99decb99ee727bcd5 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 21 Oct 2020 21:42:59 +0200
Subject: [PATCH 372/540] Add convergence tables on new ds boundary

Centered seems to converge nicely but dss only goes at 1.5 at best
---
 doc/related_pages/parallel/ds_curv1000.tex    | 22 ++++++++---------
 .../parallel/ds_cylindrical_dirichlet1.tex    | 24 +++++++++----------
 .../parallel/ds_cylindrical_dirichlet10.tex   | 23 ++++++++----------
 doc/related_pages/parallel/ds_guenther10.tex  | 22 ++++++++---------
 doc/related_pages/parallel/parallel.tex       | 15 ++++--------
 inc/geometries/ds_guenther_mpit.cu            |  2 +-
 inc/geometries/ds_guenther_t.cu               |  2 +-
 inc/geometries/ds_t.cu                        | 14 +++++------
 8 files changed, 54 insertions(+), 70 deletions(-)

diff --git a/doc/related_pages/parallel/ds_curv1000.tex b/doc/related_pages/parallel/ds_curv1000.tex
index ef53ac06f..c9d982f8a 100644
--- a/doc/related_pages/parallel/ds_curv1000.tex
+++ b/doc/related_pages/parallel/ds_curv1000.tex
@@ -1,20 +1,18 @@
 \begin{tabular}{lllllllllllllll}
 \toprule
   &     &     & \multicolumn{2}{l}{$\nabla_\parallel f$
-  Eq.~\eqref{eq:paralleldis}} & \multicolumn{2}{l}{$(\nabla\cdot\bhat) f +
-  \nabla_\parallel f$} &
+  Eq.~\eqref{eq:paralleldis}} &
   \multicolumn{2}{l}{$\nabla_\parallel^2 f$ Eq.~\eqref{eq:second_order}} &
-  \multicolumn{2}{l}{$(\nabla\cdot\bhat)\nabla_\parallel f + \nabla_\parallel^2
-  f$} & \multicolumn{2}{l}{$\nabla\cdot(\bhat f)$} &
+  \multicolumn{2}{l}{$\nabla\cdot(\bhat f)$} &
   \multicolumn{2}{l}{$\Delta_\parallel^{-1}f $} \\
-     &     &     &     error & order &             error & order &     error & order &     error & order &       error &  order &          error & order \\
-\textbf{m} & \textbf{N} & \textbf{Nz} &           &       &                   &       &           &       &           &       &             &        &                &       \\
+     &     &     &     error & order &     error & order &       error &  order &          error & order \\
+\textbf{m} & \textbf{N} & \textbf{Nz} &           &       &           &       &             &        &                &       \\
 \midrule
-\textbf{1000} & \textbf{30 } & \textbf{5  } &  4.88e-01 &   n/a &          4.77e-01 &   n/a &  3.83e-01 &   n/a &  3.91e-01 &   n/a &    5.41e-01 &    n/a &       2.11e-06 &   n/a \\
-     & \textbf{60 } & \textbf{10 } &  1.64e-01 &  1.57 &          1.60e-01 &  1.57 &  1.26e-01 &  1.60 &  1.28e-01 &  1.61 &    1.84e-01 &   1.56 &       1.09e-06 &  0.95 \\
-     & \textbf{90 } & \textbf{20 } &  4.48e-02 &  1.87 &          4.37e-02 &  1.87 &  3.42e-02 &  1.88 &  3.45e-02 &  1.89 &    5.01e-02 &   1.88 &       3.62e-07 &  1.59 \\
-     & \textbf{120} & \textbf{40 } &  1.15e-02 &  1.97 &          1.12e-02 &  1.97 &  8.75e-03 &  1.97 &  8.84e-03 &  1.97 &    1.31e-02 &   1.93 &       9.85e-08 &  1.88 \\
-     & \textbf{180} & \textbf{80 } &  2.88e-03 &  1.99 &          2.81e-03 &  1.99 &  2.26e-03 &  1.95 &  2.28e-03 &  1.95 &    7.11e-03 &   0.89 &       2.67e-08 &  1.88 \\
-     & \textbf{300} & \textbf{160} &  7.22e-04 &  2.00 &          7.04e-04 &  2.00 &  6.60e-04 &  1.78 &  6.60e-04 &  1.79 &    1.27e-02 &  -0.84 &       1.90e-08 &  0.49 \\
+\textbf{1000} & \textbf{30 } & \textbf{5  } &  4.88e-01 &   n/a &  3.83e-01 &   n/a &    5.41e-01 &    n/a &       2.11e-06 &   n/a \\
+              & \textbf{60 } & \textbf{10 } &  1.64e-01 &  1.57 &  1.26e-01 &  1.60 &    1.84e-01 &   1.56 &       1.09e-06 &  0.95 \\
+              & \textbf{90 } & \textbf{20 } &  4.48e-02 &  1.87 &  3.42e-02 &  1.88 &    5.01e-02 &   1.88 &       3.62e-07 &  1.59 \\
+              & \textbf{120} & \textbf{40 } &  1.15e-02 &  1.97 &  8.75e-03 &  1.97 &    1.31e-02 &   1.93 &       9.85e-08 &  1.88 \\
+              & \textbf{180} & \textbf{80 } &  2.88e-03 &  1.99 &  2.26e-03 &  1.95 &    7.11e-03 &   0.89 &       2.67e-08 &  1.88 \\
+              & \textbf{300} & \textbf{160} &  7.22e-04 &  2.00 &  6.60e-04 &  1.78 &    1.27e-02 &  -0.84 &       1.90e-08 &  0.49 \\
 \bottomrule
 \end{tabular}
diff --git a/doc/related_pages/parallel/ds_cylindrical_dirichlet1.tex b/doc/related_pages/parallel/ds_cylindrical_dirichlet1.tex
index 922c1f13a..2cf2b2484 100644
--- a/doc/related_pages/parallel/ds_cylindrical_dirichlet1.tex
+++ b/doc/related_pages/parallel/ds_cylindrical_dirichlet1.tex
@@ -1,20 +1,18 @@
-\begin{tabular}{lllllllllllllll}
+\begin{tabular}{lllllllllll}
 \toprule
   &     &     & \multicolumn{2}{l}{$\nabla_\parallel f$
-  Eq.~\eqref{eq:paralleldis}} & \multicolumn{2}{l}{$(\nabla\cdot\bhat) f +
-  \nabla_\parallel f$} &
+  Eq.~\eqref{eq:paralleldis}} &
   \multicolumn{2}{l}{$\nabla_\parallel^2 f$ Eq.~\eqref{eq:second_order}} &
-  \multicolumn{2}{l}{$(\nabla\cdot\bhat)\nabla_\parallel f + \nabla_\parallel^2
-  f$} & \multicolumn{2}{l}{$\nabla\cdot(\bhat f)$} &
+  \multicolumn{2}{l}{$\nabla\cdot(\bhat f)$} &
   \multicolumn{2}{l}{$\Delta_\parallel^{-1}f $} \\
-  &    &     &     error & order &             error & order &     error & order &     error & order &       error &  order &          error & order \\
-\textbf{m} & \textbf{N} & \textbf{Nz} &           &       &                   &       &           &       &           &       &             &        &                &       \\
+  &    &     &     error & order &     error & order &       error &  order &          error & order \\
+\textbf{m} & \textbf{N} & \textbf{Nz} &           &       &           &       &             &        &                &       \\
 \midrule
-\textbf{1} & \textbf{5 } & \textbf{5  } &  3.56e-01 &   n/a &          3.56e-01 &   n/a &  4.64e-01 &   n/a &  4.64e-01 &   n/a &    5.04e-01 &    n/a &       7.46e-03 &   n/a \\
-  & \textbf{8 } & \textbf{10 } &  1.34e-01 &  1.41 &          1.34e-01 &  1.41 &  1.64e-01 &  1.50 &  1.64e-01 &  1.50 &    7.07e-01 &  -0.49 &       6.04e-03 &  0.30 \\
-  & \textbf{13} & \textbf{20 } &  3.76e-02 &  1.83 &          3.76e-02 &  1.83 &  4.48e-02 &  1.87 &  4.48e-02 &  1.87 &    1.40e+00 &  -0.98 &       5.14e-03 &  0.23 \\
-  & \textbf{20} & \textbf{40 } &  9.68e-03 &  1.96 &          9.68e-03 &  1.96 &  1.15e-02 &  1.97 &  1.15e-02 &  1.97 &    2.95e+00 &  -1.08 &       4.17e-03 &  0.30 \\
-  & \textbf{32} & \textbf{80 } &  2.44e-03 &  1.99 &          2.44e-03 &  1.99 &  2.94e-03 &  1.96 &  2.94e-03 &  1.96 &    5.53e+00 &  -0.91 &       3.47e-03 &  0.27 \\
-  & \textbf{50} & \textbf{160} &  6.13e-04 &  1.99 &          6.13e-04 &  1.99 &  9.93e-04 &  1.57 &  9.93e-04 &  1.57 &    1.03e+01 &  -0.89 &       3.27e-03 &  0.08 \\
+\textbf{1} & \textbf{10 } & \textbf{5  } &  2.61e-01 &   n/a &  2.40e-01 &   n/a &    4.18e-01 &    n/a &       5.08e-03 &   n/a \\
+           & \textbf{16 } & \textbf{10 } &  7.84e-02 & 1.74  &  1.04e-01 & 1.21  &    6.88e-01 &  -0.72 &       3.24e-03 &  0.65 \\
+           & \textbf{26} & \textbf{20 } &  2.16e-02 & 1.86  &  4.01e-02 & 1.37  &    1.40e+00 &  -1.02 &       4.04e-03 &  -0.32 \\
+           & \textbf{40} & \textbf{40 } &  5.65e-03 & 1.93  &  1.48e-02 & 1.44  &    3.23e+00 &  -1.21 &       3.78e-03 &  0.10 \\
+           & \textbf{64} & \textbf{80 } &  1.44e-03 & 1.97  &  5.36e-03 & 1.47  &    6.35e+00 &  -0.98 &       3.04e-03 &  0.31 \\
+           & \textbf{100} & \textbf{160} &  3.67e-04 & 1.97  &  1.96e-03 & 1.45  &    1.22e+01 &  -0.94 &       2.37e-03 &  0.36 \\
 \bottomrule
 \end{tabular}
diff --git a/doc/related_pages/parallel/ds_cylindrical_dirichlet10.tex b/doc/related_pages/parallel/ds_cylindrical_dirichlet10.tex
index 2966e322e..853ea7ae4 100644
--- a/doc/related_pages/parallel/ds_cylindrical_dirichlet10.tex
+++ b/doc/related_pages/parallel/ds_cylindrical_dirichlet10.tex
@@ -1,20 +1,17 @@
 \begin{tabular}{lllllllllllllll}
 \toprule
   &     &     & \multicolumn{2}{l}{$\nabla_\parallel f$
-  Eq.~\eqref{eq:paralleldis}} & \multicolumn{2}{l}{$(\nabla\cdot\bhat) f +
-  \nabla_\parallel f$} &
-  \multicolumn{2}{l}{$\nabla_\parallel^2 f$ Eq.~\eqref{eq:second_order}} &
-  \multicolumn{2}{l}{$(\nabla\cdot\bhat)\nabla_\parallel f + \nabla_\parallel^2
-  f$} & \multicolumn{2}{l}{$\nabla\cdot(\bhat f)$} &
+  Eq.~\eqref{eq:paralleldis}}  &
+  \multicolumn{2}{l}{$\nabla_\parallel^2 f$ Eq.~\eqref{eq:second_order}} & \multicolumn{2}{l}{$\nabla\cdot(\bhat f)$} &
   \multicolumn{2}{l}{$\Delta_\parallel^{-1}f $} \\
-  &    &     &     error & order &             error & order &     error & order &     error & order &       error &  order &          error & order \\
-\textbf{m} & \textbf{N} & \textbf{Nz} &           &       &                   &       &           &       &           &       &             &        &                &       \\
+  &    &     &     error & order &     error & order &       error &  order &          error & order \\
+\textbf{m} & \textbf{N} & \textbf{Nz} &           &       &           &       &             &        &                &       \\
 \midrule
-\textbf{10} & \textbf{6 } & \textbf{5  } &  3.56e-01 &   n/a &          3.56e-01 &   n/a &  4.64e-01 &   n/a &  4.64e-01 &   n/a &    4.48e-01 &    n/a &       7.56e-03 &   n/a \\
-  & \textbf{12} & \textbf{10 } &  1.34e-01 &  1.41 &          1.34e-01 &  1.41 &  1.64e-01 &  1.50 &  1.64e-01 &  1.50 &    6.82e-01 &  -0.61 &       6.32e-03 &  0.26 \\
-  & \textbf{18} & \textbf{20 } &  3.76e-02 &  1.83 &          3.76e-02 &  1.83 &  4.48e-02 &  1.87 &  4.48e-02 &  1.87 &    1.42e+00 &  -1.05 &       5.61e-03 &  0.17 \\
-  & \textbf{24} & \textbf{40 } &  9.68e-03 &  1.96 &          9.68e-03 &  1.96 &  1.15e-02 &  1.97 &  1.15e-02 &  1.97 &    3.03e+00 &  -1.10 &       4.39e-03 &  0.35 \\
-  & \textbf{36} & \textbf{80 } &  2.44e-03 &  1.99 &          2.44e-03 &  1.99 &  2.91e-03 &  1.98 &  2.91e-03 &  1.98 &    5.85e+00 &  -0.95 &       3.48e-03 &  0.34 \\
-  & \textbf{60} & \textbf{160} &  6.12e-04 &  2.00 &          6.11e-04 &  2.00 &  8.16e-04 &  1.83 &  8.16e-04 &  1.83 &    1.08e+01 &  -0.88 &       3.03e-03 &  0.20 \\
+\textbf{10} & \textbf{10 } & \textbf{5  } &  2.61e-01 & n/a &  2.40e-01 & n/a &    2.61e-01 & n/a  &       5.07e-03 & n/a \\
+            & \textbf{16} & \textbf{10 } &  7.84e-02 & 1.74 &  1.06e-01 & 1.18 &    8.00e-02 & 1.71  &       2.44e-03 & 1.06 \\
+            & \textbf{26} & \textbf{20 } &  2.16e-02 & 1.86 &  4.01e-02 & 1.40 &    3.74e-02 & 1.10  &       1.02e-03 & 1.26 \\
+            & \textbf{40} & \textbf{40 } &  5.66e-03 & 1.93 &  1.48e-02 & 1.44 &    7.81e-02 & -1.06  &       4.16e-04 & 1.29 \\
+            & \textbf{64} & \textbf{80 } &  1.45e-03 & 1.96 &  5.38e-03 & 1.46 &    3.46e-01 & -2.15  &       2.15e-04 & 0.95 \\
+            & \textbf{100} & \textbf{160} &  3.67e-04 & 1.98 &  2.09e-03 & 1.36 &    9.81e-01 &  -1.50 &       1.89e-04 & 0.19 \\
 \bottomrule
 \end{tabular}
diff --git a/doc/related_pages/parallel/ds_guenther10.tex b/doc/related_pages/parallel/ds_guenther10.tex
index a3e6ab3ce..ed4376034 100644
--- a/doc/related_pages/parallel/ds_guenther10.tex
+++ b/doc/related_pages/parallel/ds_guenther10.tex
@@ -1,20 +1,18 @@
 \begin{tabular}{lllllllllllllll}
 \toprule
   &     &     & \multicolumn{2}{l}{$\nabla_\parallel f$
-  Eq.~\eqref{eq:paralleldis}} & \multicolumn{2}{l}{$(\nabla\cdot\bhat) f +
-  \nabla_\parallel f$} &
+  Eq.~\eqref{eq:paralleldis}} &
   \multicolumn{2}{l}{$\nabla_\parallel^2 f$ Eq.~\eqref{eq:second_order}} &
-  \multicolumn{2}{l}{$(\nabla\cdot\bhat)\nabla_\parallel f + \nabla_\parallel^2
-  f$} & \multicolumn{2}{l}{$\nabla\cdot(\bhat f)$} &
+  \multicolumn{2}{l}{$\nabla\cdot(\bhat f)$} &
   \multicolumn{2}{l}{$\Delta_\parallel^{-1}f $} \\
-   &    &     &     error & order &             error & order &     error & order &     error & order &       error &  order &          error &  order \\
-\textbf{m} & \textbf{N} & \textbf{Nz} &           &       &                   &       &           &       &           &       &             &        &                &        \\
+   &    &     &     error & order &     error & order &       error &  order &          error &  order \\
+\textbf{m} & \textbf{N} & \textbf{Nz} &           &       &           &       &             &        &                &        \\
 \midrule
-\textbf{10} & \textbf{6 } & \textbf{5  } &  2.77e-01 &   n/a &          2.77e-01 &   n/a &  1.93e-01 &   n/a &  1.93e-01 &   n/a &    2.79e-01 &    n/a &       5.28e-03 &    n/a \\
-   & \textbf{12} & \textbf{10 } &  7.80e-02 &  1.83 &          7.79e-02 &  1.83 &  5.81e-02 &  1.73 &  5.81e-02 &  1.73 &    8.24e-02 &   1.76 &       1.95e-03 &   1.43 \\
-   & \textbf{18} & \textbf{20 } &  2.02e-02 &  1.95 &          2.02e-02 &  1.95 &  1.55e-02 &  1.91 &  1.55e-02 &  1.91 &    5.20e-02 &   0.66 &       5.76e-04 &   1.76 \\
-   & \textbf{24} & \textbf{40 } &  5.10e-03 &  1.99 &          5.10e-03 &  1.99 &  3.94e-03 &  1.97 &  3.94e-03 &  1.97 &    1.10e-01 &  -1.08 &       1.56e-04 &   1.89 \\
-   & \textbf{36} & \textbf{80 } &  1.28e-03 &  2.00 &          1.28e-03 &  2.00 &  1.03e-03 &  1.93 &  1.03e-03 &  1.93 &    3.07e-01 &  -1.49 &       8.87e-05 &   0.81 \\
-   & \textbf{60} & \textbf{160} &  3.20e-04 &  2.00 &          3.20e-04 &  2.00 &  3.56e-04 &  1.53 &  3.56e-04 &  1.53 &    7.45e-01 &  -1.28 &       1.57e-04 &  -0.83 \\
+\textbf{10} & \textbf{6 } & \textbf{5  } &  2.77e-01 &   n/a &  1.93e-01 &   n/a &    2.79e-01 &    n/a &       5.28e-03 &    n/a \\
+            & \textbf{12} & \textbf{10 } &  7.80e-02 &  1.83 &  5.81e-02 &  1.73 &    8.24e-02 &   1.76 &       1.95e-03 &   1.43 \\
+            & \textbf{18} & \textbf{20 } &  2.02e-02 &  1.95 &  1.55e-02 &  1.91 &    5.20e-02 &   0.66 &       5.76e-04 &   1.76 \\
+            & \textbf{24} & \textbf{40 } &  5.10e-03 &  1.99 &  3.94e-03 &  1.97 &    1.10e-01 &  -1.08 &       1.56e-04 &   1.89 \\
+            & \textbf{36} & \textbf{80 } &  1.28e-03 &  2.00 &  1.03e-03 &  1.93 &    3.07e-01 &  -1.49 &       8.87e-05 &   0.81 \\
+            & \textbf{60} & \textbf{160} &  3.20e-04 &  2.00 &  3.56e-04 &  1.53 &    7.45e-01 &  -1.28 &       1.57e-04 &  -0.83 \\
 \bottomrule
 \end{tabular}
diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index f5fff05fa..638e120b5 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -605,16 +605,9 @@ increase the refinement.
 \begin{centering}
 \footnotesize
 \hspace*{-2cm}
-%  &     &     & \multicolumn{2}{l}{$\nabla_\parallel f$
-%  Eq.~\eqref{eq:paralleldis}} & \multicolumn{2}{l}{$(\nabla\cdot\bhat) f +
-%  \nabla_\parallel f$} &
-%  \multicolumn{2}{l}{$\nabla_\parallel^2 f$ Eq.~\eqref{eq:second_order}} &
-%  \multicolumn{2}{l}{$(\nabla\cdot\bhat)\nabla_\parallel f + \nabla_\parallel^2
-%  f$} & \multicolumn{2}{l}{$\nabla\cdot(\bhat f)$} &
-%  \multicolumn{2}{l}{$\Delta_\parallel^{-1}f $} \\
 \input{ds_cylindrical_dirichlet1.tex}
 \caption{Convergence Table for Dirichlet boundary conditions and $m=1$,
-$N_R=N_Z=N$. The table for Neumann conditions exhibits equivalent features.}
+$N_R=N_Z=N$. Centered discretization with boundary conditions along the fieldline. The table for Neumann conditions exhibits similar numbers.}
 \label{tab:ds_cylindrical_dirichlet1}
 \end{centering}
 \end{table*}
@@ -625,7 +618,7 @@ $N_R=N_Z=N$. The table for Neumann conditions exhibits equivalent features.}
 \hspace*{-2cm}
 \input{ds_cylindrical_dirichlet10.tex}
 \caption{Convergence Table for Dirichlet boundary conditions and $m=10$,
-$N_R=N_Z=N$. The table for Neumann conditions exhibits equivalent features.}
+$N_R=N_Z=N$. Centered discretization with boundary conditions along the fieldline. The table for Neumann conditions exhibits similar numbers.}
 \label{tab:ds_cylindrical_dirichlet10}
 \end{centering}
 \end{table*}
@@ -651,7 +644,7 @@ $\varphi \in [0,2\pi]$ and choose
 \footnotesize
 \hspace*{-2cm}
 \input{ds_guenther10.tex}
-\caption{Convergence Table for $m=10$ and $N_R=N_Z=N$. Centered discretizations.
+\caption{Convergence Table for $m=10$ and $N_R=N_Z=N$ and the G\"unther field. Centered discretizations, no boundary condition.
 }
 \label{tab:ds_guenther10}
 \end{centering}
@@ -666,7 +659,7 @@ grids on the ring bounded by $\psi_p=-20$ and $\psi_p=-4$.
 \footnotesize
 \hspace*{-2cm}
 \input{ds_curv1000.tex}
-\caption{Convergence Table for $m=1000$, $N_R = 2$, $N_Z=N$. Centered discretizations.
+\caption{Convergence Table for $m=1000$, $N_R = 2$, $N_Z=N$ on a curvilinear grid. Centered discretizations, no boundary condition.
 }
 \label{tab:ds_curv1000}
 \end{centering}
diff --git a/inc/geometries/ds_guenther_mpit.cu b/inc/geometries/ds_guenther_mpit.cu
index b55237c76..c704cd5b0 100644
--- a/inc/geometries/ds_guenther_mpit.cu
+++ b/inc/geometries/ds_guenther_mpit.cu
@@ -43,7 +43,7 @@ int main(int argc, char * argv[])
     const dg::geo::TokamakMagneticField mag = dg::geo::createGuentherField(R_0, I_0);
     dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds(
         mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(),
-        dg::centered, 1e-8, mx[0], mx[1]);
+        dg::centered, dg::geo::boundary::along_field, 1e-8, mx[0], mx[1]);
 
     ///##########################################################///
     const dg::MDVec fun = dg::evaluate( dg::geo::TestFunctionPsi2(mag), g3d);
diff --git a/inc/geometries/ds_guenther_t.cu b/inc/geometries/ds_guenther_t.cu
index 131ffbb45..03f8dc6b4 100644
--- a/inc/geometries/ds_guenther_t.cu
+++ b/inc/geometries/ds_guenther_t.cu
@@ -34,7 +34,7 @@ int main( )
     const dg::geo::TokamakMagneticField mag = dg::geo::createGuentherField(R_0, I_0);
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds(
         mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(),
-        dg::centered, 1e-8, mx, my);
+        dg::centered, dg::geo::boundary::along_field, 1e-8, mx, my);
 
     ///##########################################################///
     const dg::DVec fun = dg::evaluate( dg::geo::TestFunctionPsi2(mag), g3d);
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index 12cc7e127..d467a7207 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -40,14 +40,14 @@ int main(int argc, char * argv[])
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::centered, dg::geo::boundary::along_field );
     //![doxygen]
     ///##########################################################///
-    dg::DVec fun = dg::pullback( dg::geo::TestFunctionDirNeu(mag), g3d);
+    const dg::DVec fun = dg::pullback( dg::geo::TestFunctionDirNeu(mag), g3d);
     dg::DVec derivative(fun);
-    dg::DVec divb = dg::pullback( dg::geo::Divb(mag), g3d);
-    dg::DVec sol0 = dg::pullback( dg::geo::DsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
-    dg::DVec sol1 = dg::pullback( dg::geo::DssFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
-    dg::DVec sol2 = dg::pullback( dg::geo::DsDivFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
-    dg::DVec sol3 = dg::pullback( dg::geo::DsDivDsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
-    dg::DVec sol4 = dg::pullback( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    const dg::DVec divb = dg::pullback( dg::geo::Divb(mag), g3d);
+    const dg::DVec sol0 = dg::pullback( dg::geo::DsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    const dg::DVec sol1 = dg::pullback( dg::geo::DssFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    const dg::DVec sol2 = dg::pullback( dg::geo::DsDivFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    const dg::DVec sol3 = dg::pullback( dg::geo::DsDivDsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
+    const dg::DVec sol4 = dg::pullback( dg::geo::OMDsDivDsFunction<dg::geo::TestFunctionDirNeu>(mag), g3d);
     std::vector<std::pair<std::string, std::array<const dg::DVec*,2>>> names{
          {"forward",{&fun,&sol0}},          {"backward",{&fun,&sol0}},
          {"centered",{&fun,&sol0}},         {"dss",{&fun,&sol1}},
-- 
GitLab


From 6e43655d2582740ab11a131075339cd2bdd181ef Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 22 Oct 2020 22:42:56 +0200
Subject: [PATCH 373/540] Add Resistivity back to implicit part

---
 src/feltor/feltor.h   | 6 +++---
 src/feltor/implicit.h | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 6869399ac..4a3e45744 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -819,9 +819,9 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
         }
     }
     //------------------Add Resistivity--------------------------//
-    dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
-        m_fields[0][0], m_fields[0][1],
-        m_fields[1][0], m_fields[1][1], yp[1][0], yp[1][1]);
+    //dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+    //    m_fields[0][0], m_fields[0][1],
+    //    m_fields[1][0], m_fields[1][1], yp[1][0], yp[1][1]);
 }
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index f87713618..7cf537d11 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -193,10 +193,10 @@ struct ImplicitVelocity
                 dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU,
                     m_fields[1][i],  0., wp[i]);
         }
-        ////------------------Add Resistivity--------------------------//
-        //dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
-        //    m_fields[0][0], m_fields[0][1],
-        //    m_fields[1][0], m_fields[1][1], wp[0], wp[1]);
+        //------------------Add Resistivity--------------------------//
+        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+            m_fields[0][0], m_fields[0][1],
+            m_fields[1][0], m_fields[1][1], wp[0], wp[1]);
 #else
         dg::blas1::copy( 0, wp);
 #endif
-- 
GitLab


From 4ea894867f9d44f30bbc6d20530f7d79ac40dabf Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 26 Oct 2020 13:37:32 +0100
Subject: [PATCH 374/540] Add communication free ds versions

---
 inc/geometries/ds.h | 215 +++++++++++++++++++++++++++++++++++---------
 1 file changed, 175 insertions(+), 40 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 06159b2d7..f96972dd4 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -45,6 +45,45 @@ struct ComputeSymvEnd{
     }
 };
 
+struct ComputeDSForward{
+    ComputeDSForward( double alpha, double beta):m_alpha(alpha), m_beta(beta){}
+    DG_DEVICE
+    void operator()( double& dsf, double fo, double fp,
+            double hp)
+    {
+        dsf = m_alpha*( fp - fo)/hp
+            + m_beta*dsf;
+    }
+    DG_DEVICE
+    void operator()( double& dsf, double fo, double fp, double fpp,
+            double hp)
+    {
+        dsf = m_alpha*( -3.*fo + 4.*fp - fpp)/2./hp
+            + m_beta*dsf;
+    }
+    private:
+    double m_alpha, m_beta;
+};
+struct ComputeDSBackward{
+    ComputeDSBackward( double alpha, double beta):m_alpha(alpha), m_beta(beta){}
+    DG_DEVICE
+    void operator()( double& dsf, double fo, double fm,
+            double hm)
+    {
+        dsf = m_alpha*( fo - fm)/hm
+            + m_beta*dsf;
+    }
+    DG_DEVICE
+    void operator()( double& dsf, double fo, double fm, double fmm,
+            double hm)
+    {
+        dsf = m_alpha*( 3.*fo - 4.*fm + fmm)/2./hm
+            + m_beta*dsf;
+    }
+    private:
+    double m_alpha, m_beta;
+};
+
 struct ComputeDSCentered{
 
     ComputeDSCentered( double alpha, double beta):m_alpha(alpha), m_beta(beta){}
@@ -178,6 +217,12 @@ static const std::map < std::string, boundary> str2boundary {
 * @param g contains result on output (write only)
 * @note the vector sizes need to equal the grid size in the constructor
 */
+/*!@class hide_ds_fm
+ * @param fm fieldaligned()(einsMinus, f, fm)
+ */
+/*!@class hide_ds_fp
+ * @param fp fieldaligned()(einsPlus, f, fp)
+ */
 /*!@class hide_ds_dir_and_mode
  * @param dir indicate the direction in the bracket operator and in symv
  * @param mode indicate how boundary conditions should be treated: if \c dg::geo::boundary::perp then the boundary conditions are implemented by mirroring points perpendicular to the boundary, which has some drawbacks as to the numerical stability and toroidal resolution. if \c dg::geo::boundary::along_field then the boundary condition is implemented along the field-line which is numerically more desirable because it decouples the parallel direction.
@@ -310,7 +355,8 @@ struct DS
     * @copydoc hide_ds_parameters4
     */
     void forward( double alpha, const container& f, double beta, container& g){
-        do_forward( alpha, f, beta, g);
+        m_fa(einsPlus, f, m_tempP);
+        do_forward( alpha, f, m_tempP, beta, g);
     }
     /**
     * @brief backward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
@@ -319,7 +365,8 @@ struct DS
     * @copydoc hide_ds_parameters4
     */
     void backward( double alpha, const container& f, double beta, container& g){
-        do_backward( alpha, f, beta, g);
+        m_fa(einsMinus, f, m_tempM);
+        do_backward( alpha, m_tempM, f, beta, g);
     }
     /**
     * @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
@@ -328,7 +375,9 @@ struct DS
     * @copydoc hide_ds_parameters4
     */
     void centered( double alpha, const container& f, double beta, container& g){
-        do_centered( alpha, f, beta, g);
+        m_fa(einsPlus, f, m_tempP);
+        m_fa(einsMinus, f, m_tempM);
+        do_centered( alpha, m_tempM, f, m_tempP, beta, g);
     }
     /**
     * @brief backward derivative \f$ g = \vec v \cdot \nabla f \f$
@@ -337,7 +386,7 @@ struct DS
     * @copydoc hide_ds_parameters2
     */
     void backward( const container& f, container& g){
-        do_backward(1.,f,0.,g);
+        backward(1., f,0.,g);
     }
     /**
     * @brief forward derivative \f$ g = \vec v \cdot \nabla f \f$
@@ -346,7 +395,7 @@ struct DS
     * @copydoc hide_ds_parameters2
     */
     void forward( const container& f, container& g){
-        do_forward(1.,f,0.,g);
+        forward(1.,f, 0.,g);
     }
     /**
     * @brief centered derivative \f$ g = \vec v \cdot \nabla f \f$
@@ -355,7 +404,69 @@ struct DS
     * @copydoc hide_ds_parameters2
     */
     void centered( const container& f, container& g){
-        do_centered(1.,f,0.,g);
+        centered(1.,f,0.,g);
+    }
+    /**
+    * @brief forward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
+    *
+    * forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
+    * @copydoc hide_ds_parameters4
+    * @copydoc hide_ds_fp
+    */
+    void forward( double alpha, const container& f, const container& fp, double beta, container& g){
+        forward( alpha, f, fp, beta, g);
+    }
+    /**
+    * @brief backward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
+    *
+    * backward derivative \f$ g_i = \alpha \frac{1}{h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
+    * @copydoc hide_ds_parameters4
+    * @copydoc hide_ds_fm
+    */
+    void backward( double alpha, const container& fm, const container& f, double beta, container& g){
+        do_backward( alpha, fm, f, beta, g);
+    }
+    /**
+    * @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
+    *
+    * The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup
+    * @copydoc hide_ds_parameters4
+    * @copydoc hide_ds_fm
+    * @copydoc hide_ds_fp
+    */
+    void centered( double alpha, const container& fm, const container& f, const container& fp, double beta, container& g){
+        do_centered( alpha, fm, f, fp, beta, g);
+    }
+    /**
+    * @brief backward derivative \f$ g = \vec v \cdot \nabla f \f$
+    *
+    * backward derivative \f$ g_i = \frac{1}{h_z^-}(f_{i} - f_{i-1}) \f$
+    * @copydoc hide_ds_parameters2
+    * @copydoc hide_ds_fm
+    */
+    void backward( const container& fm, const container& f, container& g){
+        do_backward(1., fm, f,0.,g);
+    }
+    /**
+    * @brief forward derivative \f$ g = \vec v \cdot \nabla f \f$
+    *
+    * forward derivative \f$ g_i = \frac{1}{h_z^+}(f_{i+1} - f_{i})\f$
+    * @copydoc hide_ds_parameters2
+    * @copydoc hide_ds_fp
+    */
+    void forward( const container& f, const container& fp, container& g){
+        do_forward(1.,f, fp, 0.,g);
+    }
+    /**
+    * @brief centered derivative \f$ g = \vec v \cdot \nabla f \f$
+    *
+    * The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup
+    * @copydoc hide_ds_parameters2
+     * @copydoc hide_ds_fm
+     * @copydoc hide_ds_fp
+    */
+    void centered( const container& fm, const container& f, const container& fp, container& g){
+        do_centered(1.,fm, f, fp,0.,g);
     }
 
     ///@brief forward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
@@ -479,15 +590,41 @@ struct DS
      * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
      * @copydoc hide_ds_parameters2
      */
-    void dss( const container& f, container& g){ do_dss( 1., f, 0., g);}
+    void dss( const container& f, container& g){
+        dss( 1., f, 0., g);}
+    /**
+     * @brief Discretizes \f$ g = \alpha (\vec v\cdot \nabla)^2 f + \beta g \f$
+     *
+     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
+     * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
+     * @copydoc hide_ds_parameters4
+     */
+    void dss( double alpha, const container& f, double beta, container& g){
+        m_fa(einsPlus, f, m_tempP);
+        m_fa(einsMinus, f, m_tempM);
+        do_dss( alpha, m_tempM, f, m_tempP, beta, g);}
+    /**
+     * @brief Discretizes \f$ g = (\vec v\cdot \nabla)^2 f \f$
+     *
+     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
+     * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
+     * @copydoc hide_ds_parameters2
+     * @copydoc hide_ds_fm
+     * @copydoc hide_ds_fp
+     */
+    void dss( const container& fm, const container& f, const container& fp, container& g){
+        do_dss( 1., fm, f, fp, 0., g);}
     /**
      * @brief Discretizes \f$ g = \alpha (\vec v\cdot \nabla)^2 f + \beta g \f$
      *
      * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
      * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
      * @copydoc hide_ds_parameters4
+     * @copydoc hide_ds_fm
+     * @copydoc hide_ds_fp
      */
-    void dss( double alpha, const container& f, double beta, container& g){ do_dss( alpha, f, beta, g);}
+    void dss( double alpha, const container& fm, const container& f, const container& fp, double beta, container& g){
+        do_dss( alpha, fm, f, fp, beta, g);}
 
     const container& weights()const {
         return m_vol3d;
@@ -506,14 +643,14 @@ struct DS
     */
     const FA& fieldaligned() const{return m_fa;}
     private:
-    void do_forward(double alpha, const container& f, double beta, container& dsf);
-    void do_backward(double alpha, const container& f, double beta, container& dsf);
-    void do_centered(double alpha, const container& f, double beta, container& dsf);
+    void do_forward(double alpha, const container& f, const container& fp, double beta, container& dsf);
+    void do_backward(double alpha, const container& fm, const container& f, double beta, container& dsf);
+    void do_centered(double alpha, const container& fm, const container& f, const container& fp, double beta, container& dsf);
     void do_divForward(double alpha, const container& f, double beta, container& dsf);
     void do_divBackward(double alpha, const container& f, double beta, container& dsf);
     void do_divCentered(double alpha, const container& f, double beta, container& dsf);
     void do_symv(double alpha, const container& f, double beta, container& dsf);
-    void do_dss(double alpha, const container& f, double beta, container& dsf);
+    void do_dss(double alpha, const container& fm, const container& f, const container& fp, double beta, container& dsf);
 
 
     Fieldaligned<ProductGeometry, IMatrix, container> m_fa;
@@ -544,67 +681,67 @@ template<class G, class I, class M, class container>
 inline void DS<G,I,M,container>::ds( dg::direction dir, double alpha, const container& f, double beta, container& dsf) {
     switch( dir){
         case dg::centered:
-        return do_centered( alpha, f, beta, dsf);
+        return centered( alpha, f, beta, dsf);
         case dg::forward:
-        return do_forward( alpha, f, beta, dsf);
+        return forward( alpha, f, beta, dsf);
         case dg::backward:
-        return do_backward( alpha, f, beta, dsf);
+        return backward( alpha, f, beta, dsf);
     }
 }
 template<class G, class I, class M, class container>
 inline void DS<G,I,M,container>::div( dg::direction dir, double alpha, const container& f, double beta, container& dsf) {
     switch( dir){
         case dg::centered:
-        return do_divCentered( alpha, f, beta, dsf);
+        return divCentered( alpha, f, beta, dsf);
         case dg::forward:
-        return do_divForward( alpha, f, beta, dsf);
+        return divForward( alpha, f, beta, dsf);
         case dg::backward:
-        return do_divBackward( alpha, f, beta, dsf);
+        return divBackward( alpha, f, beta, dsf);
     }
 }
 
 template<class G, class I, class M, class container>
-void DS<G,I,M,container>::do_forward( double alpha, const container& f, double beta, container& dsf)
+void DS<G,I,M,container>::do_forward( double alpha, const container& f, const container& fp, double beta, container& dsf)
 {
     //direct
-    m_fa(einsPlus, f, m_tempP);
-    dg::blas1::pointwiseDivide( m_tempP, m_fa.hp(), m_tempP);
-    dg::blas1::pointwiseDivide(       f, m_fa.hp(), m_tempM);
-    dg::blas1::axpbypgz( alpha, m_tempP, -alpha, m_tempM, beta, dsf);
+    dg::blas1::subroutine( detail::ComputeDSForward( alpha, beta),
+            dsf, f, fp, m_fa.hp());
+    //m_fa(einsPlus, m_tempP, m_tempM);
+    //dg::blas1::subroutine( detail::ComputeDSForward( alpha, beta),
+    //        dsf, f, m_tempP, m_tempM, m_fa.hp());
 }
 template<class G,class I, class M, class container>
-void DS<G,I,M,container>::do_backward( double alpha, const container& f, double beta, container& dsf)
+void DS<G,I,M,container>::do_backward( double alpha, const container& fm, const container& f, double beta, container& dsf)
 {
     //direct
-    m_fa(einsMinus, f, m_tempM);
-    dg::blas1::pointwiseDivide( m_tempM, m_fa.hm(), m_tempM);
-    dg::blas1::pointwiseDivide(       f, m_fa.hm(), m_tempP);
-    dg::blas1::axpbypgz( alpha, m_tempP, -alpha, m_tempM, beta, dsf);
+    dg::blas1::subroutine( detail::ComputeDSBackward( alpha, beta),
+            dsf, f, fm, m_fa.hm());
+    //m_fa(einsMinus, m_tempM, m_tempP);
+    //dg::blas1::subroutine( detail::ComputeDSBackward( alpha, beta),
+    //        dsf, f, m_tempM, m_tempP, m_fa.hm());
 }
 template<class G, class I, class M, class container>
-void DS<G, I,M,container>::do_centered( double alpha, const container& f, double beta, container& dsf)
+void DS<G, I,M,container>::do_centered( double alpha, const container& fm, const container& f, const container& fp, double beta, container& dsf)
 {
     //direct discretisation
-    m_fa(einsPlus, f, m_tempP);
-    m_fa(einsMinus, f, m_tempM);
     if( boundary::along_field == m_bound_mode
             && m_fa.bcx() == dg::NEU && m_fa.bcy() == dg::NEU)
     {
         dg::blas1::subroutine( detail::ComputeDSCenteredNEU( alpha, beta),
-                dsf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
+                dsf, fm, f, fp, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
                 m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
     }
     else if( boundary::along_field == m_bound_mode
             && m_fa.bcx() == dg::DIR && m_fa.bcy() == dg::DIR)
     {
         dg::blas1::subroutine( detail::ComputeDSCenteredDIR( alpha, beta),
-                dsf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
+                dsf, fm, f, fp, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
                 m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
     }
     else
     {
         dg::blas1::subroutine( detail::ComputeDSCentered( alpha, beta),
-                dsf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp());
+                dsf, fm, f, fp, m_fa.hm(), m_fa.hp());
     }
 }
 
@@ -675,28 +812,26 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
 }
 
 template<class G,class I, class M, class container>
-void DS<G,I,M,container>::do_dss( double alpha, const container& f, double beta, container& dssf)
+void DS<G,I,M,container>::do_dss( double alpha, const container& fm,const container& f, const container& fp, double beta, container& dssf)
 {
-    m_fa(einsPlus,  f, m_tempP);
-    m_fa(einsMinus, f, m_tempM);
     if( boundary::along_field == m_bound_mode
             && m_fa.bcx() == dg::NEU && m_fa.bcy() == dg::NEU)
     {
         dg::blas1::subroutine( detail::ComputeDSSNEU( alpha, beta),
-                dssf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
+                dssf, fm, f, fp, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
                 m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
     }
     else if( boundary::along_field == m_bound_mode
             && m_fa.bcx() == dg::DIR && m_fa.bcy() == dg::DIR)
     {
         dg::blas1::subroutine( detail::ComputeDSSDIR( alpha, beta),
-                dssf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
+                dssf, fm, f, fp, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
                 m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
     }
     else
     {
         dg::blas1::subroutine( detail::ComputeDSS( alpha, beta),
-                dssf, m_tempM, f, m_tempP, m_fa.hm(), m_fa.hp());
+                dssf, fm, f, fp, m_fa.hm(), m_fa.hp());
     }
 }
 ///@endcond
-- 
GitLab


From 96c6dc8aefd93d24f0290e661a801a9a7b0ae1f7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 26 Oct 2020 15:06:19 +0100
Subject: [PATCH 375/540] Change new comm free functions to free functions

---
 inc/geometries/ds.h                | 281 ++++++++++++-----------------
 inc/geometries/ds_curv_mpit.cu     |   2 +-
 inc/geometries/ds_curv_t.cu        |   2 +-
 inc/geometries/ds_guenther_mpit.cu |   2 +-
 inc/geometries/ds_guenther_t.cu    |   2 +-
 inc/geometries/ds_mpit.cu          |   4 +-
 inc/geometries/ds_t.cu             |   4 +-
 src/heat/heat.cuh                  |   4 +-
 8 files changed, 127 insertions(+), 174 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index f96972dd4..370c3cd84 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -218,16 +218,17 @@ static const std::map < std::string, boundary> str2boundary {
 * @note the vector sizes need to equal the grid size in the constructor
 */
 /*!@class hide_ds_fm
- * @param fm fieldaligned()(einsMinus, f, fm)
+ * @param fm fieldaligned(einsMinus, f, fm)
  */
 /*!@class hide_ds_fp
- * @param fp fieldaligned()(einsPlus, f, fp)
+ * @param fp fieldaligned(einsPlus, f, fp)
  */
-/*!@class hide_ds_dir_and_mode
+/*!@class hide_ds_dir
  * @param dir indicate the direction in the bracket operator and in symv
+ */
+/*!@class hide_ds_mode
  * @param mode indicate how boundary conditions should be treated: if \c dg::geo::boundary::perp then the boundary conditions are implemented by mirroring points perpendicular to the boundary, which has some drawbacks as to the numerical stability and toroidal resolution. if \c dg::geo::boundary::along_field then the boundary condition is implemented along the field-line which is numerically more desirable because it decouples the parallel direction.
  * @attention The \c along_field boundary modes only works for \c centered and \c dss members and only if both \c bcx and \c bcy are the same (either \c dg::NEU or \c dg::DIR but not \c dg::NEU_DIR or \c dg::DIR_NEU) In all other cases \c perp mode is used by default
- *
 */
 
 /*!@class hide_ds_attention
@@ -271,7 +272,7 @@ struct DS
      * @brief Create the magnetic unit vector field and construct
 
      * @copydoc hide_fieldaligned_physics_parameters
-     * @copydoc hide_ds_dir_and_mode
+     * @copydoc hide_ds_dir
      * @copydoc hide_fieldaligned_numerics_parameters
      * @sa \c Fieldaligned
      */
@@ -281,18 +282,17 @@ struct DS
         dg::bc bcy = dg::NEU,
         Limiter limit = FullLimiter(),
         dg::direction dir = dg::centered,
-        boundary mode = boundary::perp,
         double eps = 1e-5,
         unsigned mx=10, unsigned my=10,
         double deltaPhi=-1):
-        DS( FA( vec, grid, bcx, bcy, limit, eps, mx, my, deltaPhi), dir, mode )
+        DS( FA( vec, grid, bcx, bcy, limit, eps, mx, my, deltaPhi), dir )
     {
     }
     /**
      * @brief Use the given vector field to construct
      *
      * @copydoc hide_fieldaligned_physics_parameters
-     * @copydoc hide_ds_dir_and_mode
+     * @copydoc hide_ds_dir
      * @copydoc hide_fieldaligned_numerics_parameters
      * @sa \c Fieldaligned
      */
@@ -302,20 +302,19 @@ struct DS
         dg::bc bcy = dg::NEU,
         Limiter limit = FullLimiter(),
         dg::direction dir = dg::centered,
-        boundary mode = boundary::perp,
         double eps = 1e-5,
         unsigned mx=10, unsigned my=10,
         double deltaPhi=-1):
-        DS( FA( vec, grid, bcx, bcy, limit, eps, mx, my, deltaPhi), dir, mode)
+        DS( FA( vec, grid, bcx, bcy, limit, eps, mx, my, deltaPhi), dir)
     {
     }
     /**
      * @brief Re-construct from a given \c Fieldaligned object
      *
      * @param fieldaligned this object will be used in all further member calls
-     * @copydoc hide_ds_dir_and_mode
+     * @copydoc hide_ds_dir
      */
-    DS( FA fieldaligned, dg::direction dir = dg::centered, boundary mode = boundary::perp);
+    DS( FA fieldaligned, dg::direction dir = dg::centered);
     /**
     * @brief Perfect forward parameters to one of the constructors
     * @tparam Params deduced by the compiler
@@ -356,7 +355,7 @@ struct DS
     */
     void forward( double alpha, const container& f, double beta, container& g){
         m_fa(einsPlus, f, m_tempP);
-        do_forward( alpha, f, m_tempP, beta, g);
+        ds_forward( m_fa, alpha, f, m_tempP, beta, g);
     }
     /**
     * @brief backward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
@@ -366,7 +365,7 @@ struct DS
     */
     void backward( double alpha, const container& f, double beta, container& g){
         m_fa(einsMinus, f, m_tempM);
-        do_backward( alpha, m_tempM, f, beta, g);
+        ds_backward( m_fa, alpha, m_tempM, f, beta, g);
     }
     /**
     * @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
@@ -377,7 +376,7 @@ struct DS
     void centered( double alpha, const container& f, double beta, container& g){
         m_fa(einsPlus, f, m_tempP);
         m_fa(einsMinus, f, m_tempM);
-        do_centered( alpha, m_tempM, f, m_tempP, beta, g);
+        ds_centered( m_fa, alpha, m_tempM, f, m_tempP, beta, g);
     }
     /**
     * @brief backward derivative \f$ g = \vec v \cdot \nabla f \f$
@@ -406,68 +405,6 @@ struct DS
     void centered( const container& f, container& g){
         centered(1.,f,0.,g);
     }
-    /**
-    * @brief forward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
-    *
-    * forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
-    * @copydoc hide_ds_parameters4
-    * @copydoc hide_ds_fp
-    */
-    void forward( double alpha, const container& f, const container& fp, double beta, container& g){
-        forward( alpha, f, fp, beta, g);
-    }
-    /**
-    * @brief backward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
-    *
-    * backward derivative \f$ g_i = \alpha \frac{1}{h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
-    * @copydoc hide_ds_parameters4
-    * @copydoc hide_ds_fm
-    */
-    void backward( double alpha, const container& fm, const container& f, double beta, container& g){
-        do_backward( alpha, fm, f, beta, g);
-    }
-    /**
-    * @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
-    *
-    * The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup
-    * @copydoc hide_ds_parameters4
-    * @copydoc hide_ds_fm
-    * @copydoc hide_ds_fp
-    */
-    void centered( double alpha, const container& fm, const container& f, const container& fp, double beta, container& g){
-        do_centered( alpha, fm, f, fp, beta, g);
-    }
-    /**
-    * @brief backward derivative \f$ g = \vec v \cdot \nabla f \f$
-    *
-    * backward derivative \f$ g_i = \frac{1}{h_z^-}(f_{i} - f_{i-1}) \f$
-    * @copydoc hide_ds_parameters2
-    * @copydoc hide_ds_fm
-    */
-    void backward( const container& fm, const container& f, container& g){
-        do_backward(1., fm, f,0.,g);
-    }
-    /**
-    * @brief forward derivative \f$ g = \vec v \cdot \nabla f \f$
-    *
-    * forward derivative \f$ g_i = \frac{1}{h_z^+}(f_{i+1} - f_{i})\f$
-    * @copydoc hide_ds_parameters2
-    * @copydoc hide_ds_fp
-    */
-    void forward( const container& f, const container& fp, container& g){
-        do_forward(1.,f, fp, 0.,g);
-    }
-    /**
-    * @brief centered derivative \f$ g = \vec v \cdot \nabla f \f$
-    *
-    * The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup
-    * @copydoc hide_ds_parameters2
-     * @copydoc hide_ds_fm
-     * @copydoc hide_ds_fp
-    */
-    void centered( const container& fm, const container& f, const container& fp, container& g){
-        do_centered(1.,fm, f, fp,0.,g);
-    }
 
     ///@brief forward divergence \f$ g = \alpha \nabla\cdot(\vec v f) + \beta g\f$
     ///@copydoc hide_ds_parameters4
@@ -602,29 +539,8 @@ struct DS
     void dss( double alpha, const container& f, double beta, container& g){
         m_fa(einsPlus, f, m_tempP);
         m_fa(einsMinus, f, m_tempM);
-        do_dss( alpha, m_tempM, f, m_tempP, beta, g);}
-    /**
-     * @brief Discretizes \f$ g = (\vec v\cdot \nabla)^2 f \f$
-     *
-     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
-     * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
-     * @copydoc hide_ds_parameters2
-     * @copydoc hide_ds_fm
-     * @copydoc hide_ds_fp
-     */
-    void dss( const container& fm, const container& f, const container& fp, container& g){
-        do_dss( 1., fm, f, fp, 0., g);}
-    /**
-     * @brief Discretizes \f$ g = \alpha (\vec v\cdot \nabla)^2 f + \beta g \f$
-     *
-     * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
-     * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
-     * @copydoc hide_ds_parameters4
-     * @copydoc hide_ds_fm
-     * @copydoc hide_ds_fp
-     */
-    void dss( double alpha, const container& fm, const container& f, const container& fp, double beta, container& g){
-        do_dss( alpha, fm, f, fp, beta, g);}
+        dss_centered( m_fa, alpha, m_tempM, f, m_tempP, beta, g);
+    }
 
     const container& weights()const {
         return m_vol3d;
@@ -643,15 +559,10 @@ struct DS
     */
     const FA& fieldaligned() const{return m_fa;}
     private:
-    void do_forward(double alpha, const container& f, const container& fp, double beta, container& dsf);
-    void do_backward(double alpha, const container& fm, const container& f, double beta, container& dsf);
-    void do_centered(double alpha, const container& fm, const container& f, const container& fp, double beta, container& dsf);
     void do_divForward(double alpha, const container& f, double beta, container& dsf);
     void do_divBackward(double alpha, const container& f, double beta, container& dsf);
     void do_divCentered(double alpha, const container& f, double beta, container& dsf);
     void do_symv(double alpha, const container& f, double beta, container& dsf);
-    void do_dss(double alpha, const container& fm, const container& f, const container& fp, double beta, container& dsf);
-
 
     Fieldaligned<ProductGeometry, IMatrix, container> m_fa;
     container m_temp;
@@ -659,14 +570,13 @@ struct DS
     container m_vol3d, m_inv3d, m_weights_wo_vol;
     dg::direction m_dir;
     Matrix m_jumpX, m_jumpY;
-    dg::geo::boundary m_bound_mode;
 };
 
 ///@cond
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
 template<class Geometry, class I, class M, class container>
-DS<Geometry, I, M,container>::DS( Fieldaligned<Geometry, I, container> fa, dg::direction dir, boundary mode): m_fa(fa)
+DS<Geometry, I, M,container>::DS( Fieldaligned<Geometry, I, container> fa, dg::direction dir): m_fa(fa)
 {
     dg::assign( dg::create::volume(     fa.grid()), m_vol3d);
     dg::assign( dg::create::weights(    fa.grid()), m_weights_wo_vol);
@@ -674,7 +584,6 @@ DS<Geometry, I, M,container>::DS( Fieldaligned<Geometry, I, container> fa, dg::d
     dg::blas2::transfer( dg::create::jumpX( fa.grid(), fa.bcx()), m_jumpX);
     dg::blas2::transfer( dg::create::jumpY( fa.grid(), fa.bcy()), m_jumpY);
     m_temp = m_vol3d, m_tempP = m_temp, m_temp0 = m_temp, m_tempM = m_temp;
-    m_bound_mode = mode;
 }
 
 template<class G, class I, class M, class container>
@@ -700,50 +609,6 @@ inline void DS<G,I,M,container>::div( dg::direction dir, double alpha, const con
     }
 }
 
-template<class G, class I, class M, class container>
-void DS<G,I,M,container>::do_forward( double alpha, const container& f, const container& fp, double beta, container& dsf)
-{
-    //direct
-    dg::blas1::subroutine( detail::ComputeDSForward( alpha, beta),
-            dsf, f, fp, m_fa.hp());
-    //m_fa(einsPlus, m_tempP, m_tempM);
-    //dg::blas1::subroutine( detail::ComputeDSForward( alpha, beta),
-    //        dsf, f, m_tempP, m_tempM, m_fa.hp());
-}
-template<class G,class I, class M, class container>
-void DS<G,I,M,container>::do_backward( double alpha, const container& fm, const container& f, double beta, container& dsf)
-{
-    //direct
-    dg::blas1::subroutine( detail::ComputeDSBackward( alpha, beta),
-            dsf, f, fm, m_fa.hm());
-    //m_fa(einsMinus, m_tempM, m_tempP);
-    //dg::blas1::subroutine( detail::ComputeDSBackward( alpha, beta),
-    //        dsf, f, m_tempM, m_tempP, m_fa.hm());
-}
-template<class G, class I, class M, class container>
-void DS<G, I,M,container>::do_centered( double alpha, const container& fm, const container& f, const container& fp, double beta, container& dsf)
-{
-    //direct discretisation
-    if( boundary::along_field == m_bound_mode
-            && m_fa.bcx() == dg::NEU && m_fa.bcy() == dg::NEU)
-    {
-        dg::blas1::subroutine( detail::ComputeDSCenteredNEU( alpha, beta),
-                dsf, fm, f, fp, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
-                m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
-    }
-    else if( boundary::along_field == m_bound_mode
-            && m_fa.bcx() == dg::DIR && m_fa.bcy() == dg::DIR)
-    {
-        dg::blas1::subroutine( detail::ComputeDSCenteredDIR( alpha, beta),
-                dsf, fm, f, fp, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
-                m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
-    }
-    else
-    {
-        dg::blas1::subroutine( detail::ComputeDSCentered( alpha, beta),
-                dsf, fm, f, fp, m_fa.hm(), m_fa.hp());
-    }
-}
 
 template<class G, class I, class M, class container>
 void DS<G,I,M,container>::do_divBackward( double alpha, const container& f, double beta, container& dsf)
@@ -810,31 +675,119 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
 
     dg::blas1::pointwiseDot( alpha, m_inv3d, m_weights_wo_vol, m_temp, beta, dsTdsf);
 }
+///@endcond
 
-template<class G,class I, class M, class container>
-void DS<G,I,M,container>::do_dss( double alpha, const container& fm,const container& f, const container& fp, double beta, container& dssf)
+/**
+* @brief forward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
+*
+* forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
+* @copydoc hide_ds_parameters4
+* @copydoc hide_ds_fp
+* @ingroup fieldaligned
+*/
+template<class FieldAligned, class container>
+void ds_forward(const FieldAligned& fa, double alpha, const container& f, const container& fp, double beta, container& g)
+{
+    //direct
+    dg::blas1::subroutine( detail::ComputeDSForward( alpha, beta),
+            g, f, fp, fa.hp());
+    //m_fa(einsPlus, m_tempP, m_tempM);
+    //dg::blas1::subroutine( detail::ComputeDSForward( alpha, beta),
+    //        g, f, m_tempP, m_tempM, m_fa.hp());
+}
+/**
+* @brief backward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
+*
+* backward derivative \f$ g_i = \alpha \frac{1}{h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
+ * @param fa this object will be used to get grid distances
+* @copydoc hide_ds_parameters4
+* @copydoc hide_ds_fm
+* @ingroup fieldaligned
+*/
+template<class FieldAligned, class container>
+void ds_backward( const FieldAligned& fa, double alpha, const container& fm, const container& f, double beta, container& g)
 {
-    if( boundary::along_field == m_bound_mode
-            && m_fa.bcx() == dg::NEU && m_fa.bcy() == dg::NEU)
+    //direct
+    dg::blas1::subroutine( detail::ComputeDSBackward( alpha, beta),
+            g, f, fm, fa.hm());
+    //m_fa(einsMinus, m_tempM, m_tempP);
+    //dg::blas1::subroutine( detail::ComputeDSBackward( alpha, beta),
+    //        g, f, m_tempM, m_tempP, m_fa.hm());
+}
+/**
+* @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
+*
+* The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup
+ * @param fa this object will be used to get grid distances
+* @copydoc hide_ds_parameters4
+* @copydoc hide_ds_fm
+* @copydoc hide_ds_fp
+* @copydoc hide_ds_mode
+* @ingroup fieldaligned
+*/
+template<class FieldAligned, class container>
+void ds_centered( const FieldAligned& fa, double alpha, const container& fm,
+        const container& f, const container& fp, double beta, container& g,
+        dg::geo::boundary bound_mode = dg::geo::boundary::perp)
+{
+    //direct discretisation
+    if( boundary::along_field == bound_mode
+            && fa.bcx() == dg::NEU && fa.bcy() == dg::NEU)
+    {
+        dg::blas1::subroutine( detail::ComputeDSCenteredNEU( alpha, beta),
+                g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
+                fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
+    }
+    else if( boundary::along_field == bound_mode
+            && fa.bcx() == dg::DIR && fa.bcy() == dg::DIR)
+    {
+        dg::blas1::subroutine( detail::ComputeDSCenteredDIR( alpha, beta),
+                g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
+                fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
+    }
+    else
+    {
+        dg::blas1::subroutine( detail::ComputeDSCentered( alpha, beta),
+                g, fm, f, fp, fa.hm(), fa.hp());
+    }
+}
+/**
+ * @brief Centered derivative \f$ g = \alpha (\vec v\cdot \nabla)^2 f + \beta g \f$
+ *
+ * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
+ * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
+ * @param fa this object will be used to get grid distances
+ * @copydoc hide_ds_parameters4
+* @copydoc hide_ds_fm
+* @copydoc hide_ds_fp
+* @copydoc hide_ds_mode
+* @ingroup fieldaligned
+ */
+template<class FieldAligned, class container>
+void dss_centered( const FieldAligned& fa, double alpha, const container&
+        fm,const container& f, const container& fp, double beta, container&
+        g, dg::geo::boundary bound_mode = dg::geo::boundary::perp)
+{
+    if( boundary::along_field == bound_mode
+            && fa.bcx() == dg::NEU && fa.bcy() == dg::NEU)
     {
         dg::blas1::subroutine( detail::ComputeDSSNEU( alpha, beta),
-                dssf, fm, f, fp, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
-                m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
+                g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
+                fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
-    else if( boundary::along_field == m_bound_mode
-            && m_fa.bcx() == dg::DIR && m_fa.bcy() == dg::DIR)
+    else if( boundary::along_field == bound_mode
+            && fa.bcx() == dg::DIR && fa.bcy() == dg::DIR)
     {
         dg::blas1::subroutine( detail::ComputeDSSDIR( alpha, beta),
-                dssf, fm, f, fp, m_fa.hm(), m_fa.hp(), m_fa.hbm(),
-                m_fa.hbp(), m_fa.bbm(), m_fa.bbo(), m_fa.bbp());
+                g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
+                fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
     else
     {
         dg::blas1::subroutine( detail::ComputeDSS( alpha, beta),
-                dssf, fm, f, fp, m_fa.hm(), m_fa.hp());
+                g, fm, f, fp, fa.hm(), fa.hp());
     }
 }
-///@endcond
 
 
 }//namespace geo
diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 53eea6075..9979409ac 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -57,7 +57,7 @@ int main(int argc, char * argv[])
     if(rank==0)std::cout << "# Constructing Grid..."<<std::endl;
     dg::geo::CurvilinearProductMPIGrid3d g3d(flux, n, Nx, Ny,Nz, dg::NEU, dg::PER, dg::PER, comm);
     if(rank==0)std::cout << "# Constructing Fieldlines..."<<std::endl;
-    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( mag, g3d, dg::NEU, dg::PER, dg::geo::FullLimiter(), dg::centered, dg::geo::boundary::along_field, 1e-8, mx[0], mx[1]);
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( mag, g3d, dg::NEU, dg::PER, dg::geo::FullLimiter(), dg::centered, 1e-8, mx[0], mx[1]);
 
     t.toc();
     if(rank==0)std::cout << "# Construction took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/ds_curv_t.cu b/inc/geometries/ds_curv_t.cu
index 1e4d2b797..267c7de25 100644
--- a/inc/geometries/ds_curv_t.cu
+++ b/inc/geometries/ds_curv_t.cu
@@ -46,7 +46,7 @@ int main(int argc, char * argv[])
     std::cout << "# Constructing Grid..."<<std::endl;
     dg::geo::CurvilinearProductGrid3d g3d(flux, n, Nx, Ny,Nz, dg::NEU);
     std::cout << "# Constructing Fieldlines..."<<std::endl;
-    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( mag, g3d, dg::NEU, dg::PER, dg::geo::FullLimiter(), dg::centered, dg::geo::boundary::along_field, 1e-8, mx, my);
+    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( mag, g3d, dg::NEU, dg::PER, dg::geo::FullLimiter(), dg::centered, 1e-8, mx, my);
 
     t.toc();
     std::cout << "# Construction took "<<t.diff()<<"s\n";
diff --git a/inc/geometries/ds_guenther_mpit.cu b/inc/geometries/ds_guenther_mpit.cu
index c704cd5b0..b55237c76 100644
--- a/inc/geometries/ds_guenther_mpit.cu
+++ b/inc/geometries/ds_guenther_mpit.cu
@@ -43,7 +43,7 @@ int main(int argc, char * argv[])
     const dg::geo::TokamakMagneticField mag = dg::geo::createGuentherField(R_0, I_0);
     dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds(
         mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(),
-        dg::centered, dg::geo::boundary::along_field, 1e-8, mx[0], mx[1]);
+        dg::centered, 1e-8, mx[0], mx[1]);
 
     ///##########################################################///
     const dg::MDVec fun = dg::evaluate( dg::geo::TestFunctionPsi2(mag), g3d);
diff --git a/inc/geometries/ds_guenther_t.cu b/inc/geometries/ds_guenther_t.cu
index 03f8dc6b4..131ffbb45 100644
--- a/inc/geometries/ds_guenther_t.cu
+++ b/inc/geometries/ds_guenther_t.cu
@@ -34,7 +34,7 @@ int main( )
     const dg::geo::TokamakMagneticField mag = dg::geo::createGuentherField(R_0, I_0);
     dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds(
         mag, g3d, dg::NEU, dg::NEU, dg::geo::FullLimiter(),
-        dg::centered, dg::geo::boundary::along_field, 1e-8, mx, my);
+        dg::centered, 1e-8, mx, my);
 
     ///##########################################################///
     const dg::DVec fun = dg::evaluate( dg::geo::TestFunctionPsi2(mag), g3d);
diff --git a/inc/geometries/ds_mpit.cu b/inc/geometries/ds_mpit.cu
index d5f9759aa..1f71aaa71 100644
--- a/inc/geometries/ds_mpit.cu
+++ b/inc/geometries/ds_mpit.cu
@@ -45,7 +45,7 @@ int main(int argc, char* argv[])
     const dg::geo::CylindricalVectorLvl0 bhat( (dg::geo::BHatR)(mag), (dg::geo::BHatZ)(mag), (dg::geo::BHatP)(mag));
     //create Fieldaligned object and construct DS from it
     dg::geo::Fieldaligned<dg::aProductMPIGeometry3d,dg::MIDMatrix,dg::MDVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-8, mx[0], mx[1]);
-    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( dsFA, dg::centered, dg::geo::boundary::along_field);
+    dg::geo::DS<dg::aProductMPIGeometry3d, dg::MIDMatrix, dg::MDMatrix, dg::MDVec> ds( dsFA, dg::centered);
     ///##########################################################///
     dg::MDVec fun = dg::evaluate( dg::geo::TestFunctionDirNeu(mag), g3d);
     dg::MDVec derivative(fun);
@@ -86,7 +86,7 @@ int main(int argc, char* argv[])
     ///##########################################################///
     if(rank==0)std::cout << "# Reconstruct parallel derivative!\n";
     dsFA.construct( bhat, g3d, dg::DIR, dg::DIR, dg::geo::NoLimiter(), 1e-8, mx[0], mx[1]);
-    ds.construct( dsFA, dg::centered, dg::geo::boundary::along_field);
+    ds.construct( dsFA, dg::centered);
     if(rank==0)std::cout << "# TEST DIR Boundary conditions!\n";
     ///##########################################################///
     if(rank==0)std::cout << "Dirichlet: \n";
diff --git a/inc/geometries/ds_t.cu b/inc/geometries/ds_t.cu
index d467a7207..1f4233848 100644
--- a/inc/geometries/ds_t.cu
+++ b/inc/geometries/ds_t.cu
@@ -37,7 +37,7 @@ int main(int argc, char * argv[])
     const dg::geo::CylindricalVectorLvl0 bhat = dg::geo::createBHat(mag);
     //create Fieldaligned object and construct DS from it
     dg::geo::Fieldaligned<dg::aProductGeometry3d,dg::IDMatrix,dg::DVec>  dsFA( bhat, g3d, dg::NEU, dg::NEU, dg::geo::NoLimiter(), 1e-8, mx, my);
-    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::centered, dg::geo::boundary::along_field );
+    dg::geo::DS<dg::aProductGeometry3d, dg::IDMatrix, dg::DMatrix, dg::DVec> ds( dsFA, dg::centered );
     //![doxygen]
     ///##########################################################///
     const dg::DVec fun = dg::pullback( dg::geo::TestFunctionDirNeu(mag), g3d);
@@ -79,7 +79,7 @@ int main(int argc, char * argv[])
     ///##########################################################///
     std::cout << "# Reconstruct parallel derivative!\n";
     dsFA.construct( bhat, g3d, dg::DIR, dg::DIR, dg::geo::NoLimiter(), 1e-8, mx, my);
-    ds.construct( dsFA, dg::centered, dg::geo::boundary::along_field);
+    ds.construct( dsFA, dg::centered);
     std::cout << "# TEST DIR Boundary conditions!\n";
     ///##########################################################///
     std::cout << "Dirichlet: \n";
diff --git a/src/heat/heat.cuh b/src/heat/heat.cuh
index e17989743..aee45e0be 100644
--- a/src/heat/heat.cuh
+++ b/src/heat/heat.cuh
@@ -13,7 +13,7 @@ struct Implicit
     Implicit( const Geometry& g, Parameters p,
         dg::geo::TokamakMagneticField mag):
         m_p(p),
-        m_ds( mag, g, p.bcx, p.bcy, dg::geo::NoLimiter(), dg::forward, dg::geo::boundary::perp,
+        m_ds( mag, g, p.bcx, p.bcy, dg::geo::NoLimiter(), dg::forward,
               p.rk4eps, p.mx, p.my),
         m_ellipticForward( g, dg::normed, dg::forward),
         m_ellipticBackward( g, dg::normed, dg::backward),
@@ -107,7 +107,7 @@ Explicit<Geometry,IMatrix,Matrix,container>::Explicit( const Geometry& g,
     m_Z( dg::pullback( dg::cooY3d, g)),
     m_P( dg::pullback( dg::cooZ3d, g)),
 #endif //DG_MANUFACTURED
-    m_ds( mag, g, p.bcx, p.bcy, dg::geo::NoLimiter(), dg::forward, dg::geo::boundary::perp,
+    m_ds( mag, g, p.bcx, p.bcy, dg::geo::NoLimiter(), dg::forward,
           p.rk4eps, p.mx, p.my),
     m_p(p),
     m_ellipticForward( g, dg::normed, dg::forward),
-- 
GitLab


From be3e8e500c2c8b41b1ccb73d2b9c78be7ec592cb Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 26 Oct 2020 21:15:30 +0100
Subject: [PATCH 376/540] Introduce ds damping in feltor

pass MMS on FELTORPARALLEL
and hopefully we can now run stable simulations
---
 src/feltor/feltor.cu       |  4 +-
 src/feltor/feltor.h        | 79 +++++++++++++++++++++++++++-----------
 src/feltor/feltor_hpc.cu   |  3 +-
 src/feltor/init.h          | 15 ++++++++
 src/feltor/manufactured.cu |  4 +-
 5 files changed, 78 insertions(+), 27 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index a6c523685..b58659d44 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -121,10 +121,10 @@ int main( int argc, char* argv[])
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
         return -1;
     }
-
     feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
         p.source_rate, dg::construct<DVec>(source_profile),
-        p.damping_rate, dg::construct<DVec>(damping_profile)
+        p.damping_rate, dg::construct<DVec>(damping_profile),
+        dg::construct<DVec>( feltor::detail::interior( grid, p, mag))
     );
 
 
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 4a3e45744..4c546e717 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -333,7 +333,7 @@ struct Explicit
     }
 
     //source strength, profile - 1
-    void set_source( bool fixed_profile, Container profile, double omega_source, Container source, double omega_damping, Container damping )
+    void set_source( bool fixed_profile, Container profile, double omega_source, Container source, double omega_damping, Container damping, Container interior )
     {
         m_fixed_profile = fixed_profile;
         m_profne = profile;
@@ -341,6 +341,7 @@ struct Explicit
         m_source = source;
         m_omega_damping = omega_damping;
         m_damping = damping;
+        m_interior = interior;
     }
     void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
   private:
@@ -371,7 +372,7 @@ struct Explicit
     std::array<Container,3> m_curv, m_curvKappa, m_b;
     Container m_divCurvKappa;
     Container m_bphi, m_binv, m_divb;
-    Container m_source, m_profne, m_damping;
+    Container m_source, m_profne, m_damping, m_interior;
     Container m_vol3d;
 
     Container m_apar;
@@ -384,7 +385,8 @@ struct Explicit
 
     //matrices and solvers
     Matrix m_dx_N, m_dx_U, m_dx_P, m_dy_N, m_dy_U, m_dy_P, m_dz;
-    dg::geo::DS<Geometry, IMatrix, Matrix, Container> m_ds_P, m_ds_N, m_ds_U;
+    dg::geo::Fieldaligned<Geometry, IMatrix, Container> m_fa_P, m_fa_N, m_fa_U;
+    std::array<Container,2> m_faP, m_faM;
     dg::Elliptic3d< Geometry, Matrix, Container> m_lapperpN, m_lapperpU, m_lapperpP;
     std::vector<dg::Elliptic3d< Geometry, Matrix, Container> > m_multi_pol;
     std::vector<dg::Helmholtz3d<Geometry, Matrix, Container> > m_multi_invgammaP,
@@ -459,20 +461,20 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
 {
     //in DS we take the true bhat
     auto bhat = dg::geo::createBHat( mag);
-    m_ds_N.construct( bhat, g, p.bcxN, p.bcyN, dg::geo::NoLimiter(),
-        dg::forward, dg::geo::boundary::perp, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz );
+    m_fa_N.construct( bhat, g, p.bcxN, p.bcyN, dg::geo::NoLimiter(),
+        p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz );
     if( p.bcxU == p.bcxN && p.bcyU == p.bcyN)
-        m_ds_U.construct( m_ds_N);
+        m_fa_U.construct( m_fa_N);
     else
-        m_ds_U.construct( bhat, g, p.bcxU, p.bcyU, dg::geo::NoLimiter(),
-            dg::forward, dg::geo::boundary::perp, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
+        m_fa_U.construct( bhat, g, p.bcxU, p.bcyU, dg::geo::NoLimiter(),
+            p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
     if( p.bcxP == p.bcxN && p.bcyP == p.bcyN)
-        m_ds_P.construct( m_ds_N);
+        m_fa_P.construct( m_fa_N);
     else if( p.bcxP == p.bcxU && p.bcyP == p.bcyU)
-        m_ds_P.construct( m_ds_U);
+        m_fa_P.construct( m_fa_U);
     else
-        m_ds_P.construct( bhat, g, p.bcxP, p.bcyP, dg::geo::NoLimiter(),
-            dg::forward, dg::geo::boundary::perp, p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
+        m_fa_P.construct( bhat, g, p.bcxP, p.bcyP, dg::geo::NoLimiter(),
+             p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
 
     // in Poisson we take EPhi except for the true curvmode
     bhat = dg::geo::createEPhi(+1);
@@ -569,10 +571,12 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
     m_UE2 = m_temp2 = m_temp1 = m_temp0;
     m_apar = m_temp0;
+    dg::assign( dg::evaluate( dg::one, g), m_interior );
 
     m_phi[0] = m_phi[1] = m_temp0;
     //m_dssN =
     m_dssU = m_dsN = m_dsU = m_dsP = m_phi;
+    m_faP = m_faM = m_phi;
     m_dA[0] = m_dA[1] = m_dA[2] = m_temp0;
     m_dP[0] = m_dP[1] = m_dA;
     m_dN = m_dU = m_dP;
@@ -834,10 +838,18 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
     //y[0] = N-1, y[1] = W; fields[0] = N, fields[1] = U
     for( unsigned i=0; i<2; i++)
     {
+        m_fa_N( dg::geo::einsMinus, y[0][i], m_faM[0]);
+        m_fa_N( dg::geo::einsPlus,  y[0][i], m_faP[0]);
+        m_fa_U( dg::geo::einsMinus, fields[1][i], m_faM[1]);
+        m_fa_U( dg::geo::einsPlus,  fields[1][i], m_faP[1]);
         //---------------------density--------------------------//
         //density: -Div ( NUb)
-        m_ds_N.centered( y[0][i], m_dsN[i]);
-        m_ds_U.centered( fields[1][i], m_dsU[i]);
+        dg::geo::ds_centered( m_fa_N, 1., m_faM[0], y[0][i], m_faP[0], 0., m_dsN[i]);
+        //centered in the interior, but backward outside
+        dg::geo::ds_backward( m_fa_U, 1., m_faM[1], fields[1][i], 0., m_dsU[i]);
+        dg::geo::ds_centered( m_fa_U, 1., m_faM[1], fields[1][i], m_faP[1], 0., m_temp1);
+        dg::blas1::pointwiseDot( 1., m_interior, m_temp1, -1., m_interior, m_dsU[i], +1., m_dsU[i]);
+
         dg::blas1::pointwiseDot(-1., m_dsN[i], fields[1][i],
             -1., fields[0][i], m_dsU[i], 1., yp[0][i] );
         dg::blas1::pointwiseDot( -1., fields[0][i],fields[1][i],m_divb,
@@ -845,25 +857,33 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         //////////density: + nu_par Delta_par N
         ////////dg::blas1::pointwiseDot( m_p.nu_parallel, m_divb, m_dsN[i],
         ////////                         0., m_temp0);
-        ////////m_ds_N.dss( y[0][i], m_dssN[i]);
+        ////////dg::geo::dss_centered( m_fa_N, 1., m_faM[0],  y[0][i], m_faP[0], 0., m_dssN[i]);
         ////////dg::blas1::axpby( m_p.nu_parallel, m_dssN[i], 1., m_temp0);//nu_par Delta_par N
         //////////Add to rhs, we again need it further down
         ////////dg::blas1::axpby( 1., m_temp0, 1., yp[0][i]);
         //---------------------velocity-------------------------//
         // Burgers term: -0.5 ds U^2
         //dg::blas1::pointwiseDot(fields[1][i], fields[1][i], m_temp1); //U^2
-        //m_ds_U.centered(-0.5, m_temp1, 1., yp[1][i]);
+        //m_fa_U.centered(-0.5, m_temp1, 1., yp[1][i]);
+        dg::geo::ds_centered( m_fa_U, 1., m_faM[1], fields[1][i], m_faP[1], 0., m_dsU[i]);
         dg::blas1::pointwiseDot(-1., fields[1][i], m_dsU[i], 1., yp[1][i]); //-U ds U
         // force terms: -tau/mu * ds lnN -1/mu * ds Phi
-        ////m_ds_N.centered(-m_p.tau[i]/m_p.mu[i], m_logn[i], 1.0, yp[1][i]);
-        dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_dsN[i], fields[0][i], 1., yp[1][i]);
-        //m_ds_P.centered(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
-        m_ds_P.centered( m_phi[i], m_dsP[i]);
+        dg::geo::ds_forward( m_fa_N, 1., y[0][i], m_faP[0], 0., m_temp1);
+        dg::blas1::pointwiseDot( 1., m_interior, m_dsN[i], -1., m_interior, m_temp1, +1., m_temp1);
+
+        dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_temp1, fields[0][i], 1.0, yp[1][i]);
+        //dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_dsN[i], fields[0][i], 1., yp[1][i]);
+        //m_fa_P.centered(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
+        m_fa_P( dg::geo::einsMinus, m_phi[i], m_faM[0]); //overwrite
+        m_fa_P( dg::geo::einsPlus,  m_phi[i], m_faP[0]);
+        dg::geo::ds_forward( m_fa_P, 1., m_phi[i], m_faP[0], 0., m_dsP[i]);
+        dg::geo::ds_centered( m_fa_P, 1., m_faM[0], m_phi[i], m_faP[0], 0., m_temp1);
+        dg::blas1::pointwiseDot( 1., m_interior, m_temp1, -1., m_interior, m_dsP[i], +1., m_dsP[i]);
         dg::blas1::axpby(-1./m_p.mu[i], m_dsP[i], 1.0, yp[1][i]);
         // diffusion: + nu_par Delta_par U/N - nu_par U Delta_par N/ N
         dg::blas1::pointwiseDot(m_p.nu_parallel[i], m_divb, m_dsU[i],
                                 0., m_temp1);
-        m_ds_U.dss( fields[1][i], m_dssU[i]);
+        dg::geo::dss_centered( m_fa_U, 1., m_faM[1], fields[1][i], m_faP[1], 0., m_dssU[i]);
         dg::blas1::axpby( m_p.nu_parallel[i], m_dssU[i], 1., m_temp1); //nu_par Delta_par U
         //////dg::blas1::pointwiseDot( -1., fields[1][i], m_temp0, 1., m_temp1);
         dg::blas1::pointwiseDivide( 1., m_temp1, fields[0][i], 1., yp[1][i]);
@@ -885,12 +905,25 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     dg::Timer timer;
     double accu = 0.;//accumulated time
     timer.tic();
+
+#if FELTORPERP == 1
+
     // set m_phi[0]
     compute_phi( t, y[0]);
-
     // set m_phi[1], m_dP[0], m_dP[1] and m_UE2 --- needs m_phi[0]
     compute_psi( t);
 
+#else
+
+    dg::blas1::evaluate( m_phi[0], dg::equals(), manufactured::Phie{
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
+        m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
+    dg::blas1::evaluate( m_phi[1], dg::equals(), manufactured::Phii{
+        m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
+        m_p.beta,m_p.nu_perp,m_p.nu_parallel[0],m_p.nu_parallel[1]},m_R,m_Z,m_P,t);
+
+#endif
+
     timer.toc();
     accu += timer.diff();
     #ifdef MPI_VERSION
@@ -933,7 +966,9 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
 
     // Add parallel dynamics --- needs m_logn
 #if FELTORPARALLEL == 1
+
     compute_parallel( t, y, m_fields, yp);
+
 #endif
 
     //Add source terms
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 20450b51a..9be42e78e 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -267,7 +267,8 @@ int main( int argc, char* argv[])
             fixed_profile, profile, grid, p, mag);
         feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
             p.source_rate, dg::construct<DVec>(source_profile),
-            p.damping_rate, dg::construct<DVec>(damping_profile)
+            p.damping_rate, dg::construct<DVec>(damping_profile),
+            dg::construct<DVec>( feltor::detail::interior( grid, p, mag))
         );
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)"<<std::endl;
diff --git a/src/feltor/init.h b/src/feltor/init.h
index fc987da7c..4cbe7f35a 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -76,6 +76,21 @@ HVec profile_damping(const Geometry& grid,
         profile_damping, profile_damping);
     return profile_damping;
 }
+HVec interior(const Geometry& grid,
+    const feltor::Parameters& p,
+    const dg::geo::TokamakMagneticField& mag )
+{
+    if( p.profile_alpha == 0)
+        throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
+    HVec interior = dg::pullback( dg::compose(dg::PolynomialHeaviside(
+        1.+p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)), grid);
+    //HVec interior = dg::pullback( dg::compose(dg::Heaviside(
+    //    1., -1), dg::geo::RhoP(mag)), grid);
+    //HVec interior = dg::evaluate( dg::one, grid);
+    dg::blas1::pointwiseDot( xpoint_damping(grid,p,mag),
+        interior, interior);
+    return interior;
+}
 HVec profile(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::TokamakMagneticField& mag ){
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 27d7ac9fe..33dc56874 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -9,8 +9,8 @@
 #include "parameters.h"
 #define DG_MANUFACTURED
 //Change here to selectively test parallel and perp parts
-#define FELTORPARALLEL 0
-#define FELTORPERP 1
+#define FELTORPARALLEL 1
+#define FELTORPERP 0
 
 #include "manufactured.h"
 #include "feltor.h"
-- 
GitLab


From 1260b32d112b26f0bae5a5592504d88648ee6f61 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 28 Oct 2020 14:15:06 +0100
Subject: [PATCH 377/540] Make electron parallel diffusion same as ions

---
 inc/dg/dg_doc.h                  | 3 +++
 src/feltor/geometry/compass.json | 2 +-
 src/feltor/parameters.h          | 2 ++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 6170b1b14..5e63f4003 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -13,16 +13,19 @@
  *         time integrators.
  *     @{
  *         @defgroup blas1 BLAS level 1 routines: Vector-Vector
+ *
  *             Successive calls to blas routines are executed sequentially.
  *             A manual synchronization of threads or devices is never needed
  *             in an application using these functions. All functions returning
  *             a value block until the value is ready.
  *         @defgroup blas2 BLAS level 2 routines: Matrix-Vector
+ *
  *             Successive calls to blas routines are executed sequentially.
  *             A manual synchronization of threads or devices is never needed
  *             in an application using these functions. All functions returning
  *             a value block until the value is ready.
  *         @defgroup tensor Tensor-Vector operations
+ *
  *              Although a tensor needs a topology to be well-defined mathematically,
  *              we do not need a grid to perform basic operations computationally.
  *              This is why the tensor operations can appear already in Level 1
diff --git a/src/feltor/geometry/compass.json b/src/feltor/geometry/compass.json
index 79849bea8..eceb7036d 100644
--- a/src/feltor/geometry/compass.json
+++ b/src/feltor/geometry/compass.json
@@ -20,7 +20,7 @@
 	],
    	"equilibrium": "solovev",
     "description" : "standardX",
-	"inverseaspectratio": 0.3211009174311926,
+    "inverseaspectratio": 0.3211009174311926,
     "triangularity": 0.3,
     "elongation": 1.44
 }
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 38dab5585..93330db72 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -96,6 +96,8 @@ struct Parameters
         //Init after reading in eta and mu[0]
         nu_parallel[0] = 0.73/eta;
         nu_parallel[1] = sqrt(fabs(mu[0]))*1.36/eta;
+        //Make electron diffusion smaller to avoid numerical difficulties
+        nu_parallel[0] = nu_parallel[1];
 
         initne      = file::get( mode, js, "initne", "blob").asString();
         initphi     = file::get( mode, js, "initphi", "zero").asString();
-- 
GitLab


From 9a0f725e4d0400d3d97ede76ff2add06908d58fb Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 28 Oct 2020 14:52:55 +0100
Subject: [PATCH 378/540] Add turbulent bath on profile_influx

---
 src/feltor/init.h | 73 ++++++++++++++++++++++++++---------------------
 1 file changed, 40 insertions(+), 33 deletions(-)

diff --git a/src/feltor/init.h b/src/feltor/init.h
index 4cbe7f35a..89ee0519d 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -93,7 +93,8 @@ HVec interior(const Geometry& grid,
 }
 HVec profile(const Geometry& grid,
     const feltor::Parameters& p,
-    const dg::geo::TokamakMagneticField& mag ){
+    const dg::geo::TokamakMagneticField& mag )
+{
     double RO=mag.R0(), ZO=0.;
     dg::geo::findOpoint( mag.get_psip(), RO, ZO);
     double psipO = mag.psip()( RO, ZO);
@@ -117,6 +118,27 @@ HVec source_damping(const Geometry& grid,
            source_damping, source_damping);
     return source_damping;
 }
+HVec turbulent_bath(const Geometry& grid,
+    const feltor::Parameters& p,
+    const dg::geo::TokamakMagneticField& mag )
+{
+    HVec ntilde = dg::evaluate(dg::zero,grid);
+    if( p.sigma_z == 0)
+        throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in turbulence initial condition\n");
+    dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
+    dg::BathRZ init0(16,16,grid.x0(),grid.y0(), 30.,2.,p.amp);
+    if( p.symmetric)
+        ntilde = dg::pullback( init0, grid);
+    else
+    {
+        dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
+            fieldaligned( mag, grid, p.bcxN, p.bcyN,
+            dg::geo::NoLimiter(), p.rk4eps, 2, 2);
+        //evaluate should always be used with mx,my > 1 (but this takes more memory)
+        ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
+    }
+    return ntilde;
+}
 
 
 void init_ni(
@@ -151,6 +173,16 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
     dg::geo::TokamakMagneticField& mag )
 > > initial_conditions =
 {
+    { "zero",
+        []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
+            const Geometry& grid, const feltor::Parameters& p,
+            dg::geo::TokamakMagneticField& mag )
+        {
+            std::array<std::array<DVec,2>,2> y0;
+            y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(dg::evaluate( dg::zero, grid));
+            return y0;
+        }
+    },
     { "blob",
         []( Explicit<Geometry, IDMatrix, DMatrix, DVec>& f,
             const Geometry& grid, const feltor::Parameters& p,
@@ -225,21 +257,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             std::array<std::array<DVec,2>,2> y0;
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(detail::profile(grid,p,mag));
             dg::blas1::scal( y0, p.nprofamp );
-            HVec ntilde = dg::evaluate(dg::zero,grid);
-            if( p.sigma_z == 0)
-                throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in turbulence initial condition\n");
-            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-            dg::BathRZ init0(16,16,grid.x0(),grid.y0(), 30.,2.,p.amp);
-            if( p.symmetric)
-                ntilde = dg::pullback( init0, grid);
-            else
-            {
-                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                    dg::geo::NoLimiter(), p.rk4eps, 1, 1);
-                //For turbulence the exact evaluate is maybe not so important (thus takes less memory)
-                ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
-            }
+            HVec ntilde = detail::turbulent_bath(grid,p,mag);
             dg::blas1::pointwiseDot( detail::profile_damping(grid,p,mag), ntilde, ntilde);
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0]);
             detail::init_ni( y0, f,grid,p,mag);
@@ -282,21 +300,7 @@ std::map<std::string, std::function< std::array<std::array<DVec,2>,2>(
             y0[0][0] = y0[0][1] = y0[1][0] = y0[1][1] = dg::construct<DVec>(
                 dg::pullback( prof, grid) );
 
-            HVec ntilde = dg::evaluate(dg::zero,grid);
-            if( p.sigma_z == 0)
-                throw dg::Error(dg::Message()<< "Invalid parameter: sigma_z must not be 0 in turbulence on gaussian\n");
-            dg::GaussianZ gaussianZ( 0., p.sigma_z*M_PI, 1);
-            dg::BathRZ init0(16,16,grid.x0(),grid.y0(), 30.,2.,p.amp);
-            if( p.symmetric)
-                ntilde = dg::pullback( init0, grid);
-            else
-            {
-                dg::geo::Fieldaligned<Geometry, IHMatrix, HVec>
-                    fieldaligned( mag, grid, p.bcxN, p.bcyN,
-                    dg::geo::NoLimiter(), p.rk4eps, 1, 1);
-                //For turbulence the exact evaluate is maybe not so important (thus takes less memory)
-                ntilde = fieldaligned.evaluate( init0, gaussianZ, 0, 1);
-            }
+            HVec ntilde = detail::turbulent_bath(grid,p,mag);
             dg::blas1::axpby( 1., dg::construct<DVec>(ntilde), 1., y0[0][0] );
             dg::blas1::pointwiseDot( dg::construct<DVec>(detail::circular_damping(grid,p,mag)),
                 y0[0][0], y0[0][0] );
@@ -334,9 +338,12 @@ std::map<std::string, std::function< HVec(
         dg::geo::TokamakMagneticField& mag )
         {
             fixed_profile = false;
-            ne_profile = dg::construct<HVec>( detail::profile(grid, p,mag));
+            HVec source_profile = detail::profile( grid, p,mag);
+            ne_profile = source_profile;
             dg::blas1::scal( ne_profile, p.nprofamp );
-            HVec source_profile = dg::construct<HVec> ( detail::profile( grid, p,mag));
+            HVec ntilde = detail::turbulent_bath(grid,p,mag);
+            dg::blas1::pointwiseDot( detail::profile_damping(grid,p,mag), ntilde, ntilde);
+            dg::blas1::axpby( 1., ntilde, 1., source_profile);
             return source_profile;
         }
     },
-- 
GitLab


From 0711e4ec501a203b34ae35b59ecd017244ae3bb4 Mon Sep 17 00:00:00 2001
From: aslakp <aslakp@fysik.dtu.dk>
Date: Wed, 4 Nov 2020 12:25:04 +0100
Subject: [PATCH 379/540] Added chi weighted jump factor to the elliptic
 operator.

---
 inc/dg/Makefile        |  2 +-
 inc/dg/elliptic.h      | 89 +++++++++++++++++++++++++++++++++++-------
 inc/dg/elliptic2d_b.cu |  1 +
 inc/dg/elliptic_b.cu   |  2 +
 4 files changed, 78 insertions(+), 16 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 058deb17a..1d6b1a310 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -1,4 +1,4 @@
-device=gpu
+device=omp
 
 #configure machine
 include ../../config/default.mk
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 7f697746d..2ae60d42a 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -79,8 +79,8 @@ class Elliptic
      * @note chi is assumed 1 per default
      */
     Elliptic( const Geometry& g, norm no = not_normed,
-        direction dir = forward, value_type jfactor=1.):
-        Elliptic( g, g.bcx(), g.bcy(), no, dir, jfactor)
+        direction dir = forward, value_type jfactor=1., bool chi_weight_jump = false):
+        Elliptic( g, g.bcx(), g.bcy(), no, dir, jfactor, chi_weight_jump)
     {
     }
 
@@ -98,9 +98,10 @@ class Elliptic
      */
     Elliptic( const Geometry& g, bc bcx, bc bcy,
         norm no = not_normed, direction dir = forward,
-        value_type jfactor=1.)
+        value_type jfactor=1., bool chi_weight_jump = false)
     {
         m_no=no, m_jfactor=jfactor;
+        m_chi_weight_jump = chi_weight_jump;
         dg::blas2::transfer( dg::create::dx( g, inverse( bcx), inverse(dir)), m_leftx);
         dg::blas2::transfer( dg::create::dy( g, inverse( bcy), inverse(dir)), m_lefty);
         dg::blas2::transfer( dg::create::dx( g, bcx, dir), m_rightx);
@@ -192,6 +193,16 @@ class Elliptic
      * @return  The current scale factor for jump terms
      */
     value_type get_jfactor() const {return m_jfactor;}
+    /**
+     * @brief Set the chi weighting of jump terms
+     * @param jump_weighting Switch for weighting the jump factor with chi. Either true or false.
+     */
+    void set_jump_weighting( bool jump_weighting) {m_chi_weight_jump = jump_weighting;}
+    /**
+     * @brief Get the current state of chi weighted jump terms.
+     * @return Whether the weighting of jump terms with chi is enabled. Either true or false.
+     */
+    bool get_jump_weighting() const {return m_chi_weight_jump;}
     /**
      * @brief Compute elliptic term and store in output
      *
@@ -242,8 +253,19 @@ class Elliptic
         dg::blas2::symv( -1., m_leftx, m_tempx, -1., m_temp);
 
         //add jump terms
-        dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
-        dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        if(m_chi_weight_jump)
+        {
+            dg::blas2::symv( m_jfactor, m_jumpX, x, 0., m_tempx);
+            dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
+            dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
+            dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
+        } 
+        else
+        {
+            dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
+            dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        }
+        
         if( m_no == normed)
             dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
         if( m_no == not_normed)//multiply weights without volume
@@ -285,8 +307,18 @@ class Elliptic
         //add jump terms
         if( 0 != m_jfactor )
         {
-            dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
-            dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+            if(m_chi_weight_jump)
+            {
+                dg::blas2::symv( m_jfactor, m_jumpX, x, 0., m_tempx);
+                dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
+                dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
+                dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
+            } 
+            else
+            {   
+                dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
+                dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+            }
         }
         if( m_no == normed)
             dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
@@ -309,6 +341,7 @@ class Elliptic
     SparseTensor<Container> m_chi;
     Container m_sigma, m_vol;
     value_type m_jfactor;
+    bool m_chi_weight_jump;
 };
 
 ///@copydoc Elliptic
@@ -375,8 +408,8 @@ class Elliptic3d
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      * @note chi is assumed 1 per default
      */
-    Elliptic3d( const Geometry& g, norm no = not_normed, direction dir = forward, value_type jfactor=1.):
-        Elliptic3d( g, g.bcx(), g.bcy(), g.bcz(), no, dir, jfactor)
+    Elliptic3d( const Geometry& g, norm no = not_normed, direction dir = forward, value_type jfactor=1., bool chi_weight_jump = false):
+        Elliptic3d( g, g.bcx(), g.bcy(), g.bcz(), no, dir, jfactor, chi_weight_jump)
     {
     }
 
@@ -394,9 +427,10 @@ class Elliptic3d
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      * @note chi is the metric tensor multiplied by the volume element per default
      */
-    Elliptic3d( const Geometry& g, bc bcx, bc bcy, bc bcz, norm no = not_normed, direction dir = forward, value_type jfactor = 1.)
+    Elliptic3d( const Geometry& g, bc bcx, bc bcy, bc bcz, norm no = not_normed, direction dir = forward, value_type jfactor = 1., bool chi_weight_jump = false)
     {
         m_no=no, m_jfactor=jfactor;
+        m_chi_weight_jump = chi_weight_jump;
         dg::blas2::transfer( dg::create::dx( g, inverse( bcx), inverse(dir)), m_leftx);
         dg::blas2::transfer( dg::create::dy( g, inverse( bcy), inverse(dir)), m_lefty);
         dg::blas2::transfer( dg::create::dz( g, inverse( bcz), inverse(dg::centered)), m_leftz);
@@ -479,6 +513,10 @@ class Elliptic3d
     void set_jfactor( value_type new_jfactor) {m_jfactor = new_jfactor;}
     ///@copydoc Elliptic::get_jfactor()
     value_type get_jfactor() const {return m_jfactor;}
+    ///@copydoc Elliptic::set_jump_weighting()
+    void set_jump_weighting( bool jump_weighting) {m_chi_weight_jump = jump_weighting;}
+    ///@copydoc Elliptic::get_jump_weighting()
+    bool get_jump_weighting() const {return m_chi_weight_jump;}
 
     /**
      * @brief Restrict the problem to the first 2 dimensions
@@ -521,8 +559,18 @@ class Elliptic3d
         dg::blas2::symv( -1., m_leftx, m_tempx, 1., m_temp);
 
         //add jump terms
-        dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
-        dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        if(m_chi_weight_jump)
+        {
+            dg::blas2::symv( m_jfactor, m_jumpX, x, 0., m_tempx);
+            dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
+            dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
+            dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
+        } 
+        else
+        {
+            dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
+            dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+        }
         if( m_no == normed)
             dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
         if( m_no == not_normed)//multiply weights without volume
@@ -561,8 +609,18 @@ class Elliptic3d
         //add jump terms
         if( 0 != m_jfactor )
         {
-            dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
-            dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+            if(m_chi_weight_jump)
+            {
+                dg::blas2::symv( m_jfactor, m_jumpX, x, 0., m_tempx);
+                dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
+                dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
+                dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
+            } 
+            else
+            {   
+                dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
+                dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
+            }
         }
         if( m_no == normed)
             dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
@@ -578,7 +636,7 @@ class Elliptic3d
     void set_norm( dg::norm new_norm) {
         m_no = new_norm;
     }
-
+    
     private:
     Matrix m_leftx, m_lefty, m_leftz, m_rightx, m_righty, m_rightz, m_jumpX, m_jumpY;
     Container m_weights, m_inv_weights, m_precond, m_weights_wo_vol;
@@ -588,6 +646,7 @@ class Elliptic3d
     Container m_sigma, m_vol;
     value_type m_jfactor;
     bool m_multiplyZ = true;
+    bool m_chi_weight_jump;
 };
 ///@cond
 template< class G, class M, class V>
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 31574a713..25b469d80 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -88,6 +88,7 @@ int main()
     {
         multi_pol[u].construct( multigrid.grid(u), dg::not_normed, dg::centered, jfactor);
         multi_pol[u].set_chi( multi_chi[u]);
+        multi_pol[u].set_jump_weighting(true);
     }
 
     t.toc();
diff --git a/inc/dg/elliptic_b.cu b/inc/dg/elliptic_b.cu
index 7a9964165..db72d32c7 100644
--- a/inc/dg/elliptic_b.cu
+++ b/inc/dg/elliptic_b.cu
@@ -47,6 +47,8 @@ int main()
 
     dg::Elliptic3d<dg::aGeometry3d, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
 
+    laplace.set_jump_weighting(true);
+    
     dg::CG< dg::DVec > pcg( x, n*n*Nx*Ny*Nz);
 
     const dg::DVec solution = dg::evaluate ( fct, grid);
-- 
GitLab


From 29d7de4b502f1a7a3107779783f055d54d19df01 Mon Sep 17 00:00:00 2001
From: aslakp <aslakp@fysik.dtu.dk>
Date: Wed, 4 Nov 2020 21:36:12 +0100
Subject: [PATCH 380/540] Added documentation for the chi weighting. Also, in
 the test, the weighting is now and input option.

---
 inc/dg/elliptic.h      | 8 +++++---
 inc/dg/elliptic2d_b.cu | 5 ++++-
 inc/dg/elliptic_b.cu   | 5 ++++-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 2ae60d42a..e399ffa98 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -35,14 +35,15 @@ namespace dg
  you like (in order for the operator to be invertible \f$\chi\f$ should be
  symmetric and positive definite though).
  Note that the local discontinuous Galerkin discretization adds so-called jump terms
- \f[ D^\dagger \chi D + \alpha J \f]
+ \f[ D^\dagger \chi D + \alpha \chi_{on/off} J \f]
  where \f$\alpha\f$  is a scale factor ( = jfactor), \f$ D \f$ contains the discretizations of the above derivatives, and \f$ J\f$ is a self-adjoint matrix.
  (The symmetric part of \f$J\f$ is added @b before the volume element is divided). The adjoint of a matrix is defined with respect to the volume element including dG weights.
  Usually, the default \f$ \alpha=1 \f$ is a good choice.
  However, in some cases, e.g. when \f$ \chi \f$ exhibits very large variations
- \f$ \alpha=0.1\f$ or \f$ \alpha=0.01\f$ might be better values.
+ \f$ \alpha=0.1\f$ or \f$ \alpha=0.01\f$ might be better values. 
  In a time dependent problem the value of \f$\alpha\f$ determines the
- numerical diffusion, i.e. for too low values numerical oscillations may appear.
+ numerical diffusion, i.e. for too low values numerical oscillations may appear. 
+ The \f$ \chi_{on/off} \f$ in the jump term serves to weight the jump term with \f$ \chi \f$. This can be switched either on or off with off being the default.
  Also note that a forward discretization has more diffusion than a centered discretization.
 
  The following code snippet demonstrates the use of \c Elliptic in an inversion problem
@@ -373,6 +374,7 @@ using Elliptic2d = Elliptic<Geometry, Matrix, Container>;
  \f$ \alpha=0.1\f$ or \f$ \alpha=0.01\f$ might be better values.
  In a time dependent problem the value of \f$\alpha\f$ determines the
  numerical diffusion, i.e. for too low values numerical oscillations may appear.
+ The \f$ \chi_{on/off} \f$ in the jump term serves to weight the jump term with \f$ \chi \f$. This can be switched either on or off with off being the default.
  Also note that a forward discretization has more diffusion than a centered discretization.
 
  The following code snippet demonstrates the use of \c Elliptic in an inversion problem
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 25b469d80..3d9aef5c8 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -44,6 +44,9 @@ int main()
 	/*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
     std::cin >> n >> Nx >> Ny; //more N means less iterations for same error
     std::cin >> eps >> jfactor;*/
+    bool jump_weight;
+    std::cout << "Jump weighting on or off? Type 1 for true or 0 for false: \n";
+    std::cin >> jump_weight;
     std::cout << "Computation on: "<< n <<" x "<< Nx <<" x "<< Ny << std::endl;
     //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
 
@@ -88,7 +91,7 @@ int main()
     {
         multi_pol[u].construct( multigrid.grid(u), dg::not_normed, dg::centered, jfactor);
         multi_pol[u].set_chi( multi_chi[u]);
-        multi_pol[u].set_jump_weighting(true);
+        multi_pol[u].set_jump_weighting(jump_weight);
     }
 
     t.toc();
diff --git a/inc/dg/elliptic_b.cu b/inc/dg/elliptic_b.cu
index db72d32c7..9cf9389eb 100644
--- a/inc/dg/elliptic_b.cu
+++ b/inc/dg/elliptic_b.cu
@@ -37,6 +37,9 @@ int main()
     double eps;
     std::cout << "Type epsilon! \n";
     std::cin >> eps;
+    bool jump_weight;
+    std::cout << "Jump weighting on or off? Type 1 for true or 0 for false: \n";
+    std::cin >> jump_weight;
     std::cout << "TEST CYLINDRICAL LAPLACIAN\n";
     //std::cout << "Create Laplacian\n";
     //! [invert]
@@ -47,7 +50,7 @@ int main()
 
     dg::Elliptic3d<dg::aGeometry3d, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
 
-    laplace.set_jump_weighting(true);
+    laplace.set_jump_weighting(jump_weight);
     
     dg::CG< dg::DVec > pcg( x, n*n*Nx*Ny*Nz);
 
-- 
GitLab


From fbcd9b5f274d633a3442a5bbcb64ab67e583eb98 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 30 Oct 2020 11:55:34 +0100
Subject: [PATCH 381/540] Add boundary values to along_field functions

---
 inc/geometries/ds.h | 52 +++++++++++++++++++++++++--------------------
 1 file changed, 29 insertions(+), 23 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 370c3cd84..31fee8ef9 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -100,7 +100,7 @@ struct ComputeDSCentered{
     double m_alpha, m_beta;
 };
 struct ComputeDSCenteredNEU{
-    ComputeDSCenteredNEU( double alpha, double beta):m_alpha(alpha), m_beta(beta), m_ds(1.,0.){}
+    ComputeDSCenteredNEU( double alpha, double beta, std::array<double,2> b_value):m_alpha(alpha), m_beta(beta), m_bm(b_value[0]), m_bp( b_value[1]),  m_ds(1.,0.){}
     DG_DEVICE
     void operator()( double& dsf, double fm, double fo, double fp,
             double hm, double hp, double hbm, double hbp,
@@ -108,33 +108,35 @@ struct ComputeDSCenteredNEU{
     {
         double inner=0, plus=0, minus=0, both=0;
         m_ds( inner, fm, fo, fp, hm, hp);
-        plus  = ( 1./hm - 1./( 2.*hbp + hm))*(fo-fm);
-        minus = ( 1./hp - 1./( 2.*hbm + hp))*(fp-fm);
-        both = 0.;
+        plus  = ( 1./hm - 1./( 2.*hbp + hm))*(fo-fm) + m_bp * hm /(2.*hbp + hm);
+        minus = ( 1./hp - 1./( 2.*hbm + hp))*(fp-fm) + m_bm * hp /(2.*hbm + hp);
+        both = (m_bp*hbm+hbp*m_bm)/(hbp+hbm);
         dsf = m_alpha*((1.-bpm-bpo-bpp)*inner + bpp*plus + bpm*minus + bpo*both)
             + m_beta*dsf;
     }
     private:
     double m_alpha, m_beta;
+    double m_bm, m_bp;
     ComputeDSCentered m_ds;
 };
 struct ComputeDSCenteredDIR{
-    ComputeDSCenteredDIR( double alpha, double beta):m_alpha(alpha), m_beta(beta), m_ds(1.,0.){}
+    ComputeDSCenteredDIR( double alpha, double beta, std::array<double,2> b_value):m_alpha(alpha), m_beta(beta), m_bm(b_value[0]), m_bp( b_value[1]),  m_ds(1.,0.){}
     DG_DEVICE
     void operator()( double& dsf, double fm, double fo, double fp,
             double hm, double hp, double hbm, double hbp,
             double bpm, double bpo, double bpp)
     {
         double inner=0, plus=0, minus=0, both=0;
-        m_ds( inner, fm, fo, fp, hm, hp);
-        m_ds( plus, fm, fo, 0., hm, hbp);
-        m_ds( minus, 0., fo, fp, hbm, hp);
-        m_ds( both, 0., fo, 0., hbm, hbp);
+        m_ds( inner, fm,   fo, fp,   hm,  hp);
+        m_ds( plus,  fm,   fo, m_bp, hm,  hbp);
+        m_ds( minus, m_bm, fo, fp,   hbm, hp);
+        m_ds( both,  m_bm, fo, m_bp, hbm, hbp);
         dsf = m_alpha*((1.-bpm-bpo-bpp)*inner + bpp*plus + bpm*minus + bpo*both)
             + m_beta*dsf;
     }
     private:
     double m_alpha, m_beta;
+    double m_bm, m_bp;
     ComputeDSCentered m_ds;
 };
 struct ComputeDSS{
@@ -152,7 +154,7 @@ struct ComputeDSS{
     double m_alpha, m_beta;
 };
 struct ComputeDSSNEU{
-    ComputeDSSNEU( double alpha, double beta):m_alpha(alpha), m_beta(beta), m_dss(1.,0.){}
+    ComputeDSSNEU( double alpha, double beta, std::array<double,2> b_value):m_alpha(alpha), m_beta(beta), m_bm(b_value[0]), m_bp( b_value[1]),  m_dss(1.,0.){}
     DG_DEVICE
     void operator()( double& dssf, double fm, double fo, double fp,
             double hm, double hp, double hbm, double hbp,
@@ -160,18 +162,20 @@ struct ComputeDSSNEU{
     {
         double inner=0, plus=0, minus=0, both=0;
         m_dss( inner, fm, fo, fp, hm, hp);
-        plus  = -2.*(fo-fm)/( 2.*hbp + hm)/hm;
-        minus =  2.*(fp-fo)/( 2.*hbm + hp)/hp;
-        both = 0.;
+        plus  =  2./( 2.*hbp + hm)*( m_bp - (fo-fm)/hm );
+        minus =  2./( 2.*hbm + hp)*( (fp-fo)/hp - m_bm );
+        both = (m_bp-m_bm)/(hbp+hbm);
         dssf = m_alpha*((1.-bpm-bpo-bpp)*inner + bpp*plus + bpm*minus + bpo*both)
             + m_beta*dssf;
     }
     private:
     double m_alpha, m_beta;
+    double m_bm, m_bp;
     ComputeDSS m_dss;
 };
+
 struct ComputeDSSDIR{
-    ComputeDSSDIR( double alpha, double beta):m_alpha(alpha), m_beta(beta), m_dss(1.,0.){}
+    ComputeDSSDIR( double alpha, double beta, std::array<double,2> b_value):m_alpha(alpha), m_beta(beta), m_bm(b_value[0]), m_bp( b_value[1]),  m_dss(1.,0.){}
     DG_DEVICE
     void operator()( double& dssf, double fm, double fo, double fp,
             double hm, double hp, double hbm, double hbp,
@@ -179,14 +183,15 @@ struct ComputeDSSDIR{
     {
         double inner=0, plus=0, minus=0, both=0;
         m_dss( inner, fm, fo, fp, hm, hp);
-        m_dss( plus, fm, fo, 0., hm, hbp);
-        m_dss( minus, 0., fo, fp, hbm, hp);
-        m_dss( both, 0., fo, 0., hbm, hbp);
+        m_dss( plus, fm, fo, m_bp, hm, hbp);
+        m_dss( minus, m_bm, fo, fp, hbm, hp);
+        m_dss( both, m_bm, fo, m_bp, hbm, hbp);
         dssf = m_alpha*((1.-bpm-bpo-bpp)*inner + bpp*plus + bpm*minus + bpo*both)
             +m_beta*dssf;
     }
     private:
     double m_alpha, m_beta;
+    double m_bm, m_bp;
     ComputeDSS m_dss;
 };
 
@@ -728,20 +733,20 @@ void ds_backward( const FieldAligned& fa, double alpha, const container& fm, con
 template<class FieldAligned, class container>
 void ds_centered( const FieldAligned& fa, double alpha, const container& fm,
         const container& f, const container& fp, double beta, container& g,
-        dg::geo::boundary bound_mode = dg::geo::boundary::perp)
+        dg::geo::boundary bound_mode = dg::geo::boundary::perp, std::array<double,2> boundary_value = {0,0})
 {
     //direct discretisation
     if( boundary::along_field == bound_mode
             && fa.bcx() == dg::NEU && fa.bcy() == dg::NEU)
     {
-        dg::blas1::subroutine( detail::ComputeDSCenteredNEU( alpha, beta),
+        dg::blas1::subroutine( detail::ComputeDSCenteredNEU( alpha, beta, boundary_value),
                 g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
                 fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
     else if( boundary::along_field == bound_mode
             && fa.bcx() == dg::DIR && fa.bcy() == dg::DIR)
     {
-        dg::blas1::subroutine( detail::ComputeDSCenteredDIR( alpha, beta),
+        dg::blas1::subroutine( detail::ComputeDSCenteredDIR( alpha, beta, boundary_value),
                 g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
                 fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
@@ -766,19 +771,20 @@ void ds_centered( const FieldAligned& fa, double alpha, const container& fm,
 template<class FieldAligned, class container>
 void dss_centered( const FieldAligned& fa, double alpha, const container&
         fm,const container& f, const container& fp, double beta, container&
-        g, dg::geo::boundary bound_mode = dg::geo::boundary::perp)
+        g, dg::geo::boundary bound_mode = dg::geo::boundary::perp,
+        std::array<double,2> boundary_value = {0,0})
 {
     if( boundary::along_field == bound_mode
             && fa.bcx() == dg::NEU && fa.bcy() == dg::NEU)
     {
-        dg::blas1::subroutine( detail::ComputeDSSNEU( alpha, beta),
+        dg::blas1::subroutine( detail::ComputeDSSNEU( alpha, beta, boundary_value),
                 g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
                 fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
     else if( boundary::along_field == bound_mode
             && fa.bcx() == dg::DIR && fa.bcy() == dg::DIR)
     {
-        dg::blas1::subroutine( detail::ComputeDSSDIR( alpha, beta),
+        dg::blas1::subroutine( detail::ComputeDSSDIR( alpha, beta, boundary_value),
                 g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
                 fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
-- 
GitLab


From 81ef560965f124e8f89163668cc060373be348e6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 2 Nov 2020 17:23:34 +0100
Subject: [PATCH 382/540] Edit dg and geometries modules pages

---
 inc/dg/dg_doc.h                 | 16 +++++++++-------
 inc/geometries/geometries_doc.h | 11 ++++-------
 2 files changed, 13 insertions(+), 14 deletions(-)

diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 5e63f4003..77e7eae3b 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -32,10 +32,10 @@
  *              of this library.
  *     @}
  *     @defgroup typedefs Useful Typedefs
- *          Useful type definitions for easy programming
  *     @defgroup sparsematrix Sparse matrix formats
  *     @defgroup view Vector view
  *     @defgroup mpi_structures MPI backend
+ *
  *             In this section the blas functions are implemented for the MPI+X
  *             hardware architectures, where X is e.g. CPU, GPU, accelerator
  *             cards...
@@ -45,10 +45,10 @@
  *             the chapter \ref mpi_backend in the introduction for more
  *             details.
  *     @defgroup dispatch The tag dispatch system
- *           Please read the chapter \ref dispatch in the introduction.
+ *           Read the chapter \ref dispatch in the introduction.
  * @}
  * @defgroup numerical0 Level 2: Basic numerical algorithms
- * These algorithms make use only of blas level 1 and 2 functions
+ *      Algorithms that make use only of blas level 1 and 2 functions
  * @{
  *     @defgroup time Time integrators
  *     @defgroup invert Linear and nonlinear solvers
@@ -58,8 +58,8 @@
  * @{
  *     @defgroup grid Topological grids and operations
  *
- *     Objects that store topological information (which point is neighbour of
- *     which other point) about the grid.
+ *          Objects that store topological information (which point is neighbour of
+ *          which other point) about the grid.
  *     @{
  *         @defgroup basictopology Topology base classes
  *         @defgroup evaluation evaluate
@@ -73,6 +73,7 @@
  *             The first elements of the resulting vector lie in the cell at (x0,y0) and the last
  *             in (x1, y1).
  *         @defgroup highlevel create weights
+ *
  *              overloads for the \c dg::create::weights and \c dg::create::inv_weights functions for all
  *              available topologies
  *         @defgroup creation create derivatives
@@ -91,13 +92,13 @@
  *         @defgroup pullback pullback and pushforward
  *         @defgroup metric create volume
  *         @defgroup generators Grid Generator classes
- *            The classes to perform field line integration for DS and averaging classes
  *     @}
  *     @defgroup gridtypes Useful Typedefs
  * @}
  * @defgroup numerical1 Level 4: Advanced numerical schemes
  *
- * These routines make use of both the basic operations as well as the interfaces defined in the Geometry section.
+ *      These routines make use of both the basic operations as well as the
+ *      interfaces defined in the Geometry section.
  * @{
  *     @defgroup arakawa Discretization of Poisson bracket
  *     @defgroup matrixoperators Elliptic and Helmholtz operators
@@ -112,6 +113,7 @@
  *         The functors are useful for either vector transformations or
  *         as init functions in the evaluate routines.
  *     @defgroup lowlevel Lowlevel helper functions and classes
+ *
  *         Low level helper routines.
  * @}
  *
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index 86c98641e..4c0398c0e 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -3,12 +3,13 @@
  *
  * @defgroup generators_geo 1. Grid generators
  *
-      All the grids introduced by this extension can be constructed with
-      generator classes.
+ *      All the grids introduced by this extension can be constructed with
+ *      generator classes.
  * @defgroup grids 2. New geometric grids
  * @defgroup fluxfunctions 3. New functors based on the magnetic field geometry
 
- All functors in this section model two or three-dimensional functions, i.e. they all overload the operator() like \c aCylindricalFunctor
+        All functors in this section model two or three-dimensional functions, i.e.
+        they all overload the operator() like \c aCylindricalFunctor
  * @{
       @defgroup geom 3.1 New flux functions and derivatives
       @{
@@ -24,10 +25,6 @@
  * @}
  * @defgroup fieldaligned 4. Fieldaligned derivatives
  * @defgroup misc_geo 5. Miscellaneous additions
- *
- * Objects that are used to define and integrate the magnetic field lines.
- * All objects can be used in the \c evaluation and \c pullback functions.
- *
  */
 /*! @mainpage
  * This extension adds new features to the FELTOR core dg library.
-- 
GitLab


From 74c49c0cae00b289aaacb43628b92517ec3258b2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 2 Nov 2020 17:24:49 +0100
Subject: [PATCH 383/540] Add and test WallDistance and WallDirection

these should form the basis of our sheath boundary condition
functionality
---
 inc/dg/functors.h               | 33 ++++++++++++++++++++++++++
 inc/geometries/geometry_diag.cu |  3 +++
 inc/geometries/magnetic_field.h | 41 +++++++++++++++++++++++++++++++++
 3 files changed, 77 insertions(+)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 818033e0c..f11c341d8 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -1932,6 +1932,39 @@ struct ISNAN
 #endif
 };
 
+
+/**
+ * @brief Shortest Distance to a collection of vertical and horizontal lines
+ *
+ * First determine which line is closest to given point and then determine
+ * the exact distance to it
+ */
+struct WallDistance
+{
+    /**
+     * @brief Allocate lines
+     *
+     * @param vertical walls R_0, R_1 ...  ( can be arbitrary size)
+     * @param horizonal walls Z_0, Z_1 ... ( can be arbitrary size)
+     */
+    WallDistance( std::vector<double> vertical, std::vector<double> horizontal) :
+        m_vertical(vertical), m_horizontal( horizontal) {}
+    /**
+     * @brief Distance to closest wall in a box
+     */
+    double operator() (double R, double Z) const
+    {
+        std::vector<double> dist( 1, 1e100); //fill in at least one (large) number in case vectors are empty)
+        for( auto v : m_vertical)
+            dist.push_back(fabs( R-v));
+        for( auto h : m_horizontal)
+            dist.push_back(fabs( Z-h));
+        return *std::min_element( dist.begin(), dist.end());
+    }
+    private:
+    std::vector<double> m_vertical;
+    std::vector<double> m_horizontal;
+};
 ///@cond
 namespace detail
 {
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index e3b5a5124..75df7f3a3 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -198,6 +198,9 @@ int main( int argc, char* argv[])
         {"CurvatureKappaGradPsip", "(Toroidal) Kappa curvature dot the gradient of Psip", dg::geo::ScalarProduct( dg::geo::createCurvatureKappa(mag, +1), dg::geo::createGradPsip(mag))},
         {"TrueCurvatureNablaBGradPsip", "True Nabla B curvature dot the gradient of Psip", dg::geo::ScalarProduct( dg::geo::createTrueCurvatureNablaB(mag), dg::geo::createGradPsip(mag))},
         {"TrueCurvatureKappaGradPsip", "True Kappa curvature dot the gradient of Psip", dg::geo::ScalarProduct( dg::geo::createTrueCurvatureKappa(mag), dg::geo::createGradPsip(mag))},
+        /////////////////////////////////////
+        {"WallDistance", "Distance to closest wall", dg::geo::CylindricalFunctor( dg::WallDistance( {grid2d.x0(), grid2d.x1()}, {grid2d.y0(), grid2d.y1()})) },
+        {"WallDirection", "Direction of magnetic field to closest wall", dg::geo::WallDirection(mag, {grid2d.x0(), grid2d.x1()}, {grid2d.y0(), grid2d.y1()}) },
         //////////////////////////////////
         {"Iris", "A flux aligned Iris", dg::compose( dg::Iris( 0.5, 0.7), dg::geo::RhoP(mag))},
         {"Pupil", "A flux aligned Pupil", dg::compose( dg::Pupil(0.7), dg::geo::RhoP(mag)) },
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 3f9fc4795..ed2dc4299 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -865,6 +865,47 @@ struct Hoo : public dg::geo::aCylindricalFunctor<Hoo>
     private:
     dg::geo::TokamakMagneticField m_mag;
 };
+
+///@brief Determine if field points towards or away from the nearest wall
+struct WallDirection : public dg::geo::aCylindricalFunctor<WallDirection>
+{
+    /**
+     * @brief Allocate lines
+     *
+     * @param mag Use to construct magnetic field
+     * @param vertical walls R_0, R_1 ...  ( can be arbitrary size)
+     * @param horizonal walls Z_0, Z_1 ... ( can be arbitrary size)
+     */
+    WallDirection( dg::geo::TokamakMagneticField mag, std::vector<double>
+            vertical, std::vector<double> horizontal) : m_vertical(vertical),
+        m_horizontal(horizontal), m_BR( mag), m_BZ(mag){}
+    double do_compute ( double R, double Z) const
+    {
+        std::vector<double> v_dist(1,1e100), h_dist(1,1e100);
+        for( auto v : m_vertical)
+            v_dist.push_back( R-v );
+        for( auto h : m_horizontal)
+            h_dist.push_back( Z-h );
+        double v_min = *std::min_element( v_dist.begin(), v_dist.end(),
+                [](double a, double b){ return fabs(a) < fabs(b);} );
+        double h_min = *std::min_element( h_dist.begin(), h_dist.end(),
+                [](double a, double b){ return fabs(a) < fabs(b);} );
+        if( fabs(v_min) < fabs(h_min) ) // if vertical R wall is closer
+        {
+            double br = m_BR( R,Z);
+            return v_min*br < 0 ? +1 : -1;
+        }
+        else //horizontal Z wall is closer
+        {
+            double bz = m_BZ( R,Z);
+            return h_min*bz < 0 ? +1 : -1;
+        }
+    }
+    private:
+    std::vector<double> m_vertical, m_horizontal;
+    dg::geo::BFieldR m_BR;
+    dg::geo::BFieldZ m_BZ;
+};
 ///@}
 
 } //namespace geo
-- 
GitLab


From 4522d1eb3e8434a6d091e53cea864f63d498eb78 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 2 Nov 2020 20:16:56 +0100
Subject: [PATCH 384/540] Rename Combine to SetUnion

and factor out transform in make_field function
---
 inc/geometries/make_field.h | 52 ++++++++++++++++---------------------
 inc/geometries/modified.h   | 18 ++++++++-----
 2 files changed, 34 insertions(+), 36 deletions(-)

diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
index f8985f9a3..10d3048d8 100644
--- a/inc/geometries/make_field.h
+++ b/inc/geometries/make_field.h
@@ -60,6 +60,19 @@ static inline TokamakMagneticField createMagneticField( Json::Value js, file::er
     }
 }
 
+
+void transform_psi( TokamakMagneticField mag, double psipO, double& psi0, double& alpha0, double& sign0)
+{
+    double RO=mag.R0(), ZO=0.;
+    dg::geo::findOpoint( mag.get_psip(), RO, ZO);
+    double psipO = mag.psip()( RO, ZO);
+    double damping_psi0p = (1.-psi0*psi0)*psipO;
+    double damping_alpha0p = -(2.*psi0+alpha0)*alpha0*psipO;
+    psi0 = damping_psi0p + sign0*damping_alpha0p/2.;
+    alpha0 = fabs( damping_alpha0p/2.);
+    sign0 = sign0*((psipO>0)-(psipO<0));
+}
+
 /**
  * @brief Modify Magnetic Field above or below certain Psi values according to given parameters
  *
@@ -106,19 +119,9 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
         {
             double psi0 = file::get( mode, jsmod, "damping", "boundary", 1.1 ).asDouble();
             double alpha = file::get( mode, jsmod, "damping", "alpha", 0.2 ).asDouble();
-            double sign = -1;
+            double sign = +1;
             if( desc == description::standardX)
-            {
-                double RO=mag.R0(), ZO=0.;
-                dg::geo::findOpoint( mag.get_psip(), RO, ZO);
-                double psipO = mag.psip()( RO, ZO);
-                double damping_psi0p = (1.-psi0*psi0)*psipO;
-                double damping_alphap = -(2.*psi0+alpha)*alpha*psipO;
-                //std::cout<< " damping "<< damping_psi0p << " "<<damping_alphap<<"\n";
-                psi0 = damping_psi0p + damping_alphap/2.;
-                alpha = fabs( damping_alphap/2.);
-                sign = ((psipO>0)-(psipO<0));
-            }
+                transform_psi( psipO, psi0, alpha, sign);
             else
                 sign = file::get( mode, jsmod, "damping", "sign", -1. ).asDouble();
 
@@ -141,20 +144,9 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
                     double RX = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
                     double ZX = -1.1*mag.params().elongation()*mag.params().a();
                     dg::geo::findXpoint( mag.get_psip(), RX, ZX);
-                    //and the X-point
-                    double RO=mag.R0(), ZO=0.;
-                    dg::geo::findOpoint( mag.get_psip(), RO, ZO);
-                    double psipO = mag.psip()( RO, ZO);
-                    double damping_psi0p = (1.-psi0*psi0)*psipO;
-                    double damping_alpha0p = -(2.*psi0+alpha0)*alpha0*psipO;
-                    double damping_psi1p = (1.-psi1*psi1)*psipO;
-                    double damping_alpha1p = -(2.*psi1+alpha1)*alpha1*psipO;
-                    psi0 = damping_psi0p + damping_alpha0p/2.;
-                    psi1 = damping_psi1p - damping_alpha1p/2.;
-                    alpha0 = fabs( damping_alpha0p/2.);
-                    alpha1 = fabs( damping_alpha1p/2.);
-                    double sign0 = ((psipO>0)-(psipO<0));
-                    double sign1 = -sign0;
+                    double sign0 = +1., sign1 = -1.;
+                    transform_psi( psipO, psi0, alpha0, sign0);
+                    transform_psi( psipO, psi1, alpha1, sign1);
                     mod0_psip = mod::createPsip(
                             mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
                     mod_psip = mod::createPsip(
@@ -163,8 +155,8 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
                     CylindricalFunctor transition0 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
                     CylindricalFunctor damping1 = mod::DampingRegion( mod::HeavisideZ(ZX, -1), mag.psip(), psi1, alpha1, -sign1);
                     CylindricalFunctor transition1 = mod::MagneticTransition( mod::HeavisideZ(ZX, -1), mag.psip(), psi1, alpha1, sign1);
-                    damping = mod::Combine( damping0, damping1);
-                    transition = mod::Combine( transition0, transition1);
+                    damping = mod::SetUnion( damping0, damping1);
+                    transition = mod::SetUnion( transition0, transition1);
                     break;
                 }
                 default:
@@ -179,8 +171,8 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
                     CylindricalFunctor transition0 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
                     CylindricalFunctor damping1 = mod::DampingRegion( mod::everywhere, mag.psip(), psi1, alpha1, sign1);
                     CylindricalFunctor transition1 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi1, alpha1, sign1);
-                    damping = mod::Combine( damping0, damping1);
-                    transition = mod::Combine( transition0, transition1);
+                    damping = mod::SetUnion( damping0, damping1);
+                    transition = mod::SetUnion( transition0, transition1);
                     break;
                 }
             }
diff --git a/inc/geometries/modified.h b/inc/geometries/modified.h
index 809461052..db6c6c0dd 100644
--- a/inc/geometries/modified.h
+++ b/inc/geometries/modified.h
@@ -166,9 +166,8 @@ struct DampingRegion : public aCylindricalFunctor<DampingRegion>
     { }
     double do_compute(double R, double Z) const
     {
-        double psip = m_psip(R,Z);
         if( m_pred( R,Z))
-            return m_poly( psip);
+            return m_poly( m_psip(R,Z));
         else
             return 0;
     }
@@ -195,15 +194,22 @@ struct MagneticTransition : public aCylindricalFunctor<MagneticTransition>
     std::function<double(double,double)> m_psip;
     std::function<bool(double,double)> m_pred;
 };
-//combine damping and transition regions by adding them up
-struct Combine : public aCylindricalFunctor<Combine>
+
+/**
+ * @brief \f$ f_1 + f_2 - f_1 f_2 \f$
+ *
+ * If f_1 and f_2 are functions between 0 and 1 this operation
+ * represents the union of two masking regions
+ */
+struct SetUnion : public aCylindricalFunctor<SetUnion>
 {
-    Combine( std::function<double(double,double)> fct1, std::function<double(double,double)> fct2) :
+    SetUnion( std::function<double(double,double)> fct1, std::function<double(double,double)> fct2) :
         m_fct1(fct1), m_fct2(fct2)
     { }
     double do_compute(double R, double Z) const
     {
-        return m_fct1(R,Z)+m_fct2(R,Z);
+        double f1 = m_fct1(R,Z), f2 = m_fct2( R,Z);
+        return f1 + f2 - f1*f2;
     }
     private:
     std::function<double(double,double)> m_fct1, m_fct2;
-- 
GitLab


From 3a943c2a1c52607413ad285ba85c6aaef95a16bd Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 3 Nov 2020 19:39:49 +0100
Subject: [PATCH 385/540] Add SetIntersection and SetNot operations

---
 inc/geometries/modified.h | 39 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/inc/geometries/modified.h b/inc/geometries/modified.h
index db6c6c0dd..14e90a07a 100644
--- a/inc/geometries/modified.h
+++ b/inc/geometries/modified.h
@@ -196,7 +196,7 @@ struct MagneticTransition : public aCylindricalFunctor<MagneticTransition>
 };
 
 /**
- * @brief \f$ f_1 + f_2 - f_1 f_2 \f$
+ * @brief \f$ f_1 + f_2 - f_1 f_2 \equiv f_1 \cup f_2\f$
  *
  * If f_1 and f_2 are functions between 0 and 1 this operation
  * represents the union of two masking regions
@@ -214,6 +214,43 @@ struct SetUnion : public aCylindricalFunctor<SetUnion>
     private:
     std::function<double(double,double)> m_fct1, m_fct2;
 };
+/**
+ * @brief \f$ f_1 f_2 \equiv f_1 \cap f_2\f$
+ *
+ * If f_1 and f_2 are functions between 0 and 1 this operation
+ * represents the intersection of two masking regions
+ */
+struct SetIntersection : public aCylindricalFunctor<SetIntersection>
+{
+    SetIntersection( std::function<double(double,double)> fct1, std::function<double(double,double)> fct2) :
+        m_fct1(fct1), m_fct2(fct2)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        double f1 = m_fct1(R,Z), f2 = m_fct2( R,Z);
+        return f1*f2;
+    }
+    private:
+    std::function<double(double,double)> m_fct1, m_fct2;
+};
+/**
+ * @brief \f$ 1-f \equiv \bar f\f$
+ *
+ * If f is a function between 0 and 1 this operation
+ * represents the negation of the masking region
+ */
+struct SetNot : public aCylindricalFunctor<SetNot>
+{
+    SetNot( std::function<double(double,double)> fct) :
+        m_fct(fct)
+    { }
+    double do_compute(double R, double Z) const
+    {
+        return 1-m_fct(R,Z);
+    }
+    private:
+    std::function<double(double,double)> m_fct;
+};
 
 //some possible predicates
 
-- 
GitLab


From 8e71487c270705c57328f518fc7b3847e8ece049 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 3 Nov 2020 19:40:24 +0100
Subject: [PATCH 386/540] Add doubleX descriptor

---
 inc/geometries/magnetic_field.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index ed2dc4299..10e14926f 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -43,6 +43,7 @@ enum class description
 {
     standardO, //!< closed flux surfaces centered around an O-point located near (R_0, 0); flux-aligned grids can be constructed
     standardX, //!< closed flux surfaces centered around an O-point located near (R_0, 0) and bordered by a separatrix with a single X-point; flux-aligned X-grids can be constructed
+    doubleX, //!< closed flux surfaces centered around an O-point located near (R_0, 0) and bordered by a separatrix with two X-points; flux-aligned X-grids cannnot be constructed
     none, //!< no shaping: Purely toroidal magnetic field
     square, //!< closed flux surfaces centered around an O-point and bordered by a square  with four X-points in the corners (mainly the Guenther field)
     centeredX //!< one X-point in the middle, no O-point, only open flux surfaces, X-grids cannot be constructed
@@ -64,6 +65,7 @@ static const std::map<std::string, modifier> str2modifier{
 static const std::map<std::string, description> str2description{
     {"standardO", description::standardO},
     {"standardX", description::standardX},
+    {"doubleX", description::doubleX},
     {"square", description::square},
     {"none", description::none},
     {"centeredX", description::centeredX}
-- 
GitLab


From aa599ee44511e0106531cbf39b66de93c56fd111 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 3 Nov 2020 19:54:34 +0100
Subject: [PATCH 387/540] Add Sheath region capabilities

With this we should have all ingredients to be able to run with sheath
boundary conditions enforced in the sheath region
---
 inc/geometries/geometry_diag.cu |  14 +++-
 inc/geometries/make_field.h     | 117 ++++++++++++++++++++++++++++++--
 2 files changed, 121 insertions(+), 10 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 75df7f3a3..64d4d6aac 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -117,8 +117,12 @@ int main( int argc, char* argv[])
     const Parameters p(input_js);
     p.display( std::cout);
     //Test coefficients
-    dg::geo::CylindricalFunctor damping, transition;
-    dg::geo::TokamakMagneticField mag = dg::geo::createModifiedField(geom_js, input_js, file::error::is_throw, damping, transition);
+    dg::geo::CylindricalFunctor damping, transition, sheath, direction;
+    dg::geo::TokamakMagneticField mag = dg::geo::createMagneticField(geom_js,
+            file::error::is_throw);
+    dg::geo::TokamakMagneticField mod_mag =
+        dg::geo::createModifiedField(geom_js, input_js, file::error::is_throw,
+                damping, transition);
     std::string input = input_js.toStyledString();
     std::string geom = geom_js.toStyledString();
     unsigned n, Nx, Ny, Nz;
@@ -127,6 +131,8 @@ int main( int argc, char* argv[])
     double Zmin=-p.boxscaleZm*mag.params().a()*mag.params().elongation();
     double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
     double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
+    dg::geo::createSheathRegion( input_js, file::error::is_warning,
+            mag, damping, Rmin, Rmax, Zmin, Zmax, sheath, direction);
 
     dg::geo::description mag_description = mag.params().getDescription();
     double psipO = -1.;
@@ -201,6 +207,8 @@ int main( int argc, char* argv[])
         /////////////////////////////////////
         {"WallDistance", "Distance to closest wall", dg::geo::CylindricalFunctor( dg::WallDistance( {grid2d.x0(), grid2d.x1()}, {grid2d.y0(), grid2d.y1()})) },
         {"WallDirection", "Direction of magnetic field to closest wall", dg::geo::WallDirection(mag, {grid2d.x0(), grid2d.x1()}, {grid2d.y0(), grid2d.y1()}) },
+        {"Sheath", "Sheath region", sheath},
+        {"SheathDirection", "Direction of magnetic field relative to sheath", direction},
         //////////////////////////////////
         {"Iris", "A flux aligned Iris", dg::compose( dg::Iris( 0.5, 0.7), dg::geo::RhoP(mag))},
         {"Pupil", "A flux aligned Pupil", dg::compose( dg::Pupil(0.7), dg::geo::RhoP(mag)) },
@@ -304,7 +312,7 @@ int main( int argc, char* argv[])
             "Alternative flux label rho_p = Sqrt[-psi/psimin + 1]");
         if( mag_description == dg::geo::description::standardX || mag_description == dg::geo::description::standardO)
         {
-            dg::geo::SafetyFactor qprof( mag);
+            dg::geo::SafetyFactor qprof( mod_mag); //mag can lead to dead-lock!?
             dg::HVec qprofile = dg::evaluate( qprof, grid1d);
             map1d.emplace_back("q-profile", qprofile,
                 "q-profile (Safety factor) using direct integration");
diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
index 10d3048d8..d2b229195 100644
--- a/inc/geometries/make_field.h
+++ b/inc/geometries/make_field.h
@@ -61,7 +61,9 @@ static inline TokamakMagneticField createMagneticField( Json::Value js, file::er
 }
 
 
-void transform_psi( TokamakMagneticField mag, double psipO, double& psi0, double& alpha0, double& sign0)
+///@cond
+namespace detail{
+void transform_psi( TokamakMagneticField mag, double& psi0, double& alpha0, double& sign0)
 {
     double RO=mag.R0(), ZO=0.;
     dg::geo::findOpoint( mag.get_psip(), RO, ZO);
@@ -72,6 +74,8 @@ void transform_psi( TokamakMagneticField mag, double psipO, double& psi0, double
     alpha0 = fabs( damping_alpha0p/2.);
     sign0 = sign0*((psipO>0)-(psipO<0));
 }
+}//namespace detail
+///@endcond
 
 /**
  * @brief Modify Magnetic Field above or below certain Psi values according to given parameters
@@ -120,8 +124,9 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
             double psi0 = file::get( mode, jsmod, "damping", "boundary", 1.1 ).asDouble();
             double alpha = file::get( mode, jsmod, "damping", "alpha", 0.2 ).asDouble();
             double sign = +1;
-            if( desc == description::standardX)
-                transform_psi( psipO, psi0, alpha, sign);
+            if( desc == description::standardX || desc == description::standardO ||
+                    desc == description::doubleX)
+                detail::transform_psi( mag, psi0, alpha, sign);
             else
                 sign = file::get( mode, jsmod, "damping", "sign", -1. ).asDouble();
 
@@ -132,7 +137,6 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
         }
         case modifier::sol_pfr:
         {
-            CylindricalFunctorsLvl2 mod0_psip;
             double psi0 = file::get_idx( mode, jsmod, "damping", "boundary",0, 1.1 ).asDouble();
             double alpha0 = file::get_idx( mode, jsmod, "damping", "alpha",0, 0.2 ).asDouble();
             double psi1 = file::get_idx( mode, jsmod, "damping", "boundary",1, 0.97 ).asDouble();
@@ -140,13 +144,14 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
             switch( desc){
                 case description::standardX:
                 {
+                    double sign0 = +1., sign1 = -1.;
+                    detail::transform_psi( mag, psi0, alpha0, sign0);
+                    detail::transform_psi( mag, psi1, alpha1, sign1);
                     //we can find the X-point
                     double RX = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
                     double ZX = -1.1*mag.params().elongation()*mag.params().a();
                     dg::geo::findXpoint( mag.get_psip(), RX, ZX);
-                    double sign0 = +1., sign1 = -1.;
-                    transform_psi( psipO, psi0, alpha0, sign0);
-                    transform_psi( psipO, psi1, alpha1, sign1);
+                    CylindricalFunctorsLvl2 mod0_psip;
                     mod0_psip = mod::createPsip(
                             mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
                     mod_psip = mod::createPsip(
@@ -159,10 +164,40 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
                     transition = mod::SetUnion( transition0, transition1);
                     break;
                 }
+                case description::doubleX:
+                {
+                    double sign0 = +1., sign1 = -1.;
+                    detail::transform_psi( mag, psi0, alpha0, sign0);
+                    detail::transform_psi( mag, psi1, alpha1, sign1);
+                    //we can find the X-point
+                    double RX1 = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
+                    double ZX1 = -1.1*mag.params().elongation()*mag.params().a();
+                    dg::geo::findXpoint( mag.get_psip(), RX1, ZX1);
+                    double RX2 = mag.R0()-1.1*mag.params().triangularity()*mag.params().a();
+                    double ZX2 = +1.1*mag.params().elongation()*mag.params().a();
+                    dg::geo::findXpoint( mag.get_psip(), RX2, ZX2);
+                    CylindricalFunctorsLvl2 mod0_psip, mod1_psip;
+                    mod0_psip = mod::createPsip(
+                            mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
+                    mod1_psip = mod::createPsip(
+                            mod::HeavisideZ( ZX1, -1), mod0_psip, psi1, alpha1, sign1);
+                    mod_psip = mod::createPsip(
+                            mod::HeavisideZ( ZX2, +1), mod1_psip, psi1, alpha1, sign1);
+                    CylindricalFunctor damping0 = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha0, -sign0);
+                    CylindricalFunctor damping1 = mod::DampingRegion( mod::HeavisideZ(ZX1, -1), mag.psip(), psi1, alpha1, -sign1);
+                    CylindricalFunctor damping2 = mod::DampingRegion( mod::HeavisideZ(ZX2, +1), mag.psip(), psi1, alpha1, -sign1);
+                    CylindricalFunctor transition0 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
+                    CylindricalFunctor transition1 = mod::MagneticTransition( mod::HeavisideZ(ZX1, -1), mag.psip(), psi1, alpha1, sign1);
+                    CylindricalFunctor transition2 = mod::MagneticTransition( mod::HeavisideZ(ZX2, +1), mag.psip(), psi1, alpha1, sign1);
+                    transition = mod::SetUnion( mod::SetUnion( transition0, transition1), transition2);
+                    damping = mod::SetUnion( mod::SetUnion( damping0, damping1), damping2);
+                    break;
+                }
                 default:
                 {
                     double sign0 = file::get_idx( mode, jsmod, "damping", "sign",0, -1. ).asDouble();
                     double sign1 = file::get_idx( mode, jsmod, "damping", "sign", 1, +1. ).asDouble();
+                    CylindricalFunctorsLvl2 mod0_psip;
                     mod0_psip = mod::createPsip(
                             mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
                     mod_psip = mod::createPsip(
@@ -193,6 +228,74 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
     }
 }
 
+static inline CylindricalFunctor createWallRegion( Json::Value js, Json::Value jsmod, file::error mode)
+{
+    CylindricalFunctor wall, transition;
+    TokamakMagneticField mag = createModifiedField( js, jsmod, mode, wall, transition);
+    return wall;
+}
+
+/**
+ * @brief Create the sheath region where fieldlines intersect the boundary
+ *
+ * Check if any fieldlines that are not in the wall region intersect the boundary
+ * @param jsmod must contain the field
+ * "sheath": "boundary" value where sheath region begins in units of minor radius a
+ * "sheath": "alpha" radius of the transition region where the modification acts in units of minor radius a
+ * @param mode Determines behaviour in case of an error
+ * @param mag (in) the magnetic field to find the direction of the field
+ * towards or away from the sheath
+ * @param wall (in) the penalization region that represents the actual
+ * (perpendicular) wall without the divertor
+ * @param R0 left boundary
+ * @param R1 right boundary
+ * @param Z0 bottom boundary
+ * @param Z1 top boundary
+ * @param direction (out) contains (+/-) indicating direction of magnetic field
+ * to closest sheath boundary (defined on entire box)
+ *
+ * @return sheath region
+ */
+static inline void createSheathRegion(
+        Json::Value jsmod, file::error mode, TokamakMagneticField mag,
+        CylindricalFunctor wall, double R0, double R1, double Z0, double Z1,
+        CylindricalFunctor& sheath, CylindricalFunctor& direction )
+{
+    Grid1d gR1d ( R0, R1, 1, 100);
+    Grid1d gZ1d ( Z0, Z1, 1, 100);
+    std::array<bool,2> sheathR = {false,false}, sheathZ = {false,false};
+    for ( unsigned i=0; i<100; i++)
+    {
+        if( wall( R0+i*gR1d.h(), Z0) == 0)
+            sheathR[0] = true;
+        if( wall( R0+i*gR1d.h(), Z1) == 0)
+            sheathR[1] = true;
+        if( wall( R0, Z0 + i*gZ1d.h()) == 0)
+            sheathZ[0] = true;
+        if( wall( R1, Z0 + i*gZ1d.h()) == 0)
+            sheathZ[1] = true;
+    }
+    std::vector<double> horizontal_sheath, vertical_sheath;
+    if( true == sheathR[0])
+        horizontal_sheath.push_back( Z0);
+    if( true == sheathR[1])
+        horizontal_sheath.push_back( Z1);
+    if( true == sheathZ[0])
+        vertical_sheath.push_back( R0);
+    if( true == sheathZ[1])
+        vertical_sheath.push_back( R1);
+    //direction
+    direction = dg::geo::WallDirection( mag, vertical_sheath, horizontal_sheath);
+    //sheath
+    CylindricalFunctor dist = dg::WallDistance( vertical_sheath, horizontal_sheath);
+    double boundary = file::get( mode, jsmod, "sheath", "boundary", 0.1 ).asDouble();
+    double alpha = file::get( mode, jsmod, "sheath", "alpha", 0.01 ).asDouble();
+    double a = mag.params().a();
+    dg::PolynomialHeaviside poly( boundary*a - alpha*a/2., alpha*a/2., -1);
+    sheath = dg::compose( poly, dist);
+    sheath = mod::SetIntersection( mod::SetNot( wall), sheath);
+}
+
 } //namespace geo
 }//namespace dg
 #endif //JSONCPP_VERSION_STRING
-- 
GitLab


From 174379889a21a89452d9c78d9f88a45e75c9c3a4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 3 Nov 2020 19:56:11 +0100
Subject: [PATCH 388/540] Make electron viscosity large again

---
 src/feltor/parameters.h | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 93330db72..38dab5585 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -96,8 +96,6 @@ struct Parameters
         //Init after reading in eta and mu[0]
         nu_parallel[0] = 0.73/eta;
         nu_parallel[1] = sqrt(fabs(mu[0]))*1.36/eta;
-        //Make electron diffusion smaller to avoid numerical difficulties
-        nu_parallel[0] = nu_parallel[1];
 
         initne      = file::get( mode, js, "initne", "blob").asString();
         initphi     = file::get( mode, js, "initphi", "zero").asString();
-- 
GitLab


From 6268c2b666e2ed965863eaafb7873078ac0e3d96 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 3 Nov 2020 20:09:13 +0100
Subject: [PATCH 389/540] Remove interior from feltor

in preparation for sheath boundary conditions
---
 src/feltor/feltor.cu     |  3 +--
 src/feltor/feltor.h      | 48 +++++++++-------------------------------
 src/feltor/feltor_hpc.cu |  3 +--
 src/feltor/init.h        | 15 -------------
 4 files changed, 13 insertions(+), 56 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index b58659d44..903321786 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -123,8 +123,7 @@ int main( int argc, char* argv[])
     }
     feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
         p.source_rate, dg::construct<DVec>(source_profile),
-        p.damping_rate, dg::construct<DVec>(damping_profile),
-        dg::construct<DVec>( feltor::detail::interior( grid, p, mag))
+        p.damping_rate, dg::construct<DVec>(damping_profile)
     );
 
 
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 4c546e717..be6d523e8 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -333,7 +333,7 @@ struct Explicit
     }
 
     //source strength, profile - 1
-    void set_source( bool fixed_profile, Container profile, double omega_source, Container source, double omega_damping, Container damping, Container interior )
+    void set_source( bool fixed_profile, Container profile, double omega_source, Container source, double omega_damping, Container damping)
     {
         m_fixed_profile = fixed_profile;
         m_profne = profile;
@@ -341,7 +341,6 @@ struct Explicit
         m_source = source;
         m_omega_damping = omega_damping;
         m_damping = damping;
-        m_interior = interior;
     }
     void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
   private:
@@ -372,7 +371,7 @@ struct Explicit
     std::array<Container,3> m_curv, m_curvKappa, m_b;
     Container m_divCurvKappa;
     Container m_bphi, m_binv, m_divb;
-    Container m_source, m_profne, m_damping, m_interior;
+    Container m_source, m_profne, m_damping;
     Container m_vol3d;
 
     Container m_apar;
@@ -571,7 +570,6 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
     m_UE2 = m_temp2 = m_temp1 = m_temp0;
     m_apar = m_temp0;
-    dg::assign( dg::evaluate( dg::one, g), m_interior );
 
     m_phi[0] = m_phi[1] = m_temp0;
     //m_dssN =
@@ -845,48 +843,24 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         //---------------------density--------------------------//
         //density: -Div ( NUb)
         dg::geo::ds_centered( m_fa_N, 1., m_faM[0], y[0][i], m_faP[0], 0., m_dsN[i]);
-        //centered in the interior, but backward outside
-        dg::geo::ds_backward( m_fa_U, 1., m_faM[1], fields[1][i], 0., m_dsU[i]);
-        dg::geo::ds_centered( m_fa_U, 1., m_faM[1], fields[1][i], m_faP[1], 0., m_temp1);
-        dg::blas1::pointwiseDot( 1., m_interior, m_temp1, -1., m_interior, m_dsU[i], +1., m_dsU[i]);
-
+        dg::geo::ds_centered( m_fa_U, 1., m_faM[1], fields[1][i], m_faP[1], 0., m_dsU[i]);
         dg::blas1::pointwiseDot(-1., m_dsN[i], fields[1][i],
             -1., fields[0][i], m_dsU[i], 1., yp[0][i] );
         dg::blas1::pointwiseDot( -1., fields[0][i],fields[1][i],m_divb,
             1.,yp[0][i]);
-        //////////density: + nu_par Delta_par N
-        ////////dg::blas1::pointwiseDot( m_p.nu_parallel, m_divb, m_dsN[i],
-        ////////                         0., m_temp0);
-        ////////dg::geo::dss_centered( m_fa_N, 1., m_faM[0],  y[0][i], m_faP[0], 0., m_dssN[i]);
-        ////////dg::blas1::axpby( m_p.nu_parallel, m_dssN[i], 1., m_temp0);//nu_par Delta_par N
-        //////////Add to rhs, we again need it further down
-        ////////dg::blas1::axpby( 1., m_temp0, 1., yp[0][i]);
         //---------------------velocity-------------------------//
-        // Burgers term: -0.5 ds U^2
-        //dg::blas1::pointwiseDot(fields[1][i], fields[1][i], m_temp1); //U^2
-        //m_fa_U.centered(-0.5, m_temp1, 1., yp[1][i]);
-        dg::geo::ds_centered( m_fa_U, 1., m_faM[1], fields[1][i], m_faP[1], 0., m_dsU[i]);
-        dg::blas1::pointwiseDot(-1., fields[1][i], m_dsU[i], 1., yp[1][i]); //-U ds U
+        // Burgers term: -U ds U
+        dg::blas1::pointwiseDot(-1., fields[1][i], m_dsU[i], 1., yp[1][i]);
         // force terms: -tau/mu * ds lnN -1/mu * ds Phi
-        dg::geo::ds_forward( m_fa_N, 1., y[0][i], m_faP[0], 0., m_temp1);
-        dg::blas1::pointwiseDot( 1., m_interior, m_dsN[i], -1., m_interior, m_temp1, +1., m_temp1);
-
-        dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_temp1, fields[0][i], 1.0, yp[1][i]);
-        //dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_dsN[i], fields[0][i], 1., yp[1][i]);
-        //m_fa_P.centered(-1./m_p.mu[i], m_phi[i], 1.0, yp[1][i]);
+        dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_dsN[i], fields[0][i], 1., yp[1][i]);
         m_fa_P( dg::geo::einsMinus, m_phi[i], m_faM[0]); //overwrite
         m_fa_P( dg::geo::einsPlus,  m_phi[i], m_faP[0]);
-        dg::geo::ds_forward( m_fa_P, 1., m_phi[i], m_faP[0], 0., m_dsP[i]);
-        dg::geo::ds_centered( m_fa_P, 1., m_faM[0], m_phi[i], m_faP[0], 0., m_temp1);
-        dg::blas1::pointwiseDot( 1., m_interior, m_temp1, -1., m_interior, m_dsP[i], +1., m_dsP[i]);
-        dg::blas1::axpby(-1./m_p.mu[i], m_dsP[i], 1.0, yp[1][i]);
-        // diffusion: + nu_par Delta_par U/N - nu_par U Delta_par N/ N
-        dg::blas1::pointwiseDot(m_p.nu_parallel[i], m_divb, m_dsU[i],
-                                0., m_temp1);
+        dg::geo::ds_centered( m_fa_P, -1./m_p.mu[i], m_faM[0], m_phi[i], m_faP[0], 1.0, yp[1][i]);
+        // viscosity: + nu_par Delta_par U/N = nu_par ( Div b dsU + dssU)/N
+        dg::blas1::pointwiseDot(1., m_divb, m_dsU[i], 0., m_temp1);
         dg::geo::dss_centered( m_fa_U, 1., m_faM[1], fields[1][i], m_faP[1], 0., m_dssU[i]);
-        dg::blas1::axpby( m_p.nu_parallel[i], m_dssU[i], 1., m_temp1); //nu_par Delta_par U
-        //////dg::blas1::pointwiseDot( -1., fields[1][i], m_temp0, 1., m_temp1);
-        dg::blas1::pointwiseDivide( 1., m_temp1, fields[0][i], 1., yp[1][i]);
+        dg::blas1::axpby( 1., m_dssU[i], 1., m_temp1);
+        dg::blas1::pointwiseDivide( m_p.nu_parallel[i], m_temp1, fields[0][i], 1., yp[1][i]);
     }
 }
 
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 9be42e78e..20450b51a 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -267,8 +267,7 @@ int main( int argc, char* argv[])
             fixed_profile, profile, grid, p, mag);
         feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
             p.source_rate, dg::construct<DVec>(source_profile),
-            p.damping_rate, dg::construct<DVec>(damping_profile),
-            dg::construct<DVec>( feltor::detail::interior( grid, p, mag))
+            p.damping_rate, dg::construct<DVec>(damping_profile)
         );
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)"<<std::endl;
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 89ee0519d..e1cd97fdc 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -76,21 +76,6 @@ HVec profile_damping(const Geometry& grid,
         profile_damping, profile_damping);
     return profile_damping;
 }
-HVec interior(const Geometry& grid,
-    const feltor::Parameters& p,
-    const dg::geo::TokamakMagneticField& mag )
-{
-    if( p.profile_alpha == 0)
-        throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
-    HVec interior = dg::pullback( dg::compose(dg::PolynomialHeaviside(
-        1.+p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)), grid);
-    //HVec interior = dg::pullback( dg::compose(dg::Heaviside(
-    //    1., -1), dg::geo::RhoP(mag)), grid);
-    //HVec interior = dg::evaluate( dg::one, grid);
-    dg::blas1::pointwiseDot( xpoint_damping(grid,p,mag),
-        interior, interior);
-    return interior;
-}
 HVec profile(const Geometry& grid,
     const feltor::Parameters& p,
     const dg::geo::TokamakMagneticField& mag )
-- 
GitLab


From 7e8a9d151da5a5f4139405bee2d9cbf17cf2f3f1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 4 Nov 2020 21:58:08 +0100
Subject: [PATCH 390/540] Improve documentation

---
 inc/geometries/fluxfunctions.h |  5 +++-
 inc/geometries/make_field.h    | 47 +++++++++++++++++-----------------
 2 files changed, 28 insertions(+), 24 deletions(-)

diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 3d7495853..26afd4154 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -261,11 +261,14 @@ struct CylindricalFunctorsLvl2
 /**
  * @brief This function finds critical points of psi (any point with vanishing gradient, including the X-point or O-point) via Newton iteration applied to the gradient of psi
  *
+ * Newton iteration applied to \f$ \nabla \psi (\vec x) = 0 \f$ reads
+ * \f[ \vec x_{i+1} = \vec x_i - H^{-1} \nabla \psi (\vec x_i)\f]
+ * where H is the Hessian matrix.
  * The inverse of the Hessian matrix is computed analytically
  * @param psi \f$ \psi(R,Z)\f$
  * @param RC start value on input, critical point on output
  * @param ZC start value on input, critical point on output
- * @return 0 if no critical point or Hessian is 0,
+ * @return 0 if no critical point or Hessian (determinant) is zero,
  * 1 if local minimum,
  * 2 if local maximum,
  * 3 if saddle point
diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
index d2b229195..1871f3d34 100644
--- a/inc/geometries/make_field.h
+++ b/inc/geometries/make_field.h
@@ -8,6 +8,8 @@
 
 namespace dg{
 namespace geo{
+///@addtogroup geom
+///@{
 
 /**
  * @brief Create a Magnetic field based on the given parameters
@@ -17,44 +19,43 @@ namespace geo{
  * field to generate and which parameters to expect in the file, for example if
  * "equilibrium" reads "toroidal", then only on additional parameter "R_0" is
  * read from the file and a field is constructed
- * @param js Has to contain "equilibrium" which is converted dg::geo::equilibrium,
+ * @param gs Has to contain "equilibrium" which is converted dg::geo::equilibrium,
  * i.e. "solovev", "polynomial", .... After that the respective parameters are created,
- * for example if "solovev", then the dg::geo::solovev::Parameters( js, mode) is called and forwarded to dg::geo::createSolovevField(gp); similar for the rest
+ * for example if "solovev", then the dg::geo::solovev::Parameters( gs, mode) is called and forwarded to dg::geo::createSolovevField(gp); similar for the rest
  * @param mode signifies what to do if an error occurs
  * @return A magnetic field object
- * @ingroup geom
  * @attention This function is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
  */
-static inline TokamakMagneticField createMagneticField( Json::Value js, file::error mode)
+static inline TokamakMagneticField createMagneticField( Json::Value gs, file::error mode)
 {
-    std::string e = file::get( mode, js, "equilibrium", "solovev" ).asString();
+    std::string e = file::get( mode, gs, "equilibrium", "solovev" ).asString();
     equilibrium equi = str2equilibrium.at( e);
     switch( equi){
         case equilibrium::polynomial:
         {
-            polynomial::Parameters gp( js, mode);
+            polynomial::Parameters gp( gs, mode);
             return createPolynomialField( gp);
         }
         case equilibrium::toroidal:
         {
-            double R0 = file::get( mode, js, "R_0", 10).asDouble();
+            double R0 = file::get( mode, gs, "R_0", 10).asDouble();
             return createToroidalField( R0);
         }
         case equilibrium::guenther:
         {
-            double I0 = file::get( mode, js, "I_0", 20).asDouble();
-            double R0 = file::get( mode, js, "R_0", 10).asDouble();
+            double I0 = file::get( mode, gs, "I_0", 20).asDouble();
+            double R0 = file::get( mode, gs, "R_0", 10).asDouble();
             return createGuentherField( R0, I0);
         }
         case equilibrium::circular:
         {
-            double I0 = file::get( mode, js, "I_0", 20).asDouble();
-            double R0 = file::get( mode, js, "R_0", 10).asDouble();
+            double I0 = file::get( mode, gs, "I_0", 20).asDouble();
+            double R0 = file::get( mode, gs, "R_0", 10).asDouble();
             return createCircularField( R0, I0);
         }
         default:
         {
-            solovev::Parameters gp( js, mode);
+            solovev::Parameters gp( gs, mode);
             return createSolovevField( gp);
         }
     }
@@ -85,29 +86,28 @@ void transform_psi( TokamakMagneticField mag, double& psi0, double& alpha0, doub
  * function with width alpha), i.e. we replace psi with IPolynomialHeaviside(psi).
  * This subsequently modifies all derivatives of psi and the poloidal
  * current in this region.
- * @param js forwarded to dg::geo::createMagneticField
+ * @param gs forwarded to dg::geo::createMagneticField
  * @param jsmod must contain the field "damping": "modifier" which has one of the values "none", "heaviside" then
  *  "damping": "boundary" value where psi is modified to a constant psi0
  * "damping": "alpha" radius of the transition region where the modification acts (smaller is quicker) or "sol_pfr", then "
  *  "damping": "boundary" and
  * "damping": "alpha" must be arrays of size 2 to indicate values for the SOL and the PFR respectively
  * @param mode Determines behaviour in case of an error
- * @param damping On output contains the region where the damping is applied
- * @param transition On output contains the region where the transition of Psip to a constant value occurs
+ * @param damping (out) On output contains the region where the damping is applied
+ * @param transition (out) On output contains the region where the transition of Psip to a constant value occurs
  * @note Per default the dampening happens nowhere
  * @return A magnetic field object
- * @ingroup geom
  * @attention This function is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
  */
-static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Value jsmod, file::error mode, CylindricalFunctor& damping, CylindricalFunctor& transition)
+static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Value jsmod, file::error mode, CylindricalFunctor& damping, CylindricalFunctor& transition)
 {
-    std::string e = file::get( mode, js, "equilibrium", "solovev" ).asString();
+    std::string e = file::get( mode, gs, "equilibrium", "solovev" ).asString();
     equilibrium equi = str2equilibrium.at( e);
     std::string m = file::get( mode, jsmod, "damping", "modifier", "heaviside" ).asString();
     modifier mod = str2modifier.at( m);
-    std::string d = file::get( mode, js, "description", "standardX" ).asString();
+    std::string d = file::get( mode, gs, "description", "standardX" ).asString();
     description desc = str2description.at( d);
-    TokamakMagneticField mag = createMagneticField( js, mode);
+    TokamakMagneticField mag = createMagneticField( gs, mode);
     const MagneticFieldParameters& inp = mag.params();
     MagneticFieldParameters mod_params{ inp.a(), inp.elongation(),
         inp.triangularity(), inp.getEquilibrium(), mod, inp.getDescription()};
@@ -216,7 +216,7 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
     switch( equi){
         case equilibrium::solovev:
         {
-            solovev::Parameters gp( js, mode);
+            solovev::Parameters gp( gs, mode);
             return TokamakMagneticField( gp.R_0,mod_psip,
                 solovev::createIpol( gp, mod_psip), mod_params);
         }
@@ -228,10 +228,10 @@ static inline TokamakMagneticField createModifiedField( Json::Value js, Json::Va
     }
 }
 
-static inline CylindricalFunctor createWallRegion( Json::Value js, Json::Value jsmod, file::error mode)
+static inline CylindricalFunctor createWallRegion( Json::Value gs, Json::Value jsmod, file::error mode)
 {
     CylindricalFunctor wall, transition;
-    TokamakMagneticField mag = createModifiedField( js, jsmod, mode, wall, transition);
+    TokamakMagneticField mag = createModifiedField( gs, jsmod, mode, wall, transition);
     return wall;
 }
 
@@ -296,6 +296,7 @@ static inline void createSheathRegion(
     sheath = mod::SetIntersection( mod::SetNot( wall), sheath);
 }
 
+///@}
 } //namespace geo
 }//namespace dg
 #endif //JSONCPP_VERSION_STRING
-- 
GitLab


From 2897099946e025bdcb48f9af78336dd5357dbfb2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 4 Nov 2020 22:25:43 +0100
Subject: [PATCH 391/540] Init AdamsBashforth as Karniadakis

with a single step method
---
 inc/dg/multistep.h | 49 +++++++++++++++++-----------------------------
 1 file changed, 18 insertions(+), 31 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 6f67a902e..44429efd5 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -87,11 +87,9 @@ struct AdamsBashforth
     const ContainerType& copyable()const{ return m_u;}
 
     /**
-     * @brief Initialize first step. Call before using the step function.
+     * @brief Initialize timestepper. Call before using the step function.
      *
-     * This routine initiates the first steps in the multistep method by integrating
-     * backwards in time with Euler's method. This routine has to be called
-     * before the first timestep is made.
+     * This routine has to be called before the first timestep is made.
      * @copydoc hide_rhs
      * @param rhs The rhs functor
      * @param t0 The intital time corresponding to u0
@@ -117,7 +115,7 @@ struct AdamsBashforth
     std::vector<ContainerType> m_f;
     ContainerType m_u;
     std::vector<value_type> m_ab;
-    unsigned m_k;
+    unsigned m_k, m_counter;
 };
 
 template< class ContainerType>
@@ -125,25 +123,26 @@ template< class RHS>
 void AdamsBashforth<ContainerType>::init( RHS& f, value_type t0, const ContainerType& u0, value_type dt)
 {
     m_tu = t0, m_dt = dt;
-    f( t0, u0, m_f[0]);
-    //now do k Euler steps
-    ContainerType u1(u0);
-    for( unsigned i=1; i<m_k; i++)
-    {
-        blas1::axpby( 1., u1, -dt, m_f[i-1], u1);
-        m_tu -= dt;
-        f( m_tu, u1, m_f[i]);
-    }
-    m_tu = t0;
+    f( t0, u0, m_f[0]); //f may not destroy u0
     blas1::copy(  u0, m_u);
-    //finally evaluate f at u0 once more to set state in f
-    f( m_tu, m_u, m_f[0]);
+    m_counter = 0;
 }
 
 template<class ContainerType>
 template< class RHS>
 void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType& u)
 {
+    if( m_counter < m_k-1)
+    {
+        ERKStep<ContainerType> erk( "ARK-4-2-3 (explicit)", u);
+        ContainerType tmp ( u);
+        erk.step( f, t, u, t, u, m_dt, tmp);
+        m_counter++;
+        m_tu = t;
+        blas1::copy(  u, m_u);
+        f( m_tu, m_u, m_f[m_counter]);
+        return;
+    }
     for( unsigned i=0; i<m_k; i++)
         blas1::axpby( m_dt*m_ab[i], m_f[i], 1., m_u);
     //permute m_f[k-1]  to be the new m_f[0]
@@ -241,9 +240,9 @@ struct Karniadakis
     const SolverType& solver() const { return m_solver;}
 
     /**
-     * @brief Initialize timestepper
+     * @brief Initialize timestepper. Call before using the step function.
      *
-     * This initializes the timestepper and sets the timestep for later use
+     * This routine has to be called before the first timestep is made.
      * @copydoc hide_explicit_implicit
      * @param t0 The intital time corresponding to u0
      * @param u0 The initial value of the integration
@@ -312,18 +311,6 @@ void Karniadakis<ContainerType, SolverType>::init( RHS& f, Diffusion& diff, valu
     blas1::copy(  u0, m_u[2]);
     f( t0, u0, m_f[2]); //f may not destroy u0
     m_counter = 0;
-    //m_t = t0, m_dt = dt;
-    //blas1::copy(  u0, m_u[0]);
-    //f( t0, u0, m_f[0]); //f may not destroy u0
-    //blas1::axpby( 1., m_u[0], -dt, m_f[0], m_f[1]); //Euler step
-    //detail::Implicit<Diffusion, ContainerType> implicit( -dt, t0, diff);
-    //implicit( m_f[1], m_u[1]); //explicit Euler step backwards
-    //f( t0-dt, m_u[1], m_f[1]);
-    //blas1::axpby( 1.,m_u[1], -dt, m_f[1], m_f[2]);
-    //implicit.time() = t0 - dt;
-    //implicit( m_f[2], m_u[2]);
-    //f( t0-2*dt, m_u[2], m_f[2]); //evaluate f at the latest step
-    //f( t0, u0, m_f[0]); // and set state in f to (t0,u0)
 }
 
 template<class ContainerType, class SolverType>
-- 
GitLab


From 634ee2d1e264e0b9fbf2305b07e41817373a9e6a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 4 Nov 2020 22:41:38 +0100
Subject: [PATCH 392/540] Refactor along field bc in separate function

so we have more control over what we want to call
---
 inc/geometries/ds.h | 87 +++++++++++++++++++++++++++++----------------
 1 file changed, 57 insertions(+), 30 deletions(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 31fee8ef9..fbf14675b 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -231,10 +231,6 @@ static const std::map < std::string, boundary> str2boundary {
 /*!@class hide_ds_dir
  * @param dir indicate the direction in the bracket operator and in symv
  */
-/*!@class hide_ds_mode
- * @param mode indicate how boundary conditions should be treated: if \c dg::geo::boundary::perp then the boundary conditions are implemented by mirroring points perpendicular to the boundary, which has some drawbacks as to the numerical stability and toroidal resolution. if \c dg::geo::boundary::along_field then the boundary condition is implemented along the field-line which is numerically more desirable because it decouples the parallel direction.
- * @attention The \c along_field boundary modes only works for \c centered and \c dss members and only if both \c bcx and \c bcy are the same (either \c dg::NEU or \c dg::DIR but not \c dg::NEU_DIR or \c dg::DIR_NEU) In all other cases \c perp mode is used by default
-*/
 
 /*!@class hide_ds_attention
 @attention The \c div and \c symv member functions reliably converge only if fieldlines
@@ -727,75 +723,106 @@ void ds_backward( const FieldAligned& fa, double alpha, const container& fm, con
 * @copydoc hide_ds_parameters4
 * @copydoc hide_ds_fm
 * @copydoc hide_ds_fp
-* @copydoc hide_ds_mode
 * @ingroup fieldaligned
 */
 template<class FieldAligned, class container>
 void ds_centered( const FieldAligned& fa, double alpha, const container& fm,
+        const container& f, const container& fp, double beta, container& g)
+{
+    //direct discretisation
+    dg::blas1::subroutine( detail::ComputeDSCentered( alpha, beta),
+            g, fm, f, fp, fa.hm(), fa.hp());
+}
+/**
+ * @brief Centered derivative \f$ g = \alpha (\vec v\cdot \nabla)^2 f + \beta g \f$
+ *
+ * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
+ * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
+ * the boundary conditions are implemented by
+ * mirroring points perpendicular to the boundary, which has some drawbacks as
+ * to the numerical stability and toroidal resolution.
+ * @param fa this object will be used to get grid distances
+ * @copydoc hide_ds_parameters4
+* @copydoc hide_ds_fm
+* @copydoc hide_ds_fp
+* @ingroup fieldaligned
+ */
+template<class FieldAligned, class container>
+void dss_centered( const FieldAligned& fa, double alpha, const container&
+        fm,const container& f, const container& fp, double beta, container& g)
+{
+    dg::blas1::subroutine( detail::ComputeDSS( alpha, beta),
+            g, fm, f, fp, fa.hm(), fa.hp());
+}
+
+/**
+* @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
+*
+* The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup.
+ * the boundary condition is implemented
+ * along the field-line
+ * @param fa this object will be used to get grid distances
+* @copydoc hide_ds_parameters4
+* @copydoc hide_ds_fm
+* @copydoc hide_ds_fp
+* @param bound either dg::NEU or dg::DIR (rest not implemented yet)
+* @param boundary_value first value is for incoming fieldlines, second one for outgoing
+* @ingroup fieldaligned
+*/
+template<class FieldAligned, class container>
+void ds_centered_bc_along_field( const FieldAligned& fa, double alpha, const container& fm,
         const container& f, const container& fp, double beta, container& g,
-        dg::geo::boundary bound_mode = dg::geo::boundary::perp, std::array<double,2> boundary_value = {0,0})
+        dg::bc bound, std::array<double,2> boundary_value = {0,0})
 {
     //direct discretisation
-    if( boundary::along_field == bound_mode
-            && fa.bcx() == dg::NEU && fa.bcy() == dg::NEU)
+    if( bound == dg::NEU)
     {
         dg::blas1::subroutine( detail::ComputeDSCenteredNEU( alpha, beta, boundary_value),
                 g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
                 fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
-    else if( boundary::along_field == bound_mode
-            && fa.bcx() == dg::DIR && fa.bcy() == dg::DIR)
+    else// if( bound == dg::DIR)
     {
         dg::blas1::subroutine( detail::ComputeDSCenteredDIR( alpha, beta, boundary_value),
                 g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
                 fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
-    else
-    {
-        dg::blas1::subroutine( detail::ComputeDSCentered( alpha, beta),
-                g, fm, f, fp, fa.hm(), fa.hp());
-    }
 }
 /**
  * @brief Centered derivative \f$ g = \alpha (\vec v\cdot \nabla)^2 f + \beta g \f$
  *
  * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
  * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
+ * the boundary condition is implemented
+ * along the field-line
  * @param fa this object will be used to get grid distances
  * @copydoc hide_ds_parameters4
 * @copydoc hide_ds_fm
 * @copydoc hide_ds_fp
-* @copydoc hide_ds_mode
+* @param bound either dg::NEU or dg::DIR (rest not implemented yet)
+* @param boundary_value first value is for incoming fieldlines, second one for outgoing
 * @ingroup fieldaligned
  */
 template<class FieldAligned, class container>
-void dss_centered( const FieldAligned& fa, double alpha, const container&
-        fm,const container& f, const container& fp, double beta, container&
-        g, dg::geo::boundary bound_mode = dg::geo::boundary::perp,
-        std::array<double,2> boundary_value = {0,0})
+void dss_centered_bc_along_field( const FieldAligned& fa, double alpha, const
+        container& fm,const container& f, const container& fp, double beta,
+        container& g, dg::bc bound, std::array<double,2> boundary_value =
+        {0,0})
 {
-    if( boundary::along_field == bound_mode
-            && fa.bcx() == dg::NEU && fa.bcy() == dg::NEU)
+    if( bound == dg::NEU)
     {
         dg::blas1::subroutine( detail::ComputeDSSNEU( alpha, beta, boundary_value),
                 g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
                 fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
-    else if( boundary::along_field == bound_mode
-            && fa.bcx() == dg::DIR && fa.bcy() == dg::DIR)
+    else// if( bound == dg:DIR)
     {
         dg::blas1::subroutine( detail::ComputeDSSDIR( alpha, beta, boundary_value),
                 g, fm, f, fp, fa.hm(), fa.hp(), fa.hbm(),
                 fa.hbp(), fa.bbm(), fa.bbo(), fa.bbp());
     }
-    else
-    {
-        dg::blas1::subroutine( detail::ComputeDSS( alpha, beta),
-                g, fm, f, fp, fa.hm(), fa.hp());
-    }
 }
 
-
 }//namespace geo
 
 ///@cond
-- 
GitLab


From d7a755f79dcf3310a8e4856363aa82f4662b9ff1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 5 Nov 2020 21:04:17 +0100
Subject: [PATCH 393/540] Introduce sheath in feltor

- introduce sheath rate and explicit diffusion
- create option to treat diffusion explicit
- forcing is always implicit
- forcing is on w_parallel
- Rhs is dampened out
---
 src/feltor/feltor.cu    |  62 +++++++++-------
 src/feltor/feltor.h     | 109 ++++++++++++++++------------
 src/feltor/implicit.h   | 154 ++++++++++++++++++++++++++++------------
 src/feltor/init.h       |   1 +
 src/feltor/parameters.h |  11 ++-
 5 files changed, 213 insertions(+), 124 deletions(-)

diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 903321786..15aea92ef 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -59,10 +59,11 @@ int main( int argc, char* argv[])
     const feltor::Parameters p(js);
     p.display( std::cout);
     std::cout << gs.toStyledString() << std::endl;
-    dg::geo::TokamakMagneticField mag;
-    dg::geo::CylindricalFunctor damping, transition;
+    dg::geo::TokamakMagneticField mag, mod_mag;
+    dg::geo::CylindricalFunctor wall, transition, sheath, direction;
     try{
-        mag = dg::geo::createModifiedField(gs, js, file::error::is_throw, damping, transition);
+        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
+        mod_mag = dg::geo::createModifiedField(gs, js, file::error::is_throw, wall, transition);
     }catch(std::runtime_error& e)
     {
         std::cerr << "ERROR in geometry file "<<geomfile<<std::endl;
@@ -70,25 +71,33 @@ int main( int argc, char* argv[])
         return -1;
     }
     /////////////////////////////////////////////////////////////////////////
+    //Make grid
     double Rmin=mag.R0()-p.boxscaleRm*mag.params().a();
     double Zmin=-p.boxscaleZm*mag.params().a()*mag.params().elongation();
     double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
     double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
-    //Make grid
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
+    try{
+        dg::geo::createSheathRegion( js, file::error::is_throw, mag, wall,
+                Rmin, Rmax, Zmin, Zmax, sheath, direction);
+    }catch(std::runtime_error& e)
+    {
+        std::cerr << "ERROR in geometry file "<<geomfile<<std::endl;
+        std::cerr <<e.what()<<std::endl;
+        return -1;
+    }
 
-    HVec damping_profile = dg::pullback( damping, grid);
     if( p.periodify)
         mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
 
     //create RHS
     //std::cout << "Constructing RHS...\n";
-    //feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, true);
+    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
     std::cout << "Constructing Explicit...\n";
-    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, false);
+    //feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, false);
     std::cout << "Constructing Implicit...\n";
-    feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
+    feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> implicit( grid, p, mag);
     std::cout << "Done!\n";
 
     DVec result = dg::evaluate( dg::zero, grid);
@@ -105,27 +114,27 @@ int main( int argc, char* argv[])
     double time = 0.;
     std::array<std::array<DVec,2>,2> y0;
     try{
-        y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,mag );
+        y0 = feltor::initial_conditions.at(p.initne)( feltor, grid, p,mod_mag );
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
         return -1;
     }
+    { //make the HVecs temporaries
 
     bool fixed_profile;
     HVec profile = dg::evaluate( dg::zero, grid);
     HVec source_profile;
     try{
         source_profile = feltor::source_profiles.at(p.source_type)(
-            fixed_profile, profile, grid, p,  mag);
+            fixed_profile, profile, grid, p,  mod_mag);
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
         return -1;
     }
     feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
-        p.source_rate, dg::construct<DVec>(source_profile),
-        p.damping_rate, dg::construct<DVec>(damping_profile)
+        p.source_rate, dg::construct<DVec>(source_profile)
     );
-
+    }
 
     ////////////////////////create timer and timestepper
     //
@@ -135,23 +144,23 @@ int main( int argc, char* argv[])
         feltor::FeltorSpecialSolver<
             Geometry, IDMatrix, DMatrix, DVec>
         > karniadakis( grid, p, mag);
-    //unsigned mMax = 3, restart = 3, max_iter = 100;
-    //double damping = 1e-3;
-    //dg::BDF< std::array<std::array<dg::DVec,2>,2 >,
-    //    dg::AndersonSolver< std::array<std::array<dg::DVec,2>,2> >
-    //    > bdf( 3, y0, mMax, p.rtol, max_iter, damping, restart);
-    //dg::AdamsBashforth< std::array<std::array<dg::DVec,2>,2 >
-    //    > bdf( 3, y0);
+    {
+    HVec h_wall = dg::pullback( wall, grid);
+    HVec h_sheath = dg::pullback( sheath, grid);
+    HVec h_velocity = dg::pullback( direction, grid);
+    feltor.set_wall_and_sheath( p.damping_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath), dg::construct<DVec>(h_velocity));
+    implicit.set_wall_and_sheath( p.damping_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    karniadakis.solver().set_wall_and_sheath( p.damping_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    }
 
     std::cout << "Initialize Timestepper" << std::endl;
-    karniadakis.init( feltor, im, time, y0, p.dt);
-    //bdf.init( feltor, time, y0, p.dt);
+    karniadakis.init( feltor, implicit, time, y0, p.dt);
     std::cout << "Done!" << std::endl;
 
     std::map<std::string, const dg::DVec* > v4d;
     v4d["ne-1 / "] = &y0[0][0],  v4d["ni-1 / "] = &y0[0][1];
     v4d["Ue / "]   = &feltor.velocity(0), v4d["Ui / "]   = &feltor.velocity(1);
-    v4d["Ome / "] = &feltor.potential(0); v4d["Apar / "] = &feltor.induction();
+    v4d["Phi / "] = &feltor.potential(0); v4d["Apar / "] = &feltor.induction();
     double dEdt = 0, accuracy = 0;
     double E0 = 0.;
     /////////////////////////set up transfer for glfw
@@ -181,9 +190,9 @@ int main( int argc, char* argv[])
         title << "t = "<<time<<"   ";
         for( auto pair : v4d)
         {
-            if(pair.first == "Ome / ")
+            if(pair.first == "Phi / ")
             {
-                dg::assign( feltor.lapMperpP(0), hvisual);
+                //dg::assign( feltor.lapMperpP(0), hvisual);
                 dg::assign( *pair.second, hvisual);
             }
             else if(pair.first == "ne-1 / " || pair.first == "ni-1 / ")
@@ -230,8 +239,7 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    karniadakis.step( feltor, im, time, y0);
-                    //bdf.step( feltor, time, y0);
+                    karniadakis.step( feltor, implicit, time, y0);
                 }
                 catch( dg::Fail& fail) {
                     std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index be6d523e8..62b0fa9cf 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -170,7 +170,7 @@ struct Explicit
     using vector = std::array<std::array<Container,2>,2>;
     using container = Container;
     Explicit( const Geometry& g, feltor::Parameters p,
-        dg::geo::TokamakMagneticField mag, bool full_system ); //full system means explicit AND implicit
+        dg::geo::TokamakMagneticField mag ); //full system means explicit AND implicit
 
     //Given N_i-1 initialize n_e-1 such that phi=0
     void initializene( const Container& ni, Container& ne);
@@ -333,14 +333,21 @@ struct Explicit
     }
 
     //source strength, profile - 1
-    void set_source( bool fixed_profile, Container profile, double omega_source, Container source, double omega_damping, Container damping)
+    void set_source( bool fixed_profile, Container profile, double omega_source, Container source)
     {
         m_fixed_profile = fixed_profile;
         m_profne = profile;
         m_omega_source = omega_source;
         m_source = source;
-        m_omega_damping = omega_damping;
-        m_damping = damping;
+    }
+    void set_wall_and_sheath(double wall_forcing, Container wall, double sheath_forcing, Container sheath, Container velocity_sheath)
+    {
+        m_sheath_forcing = sheath_forcing; //1/eta
+        dg::blas1::pointwiseDot( sheath, velocity_sheath, m_U_sheath);
+
+        dg::blas1::axpby( -1., wall, -1., sheath, m_masked);
+        dg::blas1::plus( m_masked, +1);
+        m_wall_forcing = wall_forcing;
     }
     void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
   private:
@@ -371,7 +378,7 @@ struct Explicit
     std::array<Container,3> m_curv, m_curvKappa, m_b;
     Container m_divCurvKappa;
     Container m_bphi, m_binv, m_divb;
-    Container m_source, m_profne, m_damping;
+    Container m_source, m_profne, m_U_sheath, m_masked;
     Container m_vol3d;
 
     Container m_apar;
@@ -397,9 +404,8 @@ struct Explicit
     dg::SparseTensor<Container> m_hh;
 
     const feltor::Parameters m_p;
-    double m_omega_source = 0., m_omega_damping = 0.;
+    double m_omega_source = 0., m_sheath_forcing = 0., m_wall_forcing = 0.;
     bool m_fixed_profile = true;
-    bool m_full_system = false;
 
 };
 
@@ -548,7 +554,7 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
 }
 template<class Grid, class IMatrix, class Matrix, class Container>
 Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
-    feltor::Parameters p, dg::geo::TokamakMagneticField mag, bool full_system):
+    feltor::Parameters p, dg::geo::TokamakMagneticField mag):
 #ifdef DG_MANUFACTURED
     m_R( dg::pullback( dg::cooX3d, g)),
     m_Z( dg::pullback( dg::cooY3d, g)),
@@ -564,11 +570,12 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     m_multigrid( g, p.stages),
     m_old_phi( 2, dg::evaluate( dg::zero, g)),
     m_old_psi( m_old_phi), m_old_gammaN( m_old_phi), m_old_apar( m_old_phi),
-    m_p(p), m_full_system(full_system)
+    m_p(p)
 {
     //--------------------------init vectors to 0-----------------//
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
-    m_UE2 = m_temp2 = m_temp1 = m_temp0;
+    m_source = m_U_sheath = m_UE2 = m_temp2 = m_temp1 = m_temp0;
+    dg::assign( dg::evaluate( dg::one, g), m_masked );
     m_apar = m_temp0;
 
     m_phi[0] = m_phi[1] = m_temp0;
@@ -820,10 +827,6 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
             );
         }
     }
-    //------------------Add Resistivity--------------------------//
-    //dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
-    //    m_fields[0][0], m_fields[0][1],
-    //    m_fields[1][0], m_fields[1][1], yp[1][0], yp[1][1]);
 }
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
@@ -944,39 +947,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     compute_parallel( t, y, m_fields, yp);
 
 #endif
-
-    //Add source terms
-    if( m_omega_source != 0 || m_omega_damping != 0)
-    {
-        if( m_fixed_profile )
-            dg::blas1::subroutine( routines::ComputeSource(), m_s[0][0], y[0][0],
-                m_profne, m_source, m_omega_source);
-        else
-            dg::blas1::axpby( m_omega_source, m_source, 0., m_s[0][0]);
-        // -w_d ~n
-        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, y[0][0], 1.,  m_s[0][0]);
-        //compute FLR corrections S_N = (1-0.5*mu*tau*Lap)*S_n
-        dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
-        dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
-        // potential part of FLR correction S_N += -div*(mu S_n grad*Phi/B^2)
-        dg::blas1::pointwiseDot( m_p.mu[1], m_s[0][0], m_binv, m_binv, 0., m_temp0);
-        m_lapperpP.multiply_sigma( 1., m_temp0, m_phi[0], 1., m_s[0][1]);
-
-        // S_U = - w_d U
-        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][0], 0.,  m_s[1][0]);
-        dg::blas1::pointwiseDot( -m_omega_damping, m_damping, m_fields[1][1], 0.,  m_s[1][1]);
-        // S_U += - U S_N/N
-        dg::blas1::pointwiseDot( -1.,  m_fields[1][0],  m_s[0][0], 0., m_temp0);
-        dg::blas1::pointwiseDot( -1.,  m_fields[1][1],  m_s[0][1], 0., m_temp1);
-        dg::blas1::pointwiseDivide( 1.,  m_temp0,  m_fields[0][0], 1., m_s[1][0]);
-        dg::blas1::pointwiseDivide( 1.,  m_temp1,  m_fields[0][1], 1., m_s[1][1]);
-
-        //Add all to the right hand side
-        dg::blas1::axpby( 1., m_s, 1.0, yp);
-    }
-
-
-    if( m_full_system)
+    if( m_p.explicit_diffusion)
     {
 #if FELTORPERP == 1
         /* y[0] := n_e - 1
@@ -1006,8 +977,52 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
                 dg::blas2::symv( -m_p.nu_perp, m_lapperpU,
                     m_fields[1][i],  1., yp[1][i]);
         }
+        //------------------Add Resistivity--------------------------//
+        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+            m_fields[0][0], m_fields[0][1],
+            m_fields[1][0], m_fields[1][1], yp[1][0], yp[1][1]);
 #endif
     }
+
+    //Add source terms
+    if( m_omega_source != 0 )
+    {
+        if( m_fixed_profile )
+            dg::blas1::subroutine( routines::ComputeSource(), m_s[0][0], y[0][0],
+                m_profne, m_source, m_omega_source);
+        else
+            dg::blas1::axpby( m_omega_source, m_source, 0., m_s[0][0]);
+        //compute FLR corrections S_N = (1-0.5*mu*tau*Lap)*S_n
+        dg::blas2::gemv( m_lapperpN, m_s[0][0], m_temp0);
+        dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
+        // potential part of FLR correction S_N += -div*(mu S_n grad*Phi/B^2)
+        dg::blas1::pointwiseDot( m_p.mu[1], m_s[0][0], m_binv, m_binv, 0., m_temp0);
+        m_lapperpP.multiply_sigma( 1., m_temp0, m_phi[0], 1., m_s[0][1]);
+
+        // S_U += - U S_N/N
+        dg::blas1::pointwiseDot( -1.,  m_fields[1][0],  m_s[0][0], 0., m_temp0);
+        dg::blas1::pointwiseDot( -1.,  m_fields[1][1],  m_s[0][1], 0., m_temp1);
+        dg::blas1::pointwiseDivide( 1.,  m_temp0,  m_fields[0][0], 1., m_s[1][0]);
+        dg::blas1::pointwiseDivide( 1.,  m_temp1,  m_fields[0][1], 1., m_s[1][1]);
+
+        //Add all to the right hand side
+        dg::blas1::axpby( 1., m_s, 1.0, yp);
+    }
+    //mask right hand side in forcing region
+    dg::blas1::pointwiseDot( m_masked, yp[0][0], yp[0][0]);
+    dg::blas1::pointwiseDot( m_masked, yp[0][1], yp[0][1]);
+    dg::blas1::pointwiseDot( m_masked, yp[1][0], yp[1][0]);
+    dg::blas1::pointwiseDot( m_masked, yp[1][1], yp[1][1]);
+    if( m_sheath_forcing != 0)
+    {
+        // explicit part of the sheath terms
+        //dg::blas1::transform( m_phi[0], m_temp0, dg::EXP<double>(1., -1.));
+        //dg::blas1::pointwiseDot( m_sheath_forcing, m_U_sheath, m_temp0, 1.,  yp[1][0]);
+        dg::blas1::axpby( m_sheath_forcing, m_U_sheath, 1.,  yp[1][0]);
+        dg::blas1::axpby( m_sheath_forcing, m_U_sheath, 1.,  yp[1][1]);
+    }
+
+
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( yp[0][0], dg::plus_equals(), manufactured::SNe{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 7cf537d11..823682055 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -38,8 +38,19 @@ struct ImplicitDensity
         m_lapM_perpN.set_chi( hh);
         if( p.curvmode != "true")
             m_lapM_perpN.set_compute_in_2d( true);
+
+        dg::assign( dg::evaluate( dg::zero, g), m_forcing );
+        dg::assign( dg::evaluate( dg::one, g), m_masked );
+    }
+    void set_wall_and_sheath( double wall_forcing, Container wall, double sheath_forcing, Container sheath)
+    {
+        dg::blas1::axpby( wall_forcing, wall, sheath_forcing, sheath, m_forcing); //1/eta_w
+
+        dg::blas1::axpby( -1., wall, -1., sheath, m_masked);
+        dg::blas1::plus( m_masked, +1);
     }
 
+    const Container& get_forcing() const { return m_forcing;}
     void operator()( double t, const std::array<Container,2>& y,
         std::array<Container,2>& yp)
     {
@@ -47,17 +58,26 @@ struct ImplicitDensity
         /* y[0] := n_e - 1
            y[1] := N_i - 1
         */
-        for( unsigned i=0; i<2; i++)
+        if( !m_p.explicit_diffusion)
         {
-            //dissipation acts on w!
-            if( m_p.perp_diff == "hyperviscous")
+            for( unsigned i=0; i<2; i++)
             {
-                dg::blas2::symv( m_lapM_perpN, y[i],      m_temp);
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, m_temp, 0., yp[i]);
+                //dissipation acts on w!
+                if( m_p.perp_diff == "hyperviscous")
+                {
+                    dg::blas2::symv( m_lapM_perpN, y[i],      m_temp);
+                    dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, m_temp, 0., yp[i]);
+                }
+                else // m_p.perp_diff == "viscous"
+                    dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, y[i],  0., yp[i]);
             }
-            else // m_p.perp_diff == "viscous"
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpN, y[i],  0., yp[i]);
+            dg::blas1::pointwiseDot( m_masked, yp[0], yp[0]);
+            dg::blas1::pointwiseDot( m_masked, yp[1], yp[1]);
         }
+        else
+            dg::blas1::copy( 0, yp);
+        dg::blas1::pointwiseDot( -1., m_forcing, y[0], 1., yp[0]);
+        dg::blas1::pointwiseDot( -1., m_forcing, y[1], 1., yp[1]);
 #else
         dg::blas1::copy( 0, yp);
 #endif
@@ -75,7 +95,7 @@ struct ImplicitDensity
 
   private:
     feltor::Parameters m_p;
-    Container m_temp;
+    Container m_temp, m_forcing, m_masked;
     dg::Elliptic3d<Geometry, Matrix, Container> m_lapM_perpN;
 };
 
@@ -131,6 +151,15 @@ struct ImplicitVelocity
         }
         m_multi_chi = m_multigrid.project( m_temp);
         m_old_apar = dg::Extrapolation<Container>( 2, dg::evaluate( dg::zero, g));
+
+        dg::assign( dg::evaluate( dg::zero, g), m_forcing );
+        dg::assign( dg::evaluate( dg::one, g), m_masked );
+    }
+    void set_wall_and_sheath( double wall_forcing, Container wall, double sheath_forcing, Container sheath)
+    {
+        dg::blas1::axpby( wall_forcing, wall, sheath_forcing, sheath, m_temp); //1/eta_w
+        m_forcing = m_temp;
+
     }
     void set_density( const std::array<Container, 2>& dens){
         dg::blas1::transform( dens, m_fields[0], dg::PLUS<double>(+1));
@@ -157,46 +186,56 @@ struct ImplicitVelocity
         /* w[0] := w_e
            w[1] := W_i
         */
-        dg::blas1::copy( w, m_fields[1]);
-        if( m_p.beta != 0){
-            //let us solve for apar
-            dg::blas1::pointwiseDot(  m_p.beta, m_fields[0][1], m_fields[1][1],
-                                     -m_p.beta, m_fields[0][0], m_fields[1][0],
-                                      0., m_temp);
-            //m_invert( m_induction, m_apar, m_temp, weights(),
-            //    inv_weights(), precond());
-            m_old_apar.extrapolate( m_apar);
-            //dg::blas1::scal( m_apar, 0.);
-            std::vector<unsigned> number = m_multigrid.direct_solve(
-                m_multi_induction, m_apar, m_temp, m_p.eps_pol[0]); //eps_pol[0] on all grids
-            //m_old_apar.update( m_apar); //don't update here: makes the solver potentially unstable
-            if(  number[0] == m_multigrid.max_iter())
-                throw dg::Fail( m_p.eps_pol[0]);
-
-            //compute u_e and U_i from w_e, W_i and apar
-            dg::blas1::axpby( 1., m_fields[1][0], -1./m_p.mu[0],
-                m_apar, m_fields[1][0]);
-            dg::blas1::axpby( 1., m_fields[1][1], -1./m_p.mu[1],
-                m_apar, m_fields[1][1]);
-        }
-        /* fields[1][0] := u_e
-           fields[1][1] := U_i
-        */
-        for( unsigned i=0; i<2; i++)
+        if( !m_p.explicit_diffusion)
         {
-            if( m_p.perp_diff == "hyperviscous")
+            dg::blas1::copy( w, m_fields[1]);
+            if( m_p.beta != 0){
+                //let us solve for apar
+                dg::blas1::pointwiseDot(  m_p.beta, m_fields[0][1], m_fields[1][1],
+                                         -m_p.beta, m_fields[0][0], m_fields[1][0],
+                                          0., m_temp);
+                //m_invert( m_induction, m_apar, m_temp, weights(),
+                //    inv_weights(), precond());
+                m_old_apar.extrapolate( m_apar);
+                //dg::blas1::scal( m_apar, 0.);
+                std::vector<unsigned> number = m_multigrid.direct_solve(
+                    m_multi_induction, m_apar, m_temp, m_p.eps_pol[0]); //eps_pol[0] on all grids
+                //m_old_apar.update( m_apar); //don't update here: makes the solver potentially unstable
+                if(  number[0] == m_multigrid.max_iter())
+                    throw dg::Fail( m_p.eps_pol[0]);
+
+                //compute u_e and U_i from w_e, W_i and apar
+                dg::blas1::axpby( 1., m_fields[1][0], -1./m_p.mu[0],
+                    m_apar, m_fields[1][0]);
+                dg::blas1::axpby( 1., m_fields[1][1], -1./m_p.mu[1],
+                    m_apar, m_fields[1][1]);
+            }
+            /* fields[1][0] := u_e
+               fields[1][1] := U_i
+            */
+            for( unsigned i=0; i<2; i++)
             {
-                dg::blas2::symv( m_lapM_perpU, m_fields[1][i],      m_temp);
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, m_temp, 0., wp[i]);
+                if( m_p.perp_diff == "hyperviscous")
+                {
+                    dg::blas2::symv( m_lapM_perpU, m_fields[1][i],      m_temp);
+                    dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU, m_temp, 0., wp[i]);
+                }
+                else // m_p.perp_diff == "viscous"
+                    dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU,
+                        m_fields[1][i],  0., wp[i]);
             }
-            else // m_p.perp_diff == "viscous"
-                dg::blas2::symv( -m_p.nu_perp, m_lapM_perpU,
-                    m_fields[1][i],  0., wp[i]);
+            //------------------Add Resistivity--------------------------//
+            dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
+                m_fields[0][0], m_fields[0][1],
+                m_fields[1][0], m_fields[1][1], wp[0], wp[1]);
+            dg::blas1::pointwiseDot( m_masked, wp[0], wp[0]);
+            dg::blas1::pointwiseDot( m_masked, wp[1], wp[1]);
         }
-        //------------------Add Resistivity--------------------------//
-        dg::blas1::subroutine( routines::AddResistivity( m_p.eta, m_p.mu),
-            m_fields[0][0], m_fields[0][1],
-            m_fields[1][0], m_fields[1][1], wp[0], wp[1]);
+        else
+            dg::blas1::copy( 0, wp);
+        //we force w_parallel not u
+        dg::blas1::pointwiseDot( -1., m_forcing, w[0], 1., wp[0]);
+        dg::blas1::pointwiseDot( -1., m_forcing, w[1], 1., wp[1]);
 #else
         dg::blas1::copy( 0, wp);
 #endif
@@ -239,7 +278,7 @@ struct ImplicitVelocity
 #endif //DG_MANUFACTURED
   private:
     feltor::Parameters m_p;
-    Container m_temp, m_apar;
+    Container m_temp, m_apar, m_forcing, m_masked;
 #ifdef DG_MANUFACTURED
     dg::Invert<Container> m_invert;
 #endif //DG_MANUFACTURED
@@ -259,6 +298,11 @@ struct Implicit
     Implicit( const Geometry& g, feltor::Parameters p,
             dg::geo::TokamakMagneticField mag):
             m_dens( g,p,mag), m_velo( g,p,mag){}
+    void set_wall_and_sheath( double wall_forcing, Container wall, double sheath_forcing, Container sheath)
+    {
+        m_dens.set_wall_and_sheath( wall_forcing, wall, sheath_forcing, sheath);
+        m_velo.set_wall_and_sheath( wall_forcing, wall, sheath_forcing, sheath);
+    }
 
     void operator()( double t, const std::array<std::array<Container,2>,2>& y,
         std::array<std::array<Container,2>,2>& yp)
@@ -266,6 +310,7 @@ struct Implicit
         m_dens( t,y[0], yp[0]);
         m_velo.set_density( y[0]);
         m_velo( t,y[1], yp[1]);
+        m_velo.update();
     }
     private:
     ImplicitDensity <Geometry, IMatrix, Matrix, Container> m_dens;
@@ -286,8 +331,8 @@ struct FeltorSpecialSolver
     FeltorSpecialSolver( const Geometry& grid, Parameters p,
         dg::geo::TokamakMagneticField mag)
     {
-#ifdef DG_MANUFACTURED
         m_p = p;
+#ifdef DG_MANUFACTURED
         m_R= dg::pullback( dg::cooX3d, grid);
         m_Z= dg::pullback( dg::cooY3d, grid);
         m_P= dg::pullback( dg::cooZ3d, grid);
@@ -307,6 +352,11 @@ struct FeltorSpecialSolver
 
         return std::array<std::array<Container,2>,2>{ m_solver.copyable(), m_solver.copyable()};
     }
+    void set_wall_and_sheath( double wall_forcing, Container wall, double sheath_forcing, Container sheath)
+    {
+        m_imdens.set_wall_and_sheath( wall_forcing, wall, sheath_forcing, sheath);
+        m_imvelo.set_wall_and_sheath( wall_forcing, wall, sheath_forcing, sheath);
+    }
 
     //Solve y + a I(t,y) = rho
     void solve( value_type alpha,
@@ -316,6 +366,16 @@ struct FeltorSpecialSolver
         const std::array<std::array<Container,2>,2>& rhs)
     {
 
+        if( m_p.explicit_diffusion)
+        {
+            //I(y) = -Chi y
+            dg::blas1::axpby( 1., 1., -alpha, m_imdens.get_forcing(), y[1][1]);
+            dg::blas1::pointwiseDivide( rhs[0][0], y[1][1], y[0][0]);
+            dg::blas1::pointwiseDivide( rhs[0][1], y[1][1], y[0][1]);
+            dg::blas1::pointwiseDivide( rhs[1][0], y[1][1], y[1][0]);
+            dg::blas1::pointwiseDivide( rhs[1][1], y[1][1], y[1][1]);
+            return;
+        }
         m_solver.solve( alpha, m_imdens, t, y[0], rhs[0]);
         m_imvelo.set_density( y[0]);
 #ifdef DG_MANUFACTURED
@@ -336,8 +396,8 @@ struct FeltorSpecialSolver
     ImplicitDensity<Geometry,IMatrix, Matrix,Container> m_imdens;
     ImplicitVelocity<Geometry,IMatrix, Matrix,Container> m_imvelo;
     value_type m_eps;
-#ifdef DG_MANUFACTURED
     feltor::Parameters m_p;
+#ifdef DG_MANUFACTURED
     std::array<Container, 2> m_rhs;
     Container m_sa, m_inv_sa;
     Container m_R, m_Z, m_P; //coordinates
diff --git a/src/feltor/init.h b/src/feltor/init.h
index e1cd97fdc..42080f97c 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -70,6 +70,7 @@ HVec profile_damping(const Geometry& grid,
 {
     if( p.profile_alpha == 0)
         throw dg::Error(dg::Message()<< "Invalid parameter: profile alpha must not be 0\n");
+    //we also need to avoid being too far in the PFR where psi can become very negative
     HVec profile_damping = dg::pullback( dg::compose(dg::PolynomialHeaviside(
         1.-p.profile_alpha/2., p.profile_alpha/2., -1), dg::geo::RhoP(mag)), grid);
     dg::blas1::pointwiseDot( xpoint_damping(grid,p,mag),
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 38dab5585..3db310f14 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -40,7 +40,7 @@ struct Parameters
     double sigma_z;
     double k_psi;
 
-    double source_rate, damping_rate;
+    double source_rate, damping_rate, sheath_rate;
     double source_alpha, profile_alpha;
     double source_boundary;
     double nprofamp;
@@ -50,7 +50,7 @@ struct Parameters
     enum dg::bc bcxN, bcyN, bcxU, bcyU, bcxP, bcyP;
     std::string initne, initphi, curvmode, perp_diff;
     std::string source_type;
-    bool symmetric, periodify;
+    bool symmetric, periodify, explicit_diffusion ;
     Parameters() = default;
     Parameters( const Json::Value& js, enum file::error mode = file::error::is_warning ) {
         //We need to check if a member is present
@@ -105,6 +105,7 @@ struct Parameters
         posY        = file::get( mode, js, "posY", 0.).asDouble();
         sigma_z     = file::get( mode, js, "sigma_z", 0.).asDouble();
         k_psi       = file::get( mode, js, "k_psi", 0.).asDouble();
+        explicit_diffusion = file::get( mode, js, "explicit_diff", false).asBool();
 
         nprofamp   = file::get( mode, js, "profile", "amp", 0.).asDouble();
         profile_alpha = file::get( mode, js, "profile", "alpha", 0.2).asDouble();
@@ -114,6 +115,7 @@ struct Parameters
         source_boundary = file::get( mode, js, "source", "boundary", 0.5).asDouble();
         source_alpha    = file::get( mode, js, "source", "alpha", 0.2).asDouble();
         damping_rate = file::get( mode, js, "damping", "rate", 0.).asDouble();
+        sheath_rate  = file::get( mode, js, "sheath", "rate", 0.).asDouble();
 
         bcxN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 0, "").asString());
         bcyN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 1, "").asString());
@@ -158,8 +160,10 @@ struct Parameters
             <<"     source_rate:                  "<<source_rate<<"\n"
             <<"     source_boundary:              "<<source_boundary<<"\n"
             <<"     source_alpha:                 "<<source_alpha<<"\n"
+            <<"     profile_alpha:                "<<profile_alpha<<"\n"
             <<"     source_type:                  "<<source_type<<"\n"
             <<"     damping_rate:                 "<<damping_rate<<"\n"
+            <<"     sheath_rate:                  "<<sheath_rate<<"\n"
             <<"     density profile amplitude:    "<<nprofamp<<"\n"
             <<"     boxscale R+:                  "<<boxscaleRp<<"\n"
             <<"     boxscale R-:                  "<<boxscaleRm<<"\n"
@@ -178,7 +182,8 @@ struct Parameters
             <<"     Accuracy Time Stepper "<<rtol<<"\n"
             <<"     Accuracy Fieldline    "<<rk4eps<<"\n"
             <<"     Periodify FCI         "<<std::boolalpha<< periodify<<"\n"
-            <<"     Refined FCI           "<<mx<<" "<<my<<"\n";
+            <<"     Refined FCI           "<<mx<<" "<<my<<"\n"
+            <<"     explicit diffusion    "<<std::boolalpha<<explicit_diffusion<<"\n";
         for( unsigned i=1; i<stages; i++)
             os <<"     Factors for Multigrid "<<i<<" "<<eps_pol[i]<<"\n";
         os << "Output parameters are: \n"
-- 
GitLab


From da0e4fe96f16398d21f6177054ea24a7d58e75ae Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 5 Nov 2020 22:18:57 +0100
Subject: [PATCH 394/540] Add sheath type parameter

distinguish between Bohm and insulating sheath
---
 src/feltor/feltor.h     | 15 +++++++++++----
 src/feltor/parameters.h |  4 +++-
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 62b0fa9cf..06cc48f33 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -1013,12 +1013,19 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     dg::blas1::pointwiseDot( m_masked, yp[0][1], yp[0][1]);
     dg::blas1::pointwiseDot( m_masked, yp[1][0], yp[1][0]);
     dg::blas1::pointwiseDot( m_masked, yp[1][1], yp[1][1]);
+    // explicit part of the sheath terms
     if( m_sheath_forcing != 0)
     {
-        // explicit part of the sheath terms
-        //dg::blas1::transform( m_phi[0], m_temp0, dg::EXP<double>(1., -1.));
-        //dg::blas1::pointwiseDot( m_sheath_forcing, m_U_sheath, m_temp0, 1.,  yp[1][0]);
-        dg::blas1::axpby( m_sheath_forcing, m_U_sheath, 1.,  yp[1][0]);
+        if( "insulating" == m_p.sheath_type)
+        {
+            dg::blas1::axpby( m_sheath_forcing, m_U_sheath, 1.,  yp[1][0]);
+        }
+        else // "bohm" == m_p.sheath_type
+        {
+            //exp(-phi)
+            dg::blas1::transform( m_phi[0], m_temp0, dg::EXP<double>(1., -1.));
+            dg::blas1::pointwiseDot( m_sheath_forcing, m_U_sheath, m_temp0, 1.,  yp[1][0]);
+        }
         dg::blas1::axpby( m_sheath_forcing, m_U_sheath, 1.,  yp[1][1]);
     }
 
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 3db310f14..3eb8ca96d 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -49,7 +49,7 @@ struct Parameters
 
     enum dg::bc bcxN, bcyN, bcxU, bcyU, bcxP, bcyP;
     std::string initne, initphi, curvmode, perp_diff;
-    std::string source_type;
+    std::string source_type, sheath_type;
     bool symmetric, periodify, explicit_diffusion ;
     Parameters() = default;
     Parameters( const Json::Value& js, enum file::error mode = file::error::is_warning ) {
@@ -112,6 +112,7 @@ struct Parameters
 
         source_rate     = file::get( mode, js, "source", "rate", 0.).asDouble();
         source_type     = file::get( mode, js, "source", "type", "profile").asString();
+        sheath_type     = file::get( mode, js, "sheath", "type", "bohm").asString();
         source_boundary = file::get( mode, js, "source", "boundary", 0.5).asDouble();
         source_alpha    = file::get( mode, js, "source", "alpha", 0.2).asDouble();
         damping_rate = file::get( mode, js, "damping", "rate", 0.).asDouble();
@@ -164,6 +165,7 @@ struct Parameters
             <<"     source_type:                  "<<source_type<<"\n"
             <<"     damping_rate:                 "<<damping_rate<<"\n"
             <<"     sheath_rate:                  "<<sheath_rate<<"\n"
+            <<"     sheath_type:                  "<<sheath_type<<"\n"
             <<"     density profile amplitude:    "<<nprofamp<<"\n"
             <<"     boxscale R+:                  "<<boxscaleRp<<"\n"
             <<"     boxscale R-:                  "<<boxscaleRm<<"\n"
-- 
GitLab


From f6d45d50afa86ad684bb81d9a57ed4109d211af3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 6 Nov 2020 00:01:02 +0100
Subject: [PATCH 395/540] Rewrite current to divergence form

in feltordiag, in order to be able to judge local conservation
---
 src/feltor/feltor.h     |  26 ++++++---
 src/feltor/feltordiag.h | 126 +++++++++++++++++++++++++---------------
 2 files changed, 98 insertions(+), 54 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 06cc48f33..a25b9832c 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -252,6 +252,18 @@ struct Explicit
         dg::blas2::symv( m_dy_N, m_s[0][i], gradS[1]);
         if(!m_p.symmetric)dg::blas2::symv( m_dz, m_s[0][i], gradS[2]);
     }
+    void divergence( const std::array<Container,3>& in, Container& out) const{
+        dg::blas1::pointwiseDot( m_detg, in[0], m_temp0);
+        dg::blas1::pointwiseDot( m_detg, in[1], m_temp1);
+        dg::blas1::pointwiseDot( m_detg, in[2], m_temp2);
+        dg::blas2::symv( m_dx_U, m_temp0, m_temp0);
+        dg::blas2::symv( m_dy_U, m_temp1, m_temp1);
+        dg::blas2::symv( m_dz, m_temp2, m_temp2);
+        dg::blas1::pointwiseDivide( m_temp0, m_detg, out);
+        dg::blas1::pointwiseDivide( 1., m_temp1, m_detg, 1., out);
+        dg::blas1::pointwiseDivide( 1., m_temp2, m_detg, 1., out);
+
+    }
     void compute_dot_induction( Container& tmp) const {
         m_old_apar.derive( tmp);
     }
@@ -286,7 +298,9 @@ struct Explicit
     const Container& bphi( ) const { return m_bphi; }
     const Container& binv( ) const { return m_binv; }
     const Container& divb( ) const { return m_divb; }
-    const Container& vol3d() const { return m_vol3d;}
+    const Container& detg() const { return m_detg;}
+    //volume with dG weights
+    const Container& vol3d() const { return m_lapperpN.weights();}
     const Container& weights() const { return m_lapperpN.weights();}
     //bhat / sqrt{g} / B
     const std::array<Container, 3> & bhatgB () const {
@@ -379,7 +393,7 @@ struct Explicit
     Container m_divCurvKappa;
     Container m_bphi, m_binv, m_divb;
     Container m_source, m_profne, m_U_sheath, m_masked;
-    Container m_vol3d;
+    Container m_detg;
 
     Container m_apar;
     std::array<Container,2> m_phi, m_dsN, m_dsU, m_dsP, m_dssU;// m_dssN;
@@ -491,11 +505,11 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
     dg::SparseTensor<Container> metric = g.metric();
     dg::tensor::inv_multiply3d( metric, m_b[0], m_b[1], m_b[2],
                                         m_b[0], m_b[1], m_b[2]);
-    Container vol = dg::tensor::volume( metric);
-    dg::blas1::pointwiseDivide( m_binv, vol, vol); //1/vol/B
+    m_detg = dg::tensor::volume( metric);
+    dg::blas1::pointwiseDivide( m_binv, m_detg, m_detg); //1/m_detg/B
     dg::assign( m_b[2], m_bphi); //save bphi for momentum conservation
     for( int i=0; i<3; i++)
-        dg::blas1::pointwiseDot( vol, m_b[i], m_b[i]); //b_i/vol/B
+        dg::blas1::pointwiseDot( m_detg, m_b[i], m_b[i]); //b_i/m_detg/B
     m_hh = dg::geo::createProjectionTensor( bhat, g);
     m_lapperpN.construct ( g, p.bcxN, p.bcyN, dg::PER, dg::normed, dg::centered),
     m_lapperpU.construct ( g, p.bcxU, p.bcyU, dg::PER, dg::normed, dg::centered),
@@ -592,8 +606,6 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     construct_mag( g, p, mag);
     construct_bhat( g, p, mag);
     construct_invert( g, p, mag);
-    //---------------------------Volume------------------------------//
-    dg::assign( dg::create::volume(g), m_vol3d);
 }
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index e695ab97e..d39ae4f20 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -25,32 +25,27 @@ struct RadialParticleFlux{
         m_tau(tau), m_mu(mu){
     }
     //jsNC
-    DG_DEVICE double operator()( double ne, double ue,
-        double d0S, double d1S, double d2S, //Psip
+    DG_DEVICE void operator()(
+        double& j0, double& j1, double& j2, //out
+        double ne, double ue,
         double curv0,       double curv1,       double curv2,
         double curvKappa0,  double curvKappa1,  double curvKappa2
         ){
-        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
-        double curvS = curv0*d0S+curv1*d1S+curv2*d2S;
-        double JPsi =
-            + ne * m_mu*ue*ue*curvKappaS
-            + ne * m_tau*curvS;
-        return JPsi;
+        j0 = ne*(m_tau * curv0 + m_mu * ue*ue*curvKappa0);
+        j1 = ne*(m_tau * curv1 + m_mu * ue*ue*curvKappa1);
+        j2 = ne*(m_tau * curv2 + m_mu * ue*ue*curvKappa2);
     }
     //jsNA
-    DG_DEVICE double operator()( double ne, double ue, double A,
+    DG_DEVICE double operator()(
+        double& j0, double& j1, double& j2, //out
+        double ne, double ue, double A,
         double d0A, double d1A, double d2A,
-        double d0S, double d1S, double d2S, //Psip
         double b_0,         double b_1,         double b_2,
         double curvKappa0,  double curvKappa1,  double curvKappa2
         ){
-        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
-        double SA = b_0*( d1S*d2A-d2S*d1A)+
-                    b_1*( d2S*d0A-d0S*d2A)+
-                    b_2*( d0S*d1A-d1S*d0A);
-        double JPsi =
-            ne*ue* (A*curvKappaS + SA );
-        return JPsi;
+        j0 = ne*ue*(A*curvKappa0 + d1A*b_2-d2A*b_1);
+        j1 = ne*ue*(A*curvKappa1 + d2A*b_0-d0A*b_2);
+        j2 = ne*ue*(A*curvKappa2 + d0A*b_1-d1A*b_0);
     }
     private:
     double m_tau, m_mu;
@@ -118,6 +113,37 @@ void dot( const std::array<Container, 3>& v,
     dg::blas1::evaluate( result, dg::equals(), dg::PairSum(),
         v[0], w[0], v[1], w[1], v[2], w[2]);
 }
+struct Times{
+    DG_DEVICE void operator()(
+            double lambda,
+        double d0P, double d1P, double d2P, //any three vectors
+        double d0S, double d1S, double d2S,
+        double& c_0, double& c_1, double& c_2)
+    {
+        c_0 = lambda*(d1P*d2S-d2P*d1S);
+        c_1 = lambda*(d2P*d0S-d0P*d2S);
+        c_2 = lambda*(d0P*d1S-d1P*d0S);
+    }
+};
+template<class Container>
+void times(
+          const std::array<Container, 3>& a,
+          const std::array<Container, 3>& b,
+          std::array<Container, 3>& c)
+{
+    dg::blas1::subroutine( Times(), 1.,
+        a[0], a[1], a[2], b[0], b[1], b[2], c[0], c[1], c[2]);
+}
+template<class Container>
+void times(
+          const Container& lambda,
+          const std::array<Container, 3>& a,
+          const std::array<Container, 3>& b,
+          std::array<Container, 3>& c)
+{
+    dg::blas1::subroutine( Times(), lambda,
+        a[0], a[1], a[2], b[0], b[1], b[2], c[0], c[1], c[2]);
+}
 struct Jacobian{
     DG_DEVICE double operator()(
         double d0P, double d1P, double d2P, //any three vectors
@@ -469,41 +495,44 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------ Density terms ------------------------//
-    {"jsneE_tt", "Radial electron particle flux: ExB contribution (Time average)", true,
+    {"divneE_tt", "Radial electron particle flux: ExB contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            // ExB Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
-            dg::blas1::pointwiseDot( result, v.f.density(0), result);
+            // ExB
+            routines::times( v.f.density(0),v.f.bhatgB(), v.f.gradP(0), v.tmp);
+            v.f.divergence( v.tmp, result);
         }
     },
-    {"jsneC_tt", "Radial electron particle flux: curvature contribution (Time average)", true,
+    {"divneC_tt", "Radial electron particle flux: curvature contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::evaluate( result, dg::equals(),
+            dg::blas1::subroutine(
                 routines::RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
+                v.tmp[0], v.tmp[1], v.tmp[2],
                 v.f.density(0), v.f.velocity(0),
-                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
+            v.f.divergence( v.tmp, result);
         }
     },
-    {"jsdiae_tt", "Radial electron particle flux: diamagnetic contribution (Time average)", true,
+    {"divdiae_tt", "Radial electron particle flux: diamagnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            // u_D Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.f.gradN(0), v.gradPsip, result);
+            // u_D
+            routines::times( v.f.bhatgB(), v.f.gradN(0), v.tmp);
+            v.f.divergence( v.tmp, result);
             dg::blas1::scal( result, v.p.tau[0]);
         }
     },
-    {"jsneA_tt", "Radial electron particle flux: induction contribution (Time average)", true,
+    {"divneA_tt", "Radial electron particle flux: magnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::evaluate( result, dg::equals(),
+            dg::blas1::subroutine(
                 routines::RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
+                v.tmp[0], v.tmp[1], v.tmp[2],
                 v.f.density(0), v.f.velocity(0), v.f.induction(),
                 v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
-                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
+            v.f.divergence( v.tmp, result);
         }
     },
     {"lneperp_tt", "Perpendicular electron diffusion (Time average)", true,
@@ -524,48 +553,51 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::copy( v.f.density_source(0), result);
         }
     },
-    {"dnepar_tt", "Divergence of Parallel velocity term for electron density (Time average)", true,
+    {"divnepar_tt", "Divergence of Parallel velocity term for electron density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.density(0), v.f.velocity(0), v.f.divb(), 0., result);
             dg::blas1::pointwiseDot( 1., v.f.density(0),  v.f.dsU(0), 1., result);
             dg::blas1::pointwiseDot( 1., v.f.velocity(0), v.f.dsN(0), 1., result);
         }
     },
-    {"jsniE_tt", "Radial ion particle flux: ExB contribution (Time average)", true,
+    {"divniE_tt", "Radial ion particle flux: ExB contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            // ExB Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.f.gradP(1), v.gradPsip, result);
-            dg::blas1::pointwiseDot( result, v.f.density(1), result);
+            // ExB
+            routines::times( v.f.density(1),v.f.bhatgB(), v.f.gradP(1), v.tmp);
+            v.f.divergence( v.tmp, result);
         }
     },
-    {"jsniC_tt", "Radial ion particle flux: curvature contribution (Time average)", true,
+    {"divniC_tt", "Radial ion particle flux: curvature contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::evaluate( result, dg::equals(),
+            dg::blas1::subroutine(
                 routines::RadialParticleFlux( v.p.tau[1], v.p.mu[1]),
+                v.tmp[0], v.tmp[1], v.tmp[2],
                 v.f.density(1), v.f.velocity(1),
-                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
+            v.f.divergence( v.tmp, result);
         }
     },
-    {"jsdiai_tt", "Radial ion particle flux: diamagnetic contribution (Time average)", true,
+    {"divdiai_tt", "Radial ion particle flux: diamagnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            // u_D Dot GradPsi
-            routines::jacobian( v.f.bhatgB(), v.f.gradN(1), v.gradPsip, result);
+            // u_D
+            routines::times( v.f.bhatgB(), v.f.gradN(1), v.tmp);
+            v.f.divergence( v.tmp, result);
             dg::blas1::scal( result, v.p.tau[1]);
         }
     },
-    {"jsniA_tt", "Radial ion particle flux: induction contribution (Time average)", true,
+    {"divniA_tt", "Radial ion particle flux: induction contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::evaluate( result, dg::equals(),
+            dg::blas1::subroutine(
                 routines::RadialParticleFlux( v.p.tau[1], v.p.mu[1]),
+                v.tmp[0], v.tmp[1], v.tmp[2],
                 v.f.density(1), v.f.velocity(1), v.f.induction(),
                 v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
-                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
+            v.f.divergence( v.tmp, result);
         }
     },
     {"lniperp_tt", "Perpendicular ion diffusion (Time average)", true,
@@ -586,7 +618,7 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::copy( v.f.density_source(1), result);
         }
     },
-    {"dnipar_tt", "Divergence of Parallel velocity term in ion density (Time average)", true,
+    {"divnipar_tt", "Divergence of Parallel velocity term in ion density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.density(1), v.f.velocity(1), v.f.divb(), 0., result);
             dg::blas1::pointwiseDot( 1., v.f.density(1),  v.f.dsU(1), 1., result);
@@ -671,7 +703,7 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------ Energy flux terms ------------------------//
-    {"jsee_tt", "Radial electron energy flux without induction contribution (Time average)", true,
+    {"jsee_tt", "Radial electron energy flux without magnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
@@ -684,7 +716,7 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
-    {"jseea_tt", "Radial electron energy flux: induction contribution (Time average)", true,
+    {"jseea_tt", "Radial electron energy flux: magnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
-- 
GitLab


From 77a3921a716a5e51ed7a9bdd571275e8c41e2399 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 9 Nov 2020 18:39:11 +0100
Subject: [PATCH 396/540] Revert div back to js

- Reason is that in reality we need all components of the current
separately, not just the divergence
- and in 2d the radial current is an interesting quantity
---
 src/feltor/feltordiag.h | 91 ++++++++++++++++++++---------------------
 1 file changed, 45 insertions(+), 46 deletions(-)

diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index d39ae4f20..1a50677ea 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -25,27 +25,32 @@ struct RadialParticleFlux{
         m_tau(tau), m_mu(mu){
     }
     //jsNC
-    DG_DEVICE void operator()(
-        double& j0, double& j1, double& j2, //out
-        double ne, double ue,
+    DG_DEVICE double operator()( double ne, double ue,
+        double d0S, double d1S, double d2S, //Psip
         double curv0,       double curv1,       double curv2,
         double curvKappa0,  double curvKappa1,  double curvKappa2
         ){
-        j0 = ne*(m_tau * curv0 + m_mu * ue*ue*curvKappa0);
-        j1 = ne*(m_tau * curv1 + m_mu * ue*ue*curvKappa1);
-        j2 = ne*(m_tau * curv2 + m_mu * ue*ue*curvKappa2);
+        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
+        double curvS = curv0*d0S+curv1*d1S+curv2*d2S;
+        double JPsi =
+            + ne * m_mu*ue*ue*curvKappaS
+            + ne * m_tau*curvS;
+        return JPsi;
     }
     //jsNA
-    DG_DEVICE double operator()(
-        double& j0, double& j1, double& j2, //out
-        double ne, double ue, double A,
+    DG_DEVICE double operator()( double ne, double ue, double A,
         double d0A, double d1A, double d2A,
+        double d0S, double d1S, double d2S, //Psip
         double b_0,         double b_1,         double b_2,
         double curvKappa0,  double curvKappa1,  double curvKappa2
         ){
-        j0 = ne*ue*(A*curvKappa0 + d1A*b_2-d2A*b_1);
-        j1 = ne*ue*(A*curvKappa1 + d2A*b_0-d0A*b_2);
-        j2 = ne*ue*(A*curvKappa2 + d0A*b_1-d1A*b_0);
+        double curvKappaS = curvKappa0*d0S+curvKappa1*d1S+curvKappa2*d2S;
+        double SA = b_0*( d1S*d2A-d2S*d1A)+
+                    b_1*( d2S*d0A-d0S*d2A)+
+                    b_2*( d0S*d1A-d1S*d0A);
+        double JPsi =
+            ne*ue* (A*curvKappaS + SA );
+        return JPsi;
     }
     private:
     double m_tau, m_mu;
@@ -495,44 +500,41 @@ std::vector<Record> diagnostics2d_list = {
         }
     },
     /// ------------------ Density terms ------------------------//
-    {"divneE_tt", "Radial electron particle flux: ExB contribution (Time average)", true,
+    {"jsneE_tt", "Radial electron particle flux: ExB contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            // ExB
-            routines::times( v.f.density(0),v.f.bhatgB(), v.f.gradP(0), v.tmp);
-            v.f.divergence( v.tmp, result);
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(0), v.gradPsip, result);
+            dg::blas1::pointwiseDot( result, v.f.density(0), result);
         }
     },
-    {"divneC_tt", "Radial electron particle flux: curvature contribution (Time average)", true,
+    {"jsneC_tt", "Radial electron particle flux: curvature contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::subroutine(
+            dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
-                v.tmp[0], v.tmp[1], v.tmp[2],
                 v.f.density(0), v.f.velocity(0),
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            v.f.divergence( v.tmp, result);
         }
     },
-    {"divdiae_tt", "Radial electron particle flux: diamagnetic contribution (Time average)", true,
+    {"jsdiae_tt", "Radial electron particle flux: diamagnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            // u_D
-            routines::times( v.f.bhatgB(), v.f.gradN(0), v.tmp);
-            v.f.divergence( v.tmp, result);
+            // u_D Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradN(0), v.gradPsip, result);
             dg::blas1::scal( result, v.p.tau[0]);
         }
     },
-    {"divneA_tt", "Radial electron particle flux: magnetic contribution (Time average)", true,
+    {"jsneA_tt", "Radial electron particle flux: magnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::subroutine(
+            dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialParticleFlux( v.p.tau[0], v.p.mu[0]),
-                v.tmp[0], v.tmp[1], v.tmp[2],
                 v.f.density(0), v.f.velocity(0), v.f.induction(),
                 v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            v.f.divergence( v.tmp, result);
         }
     },
     {"lneperp_tt", "Perpendicular electron diffusion (Time average)", true,
@@ -560,44 +562,41 @@ std::vector<Record> diagnostics2d_list = {
             dg::blas1::pointwiseDot( 1., v.f.velocity(0), v.f.dsN(0), 1., result);
         }
     },
-    {"divniE_tt", "Radial ion particle flux: ExB contribution (Time average)", true,
+    {"jsniE_tt", "Radial ion particle flux: ExB contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            // ExB
-            routines::times( v.f.density(1),v.f.bhatgB(), v.f.gradP(1), v.tmp);
-            v.f.divergence( v.tmp, result);
+            // ExB Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradP(1), v.gradPsip, result);
+            dg::blas1::pointwiseDot( result, v.f.density(1), result);
         }
     },
-    {"divniC_tt", "Radial ion particle flux: curvature contribution (Time average)", true,
+    {"jsniC_tt", "Radial ion particle flux: curvature contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::subroutine(
+            dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialParticleFlux( v.p.tau[1], v.p.mu[1]),
-                v.tmp[0], v.tmp[1], v.tmp[2],
                 v.f.density(1), v.f.velocity(1),
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.curv()[0], v.f.curv()[1], v.f.curv()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            v.f.divergence( v.tmp, result);
         }
     },
-    {"divdiai_tt", "Radial ion particle flux: diamagnetic contribution (Time average)", true,
+    {"jsdiai_tt", "Radial ion particle flux: diamagnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            // u_D
-            routines::times( v.f.bhatgB(), v.f.gradN(1), v.tmp);
-            v.f.divergence( v.tmp, result);
+            // u_D Dot GradPsi
+            routines::jacobian( v.f.bhatgB(), v.f.gradN(1), v.gradPsip, result);
             dg::blas1::scal( result, v.p.tau[1]);
         }
     },
-    {"divniA_tt", "Radial ion particle flux: induction contribution (Time average)", true,
+    {"jsniA_tt", "Radial ion particle flux: magnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::subroutine(
+            dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialParticleFlux( v.p.tau[1], v.p.mu[1]),
-                v.tmp[0], v.tmp[1], v.tmp[2],
                 v.f.density(1), v.f.velocity(1), v.f.induction(),
                 v.f.gradA()[0], v.f.gradA()[1], v.f.gradA()[2],
+                v.gradPsip[0], v.gradPsip[1], v.gradPsip[2],
                 v.f.bhatgB()[0], v.f.bhatgB()[1], v.f.bhatgB()[2],
                 v.f.curvKappa()[0], v.f.curvKappa()[1], v.f.curvKappa()[2]
             );
-            v.f.divergence( v.tmp, result);
         }
     },
     {"lniperp_tt", "Perpendicular ion diffusion (Time average)", true,
@@ -728,7 +727,7 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
-    {"jsei_tt", "Radial ion energy flux without induction contribution (Time average)", true,
+    {"jsei_tt", "Radial ion energy flux without magnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
@@ -741,7 +740,7 @@ std::vector<Record> diagnostics2d_list = {
             );
         }
     },
-    {"jseia_tt", "Radial ion energy flux: induction contribution (Time average)", true,
+    {"jseia_tt", "Radial ion energy flux: magnetic contribution (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
-- 
GitLab


From ad6b8e1170817697ae73984b47782f61554be903 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 20 Nov 2020 16:41:25 +0100
Subject: [PATCH 397/540] Upate documentation

- fix missing references
- fix elliptic documentation
- improve chapters in geometries documentation
---
 .../dg_introduction/dg_introduction.tex       | 33 ++++---
 doc/related_pages/parallel/parallel.tex       |  2 +-
 doc/related_pages/references.bib              | 88 ++++++++++++++-----
 inc/dg/elliptic.h                             |  6 +-
 inc/dg/functors.h                             |  2 +-
 inc/geometries/ds.h                           | 21 ++---
 inc/geometries/geometries_doc.h               | 20 ++---
 inc/geometries/magnetic_field.h               |  2 +-
 inc/geometries/modified.h                     | 41 +++++----
 inc/geometries/polynomial.h                   |  2 +-
 10 files changed, 134 insertions(+), 83 deletions(-)

diff --git a/doc/related_pages/dg_introduction/dg_introduction.tex b/doc/related_pages/dg_introduction/dg_introduction.tex
index 06f661131..f9d33d73a 100644
--- a/doc/related_pages/dg_introduction/dg_introduction.tex
+++ b/doc/related_pages/dg_introduction/dg_introduction.tex
@@ -9,7 +9,8 @@
 \author{M.~Wiesenberger}
 
 \section{ Introduction to discontinuous Galerkin methods} \label{sec:discretization}
-This writeup is based on Reference~\cite{WiesenbergerPhD}. 
+This writeup is based on Reference~\cite{WiesenbergerPhD}. A useful book to consult
+is Reference~\cite{NodalDG}.
 
 In recent years, discontinuous Galerkin (dG) methods have been investigated 
 as an excellent alternative to finite difference and finite volume schemes 
@@ -30,10 +31,10 @@ In contrast, dG methods allow discontinuities across cell boundaries, which adds
 For the discretization of second derivatives we discuss the so-called local discontinuous Galerkin (LDG) method~\cite{Cockburn1998}.
 The LDG method and its advantages can also be used for the discretization of 
 elliptic equations (including Poisson's equation). 
-Reference~\cite{Arnold2001} highlights the relation 
+Reference~\cite{Arnold2002} highlights the relation 
 of the method to interior penalty and other alternative methods. 
 A superconvergence result for the 
-LDG approach was proven on Cartesian grids~\cite{Cockburn2001},
+LDG approach was proven on Cartesian grids~\cite{Cockburn2002},
 where the order of convergence is $1/2$ better than on arbitrary meshes.
 Reference~\cite{Yadav2013} later showed a similar result for general, nonlinear elliptic equations.
 
@@ -54,7 +55,7 @@ First, let us consider the one-dimensional case.
 For simplicity and ease of implementation we choose an equidistant grid with $N$ cells $C_n = [x_{n-1/2},x_{n+1/2}]$ of size $h$;
 with this choice we are able to construct basis functions of $P(C_n)$, the space of 
 polynomials of degree at most $P-1$ on $C_n$, by using orthogonal Legendre polynomials\footnote{ Often, in the literature the order of the polynomials is denoted by $k$.
-Pay attention, that we use the number of polynomial coefficients $P$ instead. We have $P=k+1$}. 
+Pay attention, that we use the number of polynomial coefficients $P$ instead. We have $P=k+1$}.
 The Legendre polynomials can be recursively defined on $[-1,1]$ by setting
 $p_0(x) = 1$, $p_1(x) = x$ and (see e.g.~\cite{AS})
 \begin{align}
@@ -62,8 +63,8 @@ $p_0(x) = 1$, $p_1(x) = x$ and (see e.g.~\cite{AS})
     \label{eq:recursion}
 \end{align}
 The so constructed Legendre polynomials are orthogonal on $[-1,1]$.
-We write 
-$x^a_j$ and $w_j$, $j=0,\dots,P-1$ denoting the abscissas and weights of 
+We write
+$x^a_j$ and $w_j$, $j=0,\dots,P-1$ denoting the abscissas and weights of
 the Gauss--Legendre quadrature on the interval $[-1,1]$. Then we note that for $k,l=0, \dots, P-1$
 \begin{align}
     \int_{-1}^1 p_k(x)p_l(x) \dx = \sum_{j=0}^{P-1} w_jp_k (x^a_j)p_l(x^a_j) = \frac{2}{2k+1}\delta_{kl}, 
@@ -71,18 +72,22 @@ the Gauss--Legendre quadrature on the interval $[-1,1]$. Then we note that for $
 \end{align}
  since Gauss--Legendre quadrature is exact for polynomials of degree at most $2P-1$.
 
-The discrete completeness relation can then be written as 
+The discrete completeness relation can then be written as
 \begin{align}
     \sum_{k=0}^{P-1} \frac{2k+1}{2}w_j p_k(x^a_i)p_k(x^a_j) = \delta_{ij}.
     \label{eq:completeness}
 \end{align}
 Given a real function $f:[-1,1]\rightarrow \mathbb{R}$ we define $f_j:=f(x^a_j)$ and
 \begin{align} \label{eq:ex2}
-    \bar f^k := \frac{2k+1}{2}\sum_{j=0}^{P-1}w_j p_k(x^a_j) f_j 
+    \bar f^k := \frac{2k+1}{2}\sum_{j=0}^{P-1}w_j p_k(x^a_j) f_j
 \end{align}
-Now let us define the forward transformation matrix by $F^{kj}:=\frac{2k+1}{2}w_jp_k(x^a_j)$ and 
-the backward transformation matrix by $B_{kj}:= p_j(x^a_k)$. Then, 
-using Eq.~\eqref{eq:ex2}, we get
+Now let us define the forward transformation matrix by $F^{kj}:=\frac{2k+1}{2}w_jp_k(x^a_j)$ and
+the backward transformation matrix by $B_{kj}:= p_j(x^a_k)$.
+\begin{tcolorbox}[title=Note]
+Reference~\cite{NodalDG} identifies $B$ as the Vandermonde matrix and further denotes $f_i$ as the \textbf{nodal} representation and $\bar f^k$ as the \textbf{modal} representation of $f(x)$.
+\end{tcolorbox}
+
+Then, using Eq.~\eqref{eq:ex2}, we get
 \begin{subequations}
 \begin{align}
     \bar f^k = \sum_{j=0}^{P-1}F^{kj}f_j \\
@@ -449,11 +454,11 @@ The function product in Eq.~\eqref{eq:polarisationb} is computed pointwisely on
     \label{eq:naive}
 \end{align}
 where $D_x$ is either $D_x^+$, $D_x^-$, or $D_x^0$ with the correct boundary terms.
-Note, that~\cite{Cockburn2001} originally only proposed to use the forward or backward discretization for $D_x$. 
+Note, that~\cite{Cockburn2002} originally only proposed to use the forward or backward discretization for $D_x$. 
 Equation~\eqref{eq:naive} is indeed a self-adjoint
 discretization for the second derivative. However, it turns out that 
 Eq.~\eqref{eq:naive} is inconsistent for given $\rho(x)$. The solution does not converge. 
-This problem is solved according to~\cite{Cockburn2001} by adding a jump term
+This problem is solved according to~\cite{Cockburn2002} by adding a jump term
 to the flux of $j$: 
 \begin{align}
     \hat j(x_{n+1/2}) \rightarrow \hat j(x_{n+1/2}) + [\phi(x_{n+1/2})],
@@ -478,7 +483,7 @@ for periodic boundaries. Again we give the correct boundary terms for Dirichlet
 and Neumann boundary conditions in Table~\ref{tab:jump_terms}.
 \input{jump_terms.tex}
 Note that $J$ is symmetric, thus the overall discretization remains self-adjoint. Indeed, with $D_x = D_x^+$ Eq.~\eqref{eq:discreteelliptic} recovers
-the discretization proposed by~\cite{Cockburn2001}. 
+the discretization proposed by~\cite{Cockburn2002}. 
 In addition, we remark that the centered discretization in Eq.~\eqref{eq:discreteelliptic} is symmetric with respect to an inversion of the coordinate system $x\rightarrow -x$ even for double Dirichlet or Neumann boundaries, while the forward and backward discretization is not. 
 
 We note again that the generalization to two or more dimensions is straightforward
diff --git a/doc/related_pages/parallel/parallel.tex b/doc/related_pages/parallel/parallel.tex
index 638e120b5..94ff9fe72 100644
--- a/doc/related_pages/parallel/parallel.tex
+++ b/doc/related_pages/parallel/parallel.tex
@@ -206,7 +206,7 @@ a combination of both ( Robin).
 The issue with Neumann boundary conditions is that they usually prescribe
 derivatives perpendicular to the boundary
 while the fieldlines are in general not perpendicular to the boundary.
-The \textbf{approach we currently adopt} is to introduce ghostcells at the
+One possible approach is to introduce ghostcells at the
 places where fieldlines end. The value of the ghostcells are
 as if we Fourier transformed the fields on the simulation domain
 with the correct boundary conditions and thus have a periodic
diff --git a/doc/related_pages/references.bib b/doc/related_pages/references.bib
index 356006e70..d4121fa7d 100644
--- a/doc/related_pages/references.bib
+++ b/doc/related_pages/references.bib
@@ -30,6 +30,7 @@
   Volume                   = {21},
   Doi                      = {10.1063/1.4881466},
 }
+
 @Article{Braginskii1965,
   Title                    = {Transport processes in a plasma},
   Author                   = {Braginskii, S.I.},
@@ -41,34 +42,67 @@
   Url                      = {http://people.hao.ucar.edu/judge/homepage/PHSX515/fall2012/Braginskii1965.pdf}
 }
 
+@Article{Cockburn1998,
+  Title         = {The local discontinuous Galerkin method for time-dependent convection-diffusion systems},
+  Author        = {Cockburn, B. and Shu, C. W.},
+  Journal       = {SIAM J. Numer. Anal.},
+  Year          = {1998},
+  Number        = {6},
+  Pages         = {2440--2463},
+  Volume        = {35},
+  Doi           = {10.1137/S0036142997316712},
+}
+
 @Article{Cockburn2001runge,
-  Title                    = {{Runge--Kutta discontinuous Galerkin methods for convection-dominated problems}},
-  Author                   = {Cockburn, B. and Shu, C. W.},
-  Journal                  = {J. Sci. Comput.},
-  Year                     = {2001},
-  Number                   = {3},
-  Pages                    = {173--261},
-  Volume                   = {16},
-  Doi                      = {10.1023/a:1012873910884},
+  Title         = {{Runge--Kutta discontinuous Galerkin methods for convection-dominated problems}},
+  Author        = {Cockburn, B. and Shu, C. W.},
+  Journal       = {J. Sci. Comput.},
+  Year          = {2001},
+  Number        = {3},
+  Pages         = {173--261},
+  Volume        = {16},
+  Doi           = {10.1023/a:1012873910884},
 }
 
-@Article{Cockburn1998,
-  Title                    = {The local discontinuous Galerkin method for time-dependent convection-diffusion systems},
-  Author                   = {Cockburn, B. and Shu, C. W.},
-  Journal                  = {Siam Journal On Numerical Analysis},
-  Year                     = {1998},
-  Number                   = {6},
-  Pages                    = {2440--2463},
-  Volume                   = {35},
-  Doi                      = {10.1137/S0036142997316712},
+@Article{Arnold2002,
+  Title         = {{Unified analysis of discontinuous Galerkin methods for elliptic problems}},
+  Author        = {Arnold, D.N. and Brezzi, F. and Cockburn, B. and Marini, L.D.},
+  Journal       = {SIAM J. Numer. Anal.},
+  Year          = {2002},
+  Number        = {5},
+  Pages         = {1749--1779},
+  Volume        = {39},
+  Doi           = {10.1137/S0036142901384162},
+}
+
+@Article{Cockburn2002,
+  Title         = {{Superconvergence of the Local Discontinuous Galerkin Method for Elliptic Problems on Cartesian Grids}},
+  Author        = {Cockburn, B. and Kanschat, G. and Perugia, I. and Sch\"otzau, D.},
+  Journal       = {SIAM J. Numer. Anal.},
+  Year          = {2002},
+  Number        = {1},
+  Pages         = {264--285},
+  Volume        = {39},
+  Doi           = {10.1137/S0036142900371544},
+}
+
+@article{Yadav2013,
+  title         = {Superconvergent discontinuous Galerkin methods for nonlinear elliptic equations},
+  author        = {Yadav, S. and Pani, A. and Park, E.J.},
+  journal       = {Math. Comput.},
+  volume        = {82},
+  number        = {283},
+  pages         = {1297--1335},
+  year          = {2013},
+  doi           = {10.1090/S0025-5718-2013-02662-2},
 }
 
 @Book{haeseleer,
-  Title                    = {{Flux Coordinates and Magnetic Field Structure}},
-  Author                   = {D'haeseleer, W.D. and Hitchon, W.N.G. and Callen, J.D. and Shohet, J.L.},
-  Publisher                = {Springer-Verlag},
-  Year                     = {1991},
-  Series                   = {Springer Series in Computational Physics},
+  Title         = {{Flux Coordinates and Magnetic Field Structure}},
+  Author        = {D'haeseleer, W.D. and Hitchon, W.N.G. and Callen, J.D. and Shohet, J.L.},
+  Publisher     = {Springer-Verlag},
+  Year          = {1991},
+  Series        = {Springer Series in Computational Physics},
 }
 
 @Article{Einkemmer2014,
@@ -112,7 +146,6 @@
   Doi                      = {10.1063/1.4892405},
 }
 
-
 @Article{Kube2016,
   Title                    = {Amplitude and size scaling for interchange motions of plasma filaments},
   Author                   = {Kube, R. and Garcia, O. E. and Wiesenberger, M.},
@@ -132,6 +165,14 @@
   Edition                  = {Second},
 }
 
+@Book{NodalDG,
+  Title                    = {Nodal discontinuous Galerkin methods: algorithms, analysis, and applications},
+  Author                   = {Hesthaven, J.S. and Warburton, T.},
+  Publisher                = {Springer Media},
+  Year                     = {2008},
+  Series                   = {Texts in Applied Mathematics},
+}
+
 @Article{Stegmeir2017,
   Title                    = {Advances in the flux-coordinate independent approach },
   Author                   = {Andreas Stegmeir and Omar Maj and David Coster and Karl Lackner and Markus Held and Matthias Wiesenberger},
@@ -152,6 +193,7 @@
   Volume                   = {24},
   Doi                      = {10.1515/RJNAMM.2009.006},
 }
+
 @Article{Madsen2013,
     author      = {Madsen, J.},
     title       = {Full-F gyrofluid model},
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index e399ffa98..b717aceda 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -77,6 +77,7 @@ class Elliptic
      * @param dir Direction of the right first derivative in x and y
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
+     * @param chi_weight_jump If true, the Jump terms are multiplied with the Chi matrix, else it is ignored
      * @note chi is assumed 1 per default
      */
     Elliptic( const Geometry& g, norm no = not_normed,
@@ -95,6 +96,7 @@ class Elliptic
      * @param dir Direction of the right first derivative in x and y
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
+     * @param chi_weight_jump If true, the Jump terms are multiplied with the Chi matrix, else it is ignored
      * @note chi is assumed 1 per default
      */
     Elliptic( const Geometry& g, bc bcx, bc bcy,
@@ -366,7 +368,7 @@ using Elliptic2d = Elliptic<Geometry, Matrix, Container>;
  \right)\f]
  is discretized. Note that the local discontinuous Galerkin discretization adds so-called
  jump terms
- \f[ D^\dagger \chi D + \alpha J \f]
+ \f[ D^\dagger \chi D + \alpha\chi_{on/off} J \f]
  where \f$\alpha\f$  is a scale factor ( = jfactor), \f$ D \f$ contains the discretizations of the above derivatives, and \f$ J\f$ is a self-adjoint matrix.
  (The symmetric part of \f$J\f$ is added @b before the volume element is divided). The adjoint of a matrix is defined with respect to the volume element including dG weights.
  Usually the default \f$ \alpha=1 \f$ is a good choice.
@@ -408,6 +410,7 @@ class Elliptic3d
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * the direction of the z derivative is always \c dg::centered
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
+     * @param chi_weight_jump If true, the Jump terms are multiplied with the Chi matrix, else it is ignored
      * @note chi is assumed 1 per default
      */
     Elliptic3d( const Geometry& g, norm no = not_normed, direction dir = forward, value_type jfactor=1., bool chi_weight_jump = false):
@@ -427,6 +430,7 @@ class Elliptic3d
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * the direction of the z derivative is always \c dg::centered
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
+     * @param chi_weight_jump If true, the Jump terms are multiplied with the Chi matrix, else it is ignored
      * @note chi is the metric tensor multiplied by the volume element per default
      */
     Elliptic3d( const Geometry& g, bc bcx, bc bcy, bc bcz, norm no = not_normed, direction dir = forward, value_type jfactor = 1., bool chi_weight_jump = false)
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index f11c341d8..c98855b50 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -1945,7 +1945,7 @@ struct WallDistance
      * @brief Allocate lines
      *
      * @param vertical walls R_0, R_1 ...  ( can be arbitrary size)
-     * @param horizonal walls Z_0, Z_1 ... ( can be arbitrary size)
+     * @param horizontal walls Z_0, Z_1 ... ( can be arbitrary size)
      */
     WallDistance( std::vector<double> vertical, std::vector<double> horizontal) :
         m_vertical(vertical), m_horizontal( horizontal) {}
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index fbf14675b..73799357b 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -198,18 +198,6 @@ struct ComputeDSSDIR{
 }//namespace detail
 ///@endcond
 
-///@brief How boundary conditions are implemented in DS
-enum class boundary
-{
-    perp, //!< boundary condition is implemented perpendicular to boundary
-    along_field //!< boundary condition is implemented along the fieldline
-};
-
-static const std::map < std::string, boundary> str2boundary {
-    { "perp", boundary::perp},
-    { "along_field", boundary::perp}
-};
-
 /*!@class hide_ds_parameters2
 * @param f The vector to derive
 * @param g contains result on output (write only)
@@ -682,6 +670,7 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
 * @brief forward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
 *
 * forward derivative \f$ g_i = \alpha \frac{1}{h_z^+}(f_{i+1} - f_{i}) + \beta g_i\f$
+* @param fa this object will be used to get grid distances
 * @copydoc hide_ds_parameters4
 * @copydoc hide_ds_fp
 * @ingroup fieldaligned
@@ -700,7 +689,7 @@ void ds_forward(const FieldAligned& fa, double alpha, const container& f, const
 * @brief backward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
 *
 * backward derivative \f$ g_i = \alpha \frac{1}{h_z^-}(f_{i} - f_{i-1}) + \beta g_i \f$
- * @param fa this object will be used to get grid distances
+* @param fa this object will be used to get grid distances
 * @copydoc hide_ds_parameters4
 * @copydoc hide_ds_fm
 * @ingroup fieldaligned
@@ -719,7 +708,7 @@ void ds_backward( const FieldAligned& fa, double alpha, const container& fm, con
 * @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
 *
 * The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup
- * @param fa this object will be used to get grid distances
+* @param fa this object will be used to get grid distances
 * @copydoc hide_ds_parameters4
 * @copydoc hide_ds_fm
 * @copydoc hide_ds_fp
@@ -760,7 +749,7 @@ void dss_centered( const FieldAligned& fa, double alpha, const container&
 *
 * The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup.
  * the boundary condition is implemented
- * along the field-line
+ * along the field-line, that is the boundary condition is used as part of the polynomial interpolation.
  * @param fa this object will be used to get grid distances
 * @copydoc hide_ds_parameters4
 * @copydoc hide_ds_fm
@@ -794,7 +783,7 @@ void ds_centered_bc_along_field( const FieldAligned& fa, double alpha, const con
  * The formula used is \f[ \nabla_\parallel^2 f = 2\left(\frac{f^+}{h_z^+ h_z^0} - \frac{f^0}{h_z^- h_z^+} + \frac{f^-}{h_z^-h_z^0}\right) \f]
  * which is the second derivative of a 2nd order polynomial fitted through the plus, minus and centre points
  * the boundary condition is implemented
- * along the field-line
+ * along the field-line, that is the boundary condition is used as part of the polynomial interpolation.
  * @param fa this object will be used to get grid distances
  * @copydoc hide_ds_parameters4
 * @copydoc hide_ds_fm
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index 4c0398c0e..b3d51ff1f 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -6,22 +6,22 @@
  *      All the grids introduced by this extension can be constructed with
  *      generator classes.
  * @defgroup grids 2. New geometric grids
- * @defgroup fluxfunctions 3. New functors based on the magnetic field geometry
+ * @defgroup fluxfunctions 3. New functors surrounding the magnetic field geometry
 
         All functors in this section model two or three-dimensional functions, i.e.
         they all overload the operator() like \c aCylindricalFunctor
  * @{
-      @defgroup geom 3.1 New flux functions and derivatives
+      @defgroup geom 3.1 Creating a flux function
       @{
-        @defgroup solovev The solovev magnetic field
-        @defgroup polynomial The polynomial magnetic field
-        @defgroup taylor The Taylor state magnetic field
-        @defgroup guenther The Guenther magnetic field
-        @defgroup toroidal The Purely Toroidal magnetic field
-        @defgroup circular The Circular magnetic field
+        @defgroup solovev The solovev expansion
+        @defgroup polynomial The polynomial expansion
+        @defgroup taylor The Taylor state expansion
+        @defgroup guenther The Guenther expansion
+        @defgroup toroidal The Purely Toroidal expansion
+        @defgroup circular The Circular expansion
       @}
-      @defgroup magnetic 3.2 Magnetic field and associated functors
-      @defgroup profiles 3.3 Profile functors based on flux functions
+      @defgroup magnetic 3.2 Magnetic field, curvatures and associated functors
+      @defgroup profiles 3.3 Penalization, weight and monitor metric functors
  * @}
  * @defgroup fieldaligned 4. Fieldaligned derivatives
  * @defgroup misc_geo 5. Miscellaneous additions
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 10e14926f..1dea7eb9e 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -876,7 +876,7 @@ struct WallDirection : public dg::geo::aCylindricalFunctor<WallDirection>
      *
      * @param mag Use to construct magnetic field
      * @param vertical walls R_0, R_1 ...  ( can be arbitrary size)
-     * @param horizonal walls Z_0, Z_1 ... ( can be arbitrary size)
+     * @param horizontal walls Z_0, Z_1 ... ( can be arbitrary size)
      */
     WallDirection( dg::geo::TokamakMagneticField mag, std::vector<double>
             vertical, std::vector<double> horizontal) : m_vertical(vertical),
diff --git a/inc/geometries/modified.h b/inc/geometries/modified.h
index 14e90a07a..5fe811069 100644
--- a/inc/geometries/modified.h
+++ b/inc/geometries/modified.h
@@ -19,9 +19,14 @@ namespace dg
 {
 namespace geo
 {
+/**
+ * @brief A modification flux function
+ */
 namespace mod
 {
     //modify with a polynomial Heaviside function
+///@addtogroup mod
+///@{
 
 struct Psip: public aCylindricalFunctor<Psip>
 {
@@ -159,6 +164,9 @@ static inline dg::geo::CylindricalFunctorsLvl2 createPsip(
             mod::PsipRZ(predicate,psip.f(), psip.dfx(), psip.dfy(), psip.dfxy(), psi0, alpha, sign),
             mod::PsipZZ(predicate,psip.f(), psip.dfy(), psip.dfyy(), psi0, alpha, sign));
 }
+
+///@}
+///@cond
 struct DampingRegion : public aCylindricalFunctor<DampingRegion>
 {
     DampingRegion( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, double psi0, double alpha, double sign = -1) :
@@ -194,7 +202,24 @@ struct MagneticTransition : public aCylindricalFunctor<MagneticTransition>
     std::function<double(double,double)> m_psip;
     std::function<bool(double,double)> m_pred;
 };
+//some possible predicates
+static bool nowhere( double R, double Z){return false;}
+static bool everywhere( double R, double Z){return true;}
+struct HeavisideZ{
+    HeavisideZ( double Z_X, int side): m_ZX( Z_X), m_side(side) {}
+    bool operator()(double R, double Z){
+        if( Z < m_ZX && m_side <= 0) return true;
+        if( Z >= m_ZX && m_side > 0) return true;
+        return false;
+    }
+    private:
+    double m_ZX;
+    int m_side;
+};
+///@endcond
 
+///@addtogroup profiles
+///@{
 /**
  * @brief \f$ f_1 + f_2 - f_1 f_2 \equiv f_1 \cup f_2\f$
  *
@@ -252,21 +277,7 @@ struct SetNot : public aCylindricalFunctor<SetNot>
     std::function<double(double,double)> m_fct;
 };
 
-//some possible predicates
-
-static bool nowhere( double R, double Z){return false;}
-static bool everywhere( double R, double Z){return true;}
-struct HeavisideZ{
-    HeavisideZ( double Z_X, int side): m_ZX( Z_X), m_side(side) {}
-    bool operator()(double R, double Z){
-        if( Z < m_ZX && m_side <= 0) return true;
-        if( Z >= m_ZX && m_side > 0) return true;
-        return false;
-    }
-    private:
-    double m_ZX;
-    int m_side;
-};
+///@}
 
 } //namespace mod
 
diff --git a/inc/geometries/polynomial.h b/inc/geometries/polynomial.h
index cf86535a2..69d1f7149 100644
--- a/inc/geometries/polynomial.h
+++ b/inc/geometries/polynomial.h
@@ -22,7 +22,7 @@ namespace dg
 namespace geo
 {
 /**
- * @brief Contains a polynomial approximation type flux function
+ * @brief A polynomial approximation type flux function
  */
 namespace polynomial
 {
-- 
GitLab


From f2aaeb7c38d30163673f11bf46c6c93f4254e016 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 20 Nov 2020 16:43:11 +0100
Subject: [PATCH 398/540] Introduce sheath and wall penalization in feltor

- rename damping to wall parameters
- create and set sheath and wall
- adapt input files
- update feltor documentation
---
 inc/geometries/make_field.h        |  79 +++--
 src/feltor/Makefile                |   2 +-
 src/feltor/feltor.cu               |  24 +-
 src/feltor/feltor.h                | 120 ++++---
 src/feltor/feltor.tex              | 533 ++++++++++++++++-------------
 src/feltor/feltor_hpc.cu           |  56 +--
 src/feltor/feltordiag.h            |  74 ++--
 src/feltor/implicit.h              |  11 +-
 src/feltor/input/compass.json      |  20 +-
 src/feltor/input/default.json      |  19 +-
 src/feltor/input/manufactured.json |  20 +-
 src/feltor/input/tcv.json          |  23 +-
 src/feltor/interpolate_in_3d.cu    |  31 +-
 src/feltor/manufactured.cu         |   2 +-
 src/feltor/parameters.h            |  40 ++-
 15 files changed, 575 insertions(+), 479 deletions(-)

diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
index 1871f3d34..35b17206c 100644
--- a/inc/geometries/make_field.h
+++ b/inc/geometries/make_field.h
@@ -6,6 +6,10 @@
 #include "toroidal.h"
 #include <dg/file/json_utilities.h>
 
+/*!@file
+ *
+ * Making of penalization regions
+ */
 namespace dg{
 namespace geo{
 ///@addtogroup geom
@@ -69,10 +73,10 @@ void transform_psi( TokamakMagneticField mag, double& psi0, double& alpha0, doub
     double RO=mag.R0(), ZO=0.;
     dg::geo::findOpoint( mag.get_psip(), RO, ZO);
     double psipO = mag.psip()( RO, ZO);
-    double damping_psi0p = (1.-psi0*psi0)*psipO;
-    double damping_alpha0p = -(2.*psi0+alpha0)*alpha0*psipO;
-    psi0 = damping_psi0p + sign0*damping_alpha0p/2.;
-    alpha0 = fabs( damping_alpha0p/2.);
+    double wall_psi0p = (1.-psi0*psi0)*psipO;
+    double wall_alpha0p = -(2.*psi0+alpha0)*alpha0*psipO;
+    psi0 = wall_psi0p + sign0*wall_alpha0p/2.;
+    alpha0 = fabs( wall_alpha0p/2.);
     sign0 = sign0*((psipO>0)-(psipO<0));
 }
 }//namespace detail
@@ -87,23 +91,24 @@ void transform_psi( TokamakMagneticField mag, double& psi0, double& alpha0, doub
  * This subsequently modifies all derivatives of psi and the poloidal
  * current in this region.
  * @param gs forwarded to dg::geo::createMagneticField
- * @param jsmod must contain the field "damping": "modifier" which has one of the values "none", "heaviside" then
- *  "damping": "boundary" value where psi is modified to a constant psi0
- * "damping": "alpha" radius of the transition region where the modification acts (smaller is quicker) or "sol_pfr", then "
- *  "damping": "boundary" and
- * "damping": "alpha" must be arrays of size 2 to indicate values for the SOL and the PFR respectively
+ * @param jsmod must contain the field "wall": "type" which has one of the values "none", then no other values are required; "heaviside" then requires
+ *  "wall": "boundary" value where psi is modified to a constant psi0
+ * "wall": "alpha" radius of the transition region where the modification acts (smaller is quicker);
+ * or "sol_pfr", then requires
+ *  "wall": "boundary" and
+ * "wall": "alpha" must be arrays of size 2 to indicate values for the SOL and the PFR respectively
  * @param mode Determines behaviour in case of an error
- * @param damping (out) On output contains the region where the damping is applied
+ * @param wall (out) On output contains the region where the wall is applied
  * @param transition (out) On output contains the region where the transition of Psip to a constant value occurs
  * @note Per default the dampening happens nowhere
  * @return A magnetic field object
  * @attention This function is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
  */
-static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Value jsmod, file::error mode, CylindricalFunctor& damping, CylindricalFunctor& transition)
+static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Value jsmod, file::error mode, CylindricalFunctor& wall, CylindricalFunctor& transition)
 {
     std::string e = file::get( mode, gs, "equilibrium", "solovev" ).asString();
     equilibrium equi = str2equilibrium.at( e);
-    std::string m = file::get( mode, jsmod, "damping", "modifier", "heaviside" ).asString();
+    std::string m = file::get( mode, jsmod, "wall", "type", "heaviside" ).asString();
     modifier mod = str2modifier.at( m);
     std::string d = file::get( mode, gs, "description", "standardX" ).asString();
     description desc = str2description.at( d);
@@ -115,32 +120,32 @@ static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Va
     switch (mod) {
         default: //none
         {
-            damping = mod::DampingRegion( mod::nowhere, mag.psip(), 0, 0, 0);
+            wall = mod::DampingRegion( mod::nowhere, mag.psip(), 0, 0, 0);
             transition = mod::MagneticTransition( mod::nowhere, mag.psip(), 0, 0, 0);
             return mag;
         }
         case modifier::heaviside:
         {
-            double psi0 = file::get( mode, jsmod, "damping", "boundary", 1.1 ).asDouble();
-            double alpha = file::get( mode, jsmod, "damping", "alpha", 0.2 ).asDouble();
+            double psi0 = file::get( mode, jsmod, "wall", "boundary", 1.1 ).asDouble();
+            double alpha = file::get( mode, jsmod, "wall", "alpha", 0.2 ).asDouble();
             double sign = +1;
             if( desc == description::standardX || desc == description::standardO ||
                     desc == description::doubleX)
                 detail::transform_psi( mag, psi0, alpha, sign);
             else
-                sign = file::get( mode, jsmod, "damping", "sign", -1. ).asDouble();
+                sign = file::get( mode, jsmod, "wall", "sign", -1. ).asDouble();
 
             mod_psip = mod::createPsip( mod::everywhere, mag.get_psip(), psi0, alpha, sign);
-            damping = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha, -sign);
+            wall = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha, -sign);
             transition = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha, sign);
             break;
         }
         case modifier::sol_pfr:
         {
-            double psi0 = file::get_idx( mode, jsmod, "damping", "boundary",0, 1.1 ).asDouble();
-            double alpha0 = file::get_idx( mode, jsmod, "damping", "alpha",0, 0.2 ).asDouble();
-            double psi1 = file::get_idx( mode, jsmod, "damping", "boundary",1, 0.97 ).asDouble();
-            double alpha1 = file::get_idx( mode, jsmod, "damping", "alpha",1, 0.2 ).asDouble();
+            double psi0 = file::get_idx( mode, jsmod, "wall", "boundary",0, 1.1 ).asDouble();
+            double alpha0 = file::get_idx( mode, jsmod, "wall", "alpha",0, 0.2 ).asDouble();
+            double psi1 = file::get_idx( mode, jsmod, "wall", "boundary",1, 0.97 ).asDouble();
+            double alpha1 = file::get_idx( mode, jsmod, "wall", "alpha",1, 0.2 ).asDouble();
             switch( desc){
                 case description::standardX:
                 {
@@ -156,11 +161,11 @@ static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Va
                             mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
                     mod_psip = mod::createPsip(
                             mod::HeavisideZ( ZX, -1), mod0_psip, psi1, alpha1, sign1);
-                    CylindricalFunctor damping0 = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha0, -sign0);
+                    CylindricalFunctor wall0 = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha0, -sign0);
                     CylindricalFunctor transition0 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
-                    CylindricalFunctor damping1 = mod::DampingRegion( mod::HeavisideZ(ZX, -1), mag.psip(), psi1, alpha1, -sign1);
+                    CylindricalFunctor wall1 = mod::DampingRegion( mod::HeavisideZ(ZX, -1), mag.psip(), psi1, alpha1, -sign1);
                     CylindricalFunctor transition1 = mod::MagneticTransition( mod::HeavisideZ(ZX, -1), mag.psip(), psi1, alpha1, sign1);
-                    damping = mod::SetUnion( damping0, damping1);
+                    wall = mod::SetUnion( wall0, wall1);
                     transition = mod::SetUnion( transition0, transition1);
                     break;
                 }
@@ -183,30 +188,30 @@ static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Va
                             mod::HeavisideZ( ZX1, -1), mod0_psip, psi1, alpha1, sign1);
                     mod_psip = mod::createPsip(
                             mod::HeavisideZ( ZX2, +1), mod1_psip, psi1, alpha1, sign1);
-                    CylindricalFunctor damping0 = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha0, -sign0);
-                    CylindricalFunctor damping1 = mod::DampingRegion( mod::HeavisideZ(ZX1, -1), mag.psip(), psi1, alpha1, -sign1);
-                    CylindricalFunctor damping2 = mod::DampingRegion( mod::HeavisideZ(ZX2, +1), mag.psip(), psi1, alpha1, -sign1);
+                    CylindricalFunctor wall0 = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha0, -sign0);
+                    CylindricalFunctor wall1 = mod::DampingRegion( mod::HeavisideZ(ZX1, -1), mag.psip(), psi1, alpha1, -sign1);
+                    CylindricalFunctor wall2 = mod::DampingRegion( mod::HeavisideZ(ZX2, +1), mag.psip(), psi1, alpha1, -sign1);
                     CylindricalFunctor transition0 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
                     CylindricalFunctor transition1 = mod::MagneticTransition( mod::HeavisideZ(ZX1, -1), mag.psip(), psi1, alpha1, sign1);
                     CylindricalFunctor transition2 = mod::MagneticTransition( mod::HeavisideZ(ZX2, +1), mag.psip(), psi1, alpha1, sign1);
                     transition = mod::SetUnion( mod::SetUnion( transition0, transition1), transition2);
-                    damping = mod::SetUnion( mod::SetUnion( damping0, damping1), damping2);
+                    wall = mod::SetUnion( mod::SetUnion( wall0, wall1), wall2);
                     break;
                 }
                 default:
                 {
-                    double sign0 = file::get_idx( mode, jsmod, "damping", "sign",0, -1. ).asDouble();
-                    double sign1 = file::get_idx( mode, jsmod, "damping", "sign", 1, +1. ).asDouble();
+                    double sign0 = file::get_idx( mode, jsmod, "wall", "sign",0, -1. ).asDouble();
+                    double sign1 = file::get_idx( mode, jsmod, "wall", "sign", 1, +1. ).asDouble();
                     CylindricalFunctorsLvl2 mod0_psip;
                     mod0_psip = mod::createPsip(
                             mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
                     mod_psip = mod::createPsip(
                             mod::everywhere, mod0_psip, psi1, alpha1, sign1);
-                    CylindricalFunctor damping0 = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
+                    CylindricalFunctor wall0 = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
                     CylindricalFunctor transition0 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi0, alpha0, sign0);
-                    CylindricalFunctor damping1 = mod::DampingRegion( mod::everywhere, mag.psip(), psi1, alpha1, sign1);
+                    CylindricalFunctor wall1 = mod::DampingRegion( mod::everywhere, mag.psip(), psi1, alpha1, sign1);
                     CylindricalFunctor transition1 = mod::MagneticTransition( mod::everywhere, mag.psip(), psi1, alpha1, sign1);
-                    damping = mod::SetUnion( damping0, damping1);
+                    wall = mod::SetUnion( wall0, wall1);
                     transition = mod::SetUnion( transition0, transition1);
                     break;
                 }
@@ -227,6 +232,10 @@ static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Va
         }
     }
 }
+///@}
+
+///@addtogroup profiles
+///@{
 
 static inline CylindricalFunctor createWallRegion( Json::Value gs, Json::Value jsmod, file::error mode)
 {
@@ -239,6 +248,7 @@ static inline CylindricalFunctor createWallRegion( Json::Value gs, Json::Value j
  * @brief Create the sheath region where fieldlines intersect the boundary
  *
  * Check if any fieldlines that are not in the wall region intersect the boundary
+ * and determine whether the poloidal field points towards or away from the wall
  * @param jsmod must contain the field
  * "sheath": "boundary" value where sheath region begins in units of minor radius a
  * "sheath": "alpha" radius of the transition region where the modification acts in units of minor radius a
@@ -251,6 +261,7 @@ static inline CylindricalFunctor createWallRegion( Json::Value gs, Json::Value j
  * @param R1 right boundary
  * @param Z0 bottom boundary
  * @param Z1 top boundary
+ * @param sheath (out) contains the region recognized as sheath
  * @param direction (out) contains (+/-) indicating direction of magnetic field
  * to closest sheath boundary (defined on entire box)
  *
@@ -295,8 +306,8 @@ static inline void createSheathRegion(
     sheath = dg::compose( poly, dist);
     sheath = mod::SetIntersection( mod::SetNot( wall), sheath);
 }
-
 ///@}
+
 } //namespace geo
 }//namespace dg
 #endif //JSONCPP_VERSION_STRING
diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index 79878dbd1..bfb3abb15 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -8,7 +8,7 @@ include ../../config/devices/devices.mk
 INCLUDE+= -I../         # other src libraries
 INCLUDE+= -I../../inc   # other project libraries
 
-all: feltor_hpc feltor feltor_mpi manufactured feltordiag
+all: feltor_hpc feltor feltor_mpi manufactured feltordiag interpolate_in_3d
 
 manufactured: manufactured.cu manufactured.h feltor.h implicit.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(JSONLIB)
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 15aea92ef..98d1249d4 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -92,10 +92,8 @@ int main( int argc, char* argv[])
         mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
 
     //create RHS
-    //std::cout << "Constructing RHS...\n";
-    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
     std::cout << "Constructing Explicit...\n";
-    //feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, false);
+    feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
     std::cout << "Constructing Implicit...\n";
     feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> implicit( grid, p, mag);
     std::cout << "Done!\n";
@@ -119,22 +117,20 @@ int main( int argc, char* argv[])
         std::cerr << "Warning: initne parameter '"<<p.initne<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong initial condition so I exit! Bye Bye :)\n";
         return -1;
     }
-    { //make the HVecs temporaries
 
-    bool fixed_profile;
-    HVec profile = dg::evaluate( dg::zero, grid);
-    HVec source_profile;
     try{
+        bool fixed_profile;
+        HVec profile = dg::evaluate( dg::zero, grid);
+        HVec source_profile;
         source_profile = feltor::source_profiles.at(p.source_type)(
             fixed_profile, profile, grid, p,  mod_mag);
+        feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
+            p.source_rate, dg::construct<DVec>(source_profile)
+        );
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)\n";
         return -1;
     }
-    feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
-        p.source_rate, dg::construct<DVec>(source_profile)
-    );
-    }
 
     ////////////////////////create timer and timestepper
     //
@@ -148,9 +144,9 @@ int main( int argc, char* argv[])
     HVec h_wall = dg::pullback( wall, grid);
     HVec h_sheath = dg::pullback( sheath, grid);
     HVec h_velocity = dg::pullback( direction, grid);
-    feltor.set_wall_and_sheath( p.damping_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath), dg::construct<DVec>(h_velocity));
-    implicit.set_wall_and_sheath( p.damping_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
-    karniadakis.solver().set_wall_and_sheath( p.damping_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    feltor.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath), dg::construct<DVec>(h_velocity));
+    implicit.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    karniadakis.solver().set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
     }
 
     std::cout << "Initialize Timestepper" << std::endl;
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index a25b9832c..b8140c8a8 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -145,6 +145,16 @@ struct ComputeSource{
         result = omega_source*source*(profne - tilde_n);
     }
 };
+struct ComputeDensityBC{
+    DG_DEVICE
+        double operator()( double nminus, double nplus, double sheathDotDirection)
+    {
+        if ( sheathDotDirection > 0 )
+            return nminus*sheathDotDirection;
+        else
+            return -nplus*sheathDotDirection;
+    }
+};
 //Resistivity (consistent density dependency,
 //parallel momentum conserving, quadratic current energy conservation dependency)
 struct AddResistivity{
@@ -229,20 +239,25 @@ struct Explicit
     const std::array<Container, 3> & gradA () const {
         return m_dA;
     }
-    const Container & dsN (int i) const {
-        return m_dsN[i];
+    void compute_dsN (int i, Container& dsN) const {
+        dg::geo::ds_centered_bc_along_field( m_fa_N, 1., m_minusN[i], m_fields[0][i],
+                m_plusN[i], 0., dsN, dg::NEU, {0,0});
+    }
+    void compute_dsU (int i, Container& dsU) const {
+        dg::geo::ds_centered( m_fa_U, 1., m_minusU[i], m_fields[1][i],
+                m_plusU[i], 0., dsU);
     }
-    const Container & dsU (int i) const {
-        return m_dsU[i];
+    void compute_dsP (int i, Container& dsP) const {
+        dg::geo::ds_centered_bc_along_field( m_fa_P, 1., m_minusP[i], m_phi[i],
+                m_plusP[i], 0.0, dsP, dg::DIR, {0,0});
     }
-    const Container & dsP (int i) const {
-        return m_dsP[i];
+    void compute_dssU(int i, Container& dssU) {
+        dg::geo::dss_centered( m_fa_U, 1., m_minusU[i], m_fields[1][i], m_plusU[i], 0., dssU);
     }
-    //const Container & dssN(int i) { //2nd fieldaligned derivative
-    //    return m_dssN[i];
-    //}
-    const Container & dssU(int i) {
-        return m_dssU[i];
+    void compute_lapParU(int i, Container& lapU) {
+        compute_dsU(i, m_temp0);
+        compute_dssU(i, lapU);
+        dg::blas1::pointwiseDot( 1., m_divb, m_temp0, 1., lapU);
     }
     void compute_gradSN( int i, std::array<Container,3>& gradS) const{
         // MW: don't like this function, if we need more gradients we might
@@ -267,22 +282,6 @@ struct Explicit
     void compute_dot_induction( Container& tmp) const {
         m_old_apar.derive( tmp);
     }
-    //maybe better give these a temporary
-    const Container & compute_dppN(int i) { //2nd varphi derivative
-        dg::blas2::symv( m_dz, m_fields[0][i], m_temp0);
-        dg::blas2::symv( m_dz, m_temp0, m_temp1);
-        return m_temp1;
-    }
-    const Container & compute_dppP(int i) {
-        dg::blas2::symv( m_dz, m_phi[i], m_temp0);
-        dg::blas2::symv( m_dz, m_temp0, m_temp1);
-        return m_temp1;
-    }
-    const Container & compute_dppU(int i) {
-        dg::blas2::symv( m_dz, m_fields[1][i], m_temp0);
-        dg::blas2::symv( m_dz, m_temp0, m_temp1);
-        return m_temp1;
-    }
     const dg::SparseTensor<Container>& projection() const{
         return m_hh;
     }
@@ -361,7 +360,6 @@ struct Explicit
 
         dg::blas1::axpby( -1., wall, -1., sheath, m_masked);
         dg::blas1::plus( m_masked, +1);
-        m_wall_forcing = wall_forcing;
     }
     void compute_apar( double t, std::array<std::array<Container,2>,2>& fields);
   private:
@@ -396,7 +394,8 @@ struct Explicit
     Container m_detg;
 
     Container m_apar;
-    std::array<Container,2> m_phi, m_dsN, m_dsU, m_dsP, m_dssU;// m_dssN;
+    std::array<Container,2> m_phi;
+    std::array<Container,2> m_plusN, m_minusN, m_plusU, m_minusU, m_plusP, m_minusP;
     std::array<Container,3> m_dA;
     std::array<std::array<Container,3>,2> m_dP, m_dN, m_dU;
     std::array<std::array<Container,2>,2> m_fields, m_s; //fields, sources
@@ -406,7 +405,6 @@ struct Explicit
     //matrices and solvers
     Matrix m_dx_N, m_dx_U, m_dx_P, m_dy_N, m_dy_U, m_dy_P, m_dz;
     dg::geo::Fieldaligned<Geometry, IMatrix, Container> m_fa_P, m_fa_N, m_fa_U;
-    std::array<Container,2> m_faP, m_faM;
     dg::Elliptic3d< Geometry, Matrix, Container> m_lapperpN, m_lapperpU, m_lapperpP;
     std::vector<dg::Elliptic3d< Geometry, Matrix, Container> > m_multi_pol;
     std::vector<dg::Helmholtz3d<Geometry, Matrix, Container> > m_multi_invgammaP,
@@ -418,7 +416,7 @@ struct Explicit
     dg::SparseTensor<Container> m_hh;
 
     const feltor::Parameters m_p;
-    double m_omega_source = 0., m_sheath_forcing = 0., m_wall_forcing = 0.;
+    double m_omega_source = 0., m_sheath_forcing = 0.;
     bool m_fixed_profile = true;
 
 };
@@ -593,9 +591,7 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
     m_apar = m_temp0;
 
     m_phi[0] = m_phi[1] = m_temp0;
-    //m_dssN =
-    m_dssU = m_dsN = m_dsU = m_dsP = m_phi;
-    m_faP = m_faM = m_phi;
+    m_plusN = m_minusN = m_minusU = m_plusU = m_minusP = m_plusP = m_phi;
     m_dA[0] = m_dA[1] = m_dA[2] = m_temp0;
     m_dP[0] = m_dP[1] = m_dA;
     m_dN = m_dU = m_dP;
@@ -851,30 +847,30 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
     //y[0] = N-1, y[1] = W; fields[0] = N, fields[1] = U
     for( unsigned i=0; i<2; i++)
     {
-        m_fa_N( dg::geo::einsMinus, y[0][i], m_faM[0]);
-        m_fa_N( dg::geo::einsPlus,  y[0][i], m_faP[0]);
-        m_fa_U( dg::geo::einsMinus, fields[1][i], m_faM[1]);
-        m_fa_U( dg::geo::einsPlus,  fields[1][i], m_faP[1]);
+
+        m_fa_N( dg::geo::einsMinus, y[0][i], m_minusN[i]);
+        m_fa_N( dg::geo::einsPlus,  y[0][i], m_plusN[i]);
+        m_fa_U( dg::geo::einsMinus, fields[1][i], m_minusU[i]);
+        m_fa_U( dg::geo::einsPlus,  fields[1][i], m_plusU[i]);
+        m_fa_P( dg::geo::einsMinus, m_phi[i], m_minusP[i]);
+        m_fa_P( dg::geo::einsPlus,  m_phi[i], m_plusP[i]);
+        dg::geo::ds_centered_bc_along_field( m_fa_N, 1., m_minusN[i], y[0][i], m_plusN[i], 0., m_temp0, dg::NEU, {0,0});
+        dg::geo::ds_centered( m_fa_U, 1., m_minusU[i], fields[1][i], m_plusU[i], 0., m_temp1);
         //---------------------density--------------------------//
         //density: -Div ( NUb)
-        dg::geo::ds_centered( m_fa_N, 1., m_faM[0], y[0][i], m_faP[0], 0., m_dsN[i]);
-        dg::geo::ds_centered( m_fa_U, 1., m_faM[1], fields[1][i], m_faP[1], 0., m_dsU[i]);
-        dg::blas1::pointwiseDot(-1., m_dsN[i], fields[1][i],
-            -1., fields[0][i], m_dsU[i], 1., yp[0][i] );
+        dg::blas1::pointwiseDot(-1., m_temp0, fields[1][i],
+            -1., fields[0][i], m_temp1, 1., yp[0][i] );
         dg::blas1::pointwiseDot( -1., fields[0][i],fields[1][i],m_divb,
             1.,yp[0][i]);
         //---------------------velocity-------------------------//
         // Burgers term: -U ds U
-        dg::blas1::pointwiseDot(-1., fields[1][i], m_dsU[i], 1., yp[1][i]);
-        // force terms: -tau/mu * ds lnN -1/mu * ds Phi
-        dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_dsN[i], fields[0][i], 1., yp[1][i]);
-        m_fa_P( dg::geo::einsMinus, m_phi[i], m_faM[0]); //overwrite
-        m_fa_P( dg::geo::einsPlus,  m_phi[i], m_faP[0]);
-        dg::geo::ds_centered( m_fa_P, -1./m_p.mu[i], m_faM[0], m_phi[i], m_faP[0], 1.0, yp[1][i]);
+        dg::blas1::pointwiseDot(-1., fields[1][i], m_temp1, 1., yp[1][i]);
+        // force terms: -tau/mu * ds N/N -1/mu * ds Phi
+        dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_temp0, fields[0][i], 1., yp[1][i]);
+        dg::geo::ds_centered_bc_along_field( m_fa_P, -1./m_p.mu[i], m_minusP[i], m_phi[i], m_plusP[i], 1.0, yp[1][i], dg::DIR, {0,0});
         // viscosity: + nu_par Delta_par U/N = nu_par ( Div b dsU + dssU)/N
-        dg::blas1::pointwiseDot(1., m_divb, m_dsU[i], 0., m_temp1);
-        dg::geo::dss_centered( m_fa_U, 1., m_faM[1], fields[1][i], m_faP[1], 0., m_dssU[i]);
-        dg::blas1::axpby( 1., m_dssU[i], 1., m_temp1);
+        dg::blas1::pointwiseDot(1., m_divb, m_temp1, 0., m_temp1);
+        dg::geo::dss_centered( m_fa_U, 1., m_minusU[i], fields[1][i], m_plusU[i], 1., m_temp1);
         dg::blas1::pointwiseDivide( m_p.nu_parallel[i], m_temp1, fields[0][i], 1., yp[1][i]);
     }
 }
@@ -1025,20 +1021,30 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     dg::blas1::pointwiseDot( m_masked, yp[0][1], yp[0][1]);
     dg::blas1::pointwiseDot( m_masked, yp[1][0], yp[1][0]);
     dg::blas1::pointwiseDot( m_masked, yp[1][1], yp[1][1]);
-    // explicit part of the sheath terms
+    // sheath boundary conditions
     if( m_sheath_forcing != 0)
     {
-        if( "insulating" == m_p.sheath_type)
+        //density
+        for( unsigned i=0; i<2; i++)
+        {
+            dg::blas1::evaluate( m_temp0, dg::equals(), routines::ComputeDensityBC(),
+                m_minusN[i], m_plusN[i], m_U_sheath);
+            dg::blas1::axpby( m_sheath_forcing, m_temp0, 1.,  yp[0][i]);
+        }
+        //velocity
+        if( "insulating" == m_p.sheath_bc)
         {
-            dg::blas1::axpby( m_sheath_forcing, m_U_sheath, 1.,  yp[1][0]);
+            // u_e = +- sqrt(1+tau)
+            dg::blas1::axpby( m_sheath_forcing*sqrt(1+m_p.tau[1]), m_U_sheath, 1.,  yp[1][0]);
         }
-        else // "bohm" == m_p.sheath_type
+        else // "bohm" == m_p.sheath_bc
         {
             //exp(-phi)
             dg::blas1::transform( m_phi[0], m_temp0, dg::EXP<double>(1., -1.));
-            dg::blas1::pointwiseDot( m_sheath_forcing, m_U_sheath, m_temp0, 1.,  yp[1][0]);
+            dg::blas1::pointwiseDot( m_sheath_forcing*sqrt(1+m_p.tau[1]), m_U_sheath, m_temp0, 1.,  yp[1][0]);
         }
-        dg::blas1::axpby( m_sheath_forcing, m_U_sheath, 1.,  yp[1][1]);
+        // u_i = +- sqrt(1+tau)
+        dg::blas1::axpby( m_sheath_forcing*sqrt(1+m_p.tau[1]), m_U_sheath, 1.,  yp[1][1]);
     }
 
 
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 8ede90d28..81da07b33 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -35,9 +35,12 @@ system $\vec x :=\{x_0, x_1, x_2\}$, metric
 tensor $g_{ij}$ and volume element $\sqrt{g} := \sqrt{\det g}$.
 Given a vector field $\vec B(\vec x)$ with unit vector $\bhat(\vec x) := (\vec B/B)({\vec x})$
 we can define various differential operations.
-%We further assume that $\bhat$ is perturbed by the parallel
-%vector potential $A_\parallel$ via
-%${ \vec b }_\perp := ({\vn \times A_\parallel \bhat)}/{B}$
+\begin{table*}[htbp]
+\caption{Definitions of geometric operators with $b^i$ the contra-variant components of $\bhat$ and $g^{ij}$ the contra-variant elements of the metric tensor. We assume $(\vn\times\bhat)_\parallel = 0$. Note that $\vec K = \vec K_\kappa + \vec{ K_{\vn B}}$.
+% Explicit expressions of these quantities
+% depend on the choice of the magnetic field and the underlying coordinate system.
+}\label{tab:operators}
+\centering
 \rowcolors{2}{gray!25}{white}
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
 %\toprule
@@ -45,54 +48,55 @@ we can define various differential operations.
 \midrule
     Perpendicular Poisson bracket&
     $\left[.,.\right]_\perp$ &
-    $\left[f,g\right]_\perp := \bhat \cdot \left(\vn f \times \vn g\right) =
+    $\left[f,g\right]_\perp := \bhat \cdot \left(\vec{\vn} f \times\vn g\right) =
     b_i \varepsilon^{ijk}\partial_j f\partial_k g/\sqrt{g}$  \\
     Projection Tensor&
-    $h $ & $h^{ij} := g^{ij} - b^ib^j $\\
+    $h $ & $h^{ij} := g^{ij} - b^ib^j $   \quad \text{ Note }$ h^2=h$\\
     %Alignment Tensor&
     %$t $ & $ t^{ij} := b^ib^j$\\
     Perpendicular Gradient&
     $\np $&
     $ \np f := \bhat\times(\vn f\times \bhat ) \equiv
-    h \cn f$ \\
-    Perpendicular Laplacian&
-    $\Delta_\perp $&
-    $ \Delta_\perp f:= \vec \nc (\np f)
-    = \nc( h\cn f)$  \\
-    Curl-b Curvature &
+    h \cdot \vn f$ \\
+    Perpendicular Divergence&
+    $\np^\dagger $&
+    $ \np^\dagger \cdot \vec v := -\nc( h \cdot \vec v) = -\nc\vec v_\perp$ \\
+    Perpendicular Laplacian &
+    $\Delta_\perp $ &
+    $ \Delta_\perp f:= \nc (\np f)
+    = \nc( h\cdot\vn f) \equiv -\np^\dagger\cdot\np$  \\
+    Curl-b Curvature Operator&
     $\mathcal K_{\vn\times\bhat}$ &
-    $\mathcal K_{\vn\times\bhat}(f) := \vec{ \mathcal K_{\vn\times\bhat} }\cn f = \frac{1}{B}(\vn \times \bhat)\cn f$ \\[4pt]
-    Grad-B Curvature &
+    $\mathcal K_{\vn\times\bhat}(f) := \vec{ K_{\vn\times\bhat} }\cn f = \frac{1}{B}(\vn \times \bhat)\cn f$ \\[4pt]
+    Grad-B Curvature Operator &
     $\mathcal K_{\vn B} $ &
-    $\mathcal K_{\vn B}(f) := \vec{\mathcal K_{\vn B}} \cn f = \frac{1}{B}(\bhat \times \vn \ln B)\cn f$ \\[4pt]
-    Curvature &
+    $\mathcal K_{\vn B}(f) := \vec{ K_{\vn B}} \cn f = \frac{1}{B}(\bhat \times\vn \ln B)\cn f$ \\[4pt]
+    Curvature Operator&
     $\mathcal K$ &
-    $\mathcal{K}(f):=\vec{\mathcal K} \cn f =
-     \nc\left(\frac{\bhat\times\vn f}{B}\right)$,\\[4pt]
+    $\mathcal{K}(f):=\vec{ K} \cn f =
+     \vec{\vn}\cdot\left(\frac{\bhat\times\vec{\vn} f}{B}\right) =\vn \times \frac{\bhat}{B} \cn f$,\\[4pt]
     Parallel derivative&
     $\npar $&
-    $ \npar f := \bhat\cn f$ \\
-    %Perturbed parallel Derivative&
-    %$\bar\npar$ &
-    %$\bar\npar f := (\bhat + {\vec b }_\perp)\cn f = \npar f + A_\parallel \mathcal K_{\vn\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$ \\
-    Parallel Laplacian&
-    $\Delta_\parallel $&
-    $\Delta_\parallel f:= \nc ( \bhat\bhat\cn f )$\\
+    $ \npar f := \bhat\cdot\vn f$ \quad  Notice $\nc\bhat = -\npar\ln B$ \\
+     Parallel Laplacian&
+     $\Delta_\parallel $&
+     $\Delta_\parallel f:= \vec{\vn} \cdot ( \bhat\bhat\cdot\vec{\vn} f )$\\
 \bottomrule
 \end{longtable}
+\end{table*}
 with $b^i$ the contra- and $b_i$ the co-variant components of $\bhat$, and
 $\eps^{ijk}$ the Levi-Civita symbols.
 Explicit expressions for the above expressions
 depend on the choice of the magnetic field and the underlying coordinate system.
 Note that we have
 \begin{align}
-    \nc \vec{\mathcal K_{\vn\times\bhat}}
-    &= -\nc \vec{\mathcal K_{\vn B}} = -\vec{ \mathcal K_{\vn\times\bhat}}\cn\ln B, \\
-    \vec\nc\vec{ \mathcal K} &= 0, \\
-    \vec{\mathcal K} &=
+    \nc \vec{ K_{\vn\times\bhat}}
+&= -\nc \vec{ K_{\vn B}} = -\vec{K_{\vn\times\bhat}}\cn\ln B, \\
+    \vec\nc\vec{ K} &= 0, \\
+    \mathcal K(f) &=
      \vn\times\frac{\bhat}{B}\cn f
     = \mathcal K_{\vn\times\bhat}(f) + \mathcal K_{\vn B}(f),\\
-    \vec{ \mathcal K_{\vn\times\bhat}} - \vec{ \mathcal K_{\vn B}} &= \frac{1}{B^2} (\vn \times \vec B), \\
+    \vec{ K_{\vn\times\bhat}} - \vec{ K_{\vn B}} &= \frac{1}{B^2} (\vn \times \vec B), \\
     \npar \ln B &= -\vec\nc\bhat.
     \label{eq:curl_curvature}
 \end{align}
@@ -134,8 +138,7 @@ basis vectors and (covariant) metric tensor are:
 With the help of the metric elements we get a well behaved volume element \(\sqrt{g} = R\). However, we have a coordinate singularity at \(R=0\).
 The cylindrical coordinate basis vectors are mutually orthogonal to each other.
 
-\subsection{Solov'ev equilbrium}\label{sec:solovev}
-%Document polynomial field as well
+\subsection{The flux function}
 In cylindrical coordinates the general axisymmetric  magnetic field can be written as (dimensionless)
 \begin{align}
  \vec{B} &= \frac{R_0}{R}\left[I(\psi_p) \ehat_{\varphi} + \frac{\partial
@@ -145,18 +148,22 @@ which can obviously not be manipulated to be in Clebsch form.
 Hence we are dealing with a non-flux aligned coordinate system.
 For the sake of clarity we define the poloidal magnetic field \( \vec{B}_p = \frac{R_0}{R}\left( \frac{\partial \psi_p}{\partial Z}\ehat_R - \frac{\partial \psi_p}{\partial R}\ehat_Z\right)
 \) and the toroidal magnetic field \(\vec{B}_t =\frac{R_0I}{R} \ehat_{\varphi}\).
-Note that with a typically convex function $\psi_p$ (second derivative is
+\begin{tcolorbox}[title=Note]
+With a typically convex function $\psi_p$ (second derivative is
 positive), $I(\psi_p)>0$ and the previously defined coordinate system the field
 line winding is a {\bf left handed screw} in the positive $\ehat_\varphi$-direction.
 Also note that then $\vec B\times\vn\vec B$ points {\bf down}, towards the magnetic X-point,
-and we have the {\bf favourable} drift direction (since in experiments H-mode
+and we have the {\bf favourable} drift direction (in experiments H-mode
 is reached easier in this configuration).
+\end{tcolorbox}
 
 
 We scaled $R$, $Z$ and $R_0$ with $\rho_s = \sqrt{T_e m_i}/(eB_0)$, the
 magnetic field with $B_0$, the poloidal flux with $\psi_{p0} = B_0\rho_s \hat
 R_0$ and the poloidal equilibrium current streamfunction with $I_0 = B_0 \hat R_0$ (with $\hat R_0 =
 \rho_s R_0$ the dimensional major radius).
+\subsubsection{Solov'ev equilbrium}\label{sec:solovev}
+
 We have the equilibrium equations in toroidally symmetric, ideal MHD
 $\vn p = \vec j\times \vec B$ and $\vn\times\vec B = \beta \vec j$ normalized with $p_0 = n_0 T_0$, and $j_0 = e n_0 c_S$, where we introduce $\beta = n_0 T_0 \mu_0 /B_0^2$.
 Note that this normalization is in line with the one later chosen for the gyrofluid
@@ -236,6 +243,7 @@ $\bar{\psi}_{p11}=3 \bar{Z}\bar{R}^4 - 4\bar{Z}^3\bar{R}^2$\\
 \bottomrule
 \end{longtable}
 
+\subsubsection{Polynomial expansion}
 As an alternative and in order to better fit experimental equilibria we offer
 a polynomial expansion of the magnetic flux function
 \begin{subequations}
@@ -246,6 +254,7 @@ a polynomial expansion of the magnetic flux function
 \end{align}
 \end{subequations}
 where the number of polynomial coefficients $N_R$ and $N_Z$ can be freely chosen.
+\subsubsection{Discussion}
 Since Eqs.~\eqref{eq:solovev} and \eqref{eq:polynomial} are given analytically we can numerically evaluate $\psi_p$ and $I$
 and all their derivatives
 at arbitrary points to machine precision, which is simple to implement and fast to execute.
@@ -265,6 +274,7 @@ The scaling factors $\mathcal P_\psi$ and $\mathcal P_I$ are mainly introduced t
 If an X-point is present, we choose $c_1$ such that
 $\psi_p(R_X, Z_X) = 0$ that is the separatrix is given by $\psi_p(R,Z) = 0$.
 
+\subsection{Curvature operators and perpendicular Poisson bracket}
 Note that
 \begin{align}
     B^R&=B_R = R_0\psi_Z/R \\
@@ -291,7 +301,9 @@ The toroidal/negative toroidal field line approximation applies \(\bhat\approx \
 (e.g.: Poisson bracket, perpendicular elliptic operator and curvature operators)
 but retains the full expression for the magnetic field unit vector \(\bhat\)
 for parallel operators (\(\npar\) and \(\Delta_\parallel\)).
-(Note that we allow the negative sign $-\ehat_\varphi$ to enable a sign reversal of the magnetic field, see Section~\ref{sec:field_reversal}).
+\begin{tcolorbox}[title=Note]
+We allow the negative sign $-\ehat_\varphi$ to enable a sign reversal of the magnetic field, see Section~\ref{sec:field_reversal}.
+\end{tcolorbox}
 In cylindrical coordinates that is
 \begin{align}
 [f,g]_\perp \equiv [f,g]_{RZ} &= \pm\frac{1}{R} \left(\partial_R f\partial_Z g - \partial_Z f\partial_R g\right) \\
@@ -305,10 +317,10 @@ The curl of $\bhat$ reduces to
 %end{align}
 This simplifies the curvature operators to:
 \begin{align}
-\vec{\mathcal{K}}_{{\vn\times\bhat}}  &\approx  -  \frac{\pm 1}{B R} \ehat_Z , &
-\vec{ \mathcal{K} }_{\vn  B}  &\approx  -\frac{\pm 1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{\pm 1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
+\vec{{K}}_{{\vn\times\bhat}}  &\approx  -  \frac{\pm 1}{B R} \ehat_Z , &
+\vec{ {K} }_{\vn  B}  &\approx  -\frac{\pm 1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{\pm 1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
 %\ehat_\varphi \times \vn B, &
-\vec{ \mathcal{K} } &\approx \vec{ \mathcal{K} }_{\vn  B}  +\vec{ \mathcal{K} }_{{\vn\times\bhat}} ,
+\vec{ {K} } &\approx \vec{ {K} }_{\vn  B}  +\vec{ {K} }_{{\vn\times\bhat}} ,
 %\\
 %\mathcal{K}_{{\vn\times\bhat}}(f)   &\approx  -  \frac{1}{B R} \frac{\partial f}{\partial Z},&
 %\mathcal{K}_{\vn  B} (f)  &= \frac{1}{B} \left[\ln B, f \right]_{RZ},&
@@ -316,9 +328,9 @@ This simplifies the curvature operators to:
 \end{align}
 and
 \begin{align}
- \nc \vec{\mathcal{K}}_{{\vn\times\bhat}} &\approx \frac{\pm 1}{R B^2} \frac{\partial B}{\partial Z},
+ \nc \vec{{K}}_{{\vn\times\bhat}} &\approx \frac{\pm 1}{R B^2} \frac{\partial B}{\partial Z},
 \end{align}
-which results in a vanishing divergence of the curvature operators \( \nc \vec{ \mathcal{K} } = 0\).
+which results in a vanishing divergence of the curvature operators \( \nc \vec{ {K} } = 0\).
 
 Note that in an actual toroidal field we have
 \begin{align}
@@ -328,9 +340,9 @@ Note that in an actual toroidal field we have
 We then have $\bhat = \pm\ehat_\varphi$ and the curvature operators further
 simplify to
 \begin{align}
-  \vec{ \mathcal K_{\vn\times\bhat}} = \vec{ \mathcal K_{\vn B}} = -\frac{\pm 1}{R_0} \ehat_Z =
-\vec{ \mathcal K}/2\\
-  \nc\vec{\mathcal K_{{\vn\times\bhat}}}=
+  \vec{  K_{\vn\times\bhat}} = \vec{  K_{\vn B}} = -\frac{\pm 1}{R_0} \ehat_Z =
+\vec{  K}/2\\
+  \nc\vec{ K_{{\vn\times\bhat}}}=
     \npar \ln B = 0
     \label{}
 \end{align}
@@ -340,7 +352,7 @@ Note: the negative sign is automatically chosen in code if $I(R_0, 0)<0$.
 In this approximation we apply the toroidal field line approximation
 as in Section
 \ref{sec:torfieldlineapprox}
-but approximate the curvature operator $\mathcal K_{\vn\times\bhat} \approx \bhat\times\vec \kappa$
+but approximate the curvature operator $ \vec K_{\vn\times\bhat} \approx \bhat\times\vec \kappa$
   with
   $\vec \kappa := \bhat \cn\bhat = -\bhat \times( \vn\times \bhat)$.
 For an isotropic pressure plasma \(\vec{P} = \vec{I} P_\perp + \vec{b} \vec{b} P_\Delta \approx \vec{I} P_\perp\) and with the definition of the plasma beta parameter
@@ -355,13 +367,13 @@ In low beta plasmas \(\beta\ll1\) the curvature reduces to:
 \end{align}
 This simplifies the curvature operators to:
 \begin{align}
-\vec{\mathcal{K}_{{\vn\times\bhat}}}(f) \approx
-\vec{ \mathcal{K} }_{\vn  B}  &\approx  -\frac{1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
-\mathcal{K} (f) &\approx 2\mathcal{K}_{\vn  B} (f) , &
-    \vn\times\bhat \cdot \vec{\mathcal{K}}_{\vn  B} &= 0.
+\vec{{K}_{{\vn\times\bhat}}} \approx
+\vec{ {K} }_{\vn  B}  &\approx  -\frac{1}{B^2}\frac{\partial B}{\partial Z}\ehat_R +\frac{1}{B^2} \frac{\partial B}{\partial R}\ehat_Z &
+{K} (f) &\approx 2{K}_{\vn  B} (f) , &
+    \vn\times\bhat \cdot \vec{{K}}_{\vn  B} &= 0.
 \end{align}
-The divergence over the curvature vanishes \( \nc \vec{ \mathcal{K} } = 0\) only if \( \nc \vec{ \mathcal{K}}_{\vn  B}   = 0\).
-In general, the divergence \( \nc \vec{ \mathcal{K} } \approx 0\) is only approximately vanishing.
+The divergence over the curvature vanishes \( \nc \vec{ {K} } = 0\) only if \( \nc \vec{ {K}}_{\vn  B}   = 0\).
+In general, the divergence \( \nc \vec{ {K} } \approx 0\) is only approximately vanishing.
 \subsubsection{True perpendicular terms}
 
 Without any approximations we have
@@ -684,59 +696,6 @@ The labels $\rho_t$ and $\rho_p$ are useful because
 equidistant $\rho_p$ and $\rho_t$ values tend to translate to equidistant flux-surfaces
 in configuration space.
 
-\subsection{ Modified $\psi_p$}
-Our computational domain is a box and in particular not aligned with the
-magnetic flux surfaces. This means that particularly in the corners of
-the domain the field lines inside the domain are very short (in the
-sense that the distance between the entry point and leave point is short).
-It turns out that this behaviour is numerically disadvantageous (may
-blow up the simulation in the worst case) in the
-computation of parallel derivatives. In order to remedy this situation
-we propose to modify the flux surfaces $\psi_p$ to a constant value
-if $\psi_p$ exceeds a certain critical value. In this way the poloidal
-field component vanishes in the corners of the domain at the cost
-of introducing a strong shear layer limiting the scrape-off layer width.
-
-We define an approximation to the step function with a transition layer of radius $a$
-around the origin
-\begin{align}
-\Theta_a(x) := \begin{cases}
-    0 & \text{ for } x \leq -a  \\
-    \frac{1}{32 a^7}  \left(16 a^3-29 a^2 x+20 a x^2-5 x^3\right) (a+x)^4
-    &\text{ for } -a<x\leq a \\
-    1 & \text{ for } x > a
-\end{cases}
-    \approx H(x)
-\label{eq:approx_heaviside}
-\end{align}
-where $H(x)$ is the Heaviside step function.
-An integral of this function is
-\begin{align}
-\theta_a(x) := \begin{cases}
-    0 &\text{ for } x \leq -a \\
-    \frac{1}{256 a^7} \left(35 a^3-47 a^2 x+25 a x^2-5 x^3\right) (a+x)^5
-     &\text{ for } -a<x\leq a \\
-x &\text{ for } x > a
-\end{cases}
-    \approx x H(x)
-\end{align}
-Note that $\Theta_a(0) = 0.5$ and $\theta_a(0) = 35a/256$.
-
-We now use
-\begin{align}
-    -\theta_{\alpha/2}\left(\psi_{p,b} + \frac{\alpha}{2} - \psi \right)+\psi_{p,b}+\frac{\alpha}{2} \approx (\psi- \psi_{p,b})H(\psi_{p,b}-\psi) + \psi_{p,b}
-\label{eq:modified_psip}
-\end{align}
-instead of $\psi_p$ for the computation of the
-magnetic field, which introduces a shear layer between $\psi_{p,b}$ and $\psi_{p,b}+\alpha$ where the
-fieldlines are straightened to match $\ehat_\varphi$.
-In order to simplify the setup of this region we give $\psi_{p,b}$ and $\alpha$ in terms of
-$\rho_p$ and $\alpha_p$ via $\psi_{p,b} = (1-\rho_{p,b}^2)\psi_{p,O}$ and $\alpha = -(2\rho_{p,b} \alpha_p + \alpha_p^2)\psi_{p,O}$. In case we change the sign
-of $\psi_p$ via $\mathcal P_\psi$ (to make it concave) note that $\alpha$ becomes
-negative and $\psi_{p,O}$ is positive).
-We then need to point mirror Eq.~\eqref{eq:modified_psip} at $\psi_{p,b}+\frac{\alpha}{2}$.
-
-
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \section{The model} \label{sec:model}
 \subsection{Conservative form}
@@ -771,9 +730,9 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
 with
 \begin{align}
 \vec u_E := \frac{\bhat\times\vn\psi}{B},\quad
-\vec u_{K} := \tau \left(\vec{\mathcal K_{\vn B}} + \vec{\mathcal K_{\vn\times\bhat}}\right)=\tau\vec{\mathcal K}  ,\nonumber\\
-\vec u_C := \mu U_\parallel^2\vec{\mathcal K_{\vn\times\bhat}},\quad
-\vec u_{\vn\times\bhat} := \tau\vec{\mathcal K_{\vn\times\bhat}},\quad
+\vec u_{K} := \tau \left(\vec{ K_{\vn B}} + \vec{ K_{\vn\times\bhat}}\right)=\tau\vec{ K}  ,\nonumber\\
+\vec u_C := \mu U_\parallel^2\vec{ K_{\vn\times\bhat}},\quad
+\vec u_{\vn\times\bhat} := \tau\vec{ K_{\vn\times\bhat}},\quad
 {\vec b}_\perp = \frac{\vn\times A_\parallel \bhat}{B}.
 \label{}
 \end{align}
@@ -848,7 +807,7 @@ By dividing by $\rho_s^2 \Omega_{ci}$ we arrive at $\nu_\perp = \nu_{ii0}/\Omega
 \left(\frac{m_i}{m_H}\right)^{1/2},
 \end{align}
 
-\subsection{Boundary and initial conditions}
+\subsection{Boundary conditions: the penalization method}
 We define the simulation box as
 $[ R_{\min}, R_{\max}]\times [Z_{\min}, Z_{\max}] \times [0,2\pi]$,
 where we define
@@ -861,19 +820,102 @@ where we define
 where $a$ is the minor radius, $e$ is the elongation of the flux surfaces and
 the $\varepsilon$ are free parameters to be specified by the user.
 
-We choose boundary conditions separately on input for the variables
-$n_e$, $u_{\parallel,e}$ and $\phi$. The boundary condition for $N_i$, $U_{\parallel,i}$ and
-$\psi$ are equal to $n_e$, $u_{\parallel,e}$ and $\phi$ respectively.
-We choose $A_\parallel$ to have equal boundary conditions as $u_{\parallel,e}$ and $U_{\parallel,i}$.
-This will later enable us to treat the sum of $U_\parallel$ and $A_\parallel$
-in the same way as $U_\parallel$.
-Typically,
+The boundary conditions for the potential and the magnetic potential are not
+penalized and thus hold on the bounding box. Typically we choose
 \begin{align}
-n_e = n_0, \quad u_{\parallel,e} = \phi = 0
-\text{ or } \hat n \cn n_e = \hat n \cn u_{\parallel,e} = 0
+\phi = 0
+\text{ and }  \hat n \cn A_{\parallel} = 0
 \end{align}
 where $\hat n$ is the normal vector to the boundary.
 
+\subsubsection{The wall region}
+Being a box, our computational domain is in particular not aligned with the
+magnetic flux surfaces. This means that particularly in the corners of
+the domain the field lines inside the domain are very short (in the
+sense that the distance between the entry point and leave point is short).
+It turns out that this behaviour is numerically disadvantageous (may
+blow up the simulation in the worst case) in the
+computation of parallel derivatives.
+In order to remedy this situation
+we propose a penalization method to model the actual physical wall.
+We define an approximation to the step function with a transition layer of radius $a$
+around the origin
+\begin{align}
+\Theta_a(x) := \begin{cases}
+    0 & \text{ for } x \leq -a  \\
+    \frac{1}{32 a^7}  \left(16 a^3-29 a^2 x+20 a x^2-5 x^3\right) (a+x)^4
+    &\text{ for } -a<x\leq a \\
+    1 & \text{ for } x > a
+\end{cases}
+    \approx H(x)
+\label{eq:approx_heaviside}
+\end{align}
+where $H(x)$ is the Heaviside step function.
+%An integral of this function is
+%\begin{align}
+%\theta_a(x) := \begin{cases}
+%    0 &\text{ for } x \leq -a \\
+%    \frac{1}{256 a^7} \left(35 a^3-47 a^2 x+25 a x^2-5 x^3\right) (a+x)^5
+%     &\text{ for } -a<x\leq a \\
+%x &\text{ for } x > a
+%\end{cases}
+%    \approx x H(x)
+%\end{align}
+%Note that $\Theta_a(0) = 0.5$ and $\theta_a(0) = 35a/256$.
+%
+We now use the region defined by
+\begin{align}\label{eq:wall}
+    \chi_w(R,Z,\varphi):=\Theta_{\alpha/2}\left(\psi_{p,b} + \frac{\alpha}{2} - \psi \right) \approx H(\psi_{p,b}-\psi)
+\end{align}
+to define the wall region.
+In order to simplify the setup of this region we give $\psi_{p,b}$ and $\alpha$ in terms of
+$\rho_p$ and $\alpha_p$ via $\psi_{p,b} = (1-\rho_{p,b}^2)\psi_{p,O}$ and $\alpha = -(2\rho_{p,b} \alpha_p + \alpha_p^2)\psi_{p,O}$. In case we change the sign
+of $\psi_p$ via $\mathcal P_\psi$ (to make it concave) note that $\alpha$ becomes
+negative and $\psi_{p,O}$ is positive).
+We then need to point mirror Eq.~\eqref{eq:wall} at $\psi_{p,b}+\frac{\alpha}{2}$.
+
+Now, our idea is to dampen the density and velocity in the region defined by the
+wall to 1 or 0 respectively.
+For both electrons and ions we choose
+\begin{subequations} \label{eq:wall_penalization}
+\begin{align}
+    S^w_N(R,Z,\varphi, t) &:= -\omega_w\chi_w (N-1)\\
+    S^w_U(R,Z,\varphi, t) &:= -\omega_w\chi_w U_\parallel
+\end{align}
+\end{subequations}
+where $\omega_w \gg 1$ is the penalization parameter.
+\subsubsection{The sheath region}
+In order to define sheath boundary conditions we first define a sheath region
+and then determine whether the field lines point toward the wall or away from it.
+We define as sheath any part on the bounding box that is not included in the wall
+penalization. Then we check for each point in the box the poloidal distance to
+the sheath wall and if the poloidal field points toward or away from the wall closest
+to it.
+We then take $\theta_{\alpha/2}\left( (\eps_s + \frac{\alpha}{2})a - d(R,Z)\right)$
+and take the set intersection between that region and the ``not wall'' region to
+determine the sheath penalization region:
+\begin{align}\label{eq:sheath}
+    \chi_s := \left(1-\chi_w(R,Z,\varphi)\right) \theta_{\alpha/2}\left( (\eps_s + \frac{\alpha}{2})a - d(R,Z)\right)
+\end{align}
+Within the sheath region we penalize
+\begin{subequations} \label{eq:sheath_penalization}
+\begin{align}
+    S^s_N(R,Z,\varphi, t) &:= \omega_s \chi_s \left(N_{sh}-N\right)\\
+    S^s_{u_e}(R,Z,\varphi, t) &:= \omega_s \chi_s \left(\sqrt{1+\tau}\exp(-\phi) - u_{\parallel,e} \right)\\
+    S^s_{U_i}(R,Z,\varphi, t) &:= \omega_s \chi_s \left(\sqrt{1+\tau} - U_{\parallel,i} \right)\\
+\end{align}
+\end{subequations}
+where $\omega_s$ is the sheath penalization parameter and $N_{sh}$ is obtained by extrapolating $N$ along the magnetic field line with
+the help of the parallel derivative operators
+\begin{align}
+    N_{sh} = \Tpm N
+\end{align}
+where the sign depends again on the direction of the magnetic field line (we always extrapolate ``downstream'').
+
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\subsection{Initial conditions}
 We initialize the parallel velocity to zero
 \begin{align}
   u_{\parallel,e}(R,Z,\varphi,0) = U_{\parallel,i}(R,Z,\varphi,0) = 0
@@ -997,21 +1039,6 @@ Also note that with our definition of $\Lambda_{n_e}$ and $\Lambda_{N_i}$ and
 the polarisation equation we have $\Lambda_{n_e} = \Gamma_{1,i}\Lambda_{N_i} + \nc\left( \frac{\mu_i \Lambda_{N_i}}{B^2}\np \phi\right)$ in the long wavelength limit (swap the operators).
 This means that diffusion does not generate potential either.
 
-Now, our idea is to dampen the density and velocity in the region defined by the
-magnetic field straightening.
-The idea for the terms $S_U$ is mainly to provide more numerical stability
-in the corner regions of the domain, where the parallel derivative may lead
-to unwelcome numerical instabilities.
-For both electrons and ions we choose
-\begin{subequations} \label{eq:velocity_source}
-\begin{align}
-    S^d_{n_e}(R,Z,\varphi,t) &:= -\omega_d (n_e-1)\Theta_{\alpha_p/2}\left(\rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2}\right)\\
-    S^d_{N_i}(R,Z,\varphi,t) &= \left(1-\frac{1}{2}\mu_i \tau_i \Delta_\perp\right) S^d_{n_e} -\nc\left( \frac{\mu_i S^d_{n_e}}{B^2}\np \phi\right)\\
-    S^d_U(R,Z,\varphi, t)& := -\omega_d U_\parallel \Theta_{\alpha_p/2}\left(  \rho_p(R,Z) - \rho_{p,b} - \frac{\alpha_p}{2} \right) - \frac{U_\parallel}{N} S_N
-\end{align}
-\end{subequations}
-The last term is there to avoid introducing an unintentional parallel momentum source through the
-density sources.
 
 \subsection{Implemented form}
 The form that we implement avoids derivatives on the product of
@@ -1019,28 +1046,34 @@ two functions for which we have no boundary conditions
 \begin{subequations}
     \begin{align}
     \frac{\partial}{\partial t} N =&
-        - \frac{1}{B}[\psi, N]_{\perp}%\nonumber\\
+    \left[ - \frac{1}{B}[\psi, N]_{\perp} %\nonumber\\
         - \bar \npar \left( NU_\parallel\right)
         - NU_\parallel\left(\vec \nc\bhat+\vec \nc{\vec b}_\perp\right)
-        - \tau \mathcal K(N) \nonumber \\&
+        - \tau \mathcal K(N)\right. \nonumber \\&
+        \left.
         - N \mathcal K(\psi)
         -\mu \mathcal K_{\vn\times\bhat}(NU_\parallel^2)
-        -\mu NU_\parallel^2\nc \vec{ \mathcal K_{\vn\times\bhat}}
-        - \nu_\perp\Delta_\perp^2 N + S_N, \\
+        -\mu NU_\parallel^2\nc \vec{ K_{\vn\times\bhat}}
+    - \nu_\perp\Delta_\perp^2 N + S_N\right]
+    (1-\chi_s - \chi_w) \nonumber\\& + \omega_s\chi_s( N_{sh} - N) - \omega_w \chi_w N_w, \\
     \frac{\partial}{\partial t} W_\parallel =&
-        - \frac{1}{B}\left[\psi, U_\parallel\right]_{\perp}%& \nonumber\\
+  \left[- \frac{1}{B}\left[\psi, U_\parallel\right]_{\perp}%& \nonumber\\
         - \frac{1}{\mu} \bar \npar \psi% \nonumber\\
         - \frac{1}{2}\bar \npar U_\parallel^2
         -\frac{\tau}{\mu} \bar \npar \ln N
         - U_\parallel\mathcal K_{\vn\times\bhat}(\psi)
         - \tau \mathcal K(U_\parallel)
-        -\tau U_\parallel\nc\vec{ \mathcal K_{\vn\times\bhat}}\nonumber\\&
-        - \left(2\tau + {\mu}U_\parallel^2\right) \mathcal K_{\vn\times\bhat} (U_\parallel)
+        -\tau U_\parallel\nc\vec{ K_{\vn\times\bhat}}
+    \right.
+        \nonumber\\&
+    \left.- \left(2\tau + {\mu}U_\parallel^2\right) \mathcal K_{\vn\times\bhat} (U_\parallel)
         -2\tau U_\parallel\mathcal K_{\vn\times\bhat}(\ln N)
         - \frac{\eta}{\mu} \frac{n_e}{N}n_e(U_{\parallel,i} - u_{\parallel,e})
- \nonumber\\&
-        + \frac{\nu_\parallel}{N} \Delta_\parallel U_\parallel - \nu_\perp\Delta_\perp^2 U_\parallel
-                + S_U,
+    \right.\nonumber\\&
+    \left.+ \frac{\nu_\parallel}{N} \Delta_\parallel U_\parallel - \nu_\perp\Delta_\perp^2 U_\parallel
+    + S_U\right]
+    (1-\chi_s - \chi_w) + \omega_s\chi_s ( U_{\parallel}^{sh} - U_\parallel) -\omega_w \chi_wU_\parallel
+    ,
         \label{eq:EgyrofluidU} \\
         W_\parallel&:= \left( U_\parallel + \frac{A_\parallel}{\mu}\right)
     \end{align}
@@ -1048,7 +1081,7 @@ two functions for which we have no boundary conditions
 \end{subequations}
 together with
 $\bar\npar f = \npar f + A_\parallel \mathcal K_{\vn\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$
-and $\nc { \vec b}_\perp = A_\parallel \vec \nc\vec{ \mathcal{ K}_{\vn\times\bhat}} - \mathcal K_{\vn B}(A_\parallel) $
+and $\nc { \vec b}_\perp = A_\parallel \vec \nc\vec{ { K}_{\vn\times\bhat}} - \mathcal K_{\vn B}(A_\parallel) $
 and
 \begin{subequations} \label{eq:elliptic}
   \begin{align}
@@ -1059,7 +1092,9 @@ and
     A_\parallel &= \beta\left(N_iW_{\parallel,i}-n_e w_{\parallel,e}\right)
   \end{align}
 \end{subequations}
-Note that the negative signs make the operators in Eqs.~\eqref{eq:elliptic} positive definite.
+\begin{tcolorbox}[title=Note]
+The negative signs make the operators in Eqs.~\eqref{eq:elliptic} positive definite.
+\end{tcolorbox}
 
 In the output file we have
 \begin{longtable}{llll}
@@ -1449,11 +1484,11 @@ There seems to be no benefit in using the grid refinement technique except when
 \\
 time & Multistep "Karniadakis" & \\
 \qquad explicit & Multistep "Karniadakis" & $3$rd order explicit\\
-\qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains perp. Diffusion terms. \\
+\qquad implicit & Multistep "Karniadakis" & $2$nd order implicit, contains the penalization term and optionally the perp. Diffusion terms. \\
 \bottomrule
 \end{longtable}
-Note that the explicit resistive and damping terms lead to absolute (CFL) restriction
-on the timestep according to $\partial_t u_e \propto \frac{\eta}{\mu_e} u_e \Rightarrow \Delta t < \frac{-\mu_e}{\eta}$. This is a quite weak restriction unless the resistivity exceeds $\eta > 10^{-4}$ or the damping coefficient excceeds $1$.
+Note that the explicit resistive term leads to an absolute (CFL) restriction
+on the timestep according to $\partial_t u_e \propto \frac{\eta}{\mu_e} u_e \Rightarrow \Delta t < \frac{-\mu_e}{\eta}$. This is a quite weak restriction unless the resistivity exceeds $\eta > 10^{-4}$.
 The explicit parallel electron viscosity term leads to a CFL condition of the
 form $\Delta t < (2\pi (R_0 - a))^2/(N_z^2 \nu_{\parallel,e})$.
 
@@ -1495,9 +1530,12 @@ the output file \texttt{output.nc}.
  Both programs write unstructured human readable performance information of the running simulation
  to \texttt{std::cout}.
 
-Note that when compiled for mpi, the program \texttt{feltor\_hpc.cu} expects the
+\begin{tcolorbox}[title=Note]
+When compiled for mpi, the program \texttt{feltor\_hpc.cu} expects the
 partition of the total number of processes np into the three directions x, y and z
-as an input from the command line. Make sure that \texttt{npx*npy*npz==np} and that
+as an input from the command line.
+\end{tcolorbox}
+Make sure that \texttt{npx*npy*npz==np} and that
 they evenly divide the number of grid points in the respective direction! The
 number of stages in the multigrid algorithm and the compression parameters further
 restrict this choice. Also note that the number of processes in a direction must
@@ -1505,93 +1543,86 @@ not equal the number of grid points in that direction!
 
 
 \subsection{Input file structure} \label{sec:input_file}
-Input file format: json
+Input file format: \href{https://en.wikipedia.org/wiki/JSON}{json}
 
 %%This is a booktabs table
-\begin{longtable}{llllp{6cm}}
+\begin{longtable}{llp{2.5cm}p{7cm}}
 \toprule
-\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
-n      & integer & 3 & - &Number of Gaussian nodes in R and Z (we practically always take 3)
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example}  & \textbf{Description}  \\ \midrule
+n      & integer & 3 &Number of Gaussian nodes in R and Z (we practically always take 3)
 \\
-Nx     & integer &52& - &Number of grid points in R (increase if your simulations crash)
+Nx     & integer &52&Number of grid points in R (increase if your simulations crash)
 \\
-Ny     & integer &52& - &Number of grid points in Z (increase if your simulations crash)
+Ny     & integer &52&Number of grid points in Z (increase if your simulations crash)
 \\
-Nz     & integer &16& - &Number of grid points in $\varphi$ (determines dt
+Nz     & integer &16&Number of grid points in $\varphi$ (determines dt
 since parallel velocity dominates timestep)
 \\
-dt     & integer &1e-2& - & time stepsize in units of $c_s/\rho_s$ \\
-compression & integer[2] & [2,2] & [1,1] & Compress output file by reducing
+dt     & integer &1e-2& time stepsize in units of $c_s/\rho_s$ \\
+compression & integer[2] & [2,2] & Compress output file by reducing
 points in x and y (pojecting the polynomials onto a coarser grid): output
 contains n*Nx/c[0] points in x, (has to divde Nx evenly), and n*Ny/c[1] points
 in y, (has to divde Ny evenly). 2 or 3 are reasonable values.
 \\
-inner\_loop & integer & 2  & 1 & Number of time steps between updates to the
+inner\_loop & integer & 2  & Number of time steps between updates to the
 time integrated quantities. (Although the diagnostics is quite fast sometimes
 you need to amortize the time spent on it). Note that integrating selected
 quantities in time during the simulation is how we maintain the time-resolution
 in the file output (cf. \ref{sec:output_file}). Choose as low as you can get
 away with (between 1 and 10).
 \\
-itstp       & integer & 2  & - &{ \tt inner\_loop*itstp} is the number of
+itstp       & integer & 2  &{ \tt inner\_loop*itstp} is the number of
 timesteps between file outputs (2d and 3d quantities); Note that 1d and 0d
 quantities can only be computed post-simulation since we can't compute
 flux-integrals in parallel in MPI.
 \\
-maxout      & integer & 10 & - & Total Number of fields outputs excluding first
+maxout      & integer & 10 & Total Number of fields outputs excluding first
 (The total number of time steps is {\tt maxout$\cdot$itstp$\cdot$inner\_loop})
 If you want to let the simulation run for a certain time instead just choose
 this parameter very large and let the simulation hit the time-limit.
 \\
-eps\_time   & float & 1e-7  & - & Tolerance for solver for implicit part in
-time-stepper (if too low, you'll see oscillations in $u_{\parallel,e}$ and/or $\phi$)
-\\
-rtol  & float &1e-6   & - &Tolerance of adaptive time-stepper. (Ignored in Multistep)
+eps\_time   & float & 1e-7  & Tolerance for solver for implicit part in
+time-stepper (if too low, you'll see oscillations in $u_{\parallel,e}$ and/or $\phi$) Relevant only if diffusion is treated implicitly.
 \\
-stages      & integer & 3 & 3 & number of stages in multigrid, $2^{\text{stages-1}}$
+stages      & integer & 3 & number of stages in multigrid, $2^{\text{stages-1}}$
 has to evenly divide both $N_x$ and $N_y$
 \\
-eps\_pol    & float[stages] & [1e-6,1,1]  & - &  The first number is the tolerance for residual of the inversion of polarisation and induction Eq.. The second number is a multiplicative factor for the accuracy on the second grid in a multigrid scheme, the third for the third grid and so on.  (i.e. $\eps_0\eps_i$ is the accuracy on the i-th grid)
+eps\_pol    & float[stages] & [1e-6,1,1]  &  The first number is the tolerance for residual of the inversion of polarisation and induction Eq.. The second number is a multiplicative factor for the accuracy on the second grid in a multigrid scheme, the third for the third grid and so on.  (i.e. $\eps_0\eps_i$ is the accuracy on the i-th grid)
 Tuning those factors is a major performance tuning oppourtunity!! For saturated turbulence the suggested values are [1e-6, 2000, 100].
 \\
-jumpfactor  & float & 1 & 1 & Jumpfactor $\in \left[0.01,1\right]$ in the local DG method for the elliptic terms. (Don't touch unless you know what you're doing.
+jumpfactor  & float & 1 & Jumpfactor $\in \left[0.01,1\right]$ in the local DG method for the elliptic terms. (Don't touch unless you know what you're doing.
 \\
-eps\_gamma  & float & 1e-6  & - & Tolerance for $\Gamma_1$
+eps\_gamma  & float & 1e-6  & Tolerance for $\Gamma_1$
 \\
-FCI & dict & & & Parameters for Flux coordinate independent approach
+FCI & dict & & Parameters for Flux coordinate independent approach
 \\
-\qquad refine     & integer[2] & [1,1] & [1,1] & refinement factor in FCI approach in R- and Z-direction.
-We use [1,1], higher values take more time, but possibly stabilize the simulation.
+\qquad refine     & integer[2] & [2,2] & refinement factor in FCI approach in R- and Z-direction.
+We use [2,2], higher values take more time, but possibly stabilize the simulation.
 \\
-\qquad rk4eps     & float & 1e-6 & 1e-6 & Accuracy of fieldline integrator in FCI. The default is reasonable.
+\qquad rk4eps     & float & 1e-6 & Accuracy of fieldline integrator in FCI. The default is reasonable.
 \\
-\qquad periodify & bool & true & true & Indicate if flux function is periodified beyond grid boundaries such that the contours are perpendicular to the boundaries. This is not entirely consistent but works better for small toroidal resolution
+\qquad periodify & bool & true & Indicate if flux function is periodified beyond grid boundaries such that the contours are perpendicular to the boundaries. This is not entirely consistent but works better for small toroidal resolution
 \\
-mu         & float & -0.000272121& - & $\mu_e =-m_e/m_i$.
+mu         & float & -0.000272121& $\mu_e =-m_e/m_i$.
     One of $\left\{ -0.000544617, -0.000272121, -0.000181372 \right\}$
 \\
-tau        & float &1      & - & $\tau = T_i/T_e$
+tau        & float &1      & $\tau = T_i/T_e$
 \\
-beta       & float & 5e-6  & 0 & Plasma beta $5\cdot 10^{-6}$ (TJK), $4\cdot
+beta       & float & 5e-6  & Plasma beta $5\cdot 10^{-6}$ (TJK), $4\cdot
 10^{-3}$ (Compass), If $0$, then the model is electrostatic
 \\
-nu\_perp   & float &1e-3   & - & perpendicular viscosity $\nu_\perp$, increase
+nu\_perp   & float &1e-3   & perpendicular viscosity $\nu_\perp$, increase
 this or the resolution if you see vertical or horizontal oscillations (likely
 from the advection terms) in your simulation box, decrease if it dampens all
 instabilities
 \\
-perp\_diff & string & "viscous" & "viscous" & "viscous": $\Lambda_\perp\propto
+perp\_diff & string[2] & ["viscous","explicit"] & "viscous": $\Lambda_\perp\propto
 \nu_\perp\Delta_\perp$ , "hyperviscous": $\Lambda_\perp \propto
--\nu_\perp\Delta_\perp^2$
+-\nu_\perp\Delta_\perp^2$, the second entry indicates whether the perpendicular diffusion is to be treated explicit or implicit (we recommend explicit since in 3d the parallel dynamics restricts the timestep)
 \\
-%nu\_parallel & float &1e-1 & - & parallel viscosity $\nu_\parallel$
-%(dimensional analysis reveals there can be a factor $(R_0/\rho_s)^2$ between
-%$\nu_\perp $n and $\nu_\parallel$ for $\nu_\parallel$ to become relevant for
-%the dynamics)
-%\\
-resistivity & float &1e-4  & - & parallel resistivity parameter Eq.~\eqref{eq:resistivity}
+resistivity & float &1e-4  & parallel resistivity parameter Eq.~\eqref{eq:resistivity}
 \\
-curvmode  & string & "low beta"  & "toroidal" &
+curvmode  & string & "toroidal" &
 curvature mode (
 "low beta",
 "true": no approximation - requires significantly more resolution in Nz,
@@ -1599,118 +1630,131 @@ curvature mode (
 communication in z
 )
 \\
-symmetric & bool & false & false & If true, initialize all quantities symmetric
+symmetric & bool & false & If true, initialize all quantities symmetric
 in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used
 to construct the parallel derivatives and then overwritten to $N_z\equiv 1$.
 \\
-bc & dict & & & Boundary conditions (note that $A_\parallel$ has the same bc as $U_\parallel$) \ldots\\
-\qquad density   & char[2] & [DIR,DIR] & -  & boundary conditions in x and y
+bc & dict & & Boundary conditions (note that $A_\parallel$ has the same bc as $U_\parallel$) \ldots\\
+\qquad density   & char[2] & [DIR,DIR] & boundary conditions in x and y
 for $n_e$ and $N_i$, DIR (density 1 on boundary) means both convective and
     diffusive outflow while NEU (gradient 0) means no outflow by diffusion
 \\
-\qquad velocity  & char[2] & [NEU,NEU] & - & boundary conditions in x and y for
+\qquad velocity  & char[2] & [NEU,NEU] & boundary conditions in x and y for
 $u_{\parallel,e}$ and $U_{\parallel,i}$ and $A_\parallel$, DIR is in general not very stable, NEU works
 better\\
-\qquad potential & char[2] & [DIR,DIR] & - & boundary conditions in x and y for
+\qquad potential & char[2] & [DIR,DIR] & boundary conditions in x and y for
 $\phi$ and $\psi$, DIR means that the $v_{E,\perp}=0$ on the boundary (i.e. no
-outflow by \ExB drift), NEU can can have a detrimental effect on timestep \\
-box & dict & & & Bounding box \\
-    \qquad scaleR  & float[2] & [1.1,1.1]     & [1.05,1.05] & $[\varepsilon_{R-}, \varepsilon_{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
-    \qquad scaleZ  & float[2] & [1.2,1.1]     & [1.05,1.05] & $\varepsilon_{Z-}, \varepsilon_{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box}
+outflow by \ExB drift), NEU can have a detrimental effect on timestep \\
+box & dict & & Bounding box \\
+    \qquad scaleR  & float[2] & [1.1,1.1]     & $[\varepsilon_{R-}, \varepsilon_{R+}]$ scale left and right boundary in units of $a$ Eq.~\eqref{eq:box}\\
+    \qquad scaleZ  & float[2] & [1.2,1.1]     & $\varepsilon_{Z-}, \varepsilon_{Z+}$ scale lower and upper boundary in units of $ae$ Eq.~\eqref{eq:box}
 \\
-initne    & string & "turbulence"     & "blob"  & initial condition for the
+initne    & string & "turbulence"     & initial condition for the
 perturbation $\tilde n$ in \eqref{eq:initial_ne}. "zonal" (Eq.~\eqref{eq:initial_zonal_flow}),
+    "zero" = no perturbation,
     "blob" = blob simulations (several rounds fieldaligned),
     "straight blob" = straight blob simulation( 1 round fieldaligned),
     "turbulence" = turbulence simulations ( 1 round fieldaligned, Eq.~\eqref{eq:initial_turbulent})
     "turbulence on gaussian" = Gaussian bg. profile with turbulence perturbation Eq.~\eqref{eq:turbulence_on_gaussian}
     See the file {\tt init.h} to add your own custom condition.
 \\
-initphi   & string & "zero"  & "balance" & (ignored if $\tau_i = 0$, then $\phi=0$) initial condition for $\phi$ and thus $N_i$ (Eq.~\eqref{eq:initphi}: "zero" : $\phi = 0$, vanishing
+initphi   & string & "zero"  & (ignored if $\tau_i = 0$, then $\phi=0$) initial condition for $\phi$ and thus $N_i$ (Eq.~\eqref{eq:initphi}: "zero" : $\phi = 0$, vanishing
 electric potential, "balance": ExB vorticity equals ion diamagnetic vorticity
 \\
-amplitude  & float &0.01   & - & amplitude $A$ of initial perturbation (blob, turbulent bath or zonal flow)  \\
-sigma      & float &2      & - & Gaussian variance in units of $\rho_s$ \\
-posX       & float &0.3    & - & Gaussian R-position in units of $a$\\
-posY       & float &0.0    & - & Gaussian Z-position in units of $a$ \\
-sigma\_z    & float &0.25   & - & toroidal variance in units of $\pi$ of the fieldline-following initialization \\
-k\_psi     & float &0    & - & zonal mode wave number (only for "zonal" initial condition)  \\
-profile & Dict & & & Density profile \\
-\qquad amp& float &4   & 0 & Profile amplitude $\triangle n_{peak}$ in
+amplitude  & float &0.01   & amplitude $A$ of initial perturbation (blob, turbulent bath or zonal flow)  \\
+sigma      & float &2      & Gaussian variance in units of $\rho_s$ \\
+posX       & float &0.3    & Gaussian R-position in units of $a$\\
+posY       & float &0.0    & Gaussian Z-position in units of $a$ \\
+sigma\_z    & float &0.25  & toroidal variance in units of $\pi$ of the fieldline-following initialization \\
+k\_psi     & float &0    & zonal mode wave number (only for "zonal" initial condition)  \\
+profile & Dict & & Density profile \\
+\qquad amp& float &4   & Profile amplitude $\triangle n_{peak}$ in
 Eq.~\eqref{eq:density_profile} and Eq.~\eqref{eq:turbulence_on_gaussian}
 \\
-\qquad alpha  & float & 0.2 & 0.2 & Transition width $\alpha_p$ in the Heaviside
+\qquad alpha  & float & 0.2 & Transition width $\alpha_p$ in the Heaviside
 at the separatrix (must not be zero - even if amp is zero - it is also used for the perturbation)
 \\
-source & dict & & & Density source, cf. the output \texttt{sne\_tt\_ifs} in \texttt{feltordiag} (or \texttt{SourceProfile\_ifs} in \texttt{geometry\_diag}) to see how much mass the source with the parameters below generates and compare to \texttt{jsne\_tt\_fsa} to see how much mass is lost.  \\
-\qquad rate & float & 0    & 0 & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}.
+source & dict & & Density source, cf. the output \texttt{sne\_tt\_ifs} in \texttt{feltordiag} (or \texttt{SourceProfile\_ifs} in \texttt{geometry\_diag}) to see how much mass the source with the parameters below generates and compare to \texttt{jsne\_tt\_fsa} to see how much mass is lost.  \\
+\qquad rate & float & 0    & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}.
 \\
-\qquad type & string & "profile" & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
+\qquad type & string & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
 "influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx},
 "torpex": Torpex inspired source profile Eq.~\eqref{eq:electron_source_torpex},
 "gaussian": Gaussian shaped source profile - uses \texttt{posX}, \texttt{posY} and \texttt{sigma},
     See the file {\tt init.h} to add your own custom source.
 \\
-\qquad boundary & float & 0.2  & 0.5 & Source region boundary $\rho_{p,b}$: yields in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
-\qquad alpha  & float & 0.2 & 0.2 & Transition width $\alpha_p$ in the Heaviside
+\qquad boundary & float & 0.2  & Source region boundary $\rho_{p,b}$: yields in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
+\qquad alpha  & float & 0.2 & Transition width $\alpha_p$ in the Heaviside
 in the density Eq.~\eqref{eq:density_profile} (with $\rho_{p,b}=0$ and source profiles Eq.~\eqref{eq:electron_source} (should be
 small but cannot be too small if $\tau_i > 0$ else $\Delta_\perp n_e$ explodes, must not be zero)
 \\
-damping & dict & & & magnetic and density damping region \\
-\qquad modifier & string & "sol\_pfr" & "sol\_pfr" & One of ``none'', ``heaviside'' or ``sol\_pfr'' \\
-\qquad rate & float & 0    & 0   & Friction coefficient $\omega_d$ in density and velocity damping Eq.~\eqref{eq:velocity_source} \\
-\qquad boundary & float[2] & [1.2,0.8]  & [1.2,0.8] & Modification region boundary $\rho_{p,b}$: yields $\psi_0 = (1-\rho_{p,b}^2)\psi_{p,O}$ in Eq.~\eqref{eq:modified_psip}.
+wall & dict & & magnetic and density damping region \\
+\qquad type & string & "sol\_pfr" & One of ``none'', ``heaviside'' or ``sol\_pfr'' \\
+\qquad penalization & float & 1    & penalization coefficient $\omega_w$ in density and velocity damping Eq.~\eqref{eq:wall_penalization} \\
+\qquad boundary & float[2] & [1.2,0.8]  & Wall region boundary $\rho_{p,b}$: yields $\psi_0 = (1-\rho_{p,b}^2)\psi_{p,O}$ in Eq.~\eqref{eq:wall}.
+\\
+\qquad alpha   & float[2] & [0.25,0.25] & Transition width $\alpha_p$: yields
+$\alpha=-2\rho_{p,b}\alpha_p+\alpha_p^2)\psi_{p,O}$ for the Heaviside in the wall function \eqref{eq:wall}. If zero, we do not have a wall.
+\\
+sheath & dict & & Sheath region and boundary condition \\
+\qquad bc & string & "bohm" & One of ``bohm'',  or ``insulation'' \\
+\qquad penalization & float & 1    & penalization coefficient $\omega_s$ in density and velocity damping Eq.~\eqref{eq:sheath_penalization} \\
+\qquad boundary & float & 0.3  & Wall region boundary $\rho_{p,b}$: yields $\psi_0 = (1-\rho_{p,b}^2)\psi_{p,O}$ in Eq.~\eqref{eq:sheath}.
+\\
+\qquad alpha   & float & 0.2 & Transition width $\alpha_p$: yields
+$\alpha=-2\rho_{p,b}\alpha_p+\alpha_p^2)\psi_{p,O}$ for the Heaviside in the wall function \eqref{eq:sheath}.
 \\
-\qquad alpha   & float[2] & [0.25,0.25] & 0 & Transition width $\alpha_p$: yields
-$\alpha=-2\rho_{p,b}\alpha_p+\alpha_p^2)\psi_{p,O}$ for the Heaviside in the modified
-$\psi_p$ function \eqref{eq:modified_psip}. If zero, then we do not modify the
-magnetic field and damping is ignored.\\
 \bottomrule
 \end{longtable}
 \subsection{Geometry file structure} \label{sec:geometry_file}
-File format: json
+File format: \href{https://en.wikipedia.org/wiki/JSON}{json}
 
 The file structure of the geometry file depends on which expansion for $\psi_p$ is chosen Eq.~\eqref{eq:solovev} or Eq.~\eqref{eq:polynomial}.
 %%This is a booktabs table
 A solovev magnetic field equilibrium
-\begin{longtable}{llll>{\RaggedRight}p{7cm}}
+\begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
-\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
-    A      & float & 0 &  0 & Solovev parameter in Eq.~\eqref{eq:solovev} \\
-    c      & float[12] &  - & - & Solovev coefficients in Eq.~\eqref{eq:solovev} \\
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Description}  \\ \midrule
+    A      & float & 0 & Solovev parameter in Eq.~\eqref{eq:solovev} \\
+    c      & float[12] &  - & Solovev coefficients in Eq.~\eqref{eq:solovev} \\
 \bottomrule
 \end{longtable}
 A polynomial magnetic field equilibrium
-\begin{longtable}{llll>{\RaggedRight}p{7cm}}
+\begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
-\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
-    M      & float & 1 &  1 & Number of polynomial coefficients in $R$ in Eq.~\eqref{eq:polynomial} \\
-    N      & float & 1 &  1 & Number of polynomial coefficients in $Z$ in Eq.~\eqref{eq:polynomial} \\
-    c      & float[MN] &  - & - & Polynomial coefficients in Eq.~\eqref{eq:polynomial} \\
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Description}  \\ \midrule
+    M      & float & 1 & Number of polynomial coefficients in $R$ in Eq.~\eqref{eq:polynomial} \\
+    N      & float & 1 & Number of polynomial coefficients in $Z$ in Eq.~\eqref{eq:polynomial} \\
+    c      & float[MN] &  - & Polynomial coefficients in Eq.~\eqref{eq:polynomial} \\
 \bottomrule
 \end{longtable}
 In addition both files must contain
-\begin{longtable}{llll>{\RaggedRight}p{7cm}}
+\begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
-\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
-    PP     & float & 1 &  1 & Prefactor $\mathcal P_\psi$ for $\psi_p$ \\
-    PI     & float & 1 &  1 & Prefactor $\mathcal P_I$ for $I$ \\
-    R\_0   & float & - & -  & Major radius $R_0$ in units of $\rho_s$ (This is the only geometry quantity to change if $\rho_s$ changes)\\
-    elongation    & float & 1 & - & Elongation $e$, used in determining the box size Eq.~\eqref{eq:box} and the initial guess for the location of the X-point $Z_X = -1.1 ea$ \\
-    triangularity & float & 0 & - & Triangularity $\delta$, used in the initial guess for the location of the X-point $R_X = R_0-1.1\delta a$ \\
-    inverseaspectratio & float & 0.16667 & - & minor to major radius $a/R_0$ (used to compute $a$ from $R_0$) \\
-    equilibrium & string & polynomial & solovev & Tells the magnetic field generation which type of expansion to use for the flux function \\
-    description & string & standardX & standardX & Tells the magnetic field modifier
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Description}  \\ \midrule
+    PP     & float & 1 & Prefactor $\mathcal P_\psi$ for $\psi_p$ \\
+    PI     & float & 1 & Prefactor $\mathcal P_I$ for $I$ \\
+    R\_0   & float & - & Major radius $R_0$ in units of $\rho_s$ (This is the only geometry quantity to change if $\rho_s$ changes)\\
+    elongation    & float & 1 & Elongation $e$, used in determining the box size Eq.~\eqref{eq:box} and the initial guess for the location of the X-point $Z_X = -1.1 ea$ \\
+    triangularity & float & 0 & Triangularity $\delta$, used in the initial guess for the location of the X-point $R_X = R_0-1.1\delta a$ \\
+    inverseaspectratio & float & 0.16667& minor to major radius $a/R_0$ (used to compute $a$ from $R_0$) \\
+    equilibrium & string & solovev & Tells the magnetic field generation which type of expansion to use for the flux function \\
+    description & string & standardX & Tells the magnetic field modifier
     where to look for the SOL and PFR regions \\
 \bottomrule
 \end{longtable}
 
 \subsection{Output} \label{sec:output_file}
-Output file format: netcdf-4/hdf5; A \textit{coordinate variable (Coord. Var.)} is a Dataset with the same name as a dimension.
-We follow \textbf{CF Conventions CF-1.7}
-\url{http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html}
+Output file format: \href{https://www.unidata.ucar.edu/software/netcdf/docs/}{netcdf-4/hdf5};
+
+A \textit{coordinate variable (Coord. Var.)} is a Dataset with the same name as a dimension.
+We follow
+\href{http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html}{CF Conventions CF-1.7}
 and write according attributes into the file.
+
+\begin{tcolorbox}[title=Note]
 The command \texttt{ncdump -h output.nc} gives a full list of what a file contains.
+\end{tcolorbox}
 Here, we list the content without attributes
 since the internal netcdf information does not display equations.
 %
@@ -1808,7 +1852,10 @@ Usage \\
 errors or other careless mistakes.
 \end{tcolorbox}
 
-Output file format: netcdf-4/hdf5, Conventions: CF-1.7; A \textit{coordinate variable (Coord. Var.)} is a Dataset with the same name as a dimension.
+Output file format: \href{https://www.unidata.ucar.edu/software/netcdf/docs/}{netcdf-4/hdf5};
+\href{http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html}{CF Conventions CF-1.7}
+
+A \textit{coordinate variable (Coord. Var.)} is a Dataset with the same name as a dimension.
 
 \begin{longtable}{lll>{\RaggedRight}p{7cm}}
 \toprule
@@ -1856,7 +1903,7 @@ Usage \\
 \texttt{./geometry\_diag input.json geometry.json diag\_geometry.nc} \\
 The program outputs a host of static 1d, 2d and 3d geometric quantities.
 The output file is for example useful in connection with the ``Group Datasets'' filter in paraview, which merges Datasets from different files into one using shallow copy only.
-\section{Error conditions}
+\section{Troubleshooting}
 All previously mentioned codes can crash for various reasons. Here,
 we list and describe situations, which generally may lead to program
 termination
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 20450b51a..1fe6077fc 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -144,12 +144,13 @@ int main( int argc, char* argv[])
     }
     const feltor::Parameters p( js);
     MPI_OUT p.display( std::cout);
-    std::string input = js.toStyledString(), geom = gs.toStyledString();
-    MPI_OUT std::cout << geom << std::endl;
-    dg::geo::TokamakMagneticField mag;
-    dg::geo::CylindricalFunctor damping, transition;
+    std::string inputfile = js.toStyledString(), geomfile = gs.toStyledString();
+    MPI_OUT std::cout << geomfile << std::endl;
+    dg::geo::TokamakMagneticField mag, mod_mag;
+    dg::geo::CylindricalFunctor wall, transition, sheath, direction;
     try{
-        mag = dg::geo::createModifiedField(gs, js, file::error::is_throw, damping, transition);
+        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
+        mod_mag = dg::geo::createModifiedField(gs, js, file::error::is_throw, wall, transition);
     }catch(std::runtime_error& e)
     {
         std::cerr << "ERROR in geometry file "<<argv[2]<<std::endl;
@@ -168,11 +169,11 @@ int main( int argc, char* argv[])
     }
 #endif //FELTOR_MPI
     ////////////////////////////////set up computations///////////////////////////
+    //Make grids
     double Rmin=mag.R0()-p.boxscaleRm*mag.params().a();
     double Zmin=-p.boxscaleZm*mag.params().a()*mag.params().elongation();
     double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
     double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
-    //Make grids
     Geometry grid( Rmin, Rmax, Zmin, Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER
         #ifdef FELTOR_MPI
@@ -192,17 +193,23 @@ int main( int argc, char* argv[])
     unsigned local_size2d = g2d_out_ptr->size();
 #endif
 
-    HVec damping_profile = dg::pullback( damping, grid);
+    try{
+        dg::geo::createSheathRegion( js, file::error::is_throw, mag, wall,
+                Rmin, Rmax, Zmin, Zmax, sheath, direction);
+    }catch(std::runtime_error& e)
+    {
+        MPI_OUT std::cerr << "ERROR in geometry file "<<geomfile<<std::endl;
+        MPI_OUT std::cerr <<e.what()<<std::endl;
+        return -1;
+    }
     if( p.periodify)
         mag = dg::geo::periodify( mag, Rmin, Rmax, Zmin, Zmax, dg::NEU, dg::NEU);
 
     //create RHS
-    //MPI_OUT std::cout << "Constructing RHS...\n";
-    //feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, true);
     MPI_OUT std::cout << "Constructing Explicit...\n";
-    feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag, false);
+    feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
     MPI_OUT std::cout << "Constructing Implicit...\n";
-    feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> im( grid, p, mag);
+    feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> implicit( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
     // helper variables for output computations
@@ -266,8 +273,7 @@ int main( int argc, char* argv[])
         source_profile = feltor::source_profiles.at(p.source_type)(
             fixed_profile, profile, grid, p, mag);
         feltor.set_source( fixed_profile, dg::construct<DVec>(profile),
-            p.source_rate, dg::construct<DVec>(source_profile),
-            p.damping_rate, dg::construct<DVec>(damping_profile)
+            p.source_rate, dg::construct<DVec>(source_profile)
         );
     }catch ( std::out_of_range& error){
         std::cerr << "Warning: source_type parameter '"<<p.source_type<<"' not recognized! Is there a spelling error? I assume you do not want to continue with the wrong source so I exit! Bye Bye :)"<<std::endl;
@@ -308,8 +314,8 @@ int main( int argc, char* argv[])
     att["comment"] = "Find more info in feltor/src/feltor.tex";
     att["source"] = "FELTOR";
     att["references"] = "https://github.com/feltor-dev/feltor";
-    att["inputfile"] = input;
-    att["geomfile"] = geom;
+    att["inputfile"] = inputfile;
+    att["geomfile"] = geomfile;
     for( auto pair : att)
         MPI_OUT err = nc_put_att_text( ncid, NC_GLOBAL,
             pair.first.data(), pair.second.size(), pair.second.data());
@@ -468,13 +474,17 @@ int main( int argc, char* argv[])
         feltor::FeltorSpecialSolver<
             Geometry, IDMatrix, DMatrix, DVec>
         > karniadakis( grid, p, mag);
-    karniadakis.init( feltor, im, time, y0, p.dt);
-    //unsigned mMax = 3, restart = 3, max_iter = 100;
-    //double damping = 1e-3;
-    //dg::BDF< std::array<std::array<DVec,2>,2 >,
-    //    dg::AndersonSolver< std::array<std::array<DVec,2>,2> >
-    //    > bdf( 3, y0, mMax, p.rtol, max_iter, damping, restart);
-    //bdf.init( feltor, time, y0, p.dt);
+    {
+    HVec h_wall = dg::pullback( wall, grid);
+    HVec h_sheath = dg::pullback( sheath, grid);
+    HVec h_velocity = dg::pullback( direction, grid);
+    feltor.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath), dg::construct<DVec>(h_velocity));
+    implicit.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    karniadakis.solver().set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    }
+
+    std::cout << "Initialize Timestepper" << std::endl;
+    karniadakis.init( feltor, implicit, time, y0, p.dt);
     dg::Timer t;
     t.tic();
     unsigned step = 0;
@@ -489,7 +499,7 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    karniadakis.step( feltor, im, time, y0);
+                    karniadakis.step( feltor, implicit, time, y0);
                     //bdf.step( feltor, time, y0);
                 }
                 catch( dg::Fail& fail){
diff --git a/src/feltor/feltordiag.h b/src/feltor/feltordiag.h
index 1a50677ea..bafa32a76 100644
--- a/src/feltor/feltordiag.h
+++ b/src/feltor/feltordiag.h
@@ -433,19 +433,9 @@ std::vector<Record> diagnostics2d_list = {
     },
     {"dssue", "2nd parallel derivative of electron velocity", false,
         []( DVec& result, Variables& v ) {
-             dg::blas1::copy(v.f.dssU(0), result);
+            v.f.compute_dssU( 0, result);
         }
     },
-    //{"dppue", "2nd varphi derivative of electron velocity", false,
-    //    []( DVec& result, Variables& v ) {
-    //         dg::blas1::copy(v.f.compute_dppU(0), result);
-    //    }
-    //},
-    //{"dpue2", "1st varphi derivative squared of electron velocity", false,
-    //    []( DVec& result, Variables& v ) {
-    //         dg::blas1::pointwiseDot(v.f.gradU(0)[2], v.f.gradU(0)[2], result);
-    //    }
-    //},
     {"lperpinv", "Perpendicular density gradient length scale", false,
         []( DVec& result, Variables& v ) {
             const std::array<DVec, 3>& dN = v.f.gradN(0);
@@ -468,7 +458,8 @@ std::vector<Record> diagnostics2d_list = {
     },
     {"lparallelinv", "Parallel density gradient length scale", false,
         []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot ( v.f.dsN(0), v.f.dsN(0), result);
+            v.f.compute_dsN(0, result);
+            dg::blas1::pointwiseDot ( result, result, result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
             dg::blas1::transform( result, result, dg::SQRT<double>());
@@ -476,7 +467,8 @@ std::vector<Record> diagnostics2d_list = {
     },
     {"aligned", "Parallel density alignement", false,
         []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot ( v.f.dsN(0), v.f.dsN(0), result);
+            v.f.compute_dsN(0, result);
+            dg::blas1::pointwiseDot ( result, result, result);
             dg::blas1::pointwiseDivide( result, v.f.density(0), result);
         }
     },
@@ -558,8 +550,10 @@ std::vector<Record> diagnostics2d_list = {
     {"divnepar_tt", "Divergence of Parallel velocity term for electron density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.density(0), v.f.velocity(0), v.f.divb(), 0., result);
-            dg::blas1::pointwiseDot( 1., v.f.density(0),  v.f.dsU(0), 1., result);
-            dg::blas1::pointwiseDot( 1., v.f.velocity(0), v.f.dsN(0), 1., result);
+            v.f.compute_dsU(0, v.tmp[0]);
+            dg::blas1::pointwiseDot( 1., v.f.density(0),  v.tmp[0], 1., result);
+            v.f.compute_dsN(0, v.tmp[0]);
+            dg::blas1::pointwiseDot( 1., v.f.velocity(0), v.tmp[0], 1., result);
         }
     },
     {"jsniE_tt", "Radial ion particle flux: ExB contribution (Time average)", true,
@@ -620,8 +614,10 @@ std::vector<Record> diagnostics2d_list = {
     {"divnipar_tt", "Divergence of Parallel velocity term in ion density (Time average)", true,
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( 1., v.f.density(1), v.f.velocity(1), v.f.divb(), 0., result);
-            dg::blas1::pointwiseDot( 1., v.f.density(1),  v.f.dsU(1), 1., result);
-            dg::blas1::pointwiseDot( 1., v.f.velocity(1), v.f.dsN(1), 1., result);
+            v.f.compute_dsU(1, v.tmp[1]);
+            dg::blas1::pointwiseDot( 1., v.f.density(1),  v.tmp[1], 1., result);
+            v.f.compute_dsN(1, v.tmp[1]);
+            dg::blas1::pointwiseDot( 1., v.f.velocity(1), v.tmp[1], 1., result);
         }
     },
     /// ------------------- Energy terms ------------------------//
@@ -779,13 +775,9 @@ std::vector<Record> diagnostics2d_list = {
     },
     {"leeparallel_tt", "Parallel electron energy dissipation (Time average)", true,
         []( DVec& result, Variables& v ) {
-            //dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(0),
-            //                         0., v.tmp[0]);
-            //dg::blas1::axpby( 1., v.f.dssN(0), 1., v.tmp[0]);
+            //v.f.compute_lapParN( 0, v.tmp[0]);
             dg::blas1::copy(0., v.tmp[0]);
-            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(0),
-                                     0., v.tmp[1]);
-            dg::blas1::axpby( 1., v.f.dssU(0), 1., v.tmp[1]);
+            v.f.compute_lapParU( 0, v.tmp[1]);
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[0], v.p.mu[0], -1.),
                 v.f.density(0), v.f.velocity(0), v.f.potential(0),
@@ -796,13 +788,9 @@ std::vector<Record> diagnostics2d_list = {
     },
     {"leiparallel_tt", "Parallel ion energy dissipation (Time average)", true,
         []( DVec& result, Variables& v ) {
-            //dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsN(1),
-            //                         0., v.tmp[0]);
-            //dg::blas1::axpby( 1., v.f.dssN(1), 1., v.tmp[0]);
+            //v.f.compute_lapParN( 1, v.tmp[0]);
             dg::blas1::copy(0., v.tmp[0]);
-            dg::blas1::pointwiseDot( 1., v.f.divb(), v.f.dsU(1),
-                                     0., v.tmp[1]);
-            dg::blas1::axpby( 1., v.f.dssU(1), 1., v.tmp[1]);
+            v.f.compute_lapParU( 1, v.tmp[1]);
             dg::blas1::evaluate( result, dg::equals(),
                 routines::RadialEnergyFlux( v.p.tau[1], v.p.mu[1], 1.),
                 v.f.density(1), v.f.velocity(1), v.f.potential(1),
@@ -983,9 +971,8 @@ std::vector<Record> diagnostics2d_list = {
 
             v.f.compute_diffusive_lapMperpN( v.f.density(0), v.tmp[0], v.tmp[1]);
             dg::blas1::scal( v.tmp[1], -v.p.nu_perp);
-            //dg::blas1::pointwiseDot( v.p.nu_parallel, v.f.divb(), v.f.dsN(0),
-            //                         0., v.tmp[2]);
-            //dg::blas1::axpby( v.p.nu_parallel, v.f.dssN(0), 1., v.tmp[2]);
+            //v.f.compute_lapParN( 0, v.tmp[2]);
+            //dg::blas1::scal( v.tmp[2], v.p.nu_parallel);
             dg::blas1::copy( 0., v.tmp[2]);
             dg::blas1::axpby( 1., v.tmp[1], 1., v.tmp[2]); //Lambda_ne
             dg::blas1::pointwiseDot( v.tmp[2], result, result);
@@ -1116,8 +1103,10 @@ std::vector<Record> diagnostics2d_list = {
         []( DVec& result, Variables& v ) {
             dg::blas1::pointwiseDot( v.f.velocity(1), v.f.velocity(1), v.tmp[1]);
             dg::blas1::pointwiseDot( v.p.mu[1], v.f.density(1), v.tmp[1], v.f.divb(), 0., result);
-            dg::blas1::pointwiseDot( 0.5*v.p.mu[1], v.f.density(1),  v.f.velocity(1), v.f.dsU(1), 1., result);
-            dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[1], v.f.dsN(1), 1., result);
+            v.f.compute_dsU( 1, v.tmp[2]);
+            dg::blas1::pointwiseDot( 0.5*v.p.mu[1], v.f.density(1),  v.f.velocity(1), v.tmp[2], 1., result);
+            v.f.compute_dsN( 1, v.tmp[2]);
+            dg::blas1::pointwiseDot( v.p.mu[1], v.tmp[1], v.tmp[2], 1., result);
         }
     },
     //not so important
@@ -1152,9 +1141,8 @@ std::vector<Record> diagnostics2d_list = {
     //should be zero
     {"lparpar_tt", "Parallel momentum dissipation by parallel diffusion", true,
         []( DVec& result, Variables& v ) {
-            dg::blas1::pointwiseDot( v.p.nu_parallel[1], v.f.divb(), v.f.dsU(1),
-                                     0., result);
-            dg::blas1::axpby( v.p.nu_parallel[1], v.f.dssU(1), 1., result);
+            v.f.compute_lapParU( 1, result);
+            dg::blas1::scal( result, v.p.nu_parallel[1]);
         }
     },
     {"lparperp_tt", "Parallel momentum dissipation by perp diffusion", true,
@@ -1168,7 +1156,8 @@ std::vector<Record> diagnostics2d_list = {
     {"sparmirrore_tt", "Mirror force term with electron density (Time average)", true,
         []( DVec& result, Variables& v){
             //dg::blas1::pointwiseDot( -v.p.tau[0], v.f.divb(), v.f.density(0), 0., result);
-            dg::blas1::axpby( v.p.tau[0], v.f.dsN(0), 0., result);
+            v.f.compute_dsN(0, result);
+            dg::blas1::scal( result, v.p.tau[0]);
         }
     },
     {"sparmirrorAe_tt", "Apar Mirror force term with electron density (Time average)", true,
@@ -1180,13 +1169,15 @@ std::vector<Record> diagnostics2d_list = {
     {"sparmirrori_tt", "Mirror force term with ion density (Time average)", true,
         []( DVec& result, Variables& v){
             //dg::blas1::pointwiseDot( v.p.tau[1], v.f.divb(), v.f.density(1), 0., result);
-            dg::blas1::axpby( -v.p.tau[1], v.f.dsN(1), 0., result);
+            v.f.compute_dsN(1, result);
+            dg::blas1::scal( result, -v.p.tau[1]);
         }
     },
     //electric force balance usually well-fulfilled
     {"sparphie_tt", "Electric force in electron momentum density (Time average)", true,
         []( DVec& result, Variables& v){
-            dg::blas1::pointwiseDot( 1., v.f.dsP(0), v.f.density(0), 0., result);
+            v.f.compute_dsP(0, result);
+            dg::blas1::pointwiseDot( 1., result, v.f.density(0), 0., result);
         }
     },
     {"sparphiAe_tt", "Apar Electric force in electron momentum density (Time average)", true,
@@ -1204,7 +1195,8 @@ std::vector<Record> diagnostics2d_list = {
     //These two should be almost the same
     {"sparphii_tt", "Electric force term in ion momentum density (Time average)", true,
         []( DVec& result, Variables& v){
-            dg::blas1::pointwiseDot( -1., v.f.dsP(1), v.f.density(1), 0., result);
+            v.f.compute_dsP(1, result);
+            dg::blas1::pointwiseDot( -1., result, v.f.density(1), 0., result);
         }
     },
     {"friction_tt", "Friction force in momentum density (Time average)", true,
diff --git a/src/feltor/implicit.h b/src/feltor/implicit.h
index 823682055..9b017ac09 100644
--- a/src/feltor/implicit.h
+++ b/src/feltor/implicit.h
@@ -44,8 +44,7 @@ struct ImplicitDensity
     }
     void set_wall_and_sheath( double wall_forcing, Container wall, double sheath_forcing, Container sheath)
     {
-        dg::blas1::axpby( wall_forcing, wall, sheath_forcing, sheath, m_forcing); //1/eta_w
-
+        dg::blas1::axpby( wall_forcing, wall, sheath_forcing, sheath, m_forcing);
         dg::blas1::axpby( -1., wall, -1., sheath, m_masked);
         dg::blas1::plus( m_masked, +1);
     }
@@ -155,6 +154,7 @@ struct ImplicitVelocity
         dg::assign( dg::evaluate( dg::zero, g), m_forcing );
         dg::assign( dg::evaluate( dg::one, g), m_masked );
     }
+    const Container& get_forcing() const { return m_forcing;}
     void set_wall_and_sheath( double wall_forcing, Container wall, double sheath_forcing, Container sheath)
     {
         dg::blas1::axpby( wall_forcing, wall, sheath_forcing, sheath, m_temp); //1/eta_w
@@ -369,9 +369,10 @@ struct FeltorSpecialSolver
         if( m_p.explicit_diffusion)
         {
             //I(y) = -Chi y
-            dg::blas1::axpby( 1., 1., -alpha, m_imdens.get_forcing(), y[1][1]);
-            dg::blas1::pointwiseDivide( rhs[0][0], y[1][1], y[0][0]);
-            dg::blas1::pointwiseDivide( rhs[0][1], y[1][1], y[0][1]);
+            dg::blas1::axpby( 1., 1., -alpha, m_imdens.get_forcing(), y[0][1]);
+            dg::blas1::pointwiseDivide( rhs[0][0], y[0][1], y[0][0]);
+            dg::blas1::pointwiseDivide( rhs[0][1], y[0][1], y[0][1]);
+            dg::blas1::axpby( 1., 1., -alpha, m_imvelo.get_forcing(), y[1][1]);
             dg::blas1::pointwiseDivide( rhs[1][0], y[1][1], y[1][0]);
             dg::blas1::pointwiseDivide( rhs[1][1], y[1][1], y[1][1]);
             return;
diff --git a/src/feltor/input/compass.json b/src/feltor/input/compass.json
index 0f83cedf8..36e87c934 100644
--- a/src/feltor/input/compass.json
+++ b/src/feltor/input/compass.json
@@ -19,12 +19,11 @@
     "jumpfactor" : 1,
     "eps_gamma"  : 1e-6,
     "eps_time"   : 1e-10,
-    "rtol"       : 1e-5,
     "mu"          : -0.000272121,
     "tau"         : 1.0,
     "beta"        : 0e-4,
     "nu_perp"     : 2e-3,
-    "perp_diff"   : "hyperviscous",
+    "perp_diff"   : ["hyperviscous", "explicit"],
     "resistivity" : 1e-4,
     "curvmode"   : "toroidal",
     "symmetric"  : false,
@@ -59,11 +58,18 @@
         "boundary": 0.55,
         "alpha" : 0.2
     },
-    "damping":
+    "wall":
     {
-        "modifier": "sol_pfr",
-        "rate" : 1e-2,
-        "boundary": [1.1,0.988],
-        "alpha": [0.25,0.15]
+        "type": "sol_pfr",
+        "penalization" : 1e+0,
+        "boundary": [1.1,0.998],
+        "alpha": [0.10,0.10]
+    },
+    "sheath":
+    {
+        "bc": "bohm",
+        "penalization" : 1e+0,
+        "boundary": 0.30,
+        "alpha": 0.2
     }
 }
diff --git a/src/feltor/input/default.json b/src/feltor/input/default.json
index b8e7407f6..ff1478578 100644
--- a/src/feltor/input/default.json
+++ b/src/feltor/input/default.json
@@ -19,13 +19,11 @@
     "jumpfactor" : 1,
     "eps_gamma"  : 1e-6,
     "eps_time"   : 1e-7,
-    "rtol"       : 1e-4,
     "mu"          : -0.000272121,
     "tau"         : 0.5,
     "beta"        : 0,
     "nu_perp"     : 1e-3,
-    "perp_diff"   : "hyperviscous",
-    "nu_parallel" : 10,
+    "perp_diff"   : ["hyperviscous","explicit"],
     "resistivity" : 1e-4,
     "curvmode"   : "toroidal",
     "symmetric"  : false,
@@ -60,11 +58,18 @@
         "boundary": 0.55,
         "alpha": 0.2
     },
-    "damping":
+    "wall":
     {
-        "modifier": "heaviside",
-        "rate": 1e-2,
-        "boundary": 1.1,
+        "type": "sol_pfr",
+        "penalization" : 1e+0,
+        "boundary": [1.1,0.998],
+        "alpha": [0.10,0.10]
+    },
+    "sheath":
+    {
+        "bc": "bohm",
+        "penalization" : 1e+0,
+        "boundary": 0.30,
         "alpha": 0.2
     }
 }
diff --git a/src/feltor/input/manufactured.json b/src/feltor/input/manufactured.json
index 809324237..f89d8506d 100644
--- a/src/feltor/input/manufactured.json
+++ b/src/feltor/input/manufactured.json
@@ -19,12 +19,11 @@
     "eps_gamma"  : 1e-5,
     "stages"     : 3,
     "eps_time"   : 1e-7,
-    "rtol"       : 1e-4,
     "mu"          : -0.000272121,
     "tau"         : 0,
     "beta"        : 0,
     "nu_perp"     : 0.1,
-    "perp_diff"   : "viscous",
+    "perp_diff"   : ["viscous","explicit"],
     "resistivity" : 0.1,
     "curvmode"   : "toroidal",
     "symmetric"  : false,
@@ -59,11 +58,18 @@
         "boundary": 0.5,
         "alpha": 0.2
     },
-    "damping":
+    "wall":
     {
-        "modifier": "heaviside",
-        "rate": 1e-2,
-        "boundary": 1.1,
-        "alpha": 0.1
+        "type": "sol_pfr",
+        "penalization" : 1e+0,
+        "boundary": [1.1,0.998],
+        "alpha": [0.10,0.10]
+    },
+    "sheath":
+    {
+        "bc": "bohm",
+        "penalization" : 1e+0,
+        "boundary": 0.30,
+        "alpha": 0.2
     }
 }
diff --git a/src/feltor/input/tcv.json b/src/feltor/input/tcv.json
index 3fa7b91f0..00034a91b 100644
--- a/src/feltor/input/tcv.json
+++ b/src/feltor/input/tcv.json
@@ -14,18 +14,16 @@
     "inner_loop": 4,
     "itstp": 2,
     "maxout": 10,
+    "stages"     : 3,
     "eps_pol"    : [1e-6,1,1],
     "jumpfactor" : 1,
     "eps_gamma"  : 1e-6,
-    "stages"     : 3,
     "eps_time"   : 1e-10,
-    "rtol"       : 1e-5,
     "mu"          : -0.000272121,
     "tau"         : 0.0,
     "beta"        : 1e-4,
     "nu_perp"     : 1e-2,
-    "perp_diff"   : "hyperviscous",
-    "nu_parallel" : 1e3,
+    "perp_diff"   : ["hyperviscous","explicit"],
     "resistivity" : 3.43e-6,
     "curvmode"   : "toroidal",
     "symmetric"  : false,
@@ -60,11 +58,18 @@
         "boundary": 0.55,
         "alpha" : 0.2
     },
-    "damping":
+    "wall":
     {
-        "modifier": "sol_pfr",
-        "rate" : 1e-2,
-        "boundary": [1.03,0.99],
-        "alpha": [0.20, 0.1]
+        "type": "sol_pfr",
+        "penalization" : 1e+0,
+        "boundary": [1.1,0.998],
+        "alpha": [0.10,0.10]
+    },
+    "sheath":
+    {
+        "bc": "bohm",
+        "penalization" : 1e+0,
+        "boundary": 0.30,
+        "alpha": 0.2
     }
 }
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index 2fac50913..e84ce0fd1 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -56,9 +56,17 @@ int main( int argc, char* argv[])
     file::string2Json(inputfile, js, file::comments::are_forbidden);
     file::string2Json(geomfile, gs, file::comments::are_forbidden);
     const feltor::Parameters p(js, file::error::is_warning);
-    const dg::geo::solovev::Parameters gp(gs);
     p.display();
-    gp.display();
+    std::cout << gs.toStyledString() << std::endl;
+    dg::geo::TokamakMagneticField mag;
+    try{
+        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
+    }catch(std::runtime_error& e)
+    {
+        std::cerr << "ERROR in geometry file "<<geomfile<<std::endl;
+        std::cerr <<e.what()<<std::endl;
+        return -1;
+    }
 
     //-----------------Create Netcdf output file with attributes----------//
     int ncid_out;
@@ -87,10 +95,10 @@ int main( int argc, char* argv[])
 
     //-------------------Construct grids-------------------------------------//
 
-    const double Rmin=gp.R_0-p.boxscaleRm*gp.a;
-    const double Zmin=-p.boxscaleZm*gp.a*gp.elongation;
-    const double Rmax=gp.R_0+p.boxscaleRp*gp.a;
-    const double Zmax=p.boxscaleZp*gp.a*gp.elongation;
+    const double Rmin=mag.R0()-p.boxscaleRm*mag.params().a();
+    const double Zmin=-p.boxscaleZm*mag.params().a()*mag.params().elongation();
+    const double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
+    const double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
     const unsigned FACTOR = 6;
 
     dg::RealCylindricalGrid3d<double> g3d_in( Rmin,Rmax, Zmin,Zmax, 0, 2*M_PI,
@@ -116,17 +124,6 @@ int main( int argc, char* argv[])
     std::map<std::string, int> id4d;
 
     /////////////////////////////////////////////////////////////////////////
-    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
-    if( p.damping_alpha > 0.)
-    {
-        double RO=mag.R0(), ZO=0.;
-        dg::geo::findOpoint( mag.get_psip(), RO, ZO);
-        double psipO = mag.psip()( RO, ZO);
-        double damping_psi0p = (1.-p.damping_boundary*p.damping_boundary)*psipO;
-        double damping_alphap = -(2.*p.damping_boundary+p.damping_alpha)*p.damping_alpha*psipO;
-        mag = dg::geo::createModifiedSolovevField(gp, damping_psi0p+damping_alphap/2.,
-                fabs(damping_alphap/2.), ((psipO>0)-(psipO<0)));
-    }
     auto bhat = dg::geo::createBHat( mag);
     dg::geo::Fieldaligned<Geometry, IHMatrix, HVec> fieldaligned(
         bhat, g3d_out, dg::NEU, dg::NEU, dg::geo::NoLimiter(), //let's take NEU bc because N is not homogeneous
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 33dc56874..166e2ee05 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -40,7 +40,7 @@ int main( int argc, char* argv[])
     //create RHS
     std::cout << "Initialize explicit" << std::endl;
     dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
-    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, mag, false);
+    feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, mag);
     std::cout << "Initialize implicit" << std::endl;
     feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
 
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index 3eb8ca96d..f1593abe8 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -22,7 +22,6 @@ struct Parameters
     double jfactor;
     double eps_gamma;
     double eps_time;
-    double rtol;
     unsigned stages;
     unsigned mx, my;
     double rk4eps;
@@ -40,7 +39,7 @@ struct Parameters
     double sigma_z;
     double k_psi;
 
-    double source_rate, damping_rate, sheath_rate;
+    double source_rate, wall_rate, sheath_rate;
     double source_alpha, profile_alpha;
     double source_boundary;
     double nprofamp;
@@ -49,7 +48,7 @@ struct Parameters
 
     enum dg::bc bcxN, bcyN, bcxU, bcyU, bcxP, bcyP;
     std::string initne, initphi, curvmode, perp_diff;
-    std::string source_type, sheath_type;
+    std::string source_type, sheath_bc;
     bool symmetric, periodify, explicit_diffusion ;
     Parameters() = default;
     Parameters( const Json::Value& js, enum file::error mode = file::error::is_warning ) {
@@ -66,7 +65,6 @@ struct Parameters
         itstp   = file::get( mode, js, "itstp", 0).asUInt();
         maxout  = file::get( mode, js, "maxout", 0).asUInt();
         eps_time    = file::get( mode, js, "eps_time", 1e-10).asDouble();
-        rtol        = file::get( mode, js, "rtol", 1e-5).asDouble();
 
         stages      = file::get( mode, js, "stages", 3).asUInt();
         eps_pol.resize(stages);
@@ -91,7 +89,25 @@ struct Parameters
         beta        = file::get( mode, js, "beta", 0.).asDouble();
         eta         = file::get( mode, js, "resistivity", 0.).asDouble();
         nu_perp     = file::get( mode, js, "nu_perp", 0.).asDouble();
-        perp_diff   = file::get( mode, js, "perp_diff", "viscous").asString();
+        perp_diff   = file::get_idx( mode, js, "perp_diff", 0, "viscous").asString();
+        std::string temp = file::get_idx( mode, js, "perp_diff", 1, "").asString();
+        explicit_diffusion = true;
+        if(temp == "implicit")
+            explicit_diffusion = false;
+        else if(temp == "explicit")
+            explicit_diffusion = true;
+        else
+        {
+            if( file::error::is_throw == mode)
+                throw std::runtime_error( "Value "+temp+" for perp_diff[1] is invalid! Must be either explicit or implicit\n");
+            else if ( file::error::is_warning == mode)
+                std::cerr << "Value "+temp+" for perp_diff[1] is invalid!\n";
+            else
+                ;
+        }
+
+        explicit_diffusion = file::get( mode, js, "explicit_diff", false).asBool();
+
         //nu_parallel = file::get( mode, js, "nu_parallel", 0.).asDouble();
         //Init after reading in eta and mu[0]
         nu_parallel[0] = 0.73/eta;
@@ -105,18 +121,17 @@ struct Parameters
         posY        = file::get( mode, js, "posY", 0.).asDouble();
         sigma_z     = file::get( mode, js, "sigma_z", 0.).asDouble();
         k_psi       = file::get( mode, js, "k_psi", 0.).asDouble();
-        explicit_diffusion = file::get( mode, js, "explicit_diff", false).asBool();
 
         nprofamp   = file::get( mode, js, "profile", "amp", 0.).asDouble();
         profile_alpha = file::get( mode, js, "profile", "alpha", 0.2).asDouble();
 
         source_rate     = file::get( mode, js, "source", "rate", 0.).asDouble();
         source_type     = file::get( mode, js, "source", "type", "profile").asString();
-        sheath_type     = file::get( mode, js, "sheath", "type", "bohm").asString();
+        sheath_bc       = file::get( mode, js, "sheath", "bc", "bohm").asString();
         source_boundary = file::get( mode, js, "source", "boundary", 0.5).asDouble();
         source_alpha    = file::get( mode, js, "source", "alpha", 0.2).asDouble();
-        damping_rate = file::get( mode, js, "damping", "rate", 0.).asDouble();
-        sheath_rate  = file::get( mode, js, "sheath", "rate", 0.).asDouble();
+        wall_rate = file::get( mode, js, "wall", "penalization", 0.).asDouble();
+        sheath_rate  = file::get( mode, js, "sheath", "penalization", 0.).asDouble();
 
         bcxN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 0, "").asString());
         bcyN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 1, "").asString());
@@ -163,9 +178,9 @@ struct Parameters
             <<"     source_alpha:                 "<<source_alpha<<"\n"
             <<"     profile_alpha:                "<<profile_alpha<<"\n"
             <<"     source_type:                  "<<source_type<<"\n"
-            <<"     damping_rate:                 "<<damping_rate<<"\n"
-            <<"     sheath_rate:                  "<<sheath_rate<<"\n"
-            <<"     sheath_type:                  "<<sheath_type<<"\n"
+            <<"     wall_penalization:            "<<wall_rate<<"\n"
+            <<"     sheath_penalization:          "<<sheath_rate<<"\n"
+            <<"     sheath_bc:                    "<<sheath_bc<<"\n"
             <<"     density profile amplitude:    "<<nprofamp<<"\n"
             <<"     boxscale R+:                  "<<boxscaleRp<<"\n"
             <<"     boxscale R-:                  "<<boxscaleRm<<"\n"
@@ -181,7 +196,6 @@ struct Parameters
             <<"     Jump scale factor:    "<<jfactor<<"\n"
             <<"     Accuracy Gamma CG:    "<<eps_gamma<<"\n"
             <<"     Accuracy Time  CG:    "<<eps_time<<"\n"
-            <<"     Accuracy Time Stepper "<<rtol<<"\n"
             <<"     Accuracy Fieldline    "<<rk4eps<<"\n"
             <<"     Periodify FCI         "<<std::boolalpha<< periodify<<"\n"
             <<"     Refined FCI           "<<mx<<" "<<my<<"\n"
-- 
GitLab


From 0bf5ce0d8d950af2d6a989702487f0ae8782f4d1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 24 Nov 2020 18:02:19 +0100
Subject: [PATCH 399/540] Fix bug in parameters

- rename damping to wall in geometry_diag
- add some more documentation on sheath
---
 inc/geometries/geometry_diag.cu |  8 ++++----
 src/feltor/feltor.h             |  3 +++
 src/feltor/feltor.tex           | 16 +++++++++++-----
 src/feltor/feltordiag.cu        |  4 ++--
 src/feltor/parameters.h         |  2 --
 5 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index 64d4d6aac..e58a6c38d 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -117,12 +117,12 @@ int main( int argc, char* argv[])
     const Parameters p(input_js);
     p.display( std::cout);
     //Test coefficients
-    dg::geo::CylindricalFunctor damping, transition, sheath, direction;
+    dg::geo::CylindricalFunctor wall, transition, sheath, direction;
     dg::geo::TokamakMagneticField mag = dg::geo::createMagneticField(geom_js,
             file::error::is_throw);
     dg::geo::TokamakMagneticField mod_mag =
         dg::geo::createModifiedField(geom_js, input_js, file::error::is_throw,
-                damping, transition);
+                wall, transition);
     std::string input = input_js.toStyledString();
     std::string geom = geom_js.toStyledString();
     unsigned n, Nx, Ny, Nz;
@@ -132,7 +132,7 @@ int main( int argc, char* argv[])
     double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
     double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
     dg::geo::createSheathRegion( input_js, file::error::is_warning,
-            mag, damping, Rmin, Rmax, Zmin, Zmax, sheath, direction);
+            mag, wall, Rmin, Rmax, Zmin, Zmax, sheath, direction);
 
     dg::geo::description mag_description = mag.params().getDescription();
     double psipO = -1.;
@@ -218,7 +218,7 @@ int main( int argc, char* argv[])
         {"SourceProfile", "A source profile", dg::compose( dg::PolynomialHeaviside(
                     p.source_boundary-p.source_alpha/2., p.source_alpha/2., -1 ),
                 dg::geo::RhoP(mag))},
-        {"ProfileDamping", "Density profile damping", damping },
+        {"Wall", "Penalization region that acts as the wall", wall },
         {"MagneticTransition", "The region where the magnetic field is modified", transition},
         {"Nprofile", "A flux aligned profile", dg::compose( dg::LinearX( p.nprofileamp/mag.psip()(mag.R0(),0.), p.nprofileamp ), mag.psip())},
         {"Delta", "A flux aligned Gaussian peak", dg::compose( dg::GaussianX( psipO*0.2, 0.1, 1./(sqrt(2.*M_PI)*0.1)), mag.psip())},
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index b8140c8a8..a4d1c58cc 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -794,6 +794,9 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
     const std::array<std::array<Container,2>,2>& fields,
     std::array<std::array<Container,2>,2>& yp)
 {
+    //MW: in theory we have the possibility to
+    // make the implementation conservative since the perp boundaries are
+    // penalized away
     //y[0] = N-1, y[1] = W; fields[0] = N, fields[1] = U
     for( unsigned i=0; i<2; i++)
     {
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 81da07b33..4f204d2de 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -826,7 +826,9 @@ penalized and thus hold on the bounding box. Typically we choose
 \phi = 0
 \text{ and }  \hat n \cn A_{\parallel} = 0
 \end{align}
-where $\hat n$ is the normal vector to the boundary.
+where $\hat n$ is the normal vector to the boundary. The perpendicular boundary
+conditions for the remaining fields become obsolete with the following
+penalization scheme.
 
 \subsubsection{The wall region}
 Being a box, our computational domain is in particular not aligned with the
@@ -895,22 +897,26 @@ We then take $\theta_{\alpha/2}\left( (\eps_s + \frac{\alpha}{2})a - d(R,Z)\righ
 and take the set intersection between that region and the ``not wall'' region to
 determine the sheath penalization region:
 \begin{align}\label{eq:sheath}
-    \chi_s := \left(1-\chi_w(R,Z,\varphi)\right) \theta_{\alpha/2}\left( (\eps_s + \frac{\alpha}{2})a - d(R,Z)\right)
+    \chi_s := \left(1-\chi_w(R,Z,\varphi)\right) \theta_{\alpha/2}\left( \left(\eps_s + \frac{\alpha}{2}\right)a - d(R,Z)\right)
 \end{align}
 Within the sheath region we penalize
 \begin{subequations} \label{eq:sheath_penalization}
 \begin{align}
     S^s_N(R,Z,\varphi, t) &:= \omega_s \chi_s \left(N_{sh}-N\right)\\
-    S^s_{u_e}(R,Z,\varphi, t) &:= \omega_s \chi_s \left(\sqrt{1+\tau}\exp(-\phi) - u_{\parallel,e} \right)\\
-    S^s_{U_i}(R,Z,\varphi, t) &:= \omega_s \chi_s \left(\sqrt{1+\tau} - U_{\parallel,i} \right)\\
+    S^s_{U_i}(R,Z,\varphi, t) &:= \omega_s \chi_s \left(\sqrt{1+\tau} - U_{\parallel,i} \right) \\
+    S^s_{u_e}(R,Z,\varphi, t) &:= \omega_s \chi_s \left(\sqrt{1+\tau}\exp(-\phi) - u_{\parallel,e} \right) \text{ or }
+    S^s_{u_e}(R,Z,\varphi, t) := \omega_s \chi_s \left(\sqrt{1+\tau} - u_{\parallel,e} \right)
 \end{align}
 \end{subequations}
 where $\omega_s$ is the sheath penalization parameter and $N_{sh}$ is obtained by extrapolating $N$ along the magnetic field line with
 the help of the parallel derivative operators
 \begin{align}
-    N_{sh} = \Tpm N
+    N_{sh} = \Tpm N \text{ such that } \nabla_\parallel N|_{sh} = 0
 \end{align}
 where the sign depends again on the direction of the magnetic field line (we always extrapolate ``downstream'').
+The extrapolation models a parallel Neumann boundary condition for the density.
+The boundary condition for the parallel electron velocity can be either the Bohm condition containing the contribution from the electric potential or
+an insulating condition where the total current $j_s = n_e ( U_{\parallel,i} - u_{\parallel,e})$ vanishes.
 
 
 
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index 0f1015413..a2fb52f4f 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -94,9 +94,9 @@ int main( int argc, char* argv[])
     Geometry g3d_fine( Rmin, Rmax, Zmin, Zmax, 0., 2.*M_PI,
         p.n_out, p.Nx_out, p.Ny_out, FACTOR*p.Nz, p.bcxN, p.bcyN, dg::PER);
 
-    dg::geo::CylindricalFunctor damping, transition;
+    dg::geo::CylindricalFunctor wall, transition;
     dg::geo::TokamakMagneticField mag =
-        dg::geo::createModifiedField(gs, js, file::error::is_warning, damping, transition);
+        dg::geo::createModifiedField(gs, js, file::error::is_warning, wall, transition);
     dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
     // Construct weights and temporaries
 
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index f1593abe8..d2365f87b 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -106,8 +106,6 @@ struct Parameters
                 ;
         }
 
-        explicit_diffusion = file::get( mode, js, "explicit_diff", false).asBool();
-
         //nu_parallel = file::get( mode, js, "nu_parallel", 0.).asDouble();
         //Init after reading in eta and mu[0]
         nu_parallel[0] = 0.73/eta;
-- 
GitLab


From 43daf758206888837c2caf1fed3d140ede693f5f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 27 Nov 2020 10:46:43 +0100
Subject: [PATCH 400/540] Fix downstream bug in density sheath bc

---
 src/feltor/feltor.h      | 22 +++++++++++++++-------
 src/feltor/feltor.tex    |  9 +++++++++
 src/feltor/feltor_hpc.cu |  2 +-
 3 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index a4d1c58cc..26a39c026 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -417,7 +417,7 @@ struct Explicit
 
     const feltor::Parameters m_p;
     double m_omega_source = 0., m_sheath_forcing = 0.;
-    bool m_fixed_profile = true;
+    bool m_fixed_profile = true, m_reversed_field = false;
 
 };
 
@@ -429,6 +429,9 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_mag(
         throw dg::Error(dg::Message(_ping_)<<"Warning! perp_diff value '"<<p.perp_diff<<"' not recognized!! I do not know how to proceed! Exit now!");
     //due to the various approximations bhat and mag not always correspond
     dg::geo::CylindricalVectorLvl0 curvNabla, curvKappa;
+    m_reversed_field = false;
+    if( mag.ipol()( g.x0(), g.y0()) < 0)
+        m_reversed_field = true;
     if( p.curvmode == "true" )
     {
         curvNabla = dg::geo::createTrueCurvatureNablaB(mag);
@@ -438,7 +441,7 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_mag(
     }
     else if( p.curvmode == "low beta")
     {
-        if( mag.ipol()( g.x0(), g.y0()) < 0)
+        if( m_reversed_field)
             curvNabla = curvKappa = dg::geo::createCurvatureNablaB(mag, -1);
         else
             curvNabla = curvKappa = dg::geo::createCurvatureNablaB(mag, +1);
@@ -446,7 +449,7 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_mag(
     }
     else if( p.curvmode == "toroidal")
     {
-        if( mag.ipol()( g.x0(), g.y0()) < 0)
+        if( m_reversed_field)
         {
             curvNabla = dg::geo::createCurvatureNablaB(mag, -1);
             curvKappa = dg::geo::createCurvatureKappa(mag, -1);
@@ -497,7 +500,7 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
     bhat = dg::geo::createEPhi(+1);
     if( p.curvmode == "true")
         bhat = dg::geo::createBHat(mag);
-    else if( mag.ipol()( g.x0(), g.y0()) < 0)
+    else if( m_reversed_field)
         bhat = dg::geo::createEPhi(-1);
     dg::pushForward(bhat.x(), bhat.y(), bhat.z(), m_b[0], m_b[1], m_b[2], g);
     dg::SparseTensor<Container> metric = g.metric();
@@ -531,7 +534,7 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_invert(
     auto bhat = dg::geo::createEPhi(+1); //bhat = ephi except when "true"
     if( p.curvmode == "true")
         bhat = dg::geo::createBHat( mag);
-    else if( mag.ipol()( g.x0(), g.y0()) < 0)
+    else if( m_reversed_field)
         bhat = dg::geo::createEPhi(-1);
     m_multi_chi = m_multigrid.project( m_temp0);
     m_multi_pol.resize(p.stages);
@@ -1028,10 +1031,15 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     if( m_sheath_forcing != 0)
     {
         //density
+        //Here, we need to find out where "downstream" is
         for( unsigned i=0; i<2; i++)
         {
-            dg::blas1::evaluate( m_temp0, dg::equals(), routines::ComputeDensityBC(),
-                m_minusN[i], m_plusN[i], m_U_sheath);
+            if( m_reversed_field) //bphi negative (exchange + and -)
+                dg::blas1::evaluate( m_temp0, dg::equals(), routines::ComputeDensityBC(),
+                    m_plusN[i], m_minusN[i], m_U_sheath);
+            else
+                dg::blas1::evaluate( m_temp0, dg::equals(), routines::ComputeDensityBC(),
+                    m_minusN[i], m_plusN[i], m_U_sheath);
             dg::blas1::axpby( m_sheath_forcing, m_temp0, 1.,  yp[0][i]);
         }
         //velocity
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 4f204d2de..373368b4e 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -830,6 +830,14 @@ where $\hat n$ is the normal vector to the boundary. The perpendicular boundary
 conditions for the remaining fields become obsolete with the following
 penalization scheme.
 
+\begin{tcolorbox}[title=Note]
+    Computing on a box and cutting away the parts that are the wall,
+    for typical tokamak shapes incur a (loosely) estimated 50\% overhead in vector size:
+    \begin{align*}
+        \text{ number of points in box } = 1.5 \times \text{ number of points in area with plasma}
+    \end{align*}
+\end{tcolorbox}
+
 \subsubsection{The wall region}
 Being a box, our computational domain is in particular not aligned with the
 magnetic flux surfaces. This means that particularly in the corners of
@@ -1687,6 +1695,7 @@ source & dict & & Density source, cf. the output \texttt{sne\_tt\_ifs} in \textt
 "influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx},
 "torpex": Torpex inspired source profile Eq.~\eqref{eq:electron_source_torpex},
 "gaussian": Gaussian shaped source profile - uses \texttt{posX}, \texttt{posY} and \texttt{sigma},
+"profile\_influx": Copy a profile into the source function and use a constant source rate. The idea is that you can start with zero density and evolve the profile purely with the source.
     See the file {\tt init.h} to add your own custom source.
 \\
 \qquad boundary & float & 0.2  & Source region boundary $\rho_{p,b}$: yields in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 1fe6077fc..6f5c106ce 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -483,7 +483,7 @@ int main( int argc, char* argv[])
     karniadakis.solver().set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
     }
 
-    std::cout << "Initialize Timestepper" << std::endl;
+    MPI_OUT std::cout << "Initialize Timestepper" << std::endl;
     karniadakis.init( feltor, implicit, time, y0, p.dt);
     dg::Timer t;
     t.tic();
-- 
GitLab


From 2c7a428cb9ed2c688479672d528eb1e388a6a61c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 30 Nov 2020 15:48:53 +0100
Subject: [PATCH 401/540] Add MinimalProjecting Time-stepper

- is the explicit part of Karniadakis... (based on this knowledge we could
  construct Karniadakis of higher orders)
- improve test for multistep time-steppers
- fix bug in Adams Bashforth initialization
---
 inc/dg/multistep.h    | 164 +++++++++++++++++++++++++++++++++++++++-
 inc/dg/multistep_t.cu | 171 ++++++++++++++++++++----------------------
 inc/dg/runge_kutta.h  |  41 +++++-----
 3 files changed, 266 insertions(+), 110 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 44429efd5..5cc9bb8ee 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -46,6 +46,7 @@ namespace dg{
 * \f[ u^{n+1} = u^n + \Delta t\sum_{j=0}^{s-1} b_j f\left(t^n - j \Delta t, u^{n-j}\right) \f]
 *
 * with coefficients taken from https://en.wikipedia.org/wiki/Linear_multistep_method
+* @note This scheme has a smaller region of absolute stability than that of a \c MinimalProjecting method
 * @copydoc hide_note_multistep
 * @copydoc hide_ContainerType
 * @ingroup time
@@ -123,7 +124,7 @@ template< class RHS>
 void AdamsBashforth<ContainerType>::init( RHS& f, value_type t0, const ContainerType& u0, value_type dt)
 {
     m_tu = t0, m_dt = dt;
-    f( t0, u0, m_f[0]); //f may not destroy u0
+    f( t0, u0, m_f[m_k-1]); //f may not destroy u0
     blas1::copy(  u0, m_u);
     m_counter = 0;
 }
@@ -140,7 +141,7 @@ void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType&
         m_counter++;
         m_tu = t;
         blas1::copy(  u, m_u);
-        f( m_tu, m_u, m_f[m_counter]);
+        f( m_tu, m_u, m_f[m_k - 1 - m_counter]);
         return;
     }
     for( unsigned i=0; i<m_k; i++)
@@ -152,6 +153,7 @@ void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType&
     f( m_tu, m_u, m_f[0]); //evaluate f at new point
 }
 
+
 /**
 * @brief Struct for Karniadakis semi-implicit multistep time-integration
 * \f[
@@ -189,14 +191,16 @@ for a semi-implicit (first order) Euler method
 * Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$).
 * @note This scheme implements <a href = "https://dx.doi.org/10.1016/0021-9991(91)90007-8"> Karniadakis, et al. J. Comput. Phys. 97 (1991)</a>
 * @note The implicit part equals a third order backward differentiation formula (BDF) https://en.wikipedia.org/wiki/Backward_differentiation_formula
+* while the explicit part equals the minimal projection method by Alfeld (1979)
 *
 The following code example demonstrates how to implement the method of manufactured solutions on a 2d partial differential equation with the dg library:
 * @snippet multistep_t.cu function
 * In the main function:
 * @snippet multistep_t.cu karniadakis
 * @note In our experience the implicit treatment of diffusive or hyperdiffusive
-terms can significantly reduce the required number of time steps. This
+terms may significantly reduce the required number of time steps. This
 outweighs the increased computational cost of the additional matrix inversions.
+However, each PDE is different and general statements like this one should be treated with care.
 * @copydoc hide_note_multistep
 * @copydoc hide_SolverType
 * @copydoc hide_ContainerType
@@ -413,6 +417,9 @@ struct BDF
         m_u.assign( order, m_solver.copyable());
         m_f = m_solver.copyable();
     }
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_u[0];}
     ///Write access to the internal solver for the implicit part
     SolverType& solver() { return m_solver;}
     ///Read access to the internal solver for the implicit part
@@ -482,4 +489,155 @@ void BDF<ContainerType, SolverType>::step(RHS& rhs, value_type& t, container_typ
     dg::blas1::copy( u, m_u[0]);
 }
 
+/**
+* @brief Struct for Minimal Projecting explicit linear multistep time-integration
+* \f[
+* \begin{align}
+    v^{n+1} = \sum_{j=0}^{s-1} \alpha_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}\beta_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right)
+    \end{align}
+    \f]
+
+    which discretizes
+    \f[
+    \frac{\partial v}{\partial t} = \hat f(t,v)
+    \f]
+    where \f$ f \f$ contains the equations.
+    The coefficients for order 3 are given as an example:
+    \f[
+    \alpha_0 = \frac{18}{11}\ \alpha_1 = -\frac{9}{11}\ \alpha_2 = \frac{2}{11} \\
+    \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11}
+\f]
+@note This scheme needs more storage but **has a larger region of absolute stability** than an Adams-Bashforth method of the same order.
+* @note This scheme implements <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
+*
+* @copydoc hide_note_multistep
+* @copydoc hide_ContainerType
+* @ingroup time
+*/
+template<class ContainerType>
+struct MinimalProjecting
+{
+    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
+    using container_type = ContainerType; //!< the type of the vector class in use
+    ///@copydoc RungeKutta::RungeKutta()
+    MinimalProjecting(){}
+
+    ///@copydoc construct()
+    MinimalProjecting( unsigned order, const ContainerType& copyable){
+        construct( order, copyable);
+    }
+    /**
+     * @brief Reserve memory for the integration
+     *
+     * Set the coefficients \f$ \alpha_i,\ \beta_i\f$
+     * @param order (global) order (= number of steps in the multistep) of the method (Currently, one of 1 (Euler), 2, ..., or 7)
+     * @param copyable ContainerType of the size that is used in \c step
+     * @note it does not matter what values \c copyable contains, but its size is important
+     */
+    void construct( unsigned order, const ContainerType& copyable){
+        m_k = order;
+        m_f.assign( order, copyable);
+        m_u.assign( order, copyable);
+        init_coeffs(order);
+        m_counter = 0;
+    }
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_u[0];}
+
+    /**
+     * @brief Initialize timestepper. Call before using the step function.
+     *
+     * This routine has to be called before the first timestep is made.
+     * @copydoc hide_rhs
+     * @param rhs The rhs functor
+     * @param t0 The intital time corresponding to u0
+     * @param u0 The initial value of the integration
+     * @param dt The timestep saved for later use
+     * @note the implementation is such that on output the last call to the explicit part \c ex is at \c (t0,u0). This might be interesting if the call to \c ex changes its state.
+     */
+    template< class RHS>
+    void init( RHS& rhs, value_type t0, const ContainerType& u0, value_type dt);
+
+    /**
+    * @brief Advance one timestep
+    *
+    * @copydoc hide_rhs
+    * @param rhs The rhs functor
+    * @param t (write-only), contains timestep corresponding to \c u on output
+    * @param u (write-only), contains next step of time-integration on output
+    * @note the implementation is such that on output the last call to the explicit part \c ex is at the new \c (t,u). This might be interesting if the call to \c ex changes its state.
+    * @attention The first few steps after the call to the init function are performed with an explicit Runge-Kutta method
+    */
+    template< class RHS>
+    void step( RHS& rhs, value_type& t, ContainerType& u);
+
+  private:
+    void init_coeffs(unsigned order){
+        m_a.resize( order);
+        m_b.resize( order);
+        switch (order){
+            case 1: m_a = {1.};
+                    m_b = {1.}; break;
+            case 2: m_a = {4./3., -1./3.};
+                    m_b = {4./3., -2./3.}; break;
+            case 3: m_a = { 18./11., -9./11., 2./11.};
+                    m_b = { 18./11., -18./11., 6./11.}; break;
+            case 4: m_a = {48./25., -36./25., 16./25., -3./25.};
+                    m_b = {48./25.,-72./25.,48./25.,-12./25.}; break;
+            case 5: m_a = { 300./137., -300./137., 200./137., -75./137., 12./137.};
+                    m_b = {300./137.,-600./137.,600./137.,-300./137.,60./137.}; break;
+            case 6: m_a = { 360./147., -450./147., 400./147., -225./147., 72./147., -10./147.};
+                    m_b = {360./147.,-900./147.,1200./147.,-900./147.,360./147.,-60./147.}; break;
+            case 7: m_a = { 2940./1089.,-4410./1089.,4900./1089.,-3675./1089.,1764./1089.,-490./1089.,60./1089.};
+                    m_b = { 2940./1089.,-8820./1089.,14700./1089.,-14700./1089.,8820./1089.,-2940./1089.,420./1089.}; break;
+            default: throw dg::Error(dg::Message()<<"Order not implemented in MinimalProjection!");
+        }
+    }
+    std::vector<ContainerType> m_u, m_f;
+    value_type m_tu, m_dt;
+    std::vector<value_type> m_a, m_b;
+    unsigned m_k, m_counter; //counts how often step has been called after init
+};
+
+///@cond
+template< class ContainerType>
+template< class RHS>
+void MinimalProjecting<ContainerType>::init( RHS& f, value_type t0, const ContainerType& u0, value_type dt)
+{
+    m_tu = t0, m_dt = dt;
+    blas1::copy(  u0, m_u[m_k-1]);
+    f(m_tu, m_u[m_k-1], m_f[m_k-1]); //call f on new point
+    m_counter = 0;
+}
+
+template<class ContainerType>
+template< class RHS>
+void MinimalProjecting<ContainerType>::step( RHS& f, value_type& t, ContainerType& u)
+{
+    if( m_counter < m_k-1)
+    {
+        ERKStep<ContainerType> erk( "ARK-4-2-3 (explicit)", u);
+        ContainerType tmp ( u);
+        erk.step( f, t, u, t, u, m_dt, tmp);
+        m_counter++;
+        m_tu = t;
+        blas1::copy(  u, m_u[m_k-1-m_counter]);
+        f( m_tu, m_u[m_k-1-m_counter], m_f[m_k-1-m_counter]);
+        return;
+    }
+    //compute new t,u
+    t = m_tu = m_tu + m_dt;
+    dg::blas1::axpby( m_a[0], m_u[0], m_dt*m_b[0], m_f[0], u);
+    for (unsigned i = 1; i < m_k; i++){
+        dg::blas1::axpbypgz( m_a[i], m_u[i], m_dt*m_b[i], m_f[i], 1., u);
+    }
+    //permute m_f[m_k-1], m_u[m_k-1]  to be the new m_f[0], m_u[0]
+    std::rotate( m_f.rbegin(), m_f.rbegin()+1, m_f.rend());
+    std::rotate( m_u.rbegin(), m_u.rbegin()+1, m_u.rend());
+    blas1::copy( u, m_u[0]); //store result
+    f(m_tu, m_u[0], m_f[0]); //call f on new point
+}
+///@endcond
+
 } //namespace dg
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index f810b1d3d..e7dae1f71 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -9,89 +9,78 @@
 
 //![function]
 //method of manufactured solution
-struct Solution{
-    Solution(double t, double nu):t(t), nu(nu){}
-DG_DEVICE
-    double operator()(double x, double y) const{
-        return sin(t)*exp( -2.*nu*t)*sin(x)*sin(y);
-    }
-    private:
-    double t, nu;
-};
-
-struct Source{
-    Source(double t, double nu):t(t), nu(nu){}
-DG_DEVICE
-    double operator()(double x, double y) const{
-        return sin(x)*sin(y)*cos(t)*exp(-2*t*nu)*(1-sin(t));
-    }
-    private:
-    double t, nu;
-};
+std::array<double,2> solution( double t, double nu) {
+    return {exp( -nu*t) + cos(t), exp( -nu*t) + sin(t)};
+}
 
 //the explicit part contains the source Tp = S(x,y,t)
-template<class container>
 struct Explicit
 {
-    Explicit( const dg::Grid2d& g, double nu):
-        m_nu( nu),
-        m_x ( dg::evaluate(dg::cooX2d, g)),//x-coordinate
-        m_y ( dg::evaluate(dg::cooY2d, g)) //y-coordinate
-    {}
-    void operator()( double t, const container& T, container& Tp) {
-        dg::blas1::evaluate( Tp, dg::equals(), Source(t,m_nu), m_x, m_y);
+    Explicit( double nu): m_nu( nu) {}
+    void operator()( double t, const std::array<double,2>& T, std::array<double,2>& Tp)
+    {
+        Tp[0] = m_nu*cos(t) - sin(t);
+        Tp[1] = m_nu*sin(t) + cos(t);
     }
     private:
     const double m_nu;
-    const container m_x, m_y;
-
 };
 
-//the implicit part contains  Tp = nu Delta T(x,y,t) + cos(t) T(x,y,t)
-template< class Matrix, class container>
+//the implicit part contains  Tp = -nu T
 struct Implicit
 {
-    Implicit( const dg::Grid2d& g, double nu):
-        m_nu(nu),
-        m_w2d( dg::create::weights(g)),
-        m_v2d( dg::create::inv_weights(g)),
-        m_LaplacianM( g, dg::normed)
-        { }
+    Implicit( double nu): m_nu(nu) { }
 
-    void operator()( double t, const container& T, container& Tp)
+    void operator()( double t, const std::array<double,2>& T, std::array<double,2>& Tp)
+    {
+        Tp[0] = -m_nu*T[0];
+        Tp[1] = -m_nu*T[1];
+    }
+  private:
+    double m_nu;
+};
+struct ImplicitSolver
+{
+    ImplicitSolver( double nu): m_nu(nu) { }
+    std::array<double,2> copyable() const {return {0,0};}
+    template<class Implicit>
+    void solve( double alpha, Implicit& im, double t, std::array<double,2>& y, const std::array<double,2>& rhs)
     {
-        dg::blas2::gemv( m_LaplacianM, T, Tp);
-        dg::blas1::axpby( cos(t), T, -m_nu, Tp);
+        y[0] = rhs[0]/(1-alpha*m_nu);
+        y[1] = rhs[1]/(1-alpha*m_nu);
+    }
+  private:
+    double m_nu;
+};
+struct FullSolver
+{
+    FullSolver( double nu): m_nu(nu) { }
+    std::array<double,2> copyable() const {return {0,0};}
+    template<class Implicit>
+    void solve( double alpha, Implicit& im, double t, std::array<double,2>& y, const std::array<double,2>& rhs)
+    {
+        y[0] = (rhs[0]-alpha*(m_nu*cos(t) - sin(t)))/(1-alpha*m_nu);
+        y[1] = (rhs[1]-alpha*(m_nu*sin(t) + cos(t)))/(1-alpha*m_nu);
     }
-    //required by inversion in semi-implicit schemes
-    const container& inv_weights(){return m_v2d;}
-    const container& weights(){return m_w2d;}
-    const container& precond(){return m_v2d;}
   private:
     double m_nu;
-    const container m_w2d, m_v2d, m_x, m_y;
-    dg::Elliptic<dg::CartesianGrid2d, Matrix, container> m_LaplacianM;
 };
 
 //![function]
 
-template< class Matrix, class container>
 struct Full
 {
-    Full( const dg::Grid2d& g, double nu):
-        m_exp( g, nu), m_imp( g, nu), m_temp( dg::evaluate( dg::one, g))
-
-    { }
-    const container& weights(){return m_imp.weights();}
-    void operator()( double t, const container& y, container& yp) {
+    Full( double nu):
+        m_exp( nu), m_imp( nu) { }
+    void operator()( double t, const std::array<double,2>& y, std::array<double,2>& yp) {
         m_exp( t, y, yp);
         m_imp( t, y, m_temp);
         dg::blas1::axpby( 1., m_temp, 1., yp);
     }
   private:
-    Explicit<container> m_exp;
-    Implicit<Matrix, container> m_imp;
-    container m_temp;
+    Explicit m_exp;
+    Implicit m_imp;
+    std::array<double,2> m_temp;
 };
 
 
@@ -102,63 +91,67 @@ const double ly = 2.*M_PI;
 
 int main()
 {
-    unsigned n = 3, Nx = 50 , Ny = 50;
     std::cout << "Program tests Multistep and Semi-Implicit methods on a manufactured PDE\n";
-    //std::cout << "Type n (3), Nx (50), Ny (50)\n";
-    //std::cin >> n >> Nx >> Ny;
-    std::cout << "Computing on "<<n<<" x "<<Nx<<" x "<<Ny<<"\n";
-    const double T = 0.1;
-    const double NT= 40, eps = 1e-6;
+    const double T = 1;
+    const double NT= 40;
     const double dt = (T/NT);
     const double nu = 0.01;
-    //construct the grid and the explicit and implicit parts
-    dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
-
-    Full<dg::DMatrix, dg::DVec> full( grid, nu);
+    Full full( nu);
     //evaluate the initial condition
-    const dg::DVec init( dg::evaluate(Solution(0.,nu), grid));
-    dg::DVec y0(init);
+    const std::array<double,2> init( solution(0.,nu));
+    std::array<double,2> y0(init);
 
-    const dg::DVec sol = dg::evaluate( Solution(T,nu), grid);
-    const dg::DVec w2d = dg::create::weights( grid);
-    const double norm_sol = dg::blas2::dot( w2d, sol);
+    const std::array<double,2> sol = solution(T,nu);
+    const double norm_sol = dg::blas1::dot( sol, sol);
     double time = 0.;
-    dg::DVec error( sol);
+    std::array<double,2> error( sol);
     exblas::udouble res;
-    std::cout << "### Test explicit multistep methods with "<<NT<<" steps\n";
+    std::cout << "### Test Adams Bashforth methods with "<<NT<<" steps\n";
     for( unsigned s=1; s<6; s++)
     {
         time = 0., y0 = init;
-        dg::AdamsBashforth< dg::DVec > ab( s, y0);
+        dg::AdamsBashforth< std::array<double,2> > ab( s, y0);
         ab.init( full, time, y0, dt);
         //main time loop
         for( unsigned k=0; k<NT; k++)
             ab.step( full, time, y0);
         dg::blas1::axpby( -1., sol, 1., y0);
-        res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
+        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << "Relative error AB "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
     }
+    std::cout << "### Test MinimalProjecting methods with "<<NT<<" steps\n";
+    for( unsigned s=1; s<7; s++)
+    {
+        time = 0., y0 = init;
+        dg::MinimalProjecting< std::array<double,2> > ab( s, y0);
+        ab.init( full, time, y0, dt);
+        //main time loop
+        for( unsigned k=0; k<NT; k++)
+            ab.step( full, time, y0);
+        dg::blas1::axpby( -1., sol, 1., y0);
+        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
+        std::cout << "Relative error MP "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
+    }
     std::cout << "### Test implicit multistep methods with "<<NT<<" steps\n";
     for( unsigned s=1; s<7; s++)
     {
         time = 0., y0 = init;
-        dg::BDF< dg::DVec, dg::AndersonSolver<dg::DVec> > bdf( s, y0, 0, 1e-10, 100, 1, 1);
-        //dg::BDF< dg::DVec, dg::FixedPointSolver<dg::DVec> > bdf( s, y0, 10, 1e-10);
+        dg::BDF< std::array<double,2>, FullSolver > bdf( s, nu);
         bdf.init( full, time, y0, dt);
         //main time loop
         for( unsigned k=0; k<NT; k++)
             bdf.step( full, time, y0);
         dg::blas1::axpby( -1., sol, 1., y0);
-        res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
+        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << "Relative error BDF "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
     }
-    Explicit<dg::DVec> ex( grid, nu);
-    Implicit<dg::DMatrix, dg::DVec> im( grid, nu);
+    Explicit ex( nu);
+    Implicit im( nu);
     std::cout << "### Test semi-implicit Karniadakis methods with "<<NT<<" steps\n";
     //![karniadakis]
     //construct time stepper
-    dg::Karniadakis< dg::DVec > karniadakis( y0, y0.size(), eps);
-    time = 0., y0 = init; //y0 and init are of type dg::DVec and contain the initial condition
+    dg::Karniadakis< std::array<double,2>, ImplicitSolver > karniadakis( nu);
+    time = 0., y0 = init; //y0 and init are of type std::array<double,2> and contain the initial condition
     //initialize the timestepper (ex and im are objects of type Explicit and Implicit defined above)
     karniadakis.init( ex, im, time, y0, dt);
     //main time loop (NT = 20)
@@ -166,7 +159,7 @@ int main()
         karniadakis.step( ex, im, time, y0); //inplace step
     //![karniadakis]
     dg::blas1::axpby( -1., sol, 1., y0);
-    res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
+    res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
     std::cout << "Relative error Karniadakis 3 is "<< res.d<<"\t"<<res.i<<std::endl;
     for( unsigned s=2; s>0; s--)
     {
@@ -176,7 +169,7 @@ int main()
         for( unsigned i=0; i<NT; i++)
             karniadakis.step( ex, im, time, y0);
         dg::blas1::axpby( -1., sol, 1., y0);
-        res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
+        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << "Relative error Karniadakis "<<s<<" is "<< res.d<<"\t"<<res.i<<std::endl;
     }
 
@@ -188,7 +181,7 @@ int main()
     {
         //![adaptive]
         time = 0., y0 = init;
-        dg::Adaptive<dg::ARKStep<dg::DVec>> adapt( name, y0, y0.size(), eps);
+        dg::Adaptive<dg::ARKStep<std::array<double,2>, ImplicitSolver>> adapt( name, nu);
         double time = 0;
         double dt = adapt.guess_stepsize( ex, time, y0, dg::forward, dg::l2norm, rtol, atol);
         int counter=0;
@@ -201,7 +194,7 @@ int main()
         }
         //![adaptive]
         dg::blas1::axpby( -1., sol, 1., y0);
-        res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
+        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << counter <<" steps! ";
         std::cout << "Relative error "<<name<<" is "<< res.d<<"\t"<<res.i<<std::endl;
     }
@@ -222,8 +215,8 @@ int main()
     for( auto name : ex_names)
     {
         time = 0., y0 = init;
-        dg::Adaptive<dg::ERKStep<dg::DVec>> adapt( name, y0);
-        dg::ImplicitRungeKutta<dg::DVec> dirk( "Trapezoidal-2-2", y0, y0.size(), eps );
+        dg::Adaptive<dg::ERKStep<std::array<double,2>>> adapt( name, y0);
+        dg::ImplicitRungeKutta<std::array<double,2>, ImplicitSolver> dirk( "Trapezoidal-2-2", nu );
         double time = 0;
         double dt = adapt.guess_stepsize( ex, time, y0, dg::forward, dg::l2norm, rtol, atol);
         int counter=0;
@@ -240,7 +233,7 @@ int main()
             counter ++;
         }
         dg::blas1::axpby( -1., sol, 1., y0);
-        res.d = sqrt(dg::blas2::dot( w2d, y0)/norm_sol);
+        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << std::setw(4)<<counter <<" steps! ";
         std::cout << "Relative error "<<std::setw(24) <<name<<"\t"<<res.d<<"\n";
     }
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index ae9ee3e59..a87cc352d 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -271,20 +271,31 @@ struct ARKStep
      * @tparam SolverParams Type of parameters (deduced by the compiler)
      */
     template<class ...SolverParams>
-    ARKStep( std::string name,
-             SolverParams&& ...ps
-             )
+    ARKStep( std::string name, SolverParams&& ...ps) :
+         m_solver( std::forward<SolverParams>(ps)...),
+         m_rhs( m_solver.copyable())
     {
         if( name == "ARK-4-2-3" )
-            construct( "ARK-4-2-3 (explicit)", "ARK-4-2-3 (implicit)", std::forward<SolverParams>(ps)...);
+        {
+            m_rkE = ConvertsToButcherTableau<value_type>( "ARK-4-2-3 (explicit)");
+            m_rkI = ConvertsToButcherTableau<value_type>( "ARK-4-2-3 (implicit)");
+        }
         else if( name == "ARK-6-3-4" )
-            construct( "ARK-6-3-4 (explicit)", "ARK-6-3-4 (implicit)", std::forward<SolverParams>(ps)...);
+        {
+            m_rkE = ConvertsToButcherTableau<value_type>( "ARK-6-3-4 (explicit)");
+            m_rkI = ConvertsToButcherTableau<value_type>( "ARK-6-3-4 (implicit)");
+        }
         else if( name == "ARK-8-4-5" )
-            construct( "ARK-8-4-5 (explicit)", "ARK-8-4-5 (implicit)", std::forward<SolverParams>(ps)...);
+        {
+            m_rkE = ConvertsToButcherTableau<value_type>( "ARK-8-4-5 (explicit)");
+            m_rkI = ConvertsToButcherTableau<value_type>( "ARK-8-4-5 (implicit)");
+        }
         else
             throw dg::Error( dg::Message()<<"Unknown name");
+        assert( m_rkE.num_stages() == m_rkI.num_stages());
+        m_kE.assign(m_rkE.num_stages(), m_rhs);
+        m_kI.assign(m_rkI.num_stages(), m_rhs);
     }
-
     ///@copydoc construct()
     template<class ...SolverParams>
     ARKStep( ConvertsToButcherTableau<value_type> ex_tableau,
@@ -300,10 +311,6 @@ struct ARKStep
     {
         assert( m_rkE.num_stages() == m_rkI.num_stages());
     }
-    ///@brief Return an object of same size as the object used for construction
-    ///@return A copyable object; what it contains is undefined, its size is important
-    const ContainerType& copyable()const{ return m_kE[0];}
-
     /*!@brief Construct with two Butcher Tableaus
      *
      * The two Butcher Tableaus represent the parameters for the explicit
@@ -325,14 +332,12 @@ struct ARKStep
              SolverParams&& ...ps
              )
     {
-        m_rkE = ex_tableau;
-        m_rkI = im_tableau;
-        assert( m_rkE.num_stages() == m_rkI.num_stages());
-        m_solver = SolverType( std::forward<SolverParams>(ps)...);
-        m_rhs = m_solver.copyable();
-        m_kE.assign(m_rkE.num_stages(), m_rhs);
-        m_kI.assign(m_rkI.num_stages(), m_rhs);
+        *this = ARKStep( ex_tableau, im_tableau, std::forward<SolverParams>(ps)...);
     }
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_kE[0];}
+
     ///Write access to the internal solver for the implicit part
     SolverType& solver() { return m_solver;}
     ///Read access to the internal solver for the implicit part
-- 
GitLab


From 9fbafeb09cd084f9b451b1a828dc61342bfe6fd6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 30 Nov 2020 17:16:07 +0100
Subject: [PATCH 402/540] Everything is explicit in feltor

---
 src/feltor/Makefile        |  2 +-
 src/feltor/feltor.cu       | 23 +++++++++++++----------
 src/feltor/feltor.h        | 17 ++++++++++++-----
 src/feltor/feltor_hpc.cu   | 24 +++++++++++++-----------
 src/feltor/manufactured.cu | 17 +++++++++--------
 5 files changed, 48 insertions(+), 35 deletions(-)

diff --git a/src/feltor/Makefile b/src/feltor/Makefile
index bfb3abb15..e15d77108 100644
--- a/src/feltor/Makefile
+++ b/src/feltor/Makefile
@@ -33,4 +33,4 @@ feltor_mpi: feltor_hpc.cu feltordiag.h feltor.h implicit.h init.h parameters.h i
 .PHONY: clean
 
 clean:
-	rm -f feltor feltor_hpc feltor_mpi manufactured feltordiag
+	rm -f feltor feltor_hpc feltor_mpi manufactured feltordiag interpolate_in_3d
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 98d1249d4..b27935a03 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -94,8 +94,8 @@ int main( int argc, char* argv[])
     //create RHS
     std::cout << "Constructing Explicit...\n";
     feltor::Explicit<Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
-    std::cout << "Constructing Implicit...\n";
-    feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> implicit( grid, p, mag);
+    //std::cout << "Constructing Implicit...\n";
+    //feltor::Implicit<Geometry, IDMatrix, DMatrix, DVec> implicit( grid, p, mag);
     std::cout << "Done!\n";
 
     DVec result = dg::evaluate( dg::zero, grid);
@@ -136,21 +136,23 @@ int main( int argc, char* argv[])
     //
     dg::Timer t;
     unsigned step = 0;
-    dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
-        feltor::FeltorSpecialSolver<
-            Geometry, IDMatrix, DMatrix, DVec>
-        > karniadakis( grid, p, mag);
+    //dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
+    //    feltor::FeltorSpecialSolver<
+    //        Geometry, IDMatrix, DMatrix, DVec>
+    //    > karniadakis( grid, p, mag);
+    dg::MinimalProjecting< std::array<std::array<dg::DVec,2>,2 > > mp( 3, y0);
     {
     HVec h_wall = dg::pullback( wall, grid);
     HVec h_sheath = dg::pullback( sheath, grid);
     HVec h_velocity = dg::pullback( direction, grid);
     feltor.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath), dg::construct<DVec>(h_velocity));
-    implicit.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
-    karniadakis.solver().set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    //implicit.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    //karniadakis.solver().set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
     }
 
     std::cout << "Initialize Timestepper" << std::endl;
-    karniadakis.init( feltor, implicit, time, y0, p.dt);
+    //karniadakis.init( feltor, implicit, time, y0, p.dt);
+    mp.init( feltor, time, y0, p.dt);
     std::cout << "Done!" << std::endl;
 
     std::map<std::string, const dg::DVec* > v4d;
@@ -235,7 +237,8 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    karniadakis.step( feltor, implicit, time, y0);
+                    //karniadakis.step( feltor, implicit, time, y0);
+                    mp.step( feltor, time, y0);
                 }
                 catch( dg::Fail& fail) {
                     std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 26a39c026..3345d6d51 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -356,6 +356,8 @@ struct Explicit
     void set_wall_and_sheath(double wall_forcing, Container wall, double sheath_forcing, Container sheath, Container velocity_sheath)
     {
         m_sheath_forcing = sheath_forcing; //1/eta
+        m_wall_forcing = wall_forcing;
+        dg::blas1::axpby( wall_forcing, wall, sheath_forcing, sheath, m_forcing);
         dg::blas1::pointwiseDot( sheath, velocity_sheath, m_U_sheath);
 
         dg::blas1::axpby( -1., wall, -1., sheath, m_masked);
@@ -390,7 +392,7 @@ struct Explicit
     std::array<Container,3> m_curv, m_curvKappa, m_b;
     Container m_divCurvKappa;
     Container m_bphi, m_binv, m_divb;
-    Container m_source, m_profne, m_U_sheath, m_masked;
+    Container m_source, m_profne, m_forcing, m_U_sheath, m_masked;
     Container m_detg;
 
     Container m_apar;
@@ -416,7 +418,7 @@ struct Explicit
     dg::SparseTensor<Container> m_hh;
 
     const feltor::Parameters m_p;
-    double m_omega_source = 0., m_sheath_forcing = 0.;
+    double m_omega_source = 0., m_sheath_forcing = 0., m_wall_forcing = 0.;
     bool m_fixed_profile = true, m_reversed_field = false;
 
 };
@@ -589,7 +591,7 @@ Explicit<Grid, IMatrix, Matrix, Container>::Explicit( const Grid& g,
 {
     //--------------------------init vectors to 0-----------------//
     dg::assign( dg::evaluate( dg::zero, g), m_temp0 );
-    m_source = m_U_sheath = m_UE2 = m_temp2 = m_temp1 = m_temp0;
+    m_forcing = m_source = m_U_sheath = m_UE2 = m_temp2 = m_temp1 = m_temp0;
     dg::assign( dg::evaluate( dg::one, g), m_masked );
     m_apar = m_temp0;
 
@@ -875,6 +877,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_temp0, fields[0][i], 1., yp[1][i]);
         dg::geo::ds_centered_bc_along_field( m_fa_P, -1./m_p.mu[i], m_minusP[i], m_phi[i], m_plusP[i], 1.0, yp[1][i], dg::DIR, {0,0});
         // viscosity: + nu_par Delta_par U/N = nu_par ( Div b dsU + dssU)/N
+        // Maybe factor this out in an operator splitting method? To get larger timestep
         dg::blas1::pointwiseDot(1., m_divb, m_temp1, 0., m_temp1);
         dg::geo::dss_centered( m_fa_U, 1., m_minusU[i], fields[1][i], m_plusU[i], 1., m_temp1);
         dg::blas1::pointwiseDivide( m_p.nu_parallel[i], m_temp1, fields[0][i], 1., yp[1][i]);
@@ -1042,7 +1045,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
                     m_minusN[i], m_plusN[i], m_U_sheath);
             dg::blas1::axpby( m_sheath_forcing, m_temp0, 1.,  yp[0][i]);
         }
-        //velocity
+        //compute sheath velocity
+        //velocity c_s
         if( "insulating" == m_p.sheath_bc)
         {
             // u_e = +- sqrt(1+tau)
@@ -1057,7 +1061,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         // u_i = +- sqrt(1+tau)
         dg::blas1::axpby( m_sheath_forcing*sqrt(1+m_p.tau[1]), m_U_sheath, 1.,  yp[1][1]);
     }
-
+    dg::blas1::pointwiseDot( -1., m_forcing, y[0][0], 1., yp[0][0]);
+    dg::blas1::pointwiseDot( -1., m_forcing, y[0][1], 1., yp[0][1]);
+    dg::blas1::pointwiseDot( -1., m_forcing, m_fields[1][0], 1., yp[1][0]);
+    dg::blas1::pointwiseDot( -1., m_forcing, m_fields[1][1], 1., yp[1][1]);
 
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( yp[0][0], dg::plus_equals(), manufactured::SNe{
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 6f5c106ce..942553676 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -208,8 +208,8 @@ int main( int argc, char* argv[])
     //create RHS
     MPI_OUT std::cout << "Constructing Explicit...\n";
     feltor::Explicit< Geometry, IDMatrix, DMatrix, DVec> feltor( grid, p, mag);
-    MPI_OUT std::cout << "Constructing Implicit...\n";
-    feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> implicit( grid, p, mag);
+    //MPI_OUT std::cout << "Constructing Implicit...\n";
+    //feltor::Implicit< Geometry, IDMatrix, DMatrix, DVec> implicit( grid, p, mag);
     MPI_OUT std::cout << "Done!\n";
 
     // helper variables for output computations
@@ -470,21 +470,23 @@ int main( int argc, char* argv[])
     MPI_OUT err = nc_close(ncid);
     MPI_OUT std::cout << "First write successful!\n";
     ///////////////////////////////////////Timeloop/////////////////////////////////
-    dg::Karniadakis< std::array<std::array<DVec,2>,2 >,
-        feltor::FeltorSpecialSolver<
-            Geometry, IDMatrix, DMatrix, DVec>
-        > karniadakis( grid, p, mag);
+    //dg::Karniadakis< std::array<std::array<DVec,2>,2 >,
+    //    feltor::FeltorSpecialSolver<
+    //        Geometry, IDMatrix, DMatrix, DVec>
+    //    > karniadakis( grid, p, mag);
+    dg::MinimalProjecting< std::array<std::array<DVec,2>,2 > > mp( 3, y0);
     {
     HVec h_wall = dg::pullback( wall, grid);
     HVec h_sheath = dg::pullback( sheath, grid);
     HVec h_velocity = dg::pullback( direction, grid);
     feltor.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath), dg::construct<DVec>(h_velocity));
-    implicit.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
-    karniadakis.solver().set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    //implicit.set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
+    //karniadakis.solver().set_wall_and_sheath( p.wall_rate, dg::construct<DVec>( h_wall), p.sheath_rate, dg::construct<DVec>(h_sheath));
     }
 
     MPI_OUT std::cout << "Initialize Timestepper" << std::endl;
-    karniadakis.init( feltor, implicit, time, y0, p.dt);
+    //karniadakis.init( feltor, implicit, time, y0, p.dt);
+    mp.init( feltor, time, y0, p.dt);
     dg::Timer t;
     t.tic();
     unsigned step = 0;
@@ -499,8 +501,8 @@ int main( int argc, char* argv[])
             for( unsigned k=0; k<p.inner_loop; k++)
             {
                 try{
-                    karniadakis.step( feltor, implicit, time, y0);
-                    //bdf.step( feltor, time, y0);
+                    //karniadakis.step( feltor, implicit, time, y0);
+                    mp.step( feltor, time, y0);
                 }
                 catch( dg::Fail& fail){
                     MPI_OUT std::cerr << "ERROR failed to converge to "<<fail.epsilon()<<"\n";
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 166e2ee05..da76ead2c 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -41,8 +41,8 @@ int main( int argc, char* argv[])
     std::cout << "Initialize explicit" << std::endl;
     dg::geo::TokamakMagneticField mag = dg::geo::createCircularField( R_0, I_0);
     feltor::Explicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec> feltor( grid, p, mag);
-    std::cout << "Initialize implicit" << std::endl;
-    feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
+    //std::cout << "Initialize implicit" << std::endl;
+    //feltor::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
 
     feltor::manufactured::Ne ne{ p.mu[0],p.mu[1],p.tau[0],p.tau[1],p.eta,
                                  p.beta,p.nu_perp,p.nu_parallel[0],p.nu_parallel[1]};
@@ -79,16 +79,17 @@ int main( int argc, char* argv[])
     //dg::Adaptive< dg::ARKStep<std::array<std::array<dg::DVec,2>,2>> > adaptive(
     //    "ARK-4-2-3", y0, y0[0][0].size(), p.eps_time);
     //Multistep solver
-    dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
-        feltor::FeltorSpecialSolver<
-            dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
-        > karniadakis( grid, p, mag);
+    //dg::Karniadakis< std::array<std::array<dg::DVec,2>,2 >,
+    //    feltor::FeltorSpecialSolver<
+    //        dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
+    //    > karniadakis( grid, p, mag);
+    dg::MinimalProjecting< std::array<std::array<dg::DVec,2>,2 > > mp( 3, y0);
     double time = 0, TMAX = 0.1;
-    karniadakis.init( feltor, im, time, y0, p.dt);
+    mp.init( feltor, time, y0, p.dt);
     while( time < TMAX)
     {
         try{
-            karniadakis.step( feltor, im, time, y0);
+            mp.step( feltor, time, y0);
         }
         catch( dg::Fail& fail) {
             std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-- 
GitLab


From c7f332f65c851652a1bca1070629260c16745780 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 30 Nov 2020 18:31:13 +0100
Subject: [PATCH 403/540] Add colored boxes to feltor tex

---
 doc/related_pages/header.tex |  2 +-
 src/feltor/feltor.tex        | 16 ++++++++++------
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/doc/related_pages/header.tex b/doc/related_pages/header.tex
index ff0d9902c..c8178a5db 100644
--- a/doc/related_pages/header.tex
+++ b/doc/related_pages/header.tex
@@ -24,7 +24,7 @@
 \usepackage[table]{xcolor} % for alternating colors
 %\rowcolors{2}{gray!25}{white} %%% Use this line in front of longtable
 \renewcommand\arraystretch{1.3}
-\usepackage{tcolorbox}
+\usepackage[most]{tcolorbox}
 \usepackage{doi}
 \usepackage[sort,square,numbers]{natbib}
 \bibliographystyle{abbrvnat}
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 373368b4e..a5aa1e9ae 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -711,7 +711,11 @@ We introduce the dimensionless parameters
 \end{align}
 where $a\in\{e,i\}$ is the species label and $z$ is the charge number.
 Omitting the species label we arrive at (dividing the density equation by $\Omega_0n_0$ and the velocity equation by $\Omega_0 c_s$)
-\begin{align}
+\begin{tcolorbox}[ams align,
+colback=yellow!10!white, colframe=red!50!black,
+        highlight math style= {enhanced, %<-- needed for the ’remember’ options
+        colframe=red,colback=red!10!white,boxsep=0pt}, title=Model equations
+        ]
 \frac{\partial}{\partial t} N &+ \vec\nc\left( N \left(
     \vec u_E + \vec u_K + \vec u_{C} + U_\parallel\left(\bhat + {\vec b}_\perp\right)\right)\right) = \Lambda_N + S_N \\
     \mu \frac{\partial}{\partial t} \left(N U_\parallel\right) &+ \mu \nc \left( NU_\parallel \left(
@@ -722,18 +726,18 @@ Omitting the species label we arrive at (dividing the density equation by $\Omeg
     + \mu NU_\parallel\mathcal K_{\vn\times\bhat}(\psi) \nonumber\\
     =& -\tau \left(\bhat + {\vec b}_\perp\right)\cn N
     -N \left( \left(\bhat+{\vec b}_\perp\right)\cn \psi + \frac{\partial A_\parallel}{\partial t}\right)
-    - \eta n_e^2(U_{\parallel,i}-u_{\parallel,e}) 
+    - \eta n_e^2(U_{\parallel,i}-u_{\parallel,e})
     \nonumber\\
     &+ \mu \nu_\parallel \Delta_\parallel U+ \mu N\left(\Lambda_U + S_U\right) + \mu U_\parallel \left(\Lambda_N + S_N\right)
 \label{}
-\end{align}
+\end{tcolorbox}
 with
 \begin{align}
 \vec u_E := \frac{\bhat\times\vn\psi}{B},\quad
-\vec u_{K} := \tau \left(\vec{ K_{\vn B}} + \vec{ K_{\vn\times\bhat}}\right)=\tau\vec{ K}  ,\nonumber\\
-\vec u_C := \mu U_\parallel^2\vec{ K_{\vn\times\bhat}},\quad
+\vec u_{K} := \tau \left(\vec{ K_{\vn B}} + \vec{ K_{\vn\times\bhat}}\right)=\tau\vec{ K}  ,\quad  %\nonumber\\
+\vec u_C := \mu U_\parallel^2\vec{ K_{\vn\times\bhat}},\nonumber\\
 \vec u_{\vn\times\bhat} := \tau\vec{ K_{\vn\times\bhat}},\quad
-{\vec b}_\perp = \frac{\vn\times A_\parallel \bhat}{B}.
+{\vec b}_\perp = \frac{\vn\times A_\parallel \bhat}{B} = A_\parallel \vec{ K_{\vn\times\bhat}} + \frac{\vn A_\parallel \times \bhat}{B}.
 \label{}
 \end{align}
 
-- 
GitLab


From 3a21c58ec354bb748c1b2d640b4027570bf0adb3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 30 Nov 2020 18:32:36 +0100
Subject: [PATCH 404/540] Conservative density implementation

---
 src/feltor/feltor.h | 113 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 92 insertions(+), 21 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 3345d6d51..bc3b740d3 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -105,6 +105,84 @@ struct ComputePerpDrifts{
     private:
     double m_mu, m_tau;
 };
+struct ComputePerpVelocity{
+    ComputePerpVelocity( double mu, double tau):
+        m_mu(mu), m_tau(tau){}
+    DG_DEVICE
+    void operator()(
+            double N, double d0N, double d1N, double d2N,
+            double U, double d0U, double d1U, double d2U,
+            double d0P, double d1P, double d2P,
+            double b_0,         double b_1,         double b_2,
+            double curv0,       double curv1,       double curv2,
+            double curvKappa0,  double curvKappa1,  double curvKappa2,
+            double divCurvKappa, double detg,
+            double& dtNx, double& dtNy, double& dtNz, double& dtU
+        )
+    {
+        double KappaU = curvKappa0*d0U+curvKappa1*d1U+curvKappa2*d2U;
+        double KappaN = curvKappa0*d0N+curvKappa1*d1N+curvKappa2*d2N;
+        double KappaP = curvKappa0*d0P+curvKappa1*d1P+curvKappa2*d2P;
+        double KU = curv0*d0U+curv1*d1U+curv2*d2U;
+        double PU = b_0*( d1P*d2U-d2P*d1U)+
+                    b_1*( d2P*d0U-d0P*d2U)+
+                    b_2*( d0P*d1U-d1P*d0U);//ExB drift
+        dtNx =  -detg*N*( b_1*d2P - b_2*d1P + m_tau * curv0 + m_mu * U * U * curvKappa0);
+        dtNy =  -detg*N*( b_2*d0P - b_0*d2P + m_tau * curv1 + m_mu * U * U * curvKappa2);
+        dtNz =  -detg*N*( b_0*d1P - b_1*d0P + m_tau * curv2 + m_mu * U * U * curvKappa2);
+        dtU =   -PU
+                -U*KappaP
+                -m_tau * KU
+                -m_tau * U * divCurvKappa
+                -(2. * m_tau + m_mu * U * U)*KappaU
+                - 2. * m_tau * U * KappaN / N;
+    }
+    DG_DEVICE
+    void operator()(
+            double N, double d0N, double d1N, double d2N,
+            double U, double d0U, double d1U, double d2U,
+            double d0P, double d1P, double d2P,
+            double A,       double d0A, double d1A, double d2A,
+            double b_0,         double b_1,         double b_2,
+            double curv0,       double curv1,       double curv2,
+            double curvKappa0,  double curvKappa1,  double curvKappa2,
+            double divCurvKappa, double detg,
+            double& dtNx, double& dtNy, double& dtNz, double& dtU
+        )
+    {
+        //first compute the regular dynamics
+        this->operator()( N,  d0N,  d1N,  d2N,
+             U,        d0U,  d1U,  d2U,
+             d0P,  d1P,  d2P,
+             b_0,          b_1,          b_2,
+             curv0,        curv1,        curv2,
+             curvKappa0,   curvKappa1,   curvKappa2,
+             divCurvKappa, detg,
+             dtNx, dtNy, dtNz, dtU);
+        //now add the additional terms from modified parallel derivative
+        double KappaU = curvKappa0*d0U+curvKappa1*d1U+curvKappa2*d2U;
+        double KappaN = curvKappa0*d0N+curvKappa1*d1N+curvKappa2*d2N;
+        double KappaP = curvKappa0*d0P+curvKappa1*d1P+curvKappa2*d2P;
+
+        double UA = b_0*( d1U*d2A-d2U*d1A)+
+                    b_1*( d2U*d0A-d0U*d2A)+
+                    b_2*( d0U*d1A-d1U*d0A);
+        double NA = b_0*( d1N*d2A-d2N*d1A)+
+                    b_1*( d2N*d0A-d0N*d2A)+
+                    b_2*( d0N*d1A-d1N*d0A);
+        double PA = b_0*( d1P*d2A-d2P*d1A)+
+                    b_1*( d2P*d0A-d0P*d2A)+
+                    b_2*( d0P*d1A-d1P*d0A);
+        dtNx +=  -detg*N*U*( -b_1*d2A + b_2*d1A + A * curvKappa0);
+        dtNy +=  -detg*N*U*( -b_2*d0A + b_0*d2A + A * curvKappa2);
+        dtNz +=  -detg*N*U*( -b_0*d1A + b_1*d0A + A * curvKappa2);
+        dtU +=  -1./m_mu*( A*KappaP + PA)
+                -1.*U*( A*KappaU + UA)
+                -1.*m_tau/m_mu/N*(A*KappaN + NA);
+    }
+    private:
+    double m_mu, m_tau;
+};
 struct ComputeChi{
     DG_DEVICE
     void operator() ( double& chi, double tilde_Ni, double binv,
@@ -267,18 +345,6 @@ struct Explicit
         dg::blas2::symv( m_dy_N, m_s[0][i], gradS[1]);
         if(!m_p.symmetric)dg::blas2::symv( m_dz, m_s[0][i], gradS[2]);
     }
-    void divergence( const std::array<Container,3>& in, Container& out) const{
-        dg::blas1::pointwiseDot( m_detg, in[0], m_temp0);
-        dg::blas1::pointwiseDot( m_detg, in[1], m_temp1);
-        dg::blas1::pointwiseDot( m_detg, in[2], m_temp2);
-        dg::blas2::symv( m_dx_U, m_temp0, m_temp0);
-        dg::blas2::symv( m_dy_U, m_temp1, m_temp1);
-        dg::blas2::symv( m_dz, m_temp2, m_temp2);
-        dg::blas1::pointwiseDivide( m_temp0, m_detg, out);
-        dg::blas1::pointwiseDivide( 1., m_temp1, m_detg, 1., out);
-        dg::blas1::pointwiseDivide( 1., m_temp2, m_detg, 1., out);
-
-    }
     void compute_dot_induction( Container& tmp) const {
         m_old_apar.derive( tmp);
     }
@@ -389,7 +455,7 @@ struct Explicit
 #endif //DG_MANUFACTURED
 
     //these should be considered const // m_curv is full curvature
-    std::array<Container,3> m_curv, m_curvKappa, m_b;
+    std::array<Container,3> m_curv, m_curvKappa, m_b; //m_b is bhat/ sqrt(g) / B
     Container m_divCurvKappa;
     Container m_bphi, m_binv, m_divb;
     Container m_source, m_profne, m_forcing, m_U_sheath, m_masked;
@@ -508,11 +574,11 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
     dg::SparseTensor<Container> metric = g.metric();
     dg::tensor::inv_multiply3d( metric, m_b[0], m_b[1], m_b[2],
                                         m_b[0], m_b[1], m_b[2]);
-    m_detg = dg::tensor::volume( metric);
-    dg::blas1::pointwiseDivide( m_binv, m_detg, m_detg); //1/m_detg/B
     dg::assign( m_b[2], m_bphi); //save bphi for momentum conservation
+    m_detg = dg::tensor::volume( metric);
+    dg::blas1::pointwiseDivide( m_binv, m_detg, m_temp0); //1/B/m_detg
     for( int i=0; i<3; i++)
-        dg::blas1::pointwiseDot( m_detg, m_b[i], m_b[i]); //b_i/m_detg/B
+        dg::blas1::pointwiseDot( m_temp0, m_b[i], m_b[i]); //b_i/m_detg/B
     m_hh = dg::geo::createProjectionTensor( bhat, g);
     m_lapperpN.construct ( g, p.bcxN, p.bcyN, dg::PER, dg::normed, dg::centered),
     m_lapperpU.construct ( g, p.bcxU, p.bcyU, dg::PER, dg::normed, dg::centered),
@@ -813,7 +879,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
         dg::blas2::symv( m_dy_U, fields[1][i], m_dU[i][1]);
         if(!m_p.symmetric) dg::blas2::symv( m_dz, fields[1][i], m_dU[i][2]);
         if( m_p.beta == 0){
-            dg::blas1::subroutine( routines::ComputePerpDrifts(
+            dg::blas1::subroutine( routines::ComputePerpVelocity(
                 m_p.mu[i], m_p.tau[i]),
                 //species depdendent
                 fields[0][i], m_dN[i][0], m_dN[i][1], m_dN[i][2],
@@ -823,11 +889,11 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
                 m_b[0], m_b[1], m_b[2],
                 m_curv[0], m_curv[1], m_curv[2],
                 m_curvKappa[0], m_curvKappa[1], m_curvKappa[2],
-                m_divCurvKappa, yp[0][i], yp[1][i]
+                m_divCurvKappa, m_detg, m_temp0, m_temp1, m_temp2, yp[1][i]
             );
         }
         if( m_p.beta != 0){
-            dg::blas1::subroutine( routines::ComputePerpDrifts(
+            dg::blas1::subroutine( routines::ComputePerpVelocity(
                 m_p.mu[i], m_p.tau[i]),
                 //species depdendent
                 fields[0][i], m_dN[i][0], m_dN[i][1], m_dN[i][2],
@@ -839,9 +905,14 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
                 m_b[0], m_b[1], m_b[2],
                 m_curv[0], m_curv[1], m_curv[2],
                 m_curvKappa[0], m_curvKappa[1], m_curvKappa[2],
-                m_divCurvKappa, yp[0][i], yp[1][i]
+                m_divCurvKappa, m_detg, m_temp0, m_temp1, m_temp2, yp[1][i]
             );
         }
+        //compute divergence
+        dg::blas2::symv( 1., m_dx_N, m_temp0, 0., yp[0][i]);
+        dg::blas2::symv( 1., m_dy_N, m_temp1, 1., yp[0][i]);
+        if(!m_p.symmetric)dg::blas2::symv( 1., m_dz, m_temp2, 1., yp[0][i]);
+        dg::blas1::pointwiseDivide( yp[0][i], m_detg, yp[0][i]);
     }
 }
 
@@ -964,7 +1035,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     compute_parallel( t, y, m_fields, yp);
 
 #endif
-    if( m_p.explicit_diffusion)
+    if( m_p.explicit_diffusion )
     {
 #if FELTORPERP == 1
         /* y[0] := n_e - 1
-- 
GitLab


From 692a0bfbad558bc11dbd3d6103bdcdbeea6934fd Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 1 Dec 2020 10:56:29 +0100
Subject: [PATCH 405/540] Update feltor tex with implemented form

---
 src/feltor/feltor.tex | 30 ++++++++++++++++++------------
 1 file changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index a5aa1e9ae..70413cb72 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1059,21 +1059,27 @@ This means that diffusion does not generate potential either.
 
 
 \subsection{Implemented form}
-The form that we implement avoids derivatives on the product of
-two functions for which we have no boundary conditions
+We use a conservative form of the continuity equation
+in the perpendicular terms and else avoids derivatives on the product of two
+functions for which we have no boundary condition.
 \begin{subequations}
     \begin{align}
     \frac{\partial}{\partial t} N =&
-    \left[ - \frac{1}{B}[\psi, N]_{\perp} %\nonumber\\
-        - \bar \npar \left( NU_\parallel\right)
-        - NU_\parallel\left(\vec \nc\bhat+\vec \nc{\vec b}_\perp\right)
-        - \tau \mathcal K(N)\right. \nonumber \\&
-        \left.
-        - N \mathcal K(\psi)
-        -\mu \mathcal K_{\vn\times\bhat}(NU_\parallel^2)
-        -\mu NU_\parallel^2\nc \vec{ K_{\vn\times\bhat}}
+    \left[ - \frac{1}{\sqrt{g}} \partial_i \left( \sqrt{g} N \left( \frac{\epsilon^{ijk} b_j \partial_k \psi}{\sqrt{g}B}
+    + \tau K^i
+    + \mu U_\parallel^2 K_{\vn\times\bhat}^i
+    + U_\parallel\left( A_\parallel K_{\vn\times\bhat}^i +\frac{\epsilon^{ijk} \partial_j A_\parallel b_k}{\sqrt{g}B}   \right)
+    \right)\right)\right. \nonumber\\
+    &\left.- \npar \left( NU_\parallel\right)
+        - NU_\parallel\vec \nc\bhat
+        %\frac{1}{B}[\psi, N]_{\perp} %\nonumber\\
+        %- \tau \mathcal K(N)\right. \nonumber \\&
+        %\left.
+        %- N \mathcal K(\psi)
+        %-\mu \mathcal K_{\vn\times\bhat}(NU_\parallel^2)
+        %-\mu NU_\parallel^2\nc \vec{ K_{\vn\times\bhat}}
     - \nu_\perp\Delta_\perp^2 N + S_N\right]
-    (1-\chi_s - \chi_w) \nonumber\\& + \omega_s\chi_s( N_{sh} - N) - \omega_w \chi_w N_w, \\
+    (1-\chi_s - \chi_w) \nonumber\\& + \omega_s\chi_s( N_{sh} - N) + \omega_w \chi_w (1-N), \\
     \frac{\partial}{\partial t} W_\parallel =&
   \left[- \frac{1}{B}\left[\psi, U_\parallel\right]_{\perp}%& \nonumber\\
         - \frac{1}{\mu} \bar \npar \psi% \nonumber\\
@@ -1099,7 +1105,7 @@ two functions for which we have no boundary conditions
 \end{subequations}
 together with
 $\bar\npar f = \npar f + A_\parallel \mathcal K_{\vn\times\bhat}(f) + \frac{1}{B}[ f, A_\parallel]_\perp$
-and $\nc { \vec b}_\perp = A_\parallel \vec \nc\vec{ { K}_{\vn\times\bhat}} - \mathcal K_{\vn B}(A_\parallel) $
+%and $\nc { \vec b}_\perp = A_\parallel \vec \nc\vec{ { K}_{\vn\times\bhat}} - \mathcal K_{\vn B}(A_\parallel) $
 and
 \begin{subequations} \label{eq:elliptic}
   \begin{align}
-- 
GitLab


From 8ce5f3d5ceb444ada719da6dc459ef8744471a48 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 6 Dec 2020 17:46:56 +0100
Subject: [PATCH 406/540] Have only one Fieldaligned

should save some memory
---
 src/feltor/feltor.h   | 73 ++++++++++++++++++++++---------------------
 src/feltor/feltor.tex |  2 +-
 2 files changed, 39 insertions(+), 36 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index bc3b740d3..68f33f7f9 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -105,8 +105,8 @@ struct ComputePerpDrifts{
     private:
     double m_mu, m_tau;
 };
-struct ComputePerpVelocity{
-    ComputePerpVelocity( double mu, double tau):
+struct ComputePerpConservative{
+    ComputePerpConservative( double mu, double tau):
         m_mu(mu), m_tau(tau){}
     DG_DEVICE
     void operator()(
@@ -127,6 +127,7 @@ struct ComputePerpVelocity{
         double PU = b_0*( d1P*d2U-d2P*d1U)+
                     b_1*( d2P*d0U-d0P*d2U)+
                     b_2*( d0P*d1U-d1P*d0U);//ExB drift
+        //sqrt(g) times the flux
         dtNx =  -detg*N*( b_1*d2P - b_2*d1P + m_tau * curv0 + m_mu * U * U * curvKappa0);
         dtNy =  -detg*N*( b_2*d0P - b_0*d2P + m_tau * curv1 + m_mu * U * U * curvKappa2);
         dtNz =  -detg*N*( b_0*d1P - b_1*d0P + m_tau * curv2 + m_mu * U * U * curvKappa2);
@@ -318,19 +319,19 @@ struct Explicit
         return m_dA;
     }
     void compute_dsN (int i, Container& dsN) const {
-        dg::geo::ds_centered_bc_along_field( m_fa_N, 1., m_minusN[i], m_fields[0][i],
+        dg::geo::ds_centered_bc_along_field( m_fa, 1., m_minusN[i], m_fields[0][i],
                 m_plusN[i], 0., dsN, dg::NEU, {0,0});
     }
     void compute_dsU (int i, Container& dsU) const {
-        dg::geo::ds_centered( m_fa_U, 1., m_minusU[i], m_fields[1][i],
-                m_plusU[i], 0., dsU);
+        dg::geo::ds_centered_bc_along_field( m_fa, 1., m_minusU[i], m_fields[1][i],
+                m_plusU[i], 0., dsU, dg::NEU, {0,0});
     }
     void compute_dsP (int i, Container& dsP) const {
-        dg::geo::ds_centered_bc_along_field( m_fa_P, 1., m_minusP[i], m_phi[i],
+        dg::geo::ds_centered_bc_along_field( m_fa, 1., m_minusP[i], m_phi[i],
                 m_plusP[i], 0.0, dsP, dg::DIR, {0,0});
     }
     void compute_dssU(int i, Container& dssU) {
-        dg::geo::dss_centered( m_fa_U, 1., m_minusU[i], m_fields[1][i], m_plusU[i], 0., dssU);
+        dg::geo::dss_centered_bc_along_field( m_fa, 1., m_minusU[i], m_fields[1][i], m_plusU[i], 0., dssU, dg::NEU, {0,0});
     }
     void compute_lapParU(int i, Container& lapU) {
         compute_dsU(i, m_temp0);
@@ -472,7 +473,7 @@ struct Explicit
 
     //matrices and solvers
     Matrix m_dx_N, m_dx_U, m_dx_P, m_dy_N, m_dy_U, m_dy_P, m_dz;
-    dg::geo::Fieldaligned<Geometry, IMatrix, Container> m_fa_P, m_fa_N, m_fa_U;
+    dg::geo::Fieldaligned<Geometry, IMatrix, Container> m_fa;//_P, m_fa_N, m_fa_U;
     dg::Elliptic3d< Geometry, Matrix, Container> m_lapperpN, m_lapperpU, m_lapperpP;
     std::vector<dg::Elliptic3d< Geometry, Matrix, Container> > m_multi_pol;
     std::vector<dg::Helmholtz3d<Geometry, Matrix, Container> > m_multi_invgammaP,
@@ -549,20 +550,22 @@ void Explicit<Grid, IMatrix, Matrix, Container>::construct_bhat(
 {
     //in DS we take the true bhat
     auto bhat = dg::geo::createBHat( mag);
-    m_fa_N.construct( bhat, g, p.bcxN, p.bcyN, dg::geo::NoLimiter(),
+    m_fa.construct( bhat, g, p.bcxN, p.bcyN, dg::geo::NoLimiter(),
         p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz );
-    if( p.bcxU == p.bcxN && p.bcyU == p.bcyN)
-        m_fa_U.construct( m_fa_N);
-    else
-        m_fa_U.construct( bhat, g, p.bcxU, p.bcyU, dg::geo::NoLimiter(),
-            p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
-    if( p.bcxP == p.bcxN && p.bcyP == p.bcyN)
-        m_fa_P.construct( m_fa_N);
-    else if( p.bcxP == p.bcxU && p.bcyP == p.bcyU)
-        m_fa_P.construct( m_fa_U);
-    else
-        m_fa_P.construct( bhat, g, p.bcxP, p.bcyP, dg::geo::NoLimiter(),
-             p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
+    //m_fa_N.construct( bhat, g, p.bcxN, p.bcyN, dg::geo::NoLimiter(),
+    //    p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz );
+    //if( p.bcxU == p.bcxN && p.bcyU == p.bcyN)
+    //    m_fa_U.construct( m_fa_N);
+    //else
+    //    m_fa_U.construct( bhat, g, p.bcxU, p.bcyU, dg::geo::NoLimiter(),
+    //        p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
+    //if( p.bcxP == p.bcxN && p.bcyP == p.bcyN)
+    //    m_fa_P.construct( m_fa_N);
+    //else if( p.bcxP == p.bcxU && p.bcyP == p.bcyU)
+    //    m_fa_P.construct( m_fa_U);
+    //else
+    //    m_fa_P.construct( bhat, g, p.bcxP, p.bcyP, dg::geo::NoLimiter(),
+    //         p.rk4eps, p.mx, p.my, 2.*M_PI/(double)p.Nz);
 
     // in Poisson we take EPhi except for the true curvmode
     bhat = dg::geo::createEPhi(+1);
@@ -865,7 +868,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
     const std::array<std::array<Container,2>,2>& fields,
     std::array<std::array<Container,2>,2>& yp)
 {
-    //MW: in theory we have the possibility to
+    //MW: we have the possibility to
     // make the implementation conservative since the perp boundaries are
     // penalized away
     //y[0] = N-1, y[1] = W; fields[0] = N, fields[1] = U
@@ -879,7 +882,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
         dg::blas2::symv( m_dy_U, fields[1][i], m_dU[i][1]);
         if(!m_p.symmetric) dg::blas2::symv( m_dz, fields[1][i], m_dU[i][2]);
         if( m_p.beta == 0){
-            dg::blas1::subroutine( routines::ComputePerpVelocity(
+            dg::blas1::subroutine( routines::ComputePerpConservative(
                 m_p.mu[i], m_p.tau[i]),
                 //species depdendent
                 fields[0][i], m_dN[i][0], m_dN[i][1], m_dN[i][2],
@@ -893,7 +896,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
             );
         }
         if( m_p.beta != 0){
-            dg::blas1::subroutine( routines::ComputePerpVelocity(
+            dg::blas1::subroutine( routines::ComputePerpConservative(
                 m_p.mu[i], m_p.tau[i]),
                 //species depdendent
                 fields[0][i], m_dN[i][0], m_dN[i][1], m_dN[i][2],
@@ -908,7 +911,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_perp(
                 m_divCurvKappa, m_detg, m_temp0, m_temp1, m_temp2, yp[1][i]
             );
         }
-        //compute divergence
+        //compute divergence of density flux
         dg::blas2::symv( 1., m_dx_N, m_temp0, 0., yp[0][i]);
         dg::blas2::symv( 1., m_dy_N, m_temp1, 1., yp[0][i]);
         if(!m_p.symmetric)dg::blas2::symv( 1., m_dz, m_temp2, 1., yp[0][i]);
@@ -927,14 +930,14 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
     for( unsigned i=0; i<2; i++)
     {
 
-        m_fa_N( dg::geo::einsMinus, y[0][i], m_minusN[i]);
-        m_fa_N( dg::geo::einsPlus,  y[0][i], m_plusN[i]);
-        m_fa_U( dg::geo::einsMinus, fields[1][i], m_minusU[i]);
-        m_fa_U( dg::geo::einsPlus,  fields[1][i], m_plusU[i]);
-        m_fa_P( dg::geo::einsMinus, m_phi[i], m_minusP[i]);
-        m_fa_P( dg::geo::einsPlus,  m_phi[i], m_plusP[i]);
-        dg::geo::ds_centered_bc_along_field( m_fa_N, 1., m_minusN[i], y[0][i], m_plusN[i], 0., m_temp0, dg::NEU, {0,0});
-        dg::geo::ds_centered( m_fa_U, 1., m_minusU[i], fields[1][i], m_plusU[i], 0., m_temp1);
+        m_fa( dg::geo::einsMinus, y[0][i], m_minusN[i]);
+        m_fa( dg::geo::einsPlus,  y[0][i], m_plusN[i]);
+        m_fa( dg::geo::einsMinus, fields[1][i], m_minusU[i]);
+        m_fa( dg::geo::einsPlus,  fields[1][i], m_plusU[i]);
+        m_fa( dg::geo::einsMinus, m_phi[i], m_minusP[i]);
+        m_fa( dg::geo::einsPlus,  m_phi[i], m_plusP[i]);
+        dg::geo::ds_centered_bc_along_field( m_fa, 1., m_minusN[i], y[0][i], m_plusN[i], 0., m_temp0, dg::NEU, {0,0});
+        dg::geo::ds_centered_bc_along_field( m_fa, 1., m_minusU[i], fields[1][i], m_plusU[i], 0., m_temp1, dg::NEU, {0,0});
         //---------------------density--------------------------//
         //density: -Div ( NUb)
         dg::blas1::pointwiseDot(-1., m_temp0, fields[1][i],
@@ -946,11 +949,11 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_parallel(
         dg::blas1::pointwiseDot(-1., fields[1][i], m_temp1, 1., yp[1][i]);
         // force terms: -tau/mu * ds N/N -1/mu * ds Phi
         dg::blas1::pointwiseDivide( -m_p.tau[i]/m_p.mu[i], m_temp0, fields[0][i], 1., yp[1][i]);
-        dg::geo::ds_centered_bc_along_field( m_fa_P, -1./m_p.mu[i], m_minusP[i], m_phi[i], m_plusP[i], 1.0, yp[1][i], dg::DIR, {0,0});
+        dg::geo::ds_centered_bc_along_field( m_fa, -1./m_p.mu[i], m_minusP[i], m_phi[i], m_plusP[i], 1.0, yp[1][i], dg::DIR, {0,0});
         // viscosity: + nu_par Delta_par U/N = nu_par ( Div b dsU + dssU)/N
         // Maybe factor this out in an operator splitting method? To get larger timestep
         dg::blas1::pointwiseDot(1., m_divb, m_temp1, 0., m_temp1);
-        dg::geo::dss_centered( m_fa_U, 1., m_minusU[i], fields[1][i], m_plusU[i], 1., m_temp1);
+        dg::geo::dss_centered_bc_along_field( m_fa, 1., m_minusU[i], fields[1][i], m_plusU[i], 1., m_temp1, dg::NEU, {0,0});
         dg::blas1::pointwiseDivide( m_p.nu_parallel[i], m_temp1, fields[0][i], 1., yp[1][i]);
     }
 }
diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index 70413cb72..bb7ae6ce7 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1658,7 +1658,7 @@ symmetric & bool & false & If true, initialize all quantities symmetric
 in $\varphi$ (effectively reducing the problem to 2d). The input $N_z$ is used
 to construct the parallel derivatives and then overwritten to $N_z\equiv 1$.
 \\
-bc & dict & & Boundary conditions (note that $A_\parallel$ has the same bc as $U_\parallel$) \ldots\\
+bc & dict & & Perpendicular Boundary conditions (note that $A_\parallel$ has the same bc as $U_\parallel$) \ldots\\
 \qquad density   & char[2] & [DIR,DIR] & boundary conditions in x and y
 for $n_e$ and $N_i$, DIR (density 1 on boundary) means both convective and
     diffusive outflow while NEU (gradient 0) means no outflow by diffusion
-- 
GitLab


From 0334cca18cf094b0e36300ef5b89467cba4b5bba Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 8 Dec 2020 15:32:20 +0100
Subject: [PATCH 407/540] Add turbulence source profile

---
 src/feltor/feltor.tex |  6 ++++--
 src/feltor/init.h     | 16 ++++++++++++++--
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/feltor/feltor.tex b/src/feltor/feltor.tex
index bb7ae6ce7..1270671c5 100644
--- a/src/feltor/feltor.tex
+++ b/src/feltor/feltor.tex
@@ -1701,11 +1701,13 @@ at the separatrix (must not be zero - even if amp is zero - it is also used for
 source & dict & & Density source, cf. the output \texttt{sne\_tt\_ifs} in \texttt{feltordiag} (or \texttt{SourceProfile\_ifs} in \texttt{geometry\_diag}) to see how much mass the source with the parameters below generates and compare to \texttt{jsne\_tt\_fsa} to see how much mass is lost.  \\
 \qquad rate & float & 0    & profile source rate $\omega_s$ in Eq.~\eqref{eq:electron_source}.
 \\
-\qquad type & string & "profile" & The type of source to use: "profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
+\qquad type & string & "influx" & The type of source to use:
+"fixed\_profile" the source is multiplied by $(n_{prof} - n)$ to relax to the initial profile Eq.~\eqref{eq:electron_source};
 "influx" the source has a constant source rate Eq.~\eqref{eq:electron_source_influx},
 "torpex": Torpex inspired source profile Eq.~\eqref{eq:electron_source_torpex},
 "gaussian": Gaussian shaped source profile - uses \texttt{posX}, \texttt{posY} and \texttt{sigma},
-"profile\_influx": Copy a profile into the source function and use a constant source rate. The idea is that you can start with zero density and evolve the profile purely with the source.
+"profile\_influx": Copy a profile into the source function and use a constant source rate. The idea is that you can start with zero density and evolve the profile purely with the source. There is a turbulent bath on top of it.
+"turbulence" : Influx of the turbulent bath initial condition as a source, same as profile\_influx just without the profile.
     See the file {\tt init.h} to add your own custom source.
 \\
 \qquad boundary & float & 0.2  & Source region boundary $\rho_{p,b}$: yields in Eq.~\eqref{eq:electron_source} and Eq.~\eqref{eq:electron_source_influx}  \\
diff --git a/src/feltor/init.h b/src/feltor/init.h
index 42080f97c..cd6243fee 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -306,7 +306,7 @@ std::map<std::string, std::function< HVec(
     dg::geo::TokamakMagneticField& mag )
 > > source_profiles =
 {
-    {"profile",
+    {"fixed_profile",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
         dg::geo::TokamakMagneticField& mag )
@@ -325,11 +325,11 @@ std::map<std::string, std::function< HVec(
         {
             fixed_profile = false;
             HVec source_profile = detail::profile( grid, p,mag);
-            ne_profile = source_profile;
             dg::blas1::scal( ne_profile, p.nprofamp );
             HVec ntilde = detail::turbulent_bath(grid,p,mag);
             dg::blas1::pointwiseDot( detail::profile_damping(grid,p,mag), ntilde, ntilde);
             dg::blas1::axpby( 1., ntilde, 1., source_profile);
+            ne_profile = source_profile;
             return source_profile;
         }
     },
@@ -345,6 +345,18 @@ std::map<std::string, std::function< HVec(
             return source_profile;
         }
     },
+    {"turbulence",
+        []( bool& fixed_profile, HVec& ne_profile,
+        Geometry& grid, const feltor::Parameters& p,
+        dg::geo::TokamakMagneticField& mag )
+        {
+            fixed_profile = false;
+            HVec source_profile = detail::turbulent_bath(grid,p,mag);
+            dg::blas1::pointwiseDot( detail::profile_damping(grid,p,mag), source_profile, source_profile);
+            ne_profile = source_profile;
+            return source_profile;
+        }
+    },
     {"torpex",
         []( bool& fixed_profile, HVec& ne_profile,
         Geometry& grid, const feltor::Parameters& p,
-- 
GitLab


From c25241d34ffb1c595c5065d7ee54aa55b4e4bbc9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 8 Dec 2020 18:11:29 +0100
Subject: [PATCH 408/540] Fix bug in symmetric file read in feltor3d

---
 inc/dg/topology/interpolation.h  | 26 ++++++++++++++++++
 inc/dg/topology/mpi_grid.h       |  4 +--
 inc/dg/topology/mpi_projection.h |  6 +++++
 src/feltor/init_from_file.h      | 46 +++++++++++++++++++++-----------
 4 files changed, 65 insertions(+), 17 deletions(-)

diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index b154a0486..6b2dd9ddf 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -475,6 +475,32 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const aRealTo
     thrust::host_vector<real_type> pointsZ = dg::evaluate( dg::cooZ3d, g_new);
     return interpolation( pointsX, pointsY, pointsZ, g_old);
 
+}
+/**
+ * @brief Create interpolation between two grids
+ *
+ * This matrix interpolates vectors on the old grid \c g_old to the %Gaussian nodes of the new grid \c g_new. The interpolation is of the order \c g_old.n()
+ * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ *
+ * @param g_new The new grid
+ * @param g_old The old grid
+ *
+ * @return Interpolation matrix with \c g_old.size() columns and \c g_new.size() rows
+ * @note When interpolating a 2d grid to a 3d grid the third coordinate is simply ignored, i.e. the 2d vector will be trivially copied Nz times into the 3d vector
+ * @note also check the transformation matrix, which is the more general solution
+ */
+template<class real_type>
+cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const aRealTopology3d<real_type>& g_new, const aRealTopology2d<real_type>& g_old)
+{
+    //assert both grids are on the same box
+    assert( g_new.x0() >= g_old.x0());
+    assert( g_new.x1() <= g_old.x1());
+    assert( g_new.y0() >= g_old.y0());
+    assert( g_new.y1() <= g_old.y1());
+    thrust::host_vector<real_type> pointsX = dg::evaluate( dg::cooX3d, g_new);
+    thrust::host_vector<real_type> pointsY = dg::evaluate( dg::cooY3d, g_new);
+    return interpolation( pointsX, pointsY, g_old);
+
 }
 ///@}
 
diff --git a/inc/dg/topology/mpi_grid.h b/inc/dg/topology/mpi_grid.h
index d872e2084..5cb3de7fe 100644
--- a/inc/dg/topology/mpi_grid.h
+++ b/inc/dg/topology/mpi_grid.h
@@ -273,8 +273,8 @@ struct aRealMPITopology2d
     aRealMPITopology2d( real_type x0, real_type x1, real_type y0, real_type y1, unsigned n, unsigned Nx, unsigned Ny, bc bcx, bc bcy, MPI_Comm comm):
         g( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy), l(g), comm( comm)
     {
-        update_local();
         check_division( Nx, Ny, bcx, bcy);
+        update_local();
     }
     ///copydoc aTopology2d::aTopology2d(const aTopology2d&)
     aRealMPITopology2d(const aRealMPITopology2d& src) = default;
@@ -563,8 +563,8 @@ struct aRealMPITopology3d
     aRealMPITopology3d( real_type x0, real_type x1, real_type y0, real_type y1, real_type z0, real_type z1, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         g( x0, x1, y0, y1, z0, z1, n, Nx, Ny, Nz, bcx, bcy, bcz), l(g), comm( comm)
     {
-        update_local();
         check_division( Nx, Ny, Nz, bcx, bcy, bcz);
+        update_local();
         int remain_dims[] = {true,true,false}; //true true false
         MPI_Cart_sub( comm, remain_dims, &planeComm);
     }
diff --git a/inc/dg/topology/mpi_projection.h b/inc/dg/topology/mpi_projection.h
index e18ef1164..ea8452459 100644
--- a/inc/dg/topology/mpi_projection.h
+++ b/inc/dg/topology/mpi_projection.h
@@ -120,6 +120,12 @@ dg::tMIHMatrix<real_type> interpolation( const aRealMPITopology3d<real_type>& g_
 {
     return tMIHMatrix<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
+///@copydoc dg::create::interpolation(const aRealTopology3d<real_type>&,const aRealTopology2d<real_type>&)
+template<class real_type>
+dg::tMIHMatrix<real_type> interpolation( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
+{
+    return tMIHMatrix<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
+}
 
 ///@copydoc interpolationT(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index 8dffbb4f8..180c0631c 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -36,12 +36,24 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
 
     // Now read in last timestep
     Geometry grid_IN( grid.x0(), grid.x1(), grid.y0(), grid.y1(), grid.z0(), grid.z1(),
-        pINn, pINNx, pINNy, pINsymmetric ? 1 : pINNz, dg::DIR, dg::DIR, dg::PER
+        pINn, pINNx, pINNy, pINNz, dg::DIR, dg::DIR, dg::PER
         #ifdef FELTOR_MPI
         , grid.communicator()
         #endif //FELTOR_MPI
         );
-    IHMatrix interpolateIN = dg::create::interpolation( grid, grid_IN);
+    IHMatrix interpolateIN;
+    HVec transferIN;
+    if( pINsymmetric)
+    {
+        std::unique_ptr< typename Geometry::perpendicular_grid> grid_perp ( static_cast<typename Geometry::perpendicular_grid*>(grid.perp_grid()));
+        interpolateIN = dg::create::interpolation( grid, *grid_perp);
+        transferIN = dg::evaluate(dg::zero, *grid_perp);
+    }
+    else
+    {
+        interpolateIN = dg::create::interpolation( grid, grid_IN);
+        transferIN = dg::evaluate(dg::zero, grid_IN);
+    }
 
     #ifdef FELTOR_MPI
     int dimsIN[3],  coordsIN[3];
@@ -58,8 +70,12 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     size_t countIN[3] = {grid_IN.Nz(), grid_IN.n()*grid_IN.Ny(),
         grid_IN.n()*grid_IN.Nx()};
     #endif //FELTOR_MPI
-    std::vector<HVec> transferINHvec( 5, dg::evaluate( dg::zero, grid));
-    HVec transferINH( dg::evaluate(dg::zero, grid_IN));
+    if( pINsymmetric)
+    {
+        countIN[0] = 1;
+        startIN[0] = 0;
+    }
+    std::vector<HVec> transferOUTvec( 5, dg::evaluate( dg::zero, grid));
 
     std::string namesIN[5] = {"restart_electrons", "restart_ions", "restart_Ue", "restart_Ui", "restart_induction"};
 
@@ -78,26 +94,26 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
         errIN = nc_inq_varid( ncidIN, namesIN[i].data(), &dataID);
         errIN = nc_get_vara_double( ncidIN, dataID, startIN, countIN,
             #ifdef FELTOR_MPI
-                transferINH.data().data()
+                transferIN.data().data()
             #else //FELTOR_MPI
-                transferINH.data()
+                transferIN.data()
             #endif //FELTOR_MPI
             );
-        dg::blas2::gemv( interpolateIN, transferINH, transferINHvec[i]);
+        dg::blas2::gemv( interpolateIN, transferIN, transferOUTvec[i]);
     }
     errIN = nc_close(ncidIN);
     /// ///////////////Now Construct initial fields ////////////////////////
     //
     //Convert to N-1 and W
-    dg::blas1::plus( transferINHvec[0], -1.);
-    dg::blas1::plus( transferINHvec[1], -1.);
-    dg::blas1::axpby( 1., transferINHvec[2], 1./p.mu[0], transferINHvec[4], transferINHvec[2]);
-    dg::blas1::axpby( 1., transferINHvec[3], 1./p.mu[1], transferINHvec[4], transferINHvec[3]);
+    dg::blas1::plus( transferOUTvec[0], -1.);
+    dg::blas1::plus( transferOUTvec[1], -1.);
+    dg::blas1::axpby( 1., transferOUTvec[2], 1./p.mu[0], transferOUTvec[4], transferOUTvec[2]);
+    dg::blas1::axpby( 1., transferOUTvec[3], 1./p.mu[1], transferOUTvec[4], transferOUTvec[3]);
 
-    dg::assign( transferINHvec[0], y0[0][0]); //ne-1
-    dg::assign( transferINHvec[1], y0[0][1]); //Ni-1
-    dg::assign( transferINHvec[2], y0[1][0]); //We
-    dg::assign( transferINHvec[3], y0[1][1]); //Wi
+    dg::assign( transferOUTvec[0], y0[0][0]); //ne-1
+    dg::assign( transferOUTvec[1], y0[0][1]); //Ni-1
+    dg::assign( transferOUTvec[2], y0[1][0]); //We
+    dg::assign( transferOUTvec[3], y0[1][1]); //Wi
     return y0;
 }
 }//namespace feltor
-- 
GitLab


From ef2166578b79e604d87ecd704ca371d8ea974d4c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 11 Dec 2020 13:30:51 +0100
Subject: [PATCH 409/540] Minor comments in documentation on Multistep

---
 inc/dg/multistep.h  | 5 +++--
 src/feltor/feltor.h | 3 ++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 5cc9bb8ee..e10410312 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -191,7 +191,8 @@ for a semi-implicit (first order) Euler method
 * Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$).
 * @note This scheme implements <a href = "https://dx.doi.org/10.1016/0021-9991(91)90007-8"> Karniadakis, et al. J. Comput. Phys. 97 (1991)</a>
 * @note The implicit part equals a third order backward differentiation formula (BDF) https://en.wikipedia.org/wiki/Backward_differentiation_formula
-* while the explicit part equals the minimal projection method by Alfeld (1979)
+* while the explicit part equals the MinimalProjecting method by
+<a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
 *
 The following code example demonstrates how to implement the method of manufactured solutions on a 2d partial differential equation with the dg library:
 * @snippet multistep_t.cu function
@@ -507,7 +508,7 @@ void BDF<ContainerType, SolverType>::step(RHS& rhs, value_type& t, container_typ
     \alpha_0 = \frac{18}{11}\ \alpha_1 = -\frac{9}{11}\ \alpha_2 = \frac{2}{11} \\
     \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11}
 \f]
-@note This scheme needs more storage but **has a larger region of absolute stability** than an Adams-Bashforth method of the same order.
+@note This scheme needs more storage but has **a larger region of absolute stability** than an AdamsBashforth method of the same order.
 * @note This scheme implements <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
 *
 * @copydoc hide_note_multistep
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 68f33f7f9..b52aaf97b 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -1038,7 +1038,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
     compute_parallel( t, y, m_fields, yp);
 
 #endif
-    if( m_p.explicit_diffusion )
+    //right now we do not support that option i.e everything is explicit
+    //if( m_p.explicit_diffusion )
     {
 #if FELTORPERP == 1
         /* y[0] := n_e - 1
-- 
GitLab


From e455f087bb52046efa1c3707687fdf511cc9ed14 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 11 Dec 2020 17:02:17 +0100
Subject: [PATCH 410/540] Add section about modal filtering in dg_intro

---
 .../dg_introduction/dg_introduction.tex       | 73 ++++++++++++++++++-
 1 file changed, 69 insertions(+), 4 deletions(-)

diff --git a/doc/related_pages/dg_introduction/dg_introduction.tex b/doc/related_pages/dg_introduction/dg_introduction.tex
index f9d33d73a..22ce46ca6 100644
--- a/doc/related_pages/dg_introduction/dg_introduction.tex
+++ b/doc/related_pages/dg_introduction/dg_introduction.tex
@@ -531,15 +531,15 @@ To the knowledge of the author this superconvergence has not yet been observed b
 
 \section{Interpolation and Projection}
 Eq.~\eqref{eq:dgexpansion} provides a natural way to interpolate any
-dg expanded function $f_h(x)$ on any point in the interval $[a,b]$. 
-The interpolation has the same order $P$ as the expansion and 
+dg expanded function $f_h(x)$ on any point in the interval $[a,b]$.
+The interpolation has the same order $P$ as the expansion and
 can be formulated as a matrix vector multiplication
 \begin{align}
 f_h(x_0) = \sum_{n=1}^N\sum_{k=0}^{P-1} p_{nk}(x_0) \bar f^{nk} =: \sum_{n=1}^N\sum_{k=0}^{P-1}I_{nk} \bar f^{nk}
 \label{eq:basic_interpolation}
 \end{align}
-where the sparse interpolation matrix $I$ has as many rows as there are 
-points to interpolate. $I$ has $P^d$ entries per line, where $d$ is the dimensionality of the grid. 
+where the sparse interpolation matrix $I$ has as many rows as there are
+points to interpolate. $I$ has $P^d$ entries per line, where $d$ is the dimensionality of the grid.
 
 
 \subsection{Interpolation and Projection}
@@ -623,6 +623,71 @@ If the least common grid reduces to either grid (a) or (b) the
 transformation matrix reduces to either $P_{a2b}$ or $Q_{a2b}$ as expected.
 
 
+\section{Spectral and modal filtering techniques}
+It is well known from Godunov's theorem that linear high order schemes for hyperbolic (advection) problems
+are prone to oscillations. Discontinuous Galerkin methods are no different in that regard
+especially the higher order ones and if the numerical flux $f=nu$ is computed via pointwise multiplication
+of the nodal representations of $n$ and $u$ via $f_{ni} = n_{ni} u_{ni}$~\cite{NodalDG}.
+This is due to aliasing effects. Recall that pointwise multiplication is different from
+actually multiplying the two polynomials $n_h$ and $u_h$ and projecting the result back onto the
+lower polynomial space. The problem is also known in pure spectral methods like
+Fourier representations.
+
+In finite difference and Fourier codes what is often done is adding so-called hyperdiffusion
+(or \textbf{spectral viscosity}) to stabilize a purely hyperbolic system.
+\begin{align}
+    \partial_t n + \partial_x( nu) = \nu (-1)^{s+1}\partial_x^{2s} n
+\end{align}
+Since Fourier modes are Eigenfunctions of the Laplace operator this equation reads in Fourier space (disregaring the convection term)
+\begin{align}
+    \partial_t n_k = -\nu k^{2s} n_k
+\end{align}
+which when discretized in time using the Euler method reads
+\begin{align}
+    n_k^{n+1} = (1-\nu\Delta t k^{2s}) n_k^{n}
+\end{align}
+Here we see that the damping of modes is highest for large wave-numbers and leaves small wave-numbers intact.
+This can be read in a more general setting as applying a low-pass filter $\sigma(k)$ to the solution that acts on large wavenumbers
+\begin{align}
+    n_k^{n+1} = \sigma(k) n_k^{n}
+\end{align}
+where often $\sigma(k) = \exp( - \nu \Delta t k^{2s})$ is proposed. The above Euler discretization is
+then obtained by Taylor-expanding the exponential
+(an implicit Euler is obtained with the Pade approximation of the expontential).
+
+It is now possible to also discretise the hyperdiffusion directly in a dG framework, however, this
+has downsides regarding the performance since the Laplace operator is much more expensive than a
+simple pointwise multiplication as done in Fourier codes and its solution likely needs to be done implicitly
+in order to preserve the time-step restrictions.
+
+The better solution in a dG framework is to apply \textbf{modal filtering} and is motivated in much the same
+way as the spectral viscosity above.
+First we consider that the Legendre polynomials satisfy a Sturm-Liouville type differential equation.
+\begin{align}
+    \frac{d}{\d x} \left[ (1-x^2) \frac{ d }{d x} \right]p_n(x) = -n(n+1) p_n(x) \quad x \in [-1,1]
+\end{align}
+This means that the Legendre polynomials are Eigenfunctions of the Sturm-Liouville operator. If we write
+\begin{align}
+    \partial_t n = \nu (-1)^{s+1}\left[\frac{d}{\d x}  m(x) \frac{ d }{d x} \right]^s  n
+\end{align}
+where $m(x)$ is $(1-x^2)$ properly scaled onto each discrete cell, and use a dG approach to discretize this equation we get for each mode
+\begin{align}
+    \partial_t n_k = -\nu (k(k+1))^s  n_k
+\end{align}
+or in a time-discreized way
+\begin{align}
+    n_k^{n+1} = (1-\nu \Delta t (k(k+1))^s ) n_k^n
+\end{align}
+which is entirely analogous to the derivation above.
+In the same way we can read this as applying a filter functions to the Legendre modes.
+\begin{align}
+    n_k^{n+1} = \sigma(k) n_k^n
+\end{align}
+with $\sigma(k) = \exp( - \nu \Delta t (k(k+1))^s)$.
+Note that the modal filter, as the spectral filter, does not inhibit oscillations from happening (it is not shock-capturing) but it dampens grid-scale oscillations
+and can stabilize the system.
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \bibliography{../references}
 %..................................................................
 \end{document}
-- 
GitLab


From c01738ba1de0dfc68b2754faf5851201a7b7d073 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 14 Dec 2020 21:32:41 +0100
Subject: [PATCH 411/540] Add TVB multistep schemes by Ruuth and Hundsdorfer

- adding some citations to the documentation of timesteppers
- understand explicit part of Karniadakis as extrapolated BDF
---
 inc/dg/multistep.h         | 146 +++++++++++++++++++++++++++++--------
 inc/dg/multistep_t.cu      |  18 ++++-
 src/feltor/feltor.cu       |   2 +-
 src/feltor/feltor_hpc.cu   |   2 +-
 src/feltor/manufactured.cu |   2 +-
 5 files changed, 132 insertions(+), 38 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index e10410312..82f11dd96 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -46,7 +46,7 @@ namespace dg{
 * \f[ u^{n+1} = u^n + \Delta t\sum_{j=0}^{s-1} b_j f\left(t^n - j \Delta t, u^{n-j}\right) \f]
 *
 * with coefficients taken from https://en.wikipedia.org/wiki/Linear_multistep_method
-* @note This scheme has a smaller region of absolute stability than that of a \c MinimalProjecting method
+* @note This scheme has a smaller region of absolute stability than some of the \c ExplicitMultistep method
 * @copydoc hide_note_multistep
 * @copydoc hide_ContainerType
 * @ingroup time
@@ -191,8 +191,9 @@ for a semi-implicit (first order) Euler method
 * Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$).
 * @note This scheme implements <a href = "https://dx.doi.org/10.1016/0021-9991(91)90007-8"> Karniadakis, et al. J. Comput. Phys. 97 (1991)</a>
 * @note The implicit part equals a third order backward differentiation formula (BDF) https://en.wikipedia.org/wiki/Backward_differentiation_formula
-* while the explicit part equals the MinimalProjecting method by
+* while the explicit part equals the Minimal Projecting method by
 <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
+* or **extrapolated BDF** in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S. J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a>
 *
 The following code example demonstrates how to implement the method of manufactured solutions on a 2d partial differential equation with the dg library:
 * @snippet multistep_t.cu function
@@ -490,8 +491,38 @@ void BDF<ContainerType, SolverType>::step(RHS& rhs, value_type& t, container_typ
     dg::blas1::copy( u, m_u[0]);
 }
 
+
+/**
+ * @brief Identify coefficients for linear multistep methods
+ * @sa ExplicitMultistep
+*  @ingroup time
+ */
+enum multistep_identifier
+{
+    /** The family of schemes described in <a href =
+     "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S.
+     J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep
+     methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> as
+     **extrapolated BDF**  where it is found to be TVB (**total variation
+     bound**), i.e. \f$ || v^n|| \leq M ||v^0||\f$  where the norm signifies
+     the total variation semi-norm. (Note that total variation diminishing
+     (TVD) means M=1, and strong stability preserving (SSP) is the same as TVD)
+     is the same as the **Minimal Projecting** scheme
+     described in <a href =
+     "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf">
+     Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
+     * **Possible order is 1, 2,..., 7**
+
+    */
+    eBDF,
+    /** The family of schemes described in <a href="https://doi.org/10.1016/j.jcp.2005.02.029">S.J. Ruuth and W. Hundsdorfer, High-order linear multistep methods with general monotonicity and boundedness properties, Journal of Computational Physics, Volume 209, Issue 1, 2005 </a> as Total variation Bound.
+     * These schemes have larger stable step sizes than the eBDF family, for example TVB3 has 38% larger stepsize than eBDF3 and TVB4 has 109% larger stepsize than eBDF4
+     * **Possible order is 1,2,...,6**
+     */
+    TVB
+};
 /**
-* @brief Struct for Minimal Projecting explicit linear multistep time-integration
+* @brief Struct for general explicit linear multistep time-integration
 * \f[
 * \begin{align}
     v^{n+1} = \sum_{j=0}^{s-1} \alpha_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}\beta_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right)
@@ -503,43 +534,55 @@ void BDF<ContainerType, SolverType>::step(RHS& rhs, value_type& t, container_typ
     \frac{\partial v}{\partial t} = \hat f(t,v)
     \f]
     where \f$ f \f$ contains the equations.
-    The coefficients for order 3 are given as an example:
+    The coefficients for an order 3 "eBDF" scheme are given as an example:
     \f[
     \alpha_0 = \frac{18}{11}\ \alpha_1 = -\frac{9}{11}\ \alpha_2 = \frac{2}{11} \\
     \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11}
 \f]
-@note This scheme needs more storage but has **a larger region of absolute stability** than an AdamsBashforth method of the same order.
-* @note This scheme implements <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
+@sa multistep_identifier
+@note The schemes implemented here need more storage but may have **a larger region of absolute stability** than an AdamsBashforth method of the same order.
 *
 * @copydoc hide_note_multistep
 * @copydoc hide_ContainerType
 * @ingroup time
 */
 template<class ContainerType>
-struct MinimalProjecting
+struct ExplicitMultistep
 {
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
     ///@copydoc RungeKutta::RungeKutta()
-    MinimalProjecting(){}
+    ExplicitMultistep(){}
 
     ///@copydoc construct()
-    MinimalProjecting( unsigned order, const ContainerType& copyable){
-        construct( order, copyable);
+    ExplicitMultistep( std::string method, unsigned order, const ContainerType& copyable){
+        std::unordered_map < std::string, enum multistep_identifier> str2id{
+            {"eBDF", eBDF},
+            {"TVB", TVB}
+        };
+        if( str2id.find(method) == str2id.end())
+            throw dg::Error(dg::Message(_ping_)<<"Multistep coefficients for "<<method<<" not found!");
+        else
+            construct( str2id[method], order, copyable);
+    }
+    ///@copydoc construct()
+    ExplicitMultistep( enum multistep_identifier method, unsigned order, const ContainerType& copyable){
+        construct( method, order, copyable);
     }
     /**
      * @brief Reserve memory for the integration
      *
      * Set the coefficients \f$ \alpha_i,\ \beta_i\f$
-     * @param order (global) order (= number of steps in the multistep) of the method (Currently, one of 1 (Euler), 2, ..., or 7)
+     * @param method the name of the family of schemes to be used (a string can be converted to an enum with the same spelling) @sa multistep_identifier
+     * @param order (global) order (= number of steps in the multistep) of the method (Currently, possible values depend on the method
      * @param copyable ContainerType of the size that is used in \c step
      * @note it does not matter what values \c copyable contains, but its size is important
      */
-    void construct( unsigned order, const ContainerType& copyable){
+    void construct( enum multistep_identifier method, unsigned order, const ContainerType& copyable){
         m_k = order;
         m_f.assign( order, copyable);
         m_u.assign( order, copyable);
-        init_coeffs(order);
+        init_coeffs(method, order);
         m_counter = 0;
     }
     ///@brief Return an object of same size as the object used for construction
@@ -574,25 +617,64 @@ struct MinimalProjecting
     void step( RHS& rhs, value_type& t, ContainerType& u);
 
   private:
-    void init_coeffs(unsigned order){
+    void init_coeffs(enum multistep_identifier method, unsigned order){
         m_a.resize( order);
         m_b.resize( order);
-        switch (order){
-            case 1: m_a = {1.};
-                    m_b = {1.}; break;
-            case 2: m_a = {4./3., -1./3.};
-                    m_b = {4./3., -2./3.}; break;
-            case 3: m_a = { 18./11., -9./11., 2./11.};
-                    m_b = { 18./11., -18./11., 6./11.}; break;
-            case 4: m_a = {48./25., -36./25., 16./25., -3./25.};
-                    m_b = {48./25.,-72./25.,48./25.,-12./25.}; break;
-            case 5: m_a = { 300./137., -300./137., 200./137., -75./137., 12./137.};
-                    m_b = {300./137.,-600./137.,600./137.,-300./137.,60./137.}; break;
-            case 6: m_a = { 360./147., -450./147., 400./147., -225./147., 72./147., -10./147.};
-                    m_b = {360./147.,-900./147.,1200./147.,-900./147.,360./147.,-60./147.}; break;
-            case 7: m_a = { 2940./1089.,-4410./1089.,4900./1089.,-3675./1089.,1764./1089.,-490./1089.,60./1089.};
-                    m_b = { 2940./1089.,-8820./1089.,14700./1089.,-14700./1089.,8820./1089.,-2940./1089.,420./1089.}; break;
-            default: throw dg::Error(dg::Message()<<"Order not implemented in MinimalProjection!");
+        switch( method){
+            case eBDF:
+            switch (order){
+                case 1: m_a = {1.};
+                        m_b = {1.}; break;
+                case 2: m_a = {4./3., -1./3.};
+                        m_b = {4./3., -2./3.}; break;
+                case 3: m_a = { 18./11., -9./11., 2./11.};
+                        m_b = { 18./11., -18./11., 6./11.}; break; //CLM = 0.38... (% of Euler step size)
+                case 4: m_a = {48./25., -36./25., 16./25., -3./25.};
+                        m_b = {48./25.,-72./25.,48./25.,-12./25.}; break; //CLM = 0.21...
+                case 5: m_a = { 300./137., -300./137., 200./137., -75./137., 12./137.};
+                        m_b = {300./137.,-600./137.,600./137.,-300./137.,60./137.}; break;
+                case 6: m_a = { 360./147., -450./147., 400./147., -225./147., 72./147., -10./147.};
+                        m_b = {360./147.,-900./147.,1200./147.,-900./147.,360./147.,-60./147.}; break;
+                case 7: m_a = { 2940./1089.,-4410./1089.,4900./1089.,-3675./1089.,1764./1089.,-490./1089.,60./1089.};
+                        m_b = { 2940./1089.,-8820./1089.,14700./1089.,-14700./1089.,8820./1089.,-2940./1089.,420./1089.}; break;
+                default: throw dg::Error(dg::Message()<<"Order not implemented in Minimal Projecting!");
+            }
+            break;
+            case TVB:
+            switch(order){
+                case 1: m_a = {1.};
+                        m_b = {1.}; break;
+                case 2: m_a = {4./3., -1./3.};
+                        m_b = {4./3., -2./3.}; break;
+                case 3: //CLM = 0.54...
+                    m_a[0] =  1.908535476882378;     m_b[0] =  1.502575553858997;
+                    m_a[1] = -1.334951446162515;     m_b[1] = -1.654746338401493;
+                    m_a[2] = 0.426415969280137;      m_b[2] = 0.670051276940255;
+                    break;
+                case 4: //CLM = 0.45...
+                    m_a[0] = 2.628241000683208;     m_b[0] = 1.618795874276609;
+                    m_a[1] = -2.777506277494861;     m_b[1] = -3.052866947601049;
+                    m_a[2] = 1.494730011212510;     m_b[2] = 2.229909318681302;
+                    m_a[3] = -0.345464734400857;     m_b[3] = -0.620278703629274;
+                    break;
+                case 5: //CLM = 0.37...
+                    m_a[0] = 3.308891758551210;     m_b[0] = 1.747442076919292;
+                    m_a[1] = -4.653490937946655;     m_b[1] = -4.630745565661800;
+                    m_a[2] = 3.571762873789854;     m_b[2] = 5.086056171401077;
+                    m_a[3] = -1.504199914126327;     m_b[3] = -2.691494591660196;
+                    m_a[4] = 0.277036219731918;     m_b[4] = 0.574321855183372;
+                    break;
+                case 6: //CLM = 0.32...
+                    m_a[0] = 4.113382628475685;     m_b[0] = 1.825457674048542;
+                    m_a[1] = -7.345730559324184;     m_b[1] = -6.414174588309508;
+                    m_a[2] = 7.393648314992094;     m_b[2] = 9.591671249204753;
+                    m_a[3] = -4.455158576186636;     m_b[3] = -7.583521888026967;
+                    m_a[4] = 1.523638279938299;     m_b[4] = 3.147082225022105;
+                    m_a[5] = -0.229780087895259;     m_b[5] = -0.544771649561925;
+                    break;
+                default: throw dg::Error(dg::Message()<<"Order not implemented in TVB scheme!");
+            }
+            break;
         }
     }
     std::vector<ContainerType> m_u, m_f;
@@ -604,7 +686,7 @@ struct MinimalProjecting
 ///@cond
 template< class ContainerType>
 template< class RHS>
-void MinimalProjecting<ContainerType>::init( RHS& f, value_type t0, const ContainerType& u0, value_type dt)
+void ExplicitMultistep<ContainerType>::init( RHS& f, value_type t0, const ContainerType& u0, value_type dt)
 {
     m_tu = t0, m_dt = dt;
     blas1::copy(  u0, m_u[m_k-1]);
@@ -614,7 +696,7 @@ void MinimalProjecting<ContainerType>::init( RHS& f, value_type t0, const Contai
 
 template<class ContainerType>
 template< class RHS>
-void MinimalProjecting<ContainerType>::step( RHS& f, value_type& t, ContainerType& u)
+void ExplicitMultistep<ContainerType>::step( RHS& f, value_type& t, ContainerType& u)
 {
     if( m_counter < m_k-1)
     {
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index e7dae1f71..59a931b04 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -119,18 +119,30 @@ int main()
         res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << "Relative error AB "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
     }
-    std::cout << "### Test MinimalProjecting methods with "<<NT<<" steps\n";
+    std::cout << "### Test Explicit Multistep methods with "<<NT<<" steps\n";
     for( unsigned s=1; s<7; s++)
     {
         time = 0., y0 = init;
-        dg::MinimalProjecting< std::array<double,2> > ab( s, y0);
+        dg::ExplicitMultistep< std::array<double,2> > ab( "eBDF", s, y0);
         ab.init( full, time, y0, dt);
         //main time loop
         for( unsigned k=0; k<NT; k++)
             ab.step( full, time, y0);
         dg::blas1::axpby( -1., sol, 1., y0);
         res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
-        std::cout << "Relative error MP "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
+        std::cout << "Relative error eBDF "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
+    }
+    for( unsigned s=1; s<6; s++)
+    {
+        time = 0., y0 = init;
+        dg::ExplicitMultistep< std::array<double,2> > ab( "TVB", s, y0);
+        ab.init( full, time, y0, dt);
+        //main time loop
+        for( unsigned k=0; k<NT; k++)
+            ab.step( full, time, y0);
+        dg::blas1::axpby( -1., sol, 1., y0);
+        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
+        std::cout << "Relative error TVB  "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
     }
     std::cout << "### Test implicit multistep methods with "<<NT<<" steps\n";
     for( unsigned s=1; s<7; s++)
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index b27935a03..51841ac50 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -140,7 +140,7 @@ int main( int argc, char* argv[])
     //    feltor::FeltorSpecialSolver<
     //        Geometry, IDMatrix, DMatrix, DVec>
     //    > karniadakis( grid, p, mag);
-    dg::MinimalProjecting< std::array<std::array<dg::DVec,2>,2 > > mp( 3, y0);
+    dg::ExplicitMultistep< std::array<std::array<dg::DVec,2>,2 > > mp("TVB", 3, y0);
     {
     HVec h_wall = dg::pullback( wall, grid);
     HVec h_sheath = dg::pullback( sheath, grid);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 942553676..3e3e0fbd9 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -474,7 +474,7 @@ int main( int argc, char* argv[])
     //    feltor::FeltorSpecialSolver<
     //        Geometry, IDMatrix, DMatrix, DVec>
     //    > karniadakis( grid, p, mag);
-    dg::MinimalProjecting< std::array<std::array<DVec,2>,2 > > mp( 3, y0);
+    dg::ExplicitMultistep< std::array<std::array<DVec,2>,2 > > mp( "TVB", 3, y0);
     {
     HVec h_wall = dg::pullback( wall, grid);
     HVec h_sheath = dg::pullback( sheath, grid);
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index da76ead2c..0486a409e 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -83,7 +83,7 @@ int main( int argc, char* argv[])
     //    feltor::FeltorSpecialSolver<
     //        dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
     //    > karniadakis( grid, p, mag);
-    dg::MinimalProjecting< std::array<std::array<dg::DVec,2>,2 > > mp( 3, y0);
+    dg::ExplicitMultistep< std::array<std::array<dg::DVec,2>,2 > > mp("TVB", 3, y0);
     double time = 0, TMAX = 0.1;
     mp.init( feltor, time, y0, p.dt);
     while( time < TMAX)
-- 
GitLab


From 56211f8517890e7393c1e4be9357c5e787dcf514 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 15 Dec 2020 16:05:30 +0100
Subject: [PATCH 412/540] Add yet another set of multistep schemes

with b coefficients positive, maybe better?
---
 inc/dg/multistep.h    | 78 ++++++++++++++++++++++++++++++-------------
 inc/dg/multistep_t.cu | 12 +++++++
 2 files changed, 66 insertions(+), 24 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 82f11dd96..1aed1ff2b 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -502,24 +502,36 @@ enum multistep_identifier
     /** The family of schemes described in <a href =
      "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S.
      J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep
-     methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> as
-     **extrapolated BDF**  where it is found to be TVB (**total variation
+     methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a>
+     as **extrapolated BDF**  where it is found to be TVB (**total variation
      bound**), i.e. \f$ || v^n|| \leq M ||v^0||\f$  where the norm signifies
      the total variation semi-norm. (Note that total variation diminishing
-     (TVD) means M=1, and strong stability preserving (SSP) is the same as TVD)
+     (TVD) means M=1, and strong stability preserving (SSP) is the same as TVD, TVB schemes converge to the correct entropy solutions of hyperbolic conservation laws)
      is the same as the **Minimal Projecting** scheme
      described in <a href =
      "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf">
      Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
-     * **Possible order is 1, 2,..., 7**
+     * **Possible orders are 1, 2,..., 7**
 
     */
     eBDF,
     /** The family of schemes described in <a href="https://doi.org/10.1016/j.jcp.2005.02.029">S.J. Ruuth and W. Hundsdorfer, High-order linear multistep methods with general monotonicity and boundedness properties, Journal of Computational Physics, Volume 209, Issue 1, 2005 </a> as Total variation Bound.
-     * These schemes have larger stable step sizes than the eBDF family, for example TVB3 has 38% larger stepsize than eBDF3 and TVB4 has 109% larger stepsize than eBDF4
-     * **Possible order is 1,2,...,6**
+     * These schemes have larger stable step sizes than the eBDF family, for
+     * example TVB3 has 38% larger stepsize than eBDF3 and TVB4 has 109% larger
+     * stepsize than eBDF4.  **Possible orders are 1,2,...,6**, The CFL
+     * conditions in relation to forward Euler are (1-1: 1, 2-2: 0.5, 3-3: 0.54, 4-4: 0.46, 5-5:
+     * 0.38, 6-6: 0.33), we disregard the remaining schemes since their
+     * CFL conditions are worse
      */
-    TVB
+    TVB,
+    /** The family of schemes described in <a href="https://doi.org/10.1007/BF02728985">Gottlieb, S. On high order strong stability preserving runge-kutta and multi step time discretizations. J Sci Comput 25, 105–128 (2005)</a> as Strong Stability preserving.
+     * We implement the lowest order schemes for each stage. The CFL conditions
+     * in relation to forward Euler are (1-1: 1, 2-2: 0.5, 3-2: 0.5, 4-2: 0.66, 5-3:
+     * 0.5, 6-3: 0.567).  We disregard the remaining
+     * schemes since their CFL condition is worse than a TVB scheme of the same
+     * order. These schemes are noteworthy because the coefficients b_i are all positive except for the 2-2 method.
+     */
+    SSP
 };
 /**
 * @brief Struct for general explicit linear multistep time-integration
@@ -555,34 +567,35 @@ struct ExplicitMultistep
     ExplicitMultistep(){}
 
     ///@copydoc construct()
-    ExplicitMultistep( std::string method, unsigned order, const ContainerType& copyable){
+    ExplicitMultistep( std::string method, unsigned stages, const ContainerType& copyable){
         std::unordered_map < std::string, enum multistep_identifier> str2id{
             {"eBDF", eBDF},
-            {"TVB", TVB}
+            {"TVB", TVB},
+            {"SSP", SSP}
         };
         if( str2id.find(method) == str2id.end())
             throw dg::Error(dg::Message(_ping_)<<"Multistep coefficients for "<<method<<" not found!");
         else
-            construct( str2id[method], order, copyable);
+            construct( str2id[method], stages, copyable);
     }
     ///@copydoc construct()
-    ExplicitMultistep( enum multistep_identifier method, unsigned order, const ContainerType& copyable){
-        construct( method, order, copyable);
+    ExplicitMultistep( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
+        construct( method, stages, copyable);
     }
     /**
      * @brief Reserve memory for the integration
      *
      * Set the coefficients \f$ \alpha_i,\ \beta_i\f$
      * @param method the name of the family of schemes to be used (a string can be converted to an enum with the same spelling) @sa multistep_identifier
-     * @param order (global) order (= number of steps in the multistep) of the method (Currently, possible values depend on the method
+     * @param stages (global) stages (= number of steps in the multistep) of the method (Currently possible values depend on the method), does not necessarily coincide with the order of the method
      * @param copyable ContainerType of the size that is used in \c step
      * @note it does not matter what values \c copyable contains, but its size is important
      */
-    void construct( enum multistep_identifier method, unsigned order, const ContainerType& copyable){
-        m_k = order;
-        m_f.assign( order, copyable);
-        m_u.assign( order, copyable);
-        init_coeffs(method, order);
+    void construct( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
+        m_k = stages;
+        m_f.assign( stages, copyable);
+        m_u.assign( stages, copyable);
+        init_coeffs(method, stages);
         m_counter = 0;
     }
     ///@brief Return an object of same size as the object used for construction
@@ -617,12 +630,12 @@ struct ExplicitMultistep
     void step( RHS& rhs, value_type& t, ContainerType& u);
 
   private:
-    void init_coeffs(enum multistep_identifier method, unsigned order){
-        m_a.resize( order);
-        m_b.resize( order);
+    void init_coeffs(enum multistep_identifier method, unsigned stages){
+        m_a.resize( stages);
+        m_b.resize( stages);
         switch( method){
             case eBDF:
-            switch (order){
+            switch (stages){
                 case 1: m_a = {1.};
                         m_b = {1.}; break;
                 case 2: m_a = {4./3., -1./3.};
@@ -641,11 +654,11 @@ struct ExplicitMultistep
             }
             break;
             case TVB:
-            switch(order){
+            switch(stages){
                 case 1: m_a = {1.};
                         m_b = {1.}; break;
                 case 2: m_a = {4./3., -1./3.};
-                        m_b = {4./3., -2./3.}; break;
+                        m_b = {4./3., -2./3.}; break; //CLM = 0.5
                 case 3: //CLM = 0.54...
                     m_a[0] =  1.908535476882378;     m_b[0] =  1.502575553858997;
                     m_a[1] = -1.334951446162515;     m_b[1] = -1.654746338401493;
@@ -675,6 +688,23 @@ struct ExplicitMultistep
                 default: throw dg::Error(dg::Message()<<"Order not implemented in TVB scheme!");
             }
             break;
+            case SSP:
+            switch(stages){
+                case 1: m_a = {1.};
+                        m_b = {1.}; break;
+                case 2: m_a = {4./5., 1./5.};
+                        m_b = {8./5., -2./5.}; break; //CLM = 0.5 ... ,order 2
+                case 3: m_a = { 3./4., 0., 1./4.};
+                        m_b = { 3./2., 0., 0. }; break; //CLM = 0.5..., order 2
+                case 4: m_a = {8./9., 0., 0., 1./9.};
+                        m_b = {4./3., 0., 0., 0.}; break; //CLM = 0.66..., order 2
+                case 5: m_a = {25./32., 0., 0., 0., 7./32.};
+                        m_b = {25./16.,0.,0.,0.,5./16.}; break; //CLM 0.5, order 3
+                case 6: m_a = {108./125.,0.,0.,0.,0.,17./125.};
+                        m_b = {36./25.,0.,0.,0.,0.,6./25.}; break; //CLM 0.567, order 3
+                default: throw dg::Error(dg::Message()<<"Stage not implemented in SSP scheme!");
+            }
+            break;
         }
     }
     std::vector<ContainerType> m_u, m_f;
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 59a931b04..9e0a90fb6 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -144,6 +144,18 @@ int main()
         res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << "Relative error TVB  "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
     }
+    for( unsigned s=1; s<6; s++)
+    {
+        time = 0., y0 = init;
+        dg::ExplicitMultistep< std::array<double,2> > ab( "SSP", s, y0);
+        ab.init( full, time, y0, dt);
+        //main time loop
+        for( unsigned k=0; k<NT; k++)
+            ab.step( full, time, y0);
+        dg::blas1::axpby( -1., sol, 1., y0);
+        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
+        std::cout << "Relative error SSP  "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
+    }
     std::cout << "### Test implicit multistep methods with "<<NT<<" steps\n";
     for( unsigned s=1; s<7; s++)
     {
-- 
GitLab


From e9cbca6d89e7695f5f46647184de6acbd67da091 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 15 Dec 2020 23:29:02 +0100
Subject: [PATCH 413/540] Add SSPRK Runge Kutta methods

with these we can formulate the RKDG schemes presented in Shu and Cockburn
---
 inc/dg/runge_kutta_t.cu |   5 ++
 inc/dg/tableau.h        | 170 +++++++++++++++++++++++++++++++++++-----
 2 files changed, 154 insertions(+), 21 deletions(-)

diff --git a/inc/dg/runge_kutta_t.cu b/inc/dg/runge_kutta_t.cu
index ace3ab338..60d329a93 100644
--- a/inc/dg/runge_kutta_t.cu
+++ b/inc/dg/runge_kutta_t.cu
@@ -64,6 +64,11 @@ int main()
         "Midpoint-2-2",
         "Kutta-3-3",
         "Runge-Kutta-4-4",
+        "SSPRK-2-2",
+        "SSPRK-3-2",
+        "SSPRK-3-3",
+        "SSPRK-5-3",
+        "SSPRK-5-4",
         "Heun-Euler-2-1-2",
         "Bogacki-Shampine-4-2-3",
         "ARK-4-2-3 (explicit)",
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index 125e60288..6ca94645f 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -44,7 +44,7 @@ struct ButcherTableau{
      * which makes the embedded method equal to the actual method
      */
     ButcherTableau(unsigned s, unsigned order,
-                   real_type* a , real_type* b , real_type* c):
+                   const real_type* a , const real_type* b , const real_type* c):
         m_a(a, a+s*s), m_b(b, b+s), m_c(c, c+s), m_bt(b,b+s), m_q(order), m_p(order), m_s(s){}
     /*! @brief Construct an embedded tableau
      *
@@ -57,7 +57,7 @@ struct ButcherTableau{
      * @param c pointer to s real numbers interpreted as c_i=c[i]
      */
     ButcherTableau(unsigned s, unsigned embedded_order, unsigned order,
-               real_type* a, real_type* b, real_type* bt, real_type* c):
+               const real_type* a, const real_type* b, const real_type* bt, const real_type* c):
         m_a(a, a+s*s), m_b(b,b+s), m_c(c,c+s), m_bt(bt, bt+s), m_q(order), m_p(embedded_order), m_s(s), m_embedded(true){}
     ///Construct from ARKode standard format
     /*!@brief Construct embedded method from single array
@@ -67,20 +67,20 @@ struct ButcherTableau{
      */
     ButcherTableau(unsigned s, real_type* data):
         m_a(s), m_b(s), m_c(s), m_bt(s), m_s(s), m_embedded(true)
-   {
-       for( unsigned i=0; i<s; i++)
-       {
-           m_c[i] = data[i*(s+1)];
-           for( unsigned j=0; j<s; j++)
-               m_a(i,j) = data[i*(s+1)+j+1];
-       }
-       m_q = (unsigned)data[s*(s+1)];
-       for( unsigned j=0; j<s; j++)
-           m_b[j] = data[s*(s+1)+j+1];
-       m_p = (unsigned)data[(s+1)*(s+1)];
-       for( unsigned j=0; j<s; j++)
-           m_bt[j] = data[(s+1)*(s+1)+j+1];
-   }
+    {
+        for( unsigned i=0; i<s; i++)
+        {
+            m_c[i] = data[i*(s+1)];
+            for( unsigned j=0; j<s; j++)
+                m_a(i,j) = data[i*(s+1)+j+1];
+        }
+        m_q = (unsigned)data[s*(s+1)];
+        for( unsigned j=0; j<s; j++)
+            m_b[j] = data[s*(s+1)+j+1];
+        m_p = (unsigned)data[(s+1)*(s+1)];
+        for( unsigned j=0; j<s; j++)
+            m_bt[j] = data[(s+1)*(s+1)+j+1];
+    }
 
     /**
     * @brief Read the a_ij coefficients
@@ -237,8 +237,6 @@ ButcherTableau<real_type> sirk3a_im_3_3()
     return ButcherTableau<real_type>( 3,3, a,b,c);
 }
 
-
-
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%%Embedded Butcher tables%%%%%%%%%%%%%%%%%%
 //tables copied from: http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html
 template<class real_type>
@@ -816,6 +814,110 @@ ButcherTableau<real_type> ark548l2sa_dirk_8_4_5()
     };
     return ButcherTableau<real_type>( 8, data);
 }
+///////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+////                Shu-Osher form of RK methods                           ////
+///////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////
+template<class real_type>
+dg::ButcherTableau<real_type> shuosher2butcher( unsigned stages, unsigned order, std::vector<real_type> alpha_v, std::vector<real_type> beta_v)
+{
+    //first parse the input into matrices
+    dg::Operator<real_type> alpha(stages, 0), beta(stages, 0), a(stages, 0);
+    unsigned j=0;
+    for( unsigned i=0; i<stages; i++)
+        for( unsigned k=0; k<i+1; k++)
+        {
+            alpha(i,k) = alpha_v[j];
+            beta(i,k) = beta_v[j];
+            j++;
+        }
+    std::vector< real_type > b(stages), c(stages);
+    //std::cout << alpha << beta;
+
+    //now convert to Butcher tableau
+    for( unsigned i=1; i<stages; i++)
+    {
+        for( unsigned k=0; k<i; k++)
+        {
+            a( i, k) = beta ( i-1, k);
+            for( unsigned j = k+1; j<i; j++)
+                a(i,k) += alpha( i-1,j)*a(j,k);
+        }
+    }
+    for( unsigned k=0; k<stages; k++)
+    {
+        b[k] = beta( stages-1, k);
+        for( unsigned j=k+1; j<stages; j++)
+            b[k] += alpha(stages-1, j)*a( j, k);
+    }
+    for( unsigned i=0; i<stages; i++)
+    {
+        c[i] = a(i,0);
+        for( unsigned k=1; k<stages; k++)
+            c[i] += a(i,k);
+    }
+    dg::ButcherTableau<real_type> tableau(stages, order, &a.data()[0], &b[0], &c[0]);
+    return tableau;
+}
+
+template<class real_type>
+ButcherTableau<real_type> ssprk_2_2()
+{
+    unsigned stages=2, order = 2;
+    std::vector<double> alpha_v = {1., 0.5, 0.5};
+    std::vector<double> beta_v = {1., 0., 0.5};
+    return shuosher2butcher( stages, order, alpha_v, beta_v);
+}
+template<class real_type>
+ButcherTableau<real_type> ssprk_3_2()
+{
+    //CFLL = 2 -> eff = 0.66
+    unsigned stages=3, order = 2;
+    std::vector<double> alpha_v = {1., 0, 1., 1./3., 0, 2./3.};
+    std::vector<double> beta_v = {0.5, 0., 0.5, 0., 0., 1./3.};
+    return shuosher2butcher( stages, order, alpha_v, beta_v);
+}
+template<class real_type>
+ButcherTableau<real_type> ssprk_3_3()
+{
+    //CFL = 1 -> eff = 0.33
+    unsigned stages=3, order = 3;
+    std::vector<double> alpha_v = {1.,3./4.,1./4.,1./3.,0.,2./3.};
+    std::vector<double> beta_v = {1., 0., 1./4.,0.,0.,2./3.};
+    return shuosher2butcher( stages, order, alpha_v, beta_v);
+}
+template<class real_type>
+ButcherTableau<real_type> ssprk_5_3()
+{
+    //Ruuth 2005
+    //CFL = 2.6 -> eff = 0.5
+    unsigned stages=5, order = 3;
+    std::vector<double> alpha_v = {
+        1, 0, 1, 0.56656131914033, 0, 0.43343868085967, 0.09299483444413, 0.00002090369620, 0, 0.90698426185967, 0.00736132260920, 0.20127980325145, 0.00182955389682, 0, 0.78952932024253
+    };
+    std::vector<double> beta_v = {
+        0.37726891511710, 0, 0.37726891511710, 0, 0, 0.16352294089771, 0.00071997378654, 0, 0, 0.34217696850008, 0.00277719819460, 0.00001567934613, 0, 0, 0.29786487010104
+    };
+    return shuosher2butcher( stages, order, alpha_v, beta_v);
+}
+
+template<class real_type>
+ButcherTableau<real_type> ssprk_5_4()
+{
+    //Spiteri & Ruuth 2005
+    //CLM = 1.5 -> eff = 0.37 , better than 3_3
+    unsigned stages=5, order = 4;
+    std::vector<double> alpha_v = {
+        1, 0.44437049406734, 0.55562950593266, 0.62010185138540, 0, 0.37989814861460, 0.17807995410773, 0, 0, 0.82192004589227, 0.00683325884039, 0, 0.51723167208978, 0.12759831133288, 0.34833675773694
+    };
+    std::vector<double> beta_v = {
+        0.39175222700392, 0, 0.36841059262959, 0, 0, 0.25189177424738, 0, 0, 0, 0.54497475021237, 0, 0, 0, 0.08460416338212, 0.22600748319395
+    };
+    return shuosher2butcher( stages, order, alpha_v, beta_v);
+}
+
 
 }//namespace tableau
 ///@endcond
@@ -868,7 +970,13 @@ enum tableau_identifier{
     KVAERNO_5_3_4,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Kvaerno-5-3-4</a>
     ARK436L2SA_DIRK_6_3_4,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-6-3-4 (implicit)</a>
     KVAERNO_7_4_5,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Kvaerno-7-4-5</a>
-    ARK548L2SA_DIRK_8_4_5//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-8-4-5 (implicit)</a>
+    ARK548L2SA_DIRK_8_4_5,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-8-4-5 (implicit)</a>
+    // SSP RK tableaus
+    SSPRK_2_2, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
+    SSPRK_3_2, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
+    SSPRK_3_3, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
+    SSPRK_5_3, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
+    SSPRK_5_4 //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
 };
 
 ///@cond
@@ -942,6 +1050,16 @@ ButcherTableau<real_type> tableau( enum tableau_identifier id)
             return dg::tableau::kvaerno_7_4_5<real_type>();
         case ARK548L2SA_DIRK_8_4_5:
             return dg::tableau::ark548l2sa_dirk_8_4_5<real_type>();
+        case SSPRK_2_2:
+            return dg::tableau::ssprk_2_2<real_type>();
+        case SSPRK_3_2:
+            return dg::tableau::ssprk_3_2<real_type>();
+        case SSPRK_3_3:
+            return dg::tableau::ssprk_3_3<real_type>();
+        case SSPRK_5_3:
+            return dg::tableau::ssprk_5_3<real_type>();
+        case SSPRK_5_4:
+            return dg::tableau::ssprk_5_4<real_type>();
     }
     return ButcherTableau<real_type>();
 }
@@ -984,7 +1102,12 @@ ButcherTableau<real_type> tableau( std::string name)
         {"Kvaerno-5-3-4", KVAERNO_5_3_4},
         {"ARK-6-3-4 (implicit)", ARK436L2SA_DIRK_6_3_4},
         {"Kvaerno-7-4-5", KVAERNO_7_4_5},
-        {"ARK-8-4-5 (implicit)", ARK548L2SA_DIRK_8_4_5}
+        {"ARK-8-4-5 (implicit)", ARK548L2SA_DIRK_8_4_5},
+        {"SSPRK-2-2", SSPRK_2_2},
+        {"SSPRK-3-2", SSPRK_3_2},
+        {"SSPRK-3-3", SSPRK_3_3},
+        {"SSPRK-5-3", SSPRK_5_3},
+        {"SSPRK-5-4", SSPRK_5_4},
     };
     if( str2id.find(name) == str2id.end())
         throw dg::Error(dg::Message(_ping_)<<"Butcher Tableau "<<name<<" not found!");
@@ -1001,7 +1124,12 @@ ButcherTableau<real_type> tableau( std::string name)
  *   Euler                  | dg::EXPLICIT_EULER_1_1     | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Explicit Euler</a>
  *   Midpoint-2-2           | dg::MIDPOINT_2_2           | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Midpoint method</a>
  *   Kutta-3-3              | dg::KUTTA_3_3              | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Kutta-3-3</a>
- *   Runge-Kutta-4-4        | dg::CLASSIC_4_4            | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods"> "The" Runge-Kutta method</a>
+ *   Runge-Kutta-4-4        | dg::CLASSIC_4_4            | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">"The" Runge-Kutta method</a>
+ *   SSPRK-2-2              | dg::SSPRK_2_2                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (2,2)</a> CFL_eff = 0.5
+ *   SSPRK-3-2              | dg::SSPRK_3_2                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (3,2)</a> CFL_eff = 0.66
+ *   SSPRK-3-3              | dg::SSPRK_3_3                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (3,3)</a> CFL_eff = 0.33
+ *   SSPRK-5-3              | dg::SSPRK_5_3                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (5,3)</a> CFL_eff = 0.5
+ *   SSPRK-5-4              | dg::SSPRK_5_4                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (5,4)</a> CFL_eff = 0.37
  *   Heun-Euler-2-1-2       | dg::HEUN_EULER_2_1_2       | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Heun-Euler-2-1-2</a>
  *   Bogacki-Shampine-4-2-3 | dg::BOGACKI_SHAMPINE_4_2_3 | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Bogacki-Shampine</a> (fsal)
  *   ARK-4-2-3 (explicit)   | dg::ARK324L2SA_ERK_4_2_3   | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-4-2-3 (explicit)</a>
-- 
GitLab


From 3c86e2806d550bec8ed9037ee0abc00e9be5b2c6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 16 Dec 2020 16:23:11 +0100
Subject: [PATCH 414/540] Add a ModalFilter class and units tests

test on exponential filtering
---
 inc/dg/functors.h              |  27 +++++
 inc/dg/tableau.h               |   1 +
 inc/dg/topology/filter.h       | 185 +++++++++++++++++++++++++++++++++
 inc/dg/topology/filter_mpit.cu |  94 +++++++++++++++++
 inc/dg/topology/filter_t.cu    |  69 ++++++++++++
 5 files changed, 376 insertions(+)
 create mode 100644 inc/dg/topology/filter.h
 create mode 100644 inc/dg/topology/filter_mpit.cu
 create mode 100644 inc/dg/topology/filter_t.cu

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index c98855b50..ef9dfab8d 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -1101,6 +1101,33 @@ struct DPolynomialHeaviside {
     double x0, a;
 };
 
+/**
+ * @brief \f$ \begin{cases}
+    1 \text{ if } \eta < \eta_c \\
+    \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
+    0 \text{ else}
+    \end{cases}\f$
+
+    This function is continuously differentiable
+    @sa Its main use comes from the application in dg::ModalFilter
+ */
+struct ExponentialFilter
+{
+    ExponentialFilter( double alpha, double eta_c, unsigned s): m_alpha(alpha),
+                                                                m_etac(eta_c),
+                                                                m_s(s) {}
+    double operator()( double eta) const
+    {
+        if( eta < m_etac)
+            return 1.;
+        if( eta <= 1.)
+            return exp( -m_alpha*pow( (eta-m_etac)/(1.-m_etac), 2*m_s));
+        return 0;
+    }
+    private:
+    double m_alpha, m_etac;
+    unsigned m_s;
+};
 
 /**
  * @brief Exponential \f$ f(x) = A \exp(\lambda x)\f$
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index 6ca94645f..11f771433 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -820,6 +820,7 @@ ButcherTableau<real_type> ark548l2sa_dirk_8_4_5()
 ////                Shu-Osher form of RK methods                           ////
 ///////////////////////////////////////////////////////////////////////////////
 ///////////////////////////////////////////////////////////////////////////////
+// Convert between Shu-Osher representation to a Butcher tableau
 template<class real_type>
 dg::ButcherTableau<real_type> shuosher2butcher( unsigned stages, unsigned order, std::vector<real_type> alpha_v, std::vector<real_type> beta_v)
 {
diff --git a/inc/dg/topology/filter.h b/inc/dg/topology/filter.h
new file mode 100644
index 000000000..c803b90b2
--- /dev/null
+++ b/inc/dg/topology/filter.h
@@ -0,0 +1,185 @@
+#pragma once
+#include "fast_interpolation.h"
+
+/**@file
+* @brief Modal Filtering
+*/
+
+namespace dg
+{
+
+///@cond
+namespace create
+{
+    //op is evaluated at eta and returns a double
+template<class UnaryOp, class real_type>
+MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> >
+modal_filter( UnaryOp op, const RealGrid1d<real_type>& g )
+{
+    Operator<real_type> backward=g.dlt().backward();
+    Operator<real_type> forward=g.dlt().forward();
+    Operator<real_type> filter( g.n(), 0);
+    for( unsigned i=0; i<g.n(); i++)
+    {
+        real_type eta = (real_type) i / (real_type)(g.n()-1);
+        filter(i,i) = op( eta);
+    }
+    filter = backward*filter*forward;
+    //Assemble the matrix
+    EllSparseBlockMat<real_type> A(g.N(), g.N(), 1, 1, g.n());
+    A.data = filter.data();
+    for( unsigned i=0; i<g.N(); i++)
+    {
+        A.data_idx[i] = 0;
+        A.cols_idx[i] = i;
+    }
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> >
+        filter_matrix(1);
+    filter_matrix.get_matrices()[0] = A;
+    return filter_matrix;
+}
+template<class UnaryOp, class real_type>
+MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> >
+modal_filter( UnaryOp op, const aRealTopology2d<real_type>& t)
+{
+    dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
+    dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> >
+        filterX = dg::create::modal_filter( op, gx);
+    filterX.get_matrices()[0].left_size = t.n()*t.Ny();
+    filterX.get_matrices()[0].set_default_range();
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> >
+        filterY = dg::create::modal_filter( op, gy);
+    filterY.get_matrices()[0].right_size = t.n()*t.Nx();
+    filterY.get_matrices()[0].set_default_range();
+
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > filter(2);
+    filter.get_matrices()[0] = filterX.get_matrices()[0];
+    filter.get_matrices()[1] = filterY.get_matrices()[0];
+    thrust::host_vector<real_type> vec( t.size());
+    filter.get_temp()[0] = Buffer<thrust::host_vector<real_type > >(vec);
+    return filter;
+}
+template<class UnaryOp, class real_type>
+MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> >
+modal_filter( UnaryOp op, const aRealTopology3d<real_type>& t)
+{
+    dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
+    dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> >
+        filterX = dg::create::modal_filter( op, gx);
+    filterX.get_matrices()[0].left_size = t.n()*t.Ny()*t.Nz();
+    filterX.get_matrices()[0].set_default_range();
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> >
+        filterY = dg::create::modal_filter( op, gy);
+    filterY.get_matrices()[0].right_size = t.n()*t.Nx();
+    filterY.get_matrices()[0].left_size = t.Nz();
+    filterY.get_matrices()[0].set_default_range();
+
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > filter(2);
+    filter.get_matrices()[0] = filterX.get_matrices()[0];
+    filter.get_matrices()[1] = filterY.get_matrices()[0];
+    thrust::host_vector<real_type> vec( t.size());
+    filter.get_temp()[0] = Buffer<thrust::host_vector<real_type > >(vec);
+    return filter;
+}
+
+#ifdef MPI_VERSION
+//very elaborate way of telling the compiler to just apply the local matrix to the local vector
+template<class UnaryOp, class real_type>
+MultiMatrix< RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type> >, MPI_Vector<thrust::host_vector<real_type> > >
+modal_filter( UnaryOp op, const aRealMPITopology2d<real_type>& t)
+{
+    typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
+    typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::modal_filter( op, t.local());
+    MultiMatrix< Matrix, Vector > filter(2);
+    filter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
+    filter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
+    filter.get_temp()[0] = Buffer<Vector> ( Vector( temp.get_temp()[0].data(), t.communicator())  );
+    return filter;
+}
+
+template<class UnaryOp, class real_type>
+MultiMatrix< RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type> >, MPI_Vector<thrust::host_vector<real_type> > >
+modal_filter( UnaryOp op, const aRealMPITopology3d<real_type>& t)
+{
+    typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
+    typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::modal_filter( op, t.local());
+    MultiMatrix< Matrix, Vector > filter(2);
+    filter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
+    filter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
+    filter.get_temp()[0] = Buffer<Vector> ( Vector( temp.get_temp()[0].data(), t.communicator())  );
+    return filter;
+}
+
+#endif //MPI_VERSION
+
+} //namespace create
+///@endcond
+
+/**
+ * @brief Struct that applies a given modal filter to a vector
+ *
+ * \f[ y = V D V^{-1}\f]
+ * where \f$ V\f$ is the Vandermonde matrix (the backward transformation matrix)
+ * and \f$ D \f$ is a diagonal matrix with \f$ D_{ii} = \sigma(i)\f$
+ * @copydoc hide_matrix
+ * @copydoc hide_container
+ * @ingroup misc
+ */
+template<class MatrixType, class ContainerType>
+struct ModalFilter
+{
+    using real_type = get_value_type<ContainerType>;
+    ModalFilter(){}
+
+    /**
+     * @brief Create exponential filter \f$ \begin{cases}
+    1 \text{ if } \eta < \eta_c \\
+    \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
+    0 \text{ else}
+    \end{cases}\f$
+     *
+     * @tparam Topology Any grid
+     * @param eta_c cutoff frequency (0<eta_c<1), 0.5 or 0 are good starting values
+     * @param order 8 or 16 are good values
+     * @param t The topology to apply the modal filter on
+     * @sa dg::ExponentialFilter
+     */
+    template<class Topology>
+    ModalFilter( real_type eta_c, unsigned order, const Topology& t):
+        ModalFilter( ExponentialFilter( 36, eta_c, order), t)
+    { }
+    /**
+     * @brief Create arbitrary filter
+     *
+     * @tparam Topology Any grid
+     * @tparam UnaryOp Model of Unary Function \c real_type \c f(real_type) The input will be the normalized modal number \f$ \eta := \frac{i}{n-1} \f$ where \f$ i=0,...,n-1\f$ and n is the number of polynomial coefficients in use
+     * @param f The filter to evaluate on the normalized modal coefficients
+     * @param t The topology to apply the modal filter on
+     */
+    template<class UnaryOp, class Topology>
+    ModalFilter( UnaryOp f, const Topology& t) : m_filter (
+            dg::create::modal_filter( f, t)) { }
+
+    void apply( const ContainerType& x, ContainerType& y) const{ symv( 1., x, 0., y);}
+    void symv( const ContainerType& x, ContainerType& y) const{ symv( 1., x,0,y);}
+    void symv(real_type alpha, const ContainerType& x, real_type beta, ContainerType& y) const
+    {
+        m_filter.symv( alpha, x, beta, y);
+    }
+    private:
+    MultiMatrix<MatrixType, ContainerType> m_filter;
+};
+
+///@cond
+template <class M, class V>
+struct TensorTraits<ModalFilter<M, V> >
+{
+    using value_type  = get_value_type<V>;
+    using tensor_category = SelfMadeMatrixTag;
+};
+///@endcond
+}//namespace dg
diff --git a/inc/dg/topology/filter_mpit.cu b/inc/dg/topology/filter_mpit.cu
new file mode 100644
index 000000000..7781a61bc
--- /dev/null
+++ b/inc/dg/topology/filter_mpit.cu
@@ -0,0 +1,94 @@
+#include <iostream>
+#include <mpi.h>
+
+#include "dg/blas.h"
+#include "dg/functors.h"
+#include "dg/backend/mpi_init.h"
+
+#include "mpi_projection.h"
+#include "mpi_evaluation.h"
+#include "mpi_weights.h"
+#include "filter.h"
+
+double function( double x, double y){return sin(x)*sin(y);}
+double function( double x, double y, double z){return sin(x)*sin(y)*sin(z);}
+
+unsigned n = 3, Nx = 8, Ny = 10, Nz = 6;
+
+int main(int argc, char* argv[])
+{
+    MPI_Init( &argc, &argv);
+    int rank, size;
+    MPI_Comm_size( MPI_COMM_WORLD, &size);
+    if(size!=4)
+    {
+        std::cerr << "Please run with 4 processes!\n";
+        MPI_Finalize();
+        return 0;
+    }
+    MPI_Comm comm;
+    std::stringstream ss;
+    ss<< "2 2 " << n << " "<<Nx<<" "<<Ny;
+    mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm, ss);
+    MPI_Comm_rank( comm, &rank);
+
+    {
+    if(rank==0)std::cout << "Test 2d exponential filter: \n";
+    dg::MPIGrid2d g3( -M_PI, 0, -5*M_PI, -4*M_PI, 3, Nx, Ny, comm);
+    dg::MPIGrid2d g2( -M_PI, 0, -5*M_PI, -4*M_PI, 2, Nx, Ny, comm);
+
+    const dg::MDVec vec = dg::evaluate( function, g3);
+    const dg::MDVec weights = dg::create::weights( g3);
+    dg::MDVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
+    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( 0.5, 8, g3);
+    dg::MIDMatrix project = dg::create::projection( g2,g3);
+    dg::MIDMatrix interpo = dg::create::interpolation( g3,g2);
+
+    dg::blas2::symv( project, vec, projected_vec);
+    dg::blas2::symv( interpo, projected_vec, inter_vec);
+    filter.apply( vec, filtered_vec);
+    dg::blas1::axpby( 1., filtered_vec, -1., inter_vec);
+    double error = sqrt(dg::blas2::dot( inter_vec, weights, inter_vec)/ dg::blas2::dot( vec, weights, vec));
+    if(rank==0)std::cout << "Error by filtering: "<<error<<std::endl;
+
+    if(rank==0){
+    if( error > 1e-14)
+        std::cout << "2D TEST FAILED!\n";
+    else
+        std::cout << "2D TEST PASSED!\n";
+    }
+    }
+    {
+    MPI_Comm comm;
+    std::stringstream ss;
+    ss<< "2 2 1 " << n << " "<<Nx<<" "<<Ny<<" "<<Nz;
+    mpi_init3d( dg::PER, dg::PER, dg::PER, n, Nx, Ny, Nz, comm, ss);
+    MPI_Comm_rank( comm, &rank);
+    if(rank==0)std::cout << "Test 3d exponential filter: \n";
+    dg::MPIGrid3d g3( -M_PI, 0, -5*M_PI, -4*M_PI, 0., 2.*M_PI, 3, Nx, Ny, Nz, comm);
+    dg::MPIGrid3d g2( -M_PI, 0, -5*M_PI, -4*M_PI, 0., 2.*M_PI, 2, Nx, Ny, Nz, comm);
+
+    const dg::MDVec vec = dg::evaluate( function, g3);
+    const dg::MDVec weights = dg::create::weights( g3);
+    dg::MDVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
+    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( 0.5, 8, g3);
+    dg::MIDMatrix project = dg::create::projection( g2,g3);
+    dg::MIDMatrix interpo = dg::create::interpolation( g3,g2);
+
+    dg::blas2::symv( project, vec, projected_vec);
+    dg::blas2::symv( interpo, projected_vec, inter_vec);
+    filter.apply( vec, filtered_vec);
+    dg::blas1::axpby( 1., filtered_vec, -1., inter_vec);
+    double error = sqrt(dg::blas2::dot( inter_vec, weights, inter_vec)/ dg::blas2::dot( vec, weights, vec));
+    if(rank==0)std::cout << "Error by filtering: "<<error<<std::endl;
+
+    if(rank==0){
+    if( error > 1e-14)
+        std::cout << "3D TEST FAILED!\n";
+    else
+        std::cout << "3D TEST PASSED!\n";
+    }
+    }
+    MPI_Finalize();
+    return 0;
+}
diff --git a/inc/dg/topology/filter_t.cu b/inc/dg/topology/filter_t.cu
new file mode 100644
index 000000000..b72a7ab0c
--- /dev/null
+++ b/inc/dg/topology/filter_t.cu
@@ -0,0 +1,69 @@
+#include <iostream>
+
+#include "dg/blas.h"
+#include "dg/functors.h"
+
+#include "interpolation.h"
+#include "evaluation.h"
+#include "filter.h"
+
+double function( double x, double y){return sin(x)*sin(y);}
+double function( double x, double y, double z){return sin(x)*sin(y)*sin(z);}
+
+const unsigned n = 3, Nx = 8, Ny = 10, Nz = 6;
+
+int main()
+{
+
+    //We test if the Modal Filter on 3 polynomials has the same effect as the
+    //projection matrix
+    {
+    std::cout << "Test 2d exponential filter: \n";
+    dg::Grid2d g3( -M_PI, 0, -5*M_PI, -4*M_PI, 3, Nx, Ny);
+    dg::Grid2d g2( -M_PI, 0, -5*M_PI, -4*M_PI, 2, Nx, Ny);
+
+    const dg::DVec vec = dg::evaluate( function, g3);
+    const dg::DVec weights = dg::create::weights( g3);
+    dg::DVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 0.5, 8, g3);
+    dg::IDMatrix project = dg::create::projection( g2,g3);
+    dg::IDMatrix interpo = dg::create::interpolation( g3,g2);
+
+    dg::blas2::symv( project, vec, projected_vec);
+    dg::blas2::symv( interpo, projected_vec, inter_vec);
+    filter.apply( vec, filtered_vec);
+    dg::blas1::axpby( 1., filtered_vec, -1., inter_vec);
+    double error = sqrt(dg::blas2::dot( inter_vec, weights, inter_vec)/ dg::blas2::dot( vec, weights, vec));
+    std::cout << "Error by filtering: "<<error<<std::endl;
+
+    if( error > 1e-14)
+        std::cout << "2D TEST FAILED!\n";
+    else
+        std::cout << "2D TEST PASSED!\n";
+    }
+    {
+    std::cout << "Test 3d exponential filter: \n";
+    dg::Grid3d g3( -M_PI, 0, -5*M_PI, -4*M_PI, 0., 2.*M_PI, 3, Nx, Ny, Nz);
+    dg::Grid3d g2( -M_PI, 0, -5*M_PI, -4*M_PI, 0., 2.*M_PI, 2, Nx, Ny, Nz);
+
+    const dg::DVec vec = dg::evaluate( function, g3);
+    const dg::DVec weights = dg::create::weights( g3);
+    dg::DVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 0.5, 8, g3);
+    dg::IDMatrix project = dg::create::projection( g2,g3);
+    dg::IDMatrix interpo = dg::create::interpolation( g3,g2);
+
+    dg::blas2::symv( project, vec, projected_vec);
+    dg::blas2::symv( interpo, projected_vec, inter_vec);
+    filter.apply( vec, filtered_vec);
+    dg::blas1::axpby( 1., filtered_vec, -1., inter_vec);
+    double error = sqrt(dg::blas2::dot( inter_vec, weights, inter_vec)/ dg::blas2::dot( vec, weights, vec));
+    std::cout << "Error by filtering: "<<error<<std::endl;
+
+    if( error > 1e-14)
+        std::cout << "3D TEST FAILED!\n";
+    else
+        std::cout << "3D TEST PASSED!\n";
+    }
+    return 0;
+}
-- 
GitLab


From 9ea6ad89da18478487e0413aeddd4e6e9410db93 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Dec 2020 13:22:14 +0100
Subject: [PATCH 415/540] Implement Shu-Osher form for time-stepping

This is our first time-stepper that allows a limiter
to be included
---
 inc/dg/multistep.h       |  10 +-
 inc/dg/runge_kutta.h     | 161 ++++++++++++++++-
 inc/dg/runge_kutta_t.cu  |  21 +++
 inc/dg/tableau.h         | 377 +++++++++++++++++++++++++++------------
 inc/dg/topology/filter.h |   2 +-
 5 files changed, 447 insertions(+), 124 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 1aed1ff2b..ffdd18804 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -135,9 +135,8 @@ void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType&
 {
     if( m_counter < m_k-1)
     {
-        ERKStep<ContainerType> erk( "ARK-4-2-3 (explicit)", u);
-        ContainerType tmp ( u);
-        erk.step( f, t, u, t, u, m_dt, tmp);
+        RungeKutta<ContainerType> rk( "ARK-4-2-3 (explicit)", u);
+        rk.step( f, t, u, t, u, m_dt, tmp);
         m_counter++;
         m_tu = t;
         blas1::copy(  u, m_u);
@@ -730,9 +729,8 @@ void ExplicitMultistep<ContainerType>::step( RHS& f, value_type& t, ContainerTyp
 {
     if( m_counter < m_k-1)
     {
-        ERKStep<ContainerType> erk( "ARK-4-2-3 (explicit)", u);
-        ContainerType tmp ( u);
-        erk.step( f, t, u, t, u, m_dt, tmp);
+        RungeKutta<ContainerType> rk( "ARK-4-2-3 (explicit)", u);
+        rk.step( f, t, u, t, u, m_dt);
         m_counter++;
         m_tu = t;
         blas1::copy(  u, m_u[m_k-1-m_counter]);
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index a87cc352d..328b975f4 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -23,6 +23,14 @@ namespace dg{
         i.e. y' = f(t, y) translates to f(t, y, y').
         The two ContainerType arguments never alias each other in calls to the functor.
   */
+ /** @class hide_limiter
+  * @tparam Limiter The filter or limiter class to use in connection with the time-stepper
+        has a member function \c apply
+        of signature <tt> void apply( const ContainerType&, ContainerType&)</tt>
+        The first argument is the input vector, which the functor may \b not override, and the second is the output,
+        i.e. y' = L( y) translates to L.apply( y, y').
+        The two ContainerType arguments never alias each other in calls to the functor.
+  */
 
 
 /**
@@ -63,8 +71,7 @@ struct ERKStep
         m_rk = tableau;
         m_k.assign(m_rk.num_stages(), copyable);
     }
-    ///@brief Return an object of same size as the object used for construction
-    ///@return A copyable object; what it contains is undefined, its size is important
+    ///@copydoc RungeKutta::copyable()
     const ContainerType& copyable()const{ return m_k[0];}
 
     ///All subsequent calls to \c step method will ignore the first same as last property (useful if you want to implement an operator splitting)
@@ -555,6 +562,151 @@ struct RungeKutta
     ERKStep<ContainerType> m_erk;
     ContainerType m_delta;
 };
+
+/**
+ * @brief A filter that does nothing
+ */
+struct IdentityFilter
+{
+    /**
+     * @brief copy in to out
+     *
+     * @copydoc hide_ContainerType
+     * @param in (input)
+     * @param out (copied version of in)
+     */
+    template<class ContainerType0, class ContainerType1>
+    void apply( const ContainerType0& in, ContainerType1& out) const{
+        dg::blas1::copy( in, out);
+    }
+
+};
+/**
+* @brief Struct for Shu-Osher fixed-step explicit time-integration
+* \f[
+ \begin{align}
+    u_0 &= u_n \\
+    u_i &= \Lambda\Pi \left(\sum_{j=0}^{i-1}\left[ \alpha_{ij} u_j + \Delta t \beta_{ij} f( t_j, u_j)\right]\right)\\
+    u^{n+1} &= u_s
+ \end{align}
+\f]
+
+where \f$ \Lambda\Pi\f$ is the limiter, \c i=1,...,s and \c s is the number of stages (i.e. the number of times the right hand side is evaluated.
+
+The method is defined by its (explicit) ShuOsherTableau, given by
+the coefficients \c alpha and \c beta,  and \c s is the number
+of stages.
+@note the original reference for the scheme is
+ * <a href="https://doi.org/10.1016/0021-9991(88)90177-5">
+ Chi-Wang Shu, Stanley Osher,
+Efficient implementation of essentially non-oscillatory shock-capturing schemes,
+Journal of Computational Physics,
+Volume 77, Issue 2,
+1988,
+Pages 439-471</a>
+@note This struct can be used to implement the RKDG methods with slope-limiter described in
+<a href ="https://doi.org/10.1023/A:1012873910884">Cockburn, B., Shu, CW. Runge–Kutta Discontinuous Galerkin Methods for Convection-Dominated Problems. Journal of Scientific Computing 16, 173–261 (2001) </a>
+
+You can use one of our predefined methods (only the ones that are marked with "Shu-Osher-Form"):
+@copydoc hide_explicit_butcher_tableaus
+* @ingroup time
+*
+* @note Uses only \c dg::blas1 routines to integrate one step.
+* @copydoc hide_ContainerType
+*/
+template<class ContainerType>
+struct ShuOsher
+{
+    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
+    using container_type = ContainerType; //!< the type of the vector class in use
+    ///@copydoc RungeKutta::RungeKutta()
+    ShuOsher(){}
+    ///@copydoc RungeKutta::construct()
+    ShuOsher( dg::ConvertsToShuOsherTableau<value_type> tableau, const ContainerType& copyable): m_t( tableau), m_u(  m_t.num_stages(), copyable), m_k(m_u), m_temp(copyable)
+        { }
+    ///@copydoc RungeKutta::construct()
+    void construct(dg::ConvertsToShuOsherTableau<value_type> tableau, const ContainerType& copyable){
+        m_t = tableau;
+        m_u.assign(m_t.num_stages(), copyable);
+        m_k.assign(m_t.num_stages(), copyable);
+        m_temp = copyable;
+    }
+    ///@copydoc RungeKutta::copyable()
+    const ContainerType& copyable()const{ return m_u[0 ];}
+
+    ///@copydoc RungeKutta::step()
+    template<class RHS>
+    void step( RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
+        dg::IdentityFilter id;
+        regularized_step( id, rhs, t0, u0, t1, u1, dt);
+    }
+
+    /**
+    * @brief Advance one step using a limiter
+    *
+    * @copydoc hide_rhs
+    * @copydoc hide_limiter
+    * @param limiter the filter or limiter to use
+    * @param rhs right hand side subroutine
+    * @param t0 start time
+    * @param u0 value at \c t0
+    * @param t1 (write only) end time ( equals \c t0+dt on output, may alias \c t0)
+    * @param u1 (write only) contains result on output (may alias u0)
+    * @param dt timestep
+    * @note on return \c rhs(t1, u1) will be the last call to \c rhs (this is useful if \c RHS holds state, which is then updated to the current timestep)
+    */
+    template<class Limiter, class RHS>
+    void regularized_step( Limiter& limiter, RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
+        unsigned s = m_t.num_stages();
+        std::vector<value_type> ts( m_t.num_stages()+1);
+        ts[0] = t0;
+        if( t0 != m_t1 ) //this is the first time we call step
+        {
+            limiter.apply( u0, m_u[0]);
+            rhs(ts[0], m_u[0], m_k[0]); //freshly compute k_0
+        }
+        else
+            dg::blas1::copy(u0, m_u[0]);
+        for( unsigned i=1; i<=s; i++)
+        {
+
+            dg::blas1::axpbypgz( m_t.alpha(i-1,0), m_u[0], dt*m_t.beta(i-1,0), m_k[0], 0., m_temp);
+            ts[i] = m_t.alpha(i-1,0)*ts[0] + dt*m_t.beta(i-1,0);
+            for( unsigned j=1; j<i; j++)
+            {
+                //about the i-1: it is unclear to me how the ShuOsher tableau makes implicit schemes
+                dg::blas1::axpbypgz( m_t.alpha(i-1,j), m_u[j], dt*m_t.beta(i-1,j), m_k[j], 1., m_temp);
+                ts[i] += m_t.alpha(i-1,j)*ts[j] + dt*m_t.beta(i-1,j);
+
+            }
+            if(i!=s)
+            {
+                limiter.apply( m_temp, m_u[i]);
+                rhs(ts[i], m_u[i], m_k[i]);
+            }
+            else{
+                limiter.apply( m_temp, u1);
+                //make sure (t1,u1) is the last call to f
+                rhs(ts[i], u1, m_k[0]);
+            }
+        }
+        m_t1 = t1 = ts[s];
+    }
+    ///@copydoc ERKStep::order
+    unsigned order() const {
+        return m_t.order();
+    }
+    ///@copydoc ERKStep::num_stages()
+    unsigned num_stages() const{
+        return m_t.num_stages();
+    }
+  private:
+    ShuOsherTableau<value_type> m_t;
+    std::vector<ContainerType> m_u, m_k;
+    ContainerType m_temp;
+    value_type m_t1 = 1e300;
+};
+
 /*!
  * @brief Struct for diagonally implicit Runge Kutta time-step with error estimate
 * \f[
@@ -837,7 +989,10 @@ struct ImplicitRungeKutta
  * @param N number of steps
  */
 template< class RHS, class ContainerType>
-void stepperRK(ConvertsToButcherTableau<get_value_type<ContainerType>> tableau, RHS& rhs, get_value_type<ContainerType>  t_begin, const ContainerType& begin, get_value_type<ContainerType> t_end, ContainerType& end, unsigned N )
+void stepperRK(ConvertsToButcherTableau<get_value_type<ContainerType>> tableau,
+        RHS& rhs, get_value_type<ContainerType>  t_begin, const ContainerType&
+        begin, get_value_type<ContainerType> t_end, ContainerType& end,
+        unsigned N )
 {
     using value_type = get_value_type<ContainerType>;
     RungeKutta<ContainerType > rk( tableau, begin);
diff --git a/inc/dg/runge_kutta_t.cu b/inc/dg/runge_kutta_t.cu
index 60d329a93..c5a2be593 100644
--- a/inc/dg/runge_kutta_t.cu
+++ b/inc/dg/runge_kutta_t.cu
@@ -91,6 +91,27 @@ int main()
         dg::blas1::axpby( 1., sol , -1., u1);
         std::cout << "Norm of error in "<<std::setw(24) <<name<<"\t"<<sqrt(dg::blas1::dot( u1, u1))<<"\n";
     }
+    std::cout << "Shu-Osher Methods with "<<N<<" steps:\n";
+    names = std::vector<std::string> {
+        "SSPRK-2-2",
+        "SSPRK-3-2",
+        "SSPRK-3-3",
+        "SSPRK-5-3",
+        "SSPRK-5-4",
+    };
+    for( auto name : names)
+    {
+        u = solution(t_start, damping, omega_0, omega_drive);
+        std::array<double, 2> u1(u), sol = solution(t_end, damping, omega_0, omega_drive);
+        dg::ShuOsher<std::array<double,2>> rk( name, u);
+        const double dt = (t_end-t_start)/(double)N;
+        dg::blas1::copy( u, u1);
+        double t0 = t_start;
+        for( unsigned i=0; i<N; i++)
+            rk.step( functor, t0, u1, t0, u1, dt);
+        dg::blas1::axpby( 1., sol , -1., u1);
+        std::cout << "Norm of error in "<<std::setw(24) <<name<<"\t"<<sqrt(dg::blas1::dot( u1, u1))<<"\n";
+    }
     ///-------------------------------Implicit Methods----------------------//
     const unsigned N_im = 10; //we can take fewer steps
     const double dt_im = (t_end - t_start)/(double)N_im;
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index 11f771433..696463c05 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -165,6 +165,109 @@ struct ButcherTableau{
     bool m_embedded = false;
 };
 
+/**
+ * @brief Manage coefficients in Shu-Osher form
+ *
+ * Currently only explicit tables that are marked with "Shu-Osher-Form" are available in this form
+ * @copydoc hide_explicit_butcher_tableaus
+ * @note A Shu-Osher Tableau can be uniquely converted to a ButcherTableau but the converse is not true
+ *
+ * @tparam real_type type of the coefficients
+ * @sa ShuOsher
+ * @ingroup time
+ */
+template<class real_type>
+struct ShuOsherTableau
+{
+    using value_type = real_type;
+    ///No memory allocation
+    ShuOsherTableau(){}
+    /*! @brief Construct a non-embedded explicit tableau
+     * @param stages number of stages
+     * @param order (global) order of the resulting method
+     * @param alpha_v s*s/2 real numbers interpreted as a linearized triangle
+     * @param beta_v s*s/2 real numbers interpreted as a linearized triangle
+     */
+    ShuOsherTableau( unsigned stages, unsigned order, const std::vector<real_type>& alpha_v, const std::vector<real_type>& beta_v){
+        // parse the input into matrices
+        m_stages = stages;
+        m_order = order;
+        m_alpha = dg::Operator<real_type>(stages, 0);
+        m_beta = dg::Operator<real_type>(stages, 0);
+        unsigned j=0;
+        for( unsigned i=0; i<stages; i++)
+            for( unsigned k=0; k<i+1; k++)
+            {
+                m_alpha(i,k) = alpha_v[j];
+                m_beta(i,k) = beta_v[j];
+                j++;
+            }
+        //std::cout << m_alpha << m_beta;
+    }
+
+    /**
+     * @brief A Shu-Osher Tableau can be converted to a Butcher table
+     *
+     * @return the corresponding Butcher Tableau
+     */
+    operator ButcherTableau<real_type>( )const{
+        //the converse is not unique:
+        //AN  EXTENSION  AND  ANALYSISOF  THE
+        //SHU-OSHER  REPRESENTATIONOF  RUNGE-KUTTA  METHODS
+        dg::Operator<real_type> a(m_stages, 0);
+        std::vector< real_type > b(m_stages), c(m_stages);
+        //convert to Butcher tableau
+        for( unsigned i=1; i<m_stages; i++)
+        {
+            for( unsigned k=0; k<i; k++)
+            {
+                a( i, k) = m_beta ( i-1, k);
+                for( unsigned j = k+1; j<i; j++)
+                    a(i,k) += m_alpha( i-1,j)*a(j,k);
+            }
+        }
+        for( unsigned k=0; k<m_stages; k++)
+        {
+            b[k] = m_beta( m_stages-1, k);
+            for( unsigned j=k+1; j<m_stages; j++)
+                b[k] += m_alpha(m_stages-1, j)*a( j, k);
+        }
+        for( unsigned i=0; i<m_stages; i++)
+        {
+            c[i] = a(i,0);
+            for( unsigned k=1; k<m_stages; k++)
+                c[i] += a(i,k);
+        }
+        dg::ButcherTableau<real_type> tableau(m_stages, m_order, &a.data()[0], &b[0], &c[0]);
+        return tableau;
+    }
+    /**
+    * @brief Read the alpha_ij coefficients
+    * @param i row number 0<=i<s, i>=s results in undefined behaviour
+    * @param j col number 0<=j<s, j>=s results in undefined behaviour
+    * @return alpha_ij
+    */
+    real_type alpha( unsigned i, unsigned j){ return m_alpha(i,j);}
+    /**
+    * @brief Read the beta_ij coefficients
+    * @param i row number 0<=i<s, i>=s results in undefined behaviour
+    * @param j col number 0<=j<s, j>=s results in undefined behaviour
+    * @return beta_ij
+    */
+    real_type beta( unsigned i, unsigned j){ return m_beta(i,j);}
+    ///The number of stages s
+    unsigned num_stages() const  {
+        return m_stages;
+    }
+    ///global order of accuracy for the method
+    unsigned order() const {
+        return m_order;
+    }
+    private:
+    unsigned m_stages, m_order;
+    dg::Operator<real_type> m_alpha, m_beta;
+};
+
 ///@cond
 namespace tableau{
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%%Classic Butcher tables%%%%%%%%%%%%%%%%%%
@@ -824,73 +927,38 @@ ButcherTableau<real_type> ark548l2sa_dirk_8_4_5()
 template<class real_type>
 dg::ButcherTableau<real_type> shuosher2butcher( unsigned stages, unsigned order, std::vector<real_type> alpha_v, std::vector<real_type> beta_v)
 {
-    //first parse the input into matrices
-    dg::Operator<real_type> alpha(stages, 0), beta(stages, 0), a(stages, 0);
-    unsigned j=0;
-    for( unsigned i=0; i<stages; i++)
-        for( unsigned k=0; k<i+1; k++)
-        {
-            alpha(i,k) = alpha_v[j];
-            beta(i,k) = beta_v[j];
-            j++;
-        }
-    std::vector< real_type > b(stages), c(stages);
-    //std::cout << alpha << beta;
-
-    //now convert to Butcher tableau
-    for( unsigned i=1; i<stages; i++)
-    {
-        for( unsigned k=0; k<i; k++)
-        {
-            a( i, k) = beta ( i-1, k);
-            for( unsigned j = k+1; j<i; j++)
-                a(i,k) += alpha( i-1,j)*a(j,k);
-        }
-    }
-    for( unsigned k=0; k<stages; k++)
-    {
-        b[k] = beta( stages-1, k);
-        for( unsigned j=k+1; j<stages; j++)
-            b[k] += alpha(stages-1, j)*a( j, k);
-    }
-    for( unsigned i=0; i<stages; i++)
-    {
-        c[i] = a(i,0);
-        for( unsigned k=1; k<stages; k++)
-            c[i] += a(i,k);
-    }
-    dg::ButcherTableau<real_type> tableau(stages, order, &a.data()[0], &b[0], &c[0]);
-    return tableau;
+    ShuOsherTableau<real_type> shu( stages, order, alpha_v, beta_v);
+    return dg::ButcherTableau<real_type>(shu);
 }
 
 template<class real_type>
-ButcherTableau<real_type> ssprk_2_2()
+ShuOsherTableau<real_type> ssprk_2_2()
 {
     unsigned stages=2, order = 2;
     std::vector<double> alpha_v = {1., 0.5, 0.5};
     std::vector<double> beta_v = {1., 0., 0.5};
-    return shuosher2butcher( stages, order, alpha_v, beta_v);
+    return ShuOsherTableau<real_type>( stages, order, alpha_v, beta_v);
 }
 template<class real_type>
-ButcherTableau<real_type> ssprk_3_2()
+ShuOsherTableau<real_type> ssprk_3_2()
 {
     //CFLL = 2 -> eff = 0.66
     unsigned stages=3, order = 2;
     std::vector<double> alpha_v = {1., 0, 1., 1./3., 0, 2./3.};
     std::vector<double> beta_v = {0.5, 0., 0.5, 0., 0., 1./3.};
-    return shuosher2butcher( stages, order, alpha_v, beta_v);
+    return ShuOsherTableau<real_type>( stages, order, alpha_v, beta_v);
 }
 template<class real_type>
-ButcherTableau<real_type> ssprk_3_3()
+ShuOsherTableau<real_type> ssprk_3_3()
 {
     //CFL = 1 -> eff = 0.33
     unsigned stages=3, order = 3;
     std::vector<double> alpha_v = {1.,3./4.,1./4.,1./3.,0.,2./3.};
     std::vector<double> beta_v = {1., 0., 1./4.,0.,0.,2./3.};
-    return shuosher2butcher( stages, order, alpha_v, beta_v);
+    return ShuOsherTableau<real_type>( stages, order, alpha_v, beta_v);
 }
 template<class real_type>
-ButcherTableau<real_type> ssprk_5_3()
+ShuOsherTableau<real_type> ssprk_5_3()
 {
     //Ruuth 2005
     //CFL = 2.6 -> eff = 0.5
@@ -901,11 +969,11 @@ ButcherTableau<real_type> ssprk_5_3()
     std::vector<double> beta_v = {
         0.37726891511710, 0, 0.37726891511710, 0, 0, 0.16352294089771, 0.00071997378654, 0, 0, 0.34217696850008, 0.00277719819460, 0.00001567934613, 0, 0, 0.29786487010104
     };
-    return shuosher2butcher( stages, order, alpha_v, beta_v);
+    return ShuOsherTableau<real_type>( stages, order, alpha_v, beta_v);
 }
 
 template<class real_type>
-ButcherTableau<real_type> ssprk_5_4()
+ShuOsherTableau<real_type> ssprk_5_4()
 {
     //Spiteri & Ruuth 2005
     //CLM = 1.5 -> eff = 0.37 , better than 3_3
@@ -916,7 +984,7 @@ ButcherTableau<real_type> ssprk_5_4()
     std::vector<double> beta_v = {
         0.39175222700392, 0, 0.36841059262959, 0, 0, 0.25189177424738, 0, 0, 0, 0.54497475021237, 0, 0, 0, 0.08460416338212, 0.22600748319395
     };
-    return shuosher2butcher( stages, order, alpha_v, beta_v);
+    return ShuOsherTableau<real_type>( stages, order, alpha_v, beta_v);
 }
 
 
@@ -973,16 +1041,95 @@ enum tableau_identifier{
     KVAERNO_7_4_5,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Kvaerno-7-4-5</a>
     ARK548L2SA_DIRK_8_4_5,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-8-4-5 (implicit)</a>
     // SSP RK tableaus
-    SSPRK_2_2, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
-    SSPRK_3_2, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
-    SSPRK_3_3, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
-    SSPRK_5_3, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
-    SSPRK_5_4 //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a>
+    SSPRK_2_2, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a> "Shu-Osher-Form"
+    SSPRK_3_2, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a> "Shu-Osher-Form"
+    SSPRK_3_3, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a> "Shu-Osher-Form"
+    SSPRK_5_3, //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a> "Shu-Osher-Form"
+    SSPRK_5_4 //!< <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK</a> "Shu-Osher-Form"
 };
 
 ///@cond
 namespace create{
 
+static std::unordered_map<std::string, enum tableau_identifier> str2id{
+    //Explicit methods
+    {"Euler", EXPLICIT_EULER_1_1},
+    {"Midpoint-2-2", MIDPOINT_2_2},
+    {"Kutta-3-3", KUTTA_3_3},
+    {"Runge-Kutta-4-4", CLASSIC_4_4},
+    //Embedded explicit methods
+    {"Heun-Euler-2-1-2", HEUN_EULER_2_1_2},
+    {"Bogacki-Shampine-4-2-3", BOGACKI_SHAMPINE_4_2_3},
+    {"ARK-4-2-3 (explicit)", ARK324L2SA_ERK_4_2_3},
+    {"Zonneveld-5-3-4", ZONNEVELD_5_3_4},
+    {"ARK-6-3-4 (explicit)", ARK436L2SA_ERK_6_3_4},
+    {"Sayfy-Aburub-6-3-4", SAYFY_ABURUB_6_3_4},
+    {"Cash-Karp-6-4-5", CASH_KARP_6_4_5},
+    {"Fehlberg-6-4-5", FEHLBERG_6_4_5},
+    {"Dormand-Prince-7-4-5", DORMAND_PRINCE_7_4_5},
+    {"ARK-8-4-5 (explicit)", ARK548L2SA_ERK_8_4_5},
+    {"Verner-8-5-6", VERNER_8_5_6},
+    {"Fehlberg-13-7-8", FEHLBERG_13_7_8},
+    {"Feagin-17-8-10", FEAGIN_17_8_10},
+    //Implicit methods
+    {"Euler (implicit)", IMPLICIT_EULER_1_1},
+    {"Midpoint (implicit)", IMPLICIT_MIDPOINT_1_2},
+    {"Trapezoidal-2-2", TRAPEZOIDAL_2_2},
+    {"SDIRK-2-1-2", SDIRK_2_1_2},
+    {"Billington-3-3-2", BILLINGTON_3_3_2},
+    {"TRBDF2-3-3-2", TRBDF2_3_3_2},
+    {"Kvaerno-4-2-3", KVAERNO_4_2_3},
+    {"ARK-4-2-3 (implicit)", ARK324L2SA_DIRK_4_2_3},
+    {"Cash-5-2-4", CASH_5_2_4},
+    {"Cash-5-3-4", CASH_5_3_4},
+    {"SDIRK-5-3-4", SDIRK_5_3_4},
+    {"Kvaerno-5-3-4", KVAERNO_5_3_4},
+    {"ARK-6-3-4 (implicit)", ARK436L2SA_DIRK_6_3_4},
+    {"Kvaerno-7-4-5", KVAERNO_7_4_5},
+    {"ARK-8-4-5 (implicit)", ARK548L2SA_DIRK_8_4_5},
+    {"SSPRK-2-2", SSPRK_2_2},
+    {"SSPRK-3-2", SSPRK_3_2},
+    {"SSPRK-3-3", SSPRK_3_3},
+    {"SSPRK-5-3", SSPRK_5_3},
+    {"SSPRK-5-4", SSPRK_5_4},
+};
+enum tableau_identifier str2tableau( std::string name)
+{
+    if( str2id.find(name) == str2id.end())
+        throw dg::Error(dg::Message(_ping_)<<"Tableau "<<name<<" not found!");
+    else
+        return str2id[name];
+}
+std::string tableau2str( enum tableau_identifier id)
+{
+    for( auto name: str2id)
+    {
+        if( name.second == id)
+            return name.first;
+    }
+    throw dg::Error(dg::Message(_ping_)<<"Tableau conversion failed!");
+}
+
+template<class real_type>
+ShuOsherTableau<real_type> shuosher_tableau( enum tableau_identifier id)
+{
+    switch(id)
+    {
+        case SSPRK_2_2:
+            return dg::tableau::ssprk_2_2<real_type>();
+        case SSPRK_3_2:
+            return dg::tableau::ssprk_3_2<real_type>();
+        case SSPRK_3_3:
+            return dg::tableau::ssprk_3_3<real_type>();
+        case SSPRK_5_3:
+            return dg::tableau::ssprk_5_3<real_type>();
+        case SSPRK_5_4:
+            return dg::tableau::ssprk_5_4<real_type>();
+        default:
+            throw dg::Error(dg::Message(_ping_)<<"Tableau "<<tableau2str(id)<<" is not in Shu-Osher form!");
+    }
+    return ShuOsherTableau<real_type>(); //avoid compiler warning
+}
 template<class real_type>
 ButcherTableau<real_type> tableau( enum tableau_identifier id)
 {
@@ -1051,70 +1198,24 @@ ButcherTableau<real_type> tableau( enum tableau_identifier id)
             return dg::tableau::kvaerno_7_4_5<real_type>();
         case ARK548L2SA_DIRK_8_4_5:
             return dg::tableau::ark548l2sa_dirk_8_4_5<real_type>();
-        case SSPRK_2_2:
-            return dg::tableau::ssprk_2_2<real_type>();
-        case SSPRK_3_2:
-            return dg::tableau::ssprk_3_2<real_type>();
-        case SSPRK_3_3:
-            return dg::tableau::ssprk_3_3<real_type>();
-        case SSPRK_5_3:
-            return dg::tableau::ssprk_5_3<real_type>();
-        case SSPRK_5_4:
-            return dg::tableau::ssprk_5_4<real_type>();
+        default:
+            return ButcherTableau<real_type>(shuosher_tableau<real_type>(id));
     }
-    return ButcherTableau<real_type>();
+    return ButcherTableau<real_type>(); //avoid compiler warning
 }
 
+
+template<class real_type>
+ShuOsherTableau<real_type> shuosher_tableau( std::string name)
+{
+        return shuosher_tableau<real_type>( str2tableau(name));
+}
 template<class real_type>
 ButcherTableau<real_type> tableau( std::string name)
 {
-    static std::unordered_map<std::string, enum tableau_identifier> str2id{
-        //Explicit methods
-        {"Euler", EXPLICIT_EULER_1_1},
-        {"Midpoint-2-2", MIDPOINT_2_2},
-        {"Kutta-3-3", KUTTA_3_3},
-        {"Runge-Kutta-4-4", CLASSIC_4_4},
-        //Embedded explicit methods
-        {"Heun-Euler-2-1-2", HEUN_EULER_2_1_2},
-        {"Bogacki-Shampine-4-2-3", BOGACKI_SHAMPINE_4_2_3},
-        {"ARK-4-2-3 (explicit)", ARK324L2SA_ERK_4_2_3},
-        {"Zonneveld-5-3-4", ZONNEVELD_5_3_4},
-        {"ARK-6-3-4 (explicit)", ARK436L2SA_ERK_6_3_4},
-        {"Sayfy-Aburub-6-3-4", SAYFY_ABURUB_6_3_4},
-        {"Cash-Karp-6-4-5", CASH_KARP_6_4_5},
-        {"Fehlberg-6-4-5", FEHLBERG_6_4_5},
-        {"Dormand-Prince-7-4-5", DORMAND_PRINCE_7_4_5},
-        {"ARK-8-4-5 (explicit)", ARK548L2SA_ERK_8_4_5},
-        {"Verner-8-5-6", VERNER_8_5_6},
-        {"Fehlberg-13-7-8", FEHLBERG_13_7_8},
-        {"Feagin-17-8-10", FEAGIN_17_8_10},
-        //Implicit methods
-        {"Euler (implicit)", IMPLICIT_EULER_1_1},
-        {"Midpoint (implicit)", IMPLICIT_MIDPOINT_1_2},
-        {"Trapezoidal-2-2", TRAPEZOIDAL_2_2},
-        {"SDIRK-2-1-2", SDIRK_2_1_2},
-        {"Billington-3-3-2", BILLINGTON_3_3_2},
-        {"TRBDF2-3-3-2", TRBDF2_3_3_2},
-        {"Kvaerno-4-2-3", KVAERNO_4_2_3},
-        {"ARK-4-2-3 (implicit)", ARK324L2SA_DIRK_4_2_3},
-        {"Cash-5-2-4", CASH_5_2_4},
-        {"Cash-5-3-4", CASH_5_3_4},
-        {"SDIRK-5-3-4", SDIRK_5_3_4},
-        {"Kvaerno-5-3-4", KVAERNO_5_3_4},
-        {"ARK-6-3-4 (implicit)", ARK436L2SA_DIRK_6_3_4},
-        {"Kvaerno-7-4-5", KVAERNO_7_4_5},
-        {"ARK-8-4-5 (implicit)", ARK548L2SA_DIRK_8_4_5},
-        {"SSPRK-2-2", SSPRK_2_2},
-        {"SSPRK-3-2", SSPRK_3_2},
-        {"SSPRK-3-3", SSPRK_3_3},
-        {"SSPRK-5-3", SSPRK_5_3},
-        {"SSPRK-5-4", SSPRK_5_4},
-    };
-    if( str2id.find(name) == str2id.end())
-        throw dg::Error(dg::Message(_ping_)<<"Butcher Tableau "<<name<<" not found!");
-    else
-        return tableau<real_type>( str2id[name]);
+        return tableau<real_type>( str2tableau(name));
 }
+
 }//namespace create
 ///@endcond
 
@@ -1126,11 +1227,11 @@ ButcherTableau<real_type> tableau( std::string name)
  *   Midpoint-2-2           | dg::MIDPOINT_2_2           | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Midpoint method</a>
  *   Kutta-3-3              | dg::KUTTA_3_3              | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Kutta-3-3</a>
  *   Runge-Kutta-4-4        | dg::CLASSIC_4_4            | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">"The" Runge-Kutta method</a>
- *   SSPRK-2-2              | dg::SSPRK_2_2                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (2,2)</a> CFL_eff = 0.5
- *   SSPRK-3-2              | dg::SSPRK_3_2                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (3,2)</a> CFL_eff = 0.66
- *   SSPRK-3-3              | dg::SSPRK_3_3                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (3,3)</a> CFL_eff = 0.33
- *   SSPRK-5-3              | dg::SSPRK_5_3                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (5,3)</a> CFL_eff = 0.5
- *   SSPRK-5-4              | dg::SSPRK_5_4                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (5,4)</a> CFL_eff = 0.37
+ *   SSPRK-2-2              | dg::SSPRK_2_2                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (2,2)</a> CFL_eff = 0.5 "Shu-Osher-Form"
+ *   SSPRK-3-2              | dg::SSPRK_3_2                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (3,2)</a> CFL_eff = 0.66 "Shu-Osher-Form"
+ *   SSPRK-3-3              | dg::SSPRK_3_3                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (3,3)</a> CFL_eff = 0.33 "Shu-Osher-Form"
+ *   SSPRK-5-3              | dg::SSPRK_5_3                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (5,3)</a> CFL_eff = 0.5 "Shu-Osher-Form"
+ *   SSPRK-5-4              | dg::SSPRK_5_4                  | <a href="https://epubs.siam.org/doi/pdf/10.1137/S0036142901389025">SSPRK (5,4)</a> CFL_eff = 0.37 "Shu-Osher-Form"
  *   Heun-Euler-2-1-2       | dg::HEUN_EULER_2_1_2       | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Heun-Euler-2-1-2</a>
  *   Bogacki-Shampine-4-2-3 | dg::BOGACKI_SHAMPINE_4_2_3 | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Bogacki-Shampine</a> (fsal)
  *   ARK-4-2-3 (explicit)   | dg::ARK324L2SA_ERK_4_2_3   | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-4-2-3 (explicit)</a>
@@ -1219,4 +1320,52 @@ struct ConvertsToButcherTableau
     ButcherTableau<real_type> m_t;
 };
 
+/*! @brief Convert identifiers to their corresponding \c dg::ShuOsherTableau
+ *
+ * This is a helper class to simplify the interfaces of our timestepper functions and classes.
+ * The sole purpose is to implicitly convert either a ShuOsherTableau or one of
+ * the following identifiers to an instance of a ShuOsherTableau.
+ *
+ * Explicit methods (the ones that are marked with "Shu-Osher-Form"
+ * @copydoc hide_explicit_butcher_tableaus
+ * @param real_type The type of the coefficients in the ShuOsherTableau
+ * @ingroup time
+ */
+template<class real_type>
+struct ConvertsToShuOsherTableau
+{
+    using value_type = real_type;
+    ///Of course a ShuOsherTableau converts to a ShuOsherTableau
+    ///Useful if you constructed your very own coefficients
+    ConvertsToShuOsherTableau( ShuOsherTableau<real_type> tableau): m_t(tableau){}
+
+    /*! @brief Create ShuOsherTableau from \c dg::tableau_identifier
+    *
+    * The use of this constructor might be a bit awkward because you'll have to write all caps.
+    * @param id the identifier, for example \c dg::SSPRK_3_3
+    */
+    ConvertsToShuOsherTableau( enum tableau_identifier id):m_t( dg::create::shuosher_tableau<real_type>(id)){}
+    /*! @brief Create ShuOsherTableau from its name (very useful)
+    *
+    *  @note In some of the links in the Description below you might want to use the search function of your browser to find the indicated method
+    *
+    * Explicit methods
+    * @copydoc hide_explicit_butcher_tableaus
+    * Implicit methods
+    * @copydoc hide_implicit_butcher_tableaus
+    * @param name The name of the tableau as stated in the Name column above, as a string, for example "SSPRK-3-3"
+    */
+    ConvertsToShuOsherTableau( std::string name):m_t( dg::create::shuosher_tableau<real_type>(name)){}
+    ///@copydoc ConvertsToShuOsherTableau(std::string)
+    ConvertsToShuOsherTableau( const char* name):m_t( dg::create::shuosher_tableau<real_type>(std::string(name))){}
+    ///Convert to ShuOsherTableau
+    ///
+    ///which means an object can be directly assigned to a ShuOsherTableau
+    operator ShuOsherTableau<real_type>( )const{
+        return m_t;
+    }
+    private:
+    ShuOsherTableau<real_type> m_t;
+};
+
 }//namespace dg
diff --git a/inc/dg/topology/filter.h b/inc/dg/topology/filter.h
index c803b90b2..fd28c6512 100644
--- a/inc/dg/topology/filter.h
+++ b/inc/dg/topology/filter.h
@@ -126,7 +126,7 @@ modal_filter( UnaryOp op, const aRealMPITopology3d<real_type>& t)
  * where \f$ V\f$ is the Vandermonde matrix (the backward transformation matrix)
  * and \f$ D \f$ is a diagonal matrix with \f$ D_{ii} = \sigma(i)\f$
  * @copydoc hide_matrix
- * @copydoc hide_container
+ * @copydoc hide_ContainerType
  * @ingroup misc
  */
 template<class MatrixType, class ContainerType>
-- 
GitLab


From 4a9dc83539e1b4cfabfbaf2122896ef2ccc9f292 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Dec 2020 15:44:04 +0100
Subject: [PATCH 416/540] Remove regularized_step from Shu-Osher

- we always apply a filter (if not the Runge-Kutta method is entirely
the same)
- the plan is to provide separate Filter timestepper whenever needed
---
 inc/dg/runge_kutta.h    | 31 +++++++++++++++----------------
 inc/dg/runge_kutta_t.cu |  3 ++-
 inc/dg/tableau.h        |  1 +
 3 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 328b975f4..3e59c842e 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -34,7 +34,7 @@ namespace dg{
 
 
 /**
-* @brief Struct for Embedded Runge Kutta explicit time-step with error estimate
+* @brief Embedded Runge Kutta explicit time-step with error estimate
 * \f[
  \begin{align}
     k_i = f\left( t^n + c_i \Delta t, u^n + \Delta t \sum_{j=1}^{s-1} a_{ij} k_j\right) \\
@@ -102,6 +102,7 @@ struct ERKStep
     bool m_ignore_fsal = false;
 };
 
+///@cond
 template< class ContainerType>
 template< class RHS>
 void ERKStep<ContainerType>::step( RHS& f, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt, ContainerType& delta)
@@ -241,13 +242,14 @@ void ERKStep<ContainerType>::step( RHS& f, value_type t0, const ContainerType& u
         swap( m_k[0], m_k[s-1]); //enable free swap functions
     }
 }
+///@endcond
 
 
 //MW: if ever we want to change the SolverType at runtime (with an input parameter e.g.) make it a new parameter in the solve method (either abstract type or template like RHS)
 
 
 /*!
- * @brief Struct for Additive Runge Kutta (semi-implicit) time-step with error estimate
+ * @brief Additive Runge Kutta (semi-implicit) time-step with error estimate
  * following
  * <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Mathematics.html#arkstep-additive-runge-kutta-methods">The ARKode library</a>
  *
@@ -384,6 +386,7 @@ struct ARKStep
     value_type m_t1 = 1e300;
 };
 
+///@cond
 template<class ContainerType, class SolverType>
 template< class Explicit, class Implicit>
 void ARKStep<ContainerType, SolverType>::step( Explicit& ex, Implicit& im, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt, ContainerType& delta)
@@ -471,9 +474,10 @@ void ARKStep<ContainerType, SolverType>::step( Explicit& ex, Implicit& im, value
     //make sure (t1,u1) is the last call to ex
     ex(t1,u1,m_kE[0]);
 }
+///@endcond
 
 /**
-* @brief Struct for Runge-Kutta fixed-step explicit time-integration
+* @brief Runge-Kutta fixed-step explicit time-integration
 * \f[
  \begin{align}
     k_i = f\left( t^n + c_i \Delta t, u^n + \Delta t \sum_{j=1}^{s-1} a_{ij} k_j\right) \\
@@ -582,7 +586,7 @@ struct IdentityFilter
 
 };
 /**
-* @brief Struct for Shu-Osher fixed-step explicit time-integration
+* @brief Shu-Osher fixed-step explicit time-integration with Slope Limiter / Filter
 * \f[
  \begin{align}
     u_0 &= u_n \\
@@ -634,15 +638,8 @@ struct ShuOsher
     ///@copydoc RungeKutta::copyable()
     const ContainerType& copyable()const{ return m_u[0 ];}
 
-    ///@copydoc RungeKutta::step()
-    template<class RHS>
-    void step( RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
-        dg::IdentityFilter id;
-        regularized_step( id, rhs, t0, u0, t1, u1, dt);
-    }
-
     /**
-    * @brief Advance one step using a limiter
+    * @brief Advance one step
     *
     * @copydoc hide_rhs
     * @copydoc hide_limiter
@@ -656,7 +653,7 @@ struct ShuOsher
     * @note on return \c rhs(t1, u1) will be the last call to \c rhs (this is useful if \c RHS holds state, which is then updated to the current timestep)
     */
     template<class Limiter, class RHS>
-    void regularized_step( Limiter& limiter, RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
+    void step( Limiter& limiter, RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
         unsigned s = m_t.num_stages();
         std::vector<value_type> ts( m_t.num_stages()+1);
         ts[0] = t0;
@@ -708,7 +705,7 @@ struct ShuOsher
 };
 
 /*!
- * @brief Struct for diagonally implicit Runge Kutta time-step with error estimate
+ * @brief diagonally implicit Runge Kutta time-step with error estimate
 * \f[
  \begin{align}
     k_i = f\left( t^n + c_i \Delta t, u^n + \Delta t \sum_{j=1}^{s} a_{ij} k_j\right) \\
@@ -718,7 +715,7 @@ struct ShuOsher
 \f]
  *
  * So far we did not implement the use of a mass matrix \c M.
- * You can provide your own coefficients or use one of the embedded methods
+ * You can provide your own coefficients or use one of the methods
  * in the following table:
  * @copydoc hide_implicit_butcher_tableaus
  *
@@ -811,6 +808,7 @@ struct DIRKStep
     std::vector<ContainerType> m_kI;
 };
 
+///@cond
 template<class ContainerType, class SolverType>
 template< class RHS>
 void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt, ContainerType& delta)
@@ -901,8 +899,9 @@ void DIRKStep<ContainerType, SolverType>::step( RHS& rhs, value_type t0, const C
             }
     }
 }
+///@endcond
 /**
-* @brief Struct for Runge-Kutta fixed-step implicit time-integration
+* @brief Runge-Kutta fixed-step implicit time-integration
 * \f[
  \begin{align}
     k_i = f\left( t^n + c_i \Delta t, u^n + \Delta t \sum_{j=1}^{s} a_{ij} k_j\right) \\
diff --git a/inc/dg/runge_kutta_t.cu b/inc/dg/runge_kutta_t.cu
index c5a2be593..24a7cab34 100644
--- a/inc/dg/runge_kutta_t.cu
+++ b/inc/dg/runge_kutta_t.cu
@@ -104,11 +104,12 @@ int main()
         u = solution(t_start, damping, omega_0, omega_drive);
         std::array<double, 2> u1(u), sol = solution(t_end, damping, omega_0, omega_drive);
         dg::ShuOsher<std::array<double,2>> rk( name, u);
+        dg::IdentityFilter id;
         const double dt = (t_end-t_start)/(double)N;
         dg::blas1::copy( u, u1);
         double t0 = t_start;
         for( unsigned i=0; i<N; i++)
-            rk.step( functor, t0, u1, t0, u1, dt);
+            rk.step( id, functor, t0, u1, t0, u1, dt);
         dg::blas1::axpby( 1., sol , -1., u1);
         std::cout << "Norm of error in "<<std::setw(24) <<name<<"\t"<<sqrt(dg::blas1::dot( u1, u1))<<"\n";
     }
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index 696463c05..53e82b462 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -1087,6 +1087,7 @@ static std::unordered_map<std::string, enum tableau_identifier> str2id{
     {"ARK-6-3-4 (implicit)", ARK436L2SA_DIRK_6_3_4},
     {"Kvaerno-7-4-5", KVAERNO_7_4_5},
     {"ARK-8-4-5 (implicit)", ARK548L2SA_DIRK_8_4_5},
+    //Shu-Osher methods
     {"SSPRK-2-2", SSPRK_2_2},
     {"SSPRK-3-2", SSPRK_3_2},
     {"SSPRK-3-3", SSPRK_3_3},
-- 
GitLab


From 07e459dcce6aac9bfae984d548d93015981f178f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Dec 2020 15:49:15 +0100
Subject: [PATCH 417/540] Add FilteredBDF and FilteredExplicitMultistep

the first two multistep methods with filter function
---
 inc/dg/multistep.h | 311 +++++++++++++++++++++++++++++++++++++--------
 1 file changed, 259 insertions(+), 52 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index ffdd18804..1d9e1ada4 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <map>
 #include "implicit.h"
 #include "runge_kutta.h"
 
@@ -42,7 +43,7 @@ namespace dg{
 */
 
 /**
-* @brief Struct for Adams-Bashforth explicit multistep time-integration
+* @brief Adams-Bashforth explicit multistep time-integration
 * \f[ u^{n+1} = u^n + \Delta t\sum_{j=0}^{s-1} b_j f\left(t^n - j \Delta t, u^{n-j}\right) \f]
 *
 * with coefficients taken from https://en.wikipedia.org/wiki/Linear_multistep_method
@@ -108,6 +109,7 @@ struct AdamsBashforth
     * @param t (write-only) contains timestep corresponding to \c u on output
     * @param u (write-only) contains next step of the integration on output
     * @note the implementation is such that on output the last call to the rhs is at the new (t,u). This might be interesting if the call to the rhs changes its state.
+    * @attention The first few steps after the call to the init function are performed with an explicit Runge-Kutta method of the same order
     */
     template< class RHS>
     void step( RHS& rhs, value_type& t, ContainerType& u);
@@ -119,6 +121,7 @@ struct AdamsBashforth
     unsigned m_k, m_counter;
 };
 
+///@cond
 template< class ContainerType>
 template< class RHS>
 void AdamsBashforth<ContainerType>::init( RHS& f, value_type t0, const ContainerType& u0, value_type dt)
@@ -135,8 +138,15 @@ void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType&
 {
     if( m_counter < m_k-1)
     {
-        RungeKutta<ContainerType> rk( "ARK-4-2-3 (explicit)", u);
-        rk.step( f, t, u, t, u, m_dt, tmp);
+        std::map<unsigned, enum tableau_identifier> order2method{
+            {1, EXPLICIT_EULER_1_1},
+            {2, MIDPOINT_2_2},
+            {3, KUTTA_3_3},
+            {4, CLASSIC_4_4},
+            {5, CASH_KARP_6_4_5}
+        };
+        dg::RungeKutta<ContainerType> rk( order2method.at(m_k), u);
+        rk.step( f, t, u, t, u, m_dt);
         m_counter++;
         m_tu = t;
         blas1::copy(  u, m_u);
@@ -151,10 +161,11 @@ void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType&
     t = m_tu = m_tu + m_dt;
     f( m_tu, m_u, m_f[0]); //evaluate f at new point
 }
+///@endcond
 
 
 /**
-* @brief Struct for Karniadakis semi-implicit multistep time-integration
+* @brief Karniadakis semi-implicit multistep time-integration
 * \f[
 * \begin{align}
     v^{n+1} = \sum_{q=0}^2 \alpha_q v^{n-q} + \Delta t\left[\left(\sum_{q=0}^2\beta_q  \hat E(t^{n}-q\Delta t, v^{n-q})\right) + \gamma_0\hat I(t^{n}+\Delta t, v^{n+1})\right]
@@ -264,7 +275,7 @@ struct Karniadakis
     * @param t (write-only), contains timestep corresponding to \c u on output
     * @param u (write-only), contains next step of time-integration on output
     * @note the implementation is such that on output the last call to the explicit part \c ex is at the new \c (t,u). This might be interesting if the call to \c ex changes its state.
-    * @attention The first two steps after the call to the init function are performed with a semi-implicit Runge-Kutta method
+    * @attention The first few steps after the call to the init function are performed with a semi-implicit Runge-Kutta method to initialize the multistepper
     */
     template< class Explicit, class Implicit>
     void step( Explicit& ex, Implicit& im, value_type& t, ContainerType& u);
@@ -361,10 +372,11 @@ void Karniadakis<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, valu
 ///@endcond
 
 /**
-* @brief Struct for Backward differentiation formula implicit multistep time-integration
+* @brief Backward differentiation formula implicit multistep time-integration with Limiter/Filter
 * \f[
 * \begin{align}
-    v^{n+1} = \sum_{q=0}^{s-1} \alpha_q v^{n-q} + \Delta t\beta\hat I(t^{n}+\Delta t, v^{n+1})
+    \tilde v &= \sum_{q=0}^{s-1} \alpha_q v^{n-q} + \Delta t\beta\hat I(t^{n}+\Delta t, v^{n+1}) \\
+    v^{n+1} &= \Lambda\Pi\left(\tilde v\right)
     \end{align}
     \f]
 
@@ -388,19 +400,20 @@ outweighs the increased computational cost of the additional inversions.
 * @ingroup time
 */
 template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
-struct BDF
+struct FilteredBDF
 {
 
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
     ///@copydoc RungeKutta::RungeKutta()
-    BDF(){}
+    FilteredBDF(){}
 
     ///@copydoc construct()
     template<class ...SolverParams>
-    BDF( unsigned order, SolverParams&& ...ps):m_solver( std::forward<SolverParams>(ps)...), m_u( order, m_solver.copyable()), m_f(m_solver.copyable()) {
+    FilteredBDF( unsigned order, SolverParams&& ...ps):m_solver( std::forward<SolverParams>(ps)...), m_u( order, m_solver.copyable()), m_f(m_solver.copyable()) {
         init_coeffs(order);
         m_k = order;
+        m_counter = 0;
     }
 
     /*! @brief Reserve memory for integration and construct Solver
@@ -417,6 +430,7 @@ struct BDF
         m_k = order;
         m_u.assign( order, m_solver.copyable());
         m_f = m_solver.copyable();
+        m_counter = 0;
     }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
@@ -426,13 +440,13 @@ struct BDF
     ///Read access to the internal solver for the implicit part
     const SolverType& solver() const { return m_solver;}
 
-    ///@copydoc AdamsBashforth::init()
-    template<class RHS>
-    void init(RHS& rhs, value_type t0, const ContainerType& u0, value_type dt);
+    ///@copydoc FilteredExplicitMultistep::init()
+    template<class Limiter, class RHS>
+    void init(Limiter& limiter, RHS& rhs, value_type t0, const ContainerType& u0, value_type dt);
 
-    ///@copydoc AdamsBashforth::step()
-    template<class RHS>
-    void step(RHS& rhs, value_type& t, container_type& u);
+    ///@copydoc FilteredExplicitMultistep::step()
+    template<class Limiter, class RHS>
+    void step(Limiter& limiter, RHS& rhs, value_type& t, container_type& u);
     private:
     void init_coeffs(unsigned order){
         switch (order){
@@ -451,29 +465,45 @@ struct BDF
     ContainerType m_f;
     std::vector<value_type> m_bdf;
     value_type m_beta;
-    unsigned m_k;
+    unsigned m_k, m_counter = 0; //counts how often step has been called after init
 };
 
+///@cond
 template< class ContainerType, class SolverType>
-template<class RHS>
-void BDF<ContainerType, SolverType>::init(RHS& rhs, value_type t0,
+template<class Limiter, class RHS>
+void FilteredBDF<ContainerType, SolverType>::init(Limiter& l, RHS& rhs, value_type t0,
     const ContainerType& u0, value_type dt)
 {
     m_tu = t0, m_dt = dt;
-    dg::blas1::copy(u0, m_u[0]);
-    //Perform a number of backward euler steps
-    for (unsigned i = 0; i<m_k-1; i++){
-        rhs(t0-i*dt,m_u[i], m_f);
-        dg::blas1::axpby(-dt,m_f,1.,m_u[i],m_u[i+1]);
-    }
-    rhs( t0, u0, m_f); // and set state in f to (t0,u0)
+    l.apply( u0, m_u[m_k-1]);
+    //rhs(m_tu, m_u[m_k-1], m_f); //call f on new point
+    m_counter = 0;
 }
 
 template< class ContainerType, class SolverType>
-template<class RHS>
-void BDF<ContainerType, SolverType>::step(RHS& rhs, value_type& t, container_type& u)
+template<class Limiter, class RHS>
+void FilteredBDF<ContainerType, SolverType>::step(Limiter& l, RHS& rhs, value_type& t, container_type& u)
 {
-    //dg::WhichType<RHS> {};
+    if( m_counter < m_k-1)
+    {
+        std::map<unsigned, enum tableau_identifier> order2method{
+            {1, IMPLICIT_EULER_1_1},
+            {2, TRAPEZOIDAL_2_2},
+            {3, KVAERNO_4_2_3},
+            {4, SDIRK_5_3_4},
+            {5, KVAERNO_7_4_5},
+            {6, KVAERNO_7_4_5}
+        };
+        ImplicitRungeKutta<ContainerType, SolverType> dirk( order2method.at(m_k), m_solver);
+        dirk.step( rhs, t, u, t, u, m_dt);
+        m_counter++;
+        m_tu = t;
+        l.apply( u, m_u[m_k-1-m_counter]);
+        dg::blas1::copy(  m_u[m_k-1-m_counter], u);
+        //f( m_tu, m_u[m_k-1-m_counter], m_f);
+        return;
+    }
+    //compute right hand side of inversion equation
     dg::blas1::axpby( m_bdf[0], m_u[0], 0., m_f);
     for (unsigned i = 1; i < m_k; i++){
         dg::blas1::axpby( m_bdf[i], m_u[i], 1., m_f);
@@ -487,8 +517,87 @@ void BDF<ContainerType, SolverType>::step(RHS& rhs, value_type& t, container_typ
         dg::blas1::copy( m_u[0], u);
     std::rotate(m_u.rbegin(), m_u.rbegin() + 1, m_u.rend()); //Rotate 1 to the right (note the reverse iterator here!)
     m_solver.solve( -m_dt*m_beta, rhs, t, u, m_f);
-    dg::blas1::copy( u, m_u[0]);
+    l.apply( u, m_u[0]);
+    dg::blas1::copy(  m_u[0], u);
 }
+///@endcond
+
+/**
+* @brief Backward differentiation formula implicit multistep time-integration
+* \f[
+* \begin{align}
+    v^{n+1} = \sum_{q=0}^{s-1} \alpha_q v^{n-q} + \Delta t\beta\hat I(t^{n}+\Delta t, v^{n+1})
+    \end{align}
+    \f]
+
+    which discretizes
+    \f[
+    \frac{\partial v}{\partial t} = \hat I(t,v)
+    \f]
+    where \f$ \hat I \f$ represents the right hand side of the equations.
+    The coefficients for up to order 6 can be found at
+    https://en.wikipedia.org/wiki/Backward_differentiation_formula
+*
+* The necessary Inversion in the imlicit part is provided by the \c SolverType class.
+* Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$). For nonlinear right hand side we recommend the AndersonSolver
+*
+* @note In our experience the implicit treatment of diffusive or hyperdiffusive
+terms can significantly reduce the required number of time steps. This
+outweighs the increased computational cost of the additional inversions.
+* @copydoc hide_note_multistep
+* @copydoc hide_SolverType
+* @copydoc hide_ContainerType
+* @ingroup time
+*/
+template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
+struct BDF
+{
+
+    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
+    using container_type = ContainerType; //!< the type of the vector class in use
+    ///@copydoc RungeKutta::RungeKutta()
+    BDF(){}
+
+    ///@copydoc construct()
+    template<class ...SolverParams>
+    BDF( unsigned order, SolverParams&& ...ps): m_bdf( order,
+            std::forward<SolverParams>(ps)...) {}
+
+    /*! @brief Reserve memory for integration and construct Solver
+     *
+     * @param order Order of the BDF formula (1 <= order <= 6)
+     * @param ps Parameters that are forwarded to the constructor of \c SolverType
+     * @tparam SolverParams Type of parameters (deduced by the compiler)
+     */
+    template<class ...SolverParams>
+    void construct(unsigned order, SolverParams&& ...ps)
+    {
+        m_bdf.construct( order, std::forward<SolverParams>(ps)...);
+    }
+    ///@copydoc AdamsBashforth::copyable()
+    const ContainerType& copyable()const{ return m_bdf.copyable();}
+    ///Write access to the internal solver for the implicit part
+    SolverType& solver() { return m_bdf.solver();}
+    ///Read access to the internal solver for the implicit part
+    const SolverType& solver() const { return m_bdf.solver();}
+
+    ///@copydoc AdamsBashforth::init()
+    template<class RHS>
+    void init(RHS& rhs, value_type t0, const ContainerType& u0, value_type dt){
+        dg::IdentityFilter id;
+        m_bdf.init( id, rhs, t0, u0, dt);
+    }
+
+    ///@copydoc AdamsBashforth::step()
+    template<class RHS>
+    void step(RHS& rhs, value_type& t, container_type& u){
+        dg::IdentityFilter id;
+        m_bdf.step( id, rhs, t, u);
+    }
+    private:
+    FilteredBDF<ContainerType, SolverType> m_bdf;
+};
+
 
 
 /**
@@ -532,15 +641,18 @@ enum multistep_identifier
      */
     SSP
 };
+
+
 /**
-* @brief Struct for general explicit linear multistep time-integration
+* @brief General explicit linear multistep time-integration with Limiter / Filter
 * \f[
 * \begin{align}
-    v^{n+1} = \sum_{j=0}^{s-1} \alpha_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}\beta_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right)
+    \tilde v &= \sum_{j=0}^{s-1} \alpha_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}\beta_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right) \\
+    v^{n+1} &= \Lambda\Pi \left( \tilde v\right)
     \end{align}
     \f]
 
-    which discretizes
+    where \f$ \Lambda\Pi\f$ is the limiter, which discretizes
     \f[
     \frac{\partial v}{\partial t} = \hat f(t,v)
     \f]
@@ -551,6 +663,7 @@ enum multistep_identifier
     \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11}
 \f]
 @sa multistep_identifier
+@note This scheme is the same as ExplicitMultistep with the additional option to use a filter
 @note The schemes implemented here need more storage but may have **a larger region of absolute stability** than an AdamsBashforth method of the same order.
 *
 * @copydoc hide_note_multistep
@@ -558,16 +671,16 @@ enum multistep_identifier
 * @ingroup time
 */
 template<class ContainerType>
-struct ExplicitMultistep
+struct FilteredExplicitMultistep
 {
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
     ///@copydoc RungeKutta::RungeKutta()
-    ExplicitMultistep(){}
+    FilteredExplicitMultistep(){}
 
     ///@copydoc construct()
-    ExplicitMultistep( std::string method, unsigned stages, const ContainerType& copyable){
-        std::unordered_map < std::string, enum multistep_identifier> str2id{
+    FilteredExplicitMultistep( std::string method, unsigned stages, const ContainerType& copyable){
+        std::map < std::string, enum multistep_identifier> str2id{
             {"eBDF", eBDF},
             {"TVB", TVB},
             {"SSP", SSP}
@@ -578,7 +691,7 @@ struct ExplicitMultistep
             construct( str2id[method], stages, copyable);
     }
     ///@copydoc construct()
-    ExplicitMultistep( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
+    FilteredExplicitMultistep( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
         construct( method, stages, copyable);
     }
     /**
@@ -605,28 +718,32 @@ struct ExplicitMultistep
      * @brief Initialize timestepper. Call before using the step function.
      *
      * This routine has to be called before the first timestep is made.
+     * @copydoc hide_limiter
      * @copydoc hide_rhs
+     * @param limiter The limiter or filter to use
      * @param rhs The rhs functor
      * @param t0 The intital time corresponding to u0
      * @param u0 The initial value of the integration
      * @param dt The timestep saved for later use
      * @note the implementation is such that on output the last call to the explicit part \c ex is at \c (t0,u0). This might be interesting if the call to \c ex changes its state.
      */
-    template< class RHS>
-    void init( RHS& rhs, value_type t0, const ContainerType& u0, value_type dt);
+    template< class Limiter, class RHS>
+    void init( Limiter& limiter, RHS& rhs, value_type t0, const ContainerType& u0, value_type dt);
 
     /**
     * @brief Advance one timestep
     *
+    * @copydoc hide_limiter
+    * @param limiter The limiter or filter to use
     * @copydoc hide_rhs
     * @param rhs The rhs functor
     * @param t (write-only), contains timestep corresponding to \c u on output
     * @param u (write-only), contains next step of time-integration on output
     * @note the implementation is such that on output the last call to the explicit part \c ex is at the new \c (t,u). This might be interesting if the call to \c ex changes its state.
-    * @attention The first few steps after the call to the init function are performed with an explicit Runge-Kutta method
+    * @attention The first few steps after the call to the init function are performed with a Runge-Kutta method
     */
-    template< class RHS>
-    void step( RHS& rhs, value_type& t, ContainerType& u);
+    template< class Limiter, class RHS>
+    void step( Limiter& limiter, RHS& rhs, value_type& t, ContainerType& u);
 
   private:
     void init_coeffs(enum multistep_identifier method, unsigned stages){
@@ -711,26 +828,33 @@ struct ExplicitMultistep
     std::vector<value_type> m_a, m_b;
     unsigned m_k, m_counter; //counts how often step has been called after init
 };
-
 ///@cond
 template< class ContainerType>
-template< class RHS>
-void ExplicitMultistep<ContainerType>::init( RHS& f, value_type t0, const ContainerType& u0, value_type dt)
+template< class Limiter, class RHS>
+void FilteredExplicitMultistep<ContainerType>::init( Limiter& l, RHS& f, value_type t0, const ContainerType& u0, value_type dt)
 {
     m_tu = t0, m_dt = dt;
-    blas1::copy(  u0, m_u[m_k-1]);
+    l.apply( u0, m_u[m_k-1]);
     f(m_tu, m_u[m_k-1], m_f[m_k-1]); //call f on new point
     m_counter = 0;
 }
 
 template<class ContainerType>
-template< class RHS>
-void ExplicitMultistep<ContainerType>::step( RHS& f, value_type& t, ContainerType& u)
+template<class Limiter, class RHS>
+void FilteredExplicitMultistep<ContainerType>::step(Limiter& l, RHS& f, value_type& t, ContainerType& u)
 {
     if( m_counter < m_k-1)
     {
-        RungeKutta<ContainerType> rk( "ARK-4-2-3 (explicit)", u);
-        rk.step( f, t, u, t, u, m_dt);
+        std::map<unsigned, enum tableau_identifier> order2method{
+            {1, SSPRK_2_2},
+            {2, SSPRK_2_2},
+            {3, SSPRK_3_3},
+            {4, SSPRK_5_4},
+            {5, SSPRK_5_4},
+            {6, SSPRK_5_4}
+        };
+        ShuOsher<ContainerType> rk( order2method.at(m_k), u);
+        rk.step( l, f, t, u, t, u, m_dt);
         m_counter++;
         m_tu = t;
         blas1::copy(  u, m_u[m_k-1-m_counter]);
@@ -746,9 +870,92 @@ void ExplicitMultistep<ContainerType>::step( RHS& f, value_type& t, ContainerTyp
     //permute m_f[m_k-1], m_u[m_k-1]  to be the new m_f[0], m_u[0]
     std::rotate( m_f.rbegin(), m_f.rbegin()+1, m_f.rend());
     std::rotate( m_u.rbegin(), m_u.rbegin()+1, m_u.rend());
-    blas1::copy( u, m_u[0]); //store result
+    //apply limiter
+    l.apply( u, m_u[0]);
+    blas1::copy( m_u[0], u); //store result
     f(m_tu, m_u[0], m_f[0]); //call f on new point
 }
 ///@endcond
 
+/**
+* @brief General explicit linear multistep time-integration
+* \f[
+* \begin{align}
+    v^{n+1} = \sum_{j=0}^{s-1} \alpha_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}\beta_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right)
+    \end{align}
+    \f]
+
+    which discretizes
+    \f[
+    \frac{\partial v}{\partial t} = \hat f(t,v)
+    \f]
+    where \f$ f \f$ contains the equations.
+    The coefficients for an order 3 "eBDF" scheme are given as an example:
+    \f[
+    \alpha_0 = \frac{18}{11}\ \alpha_1 = -\frac{9}{11}\ \alpha_2 = \frac{2}{11} \\
+    \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11}
+\f]
+@sa multistep_identifier
+@note The schemes implemented here need more storage but may have **a larger region of absolute stability** than an AdamsBashforth method of the same order.
+*
+* @copydoc hide_note_multistep
+* @copydoc hide_ContainerType
+ @ingroup time
+*/
+template<class ContainerType>
+struct ExplicitMultistep
+{
+    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
+    using container_type = ContainerType; //!< the type of the vector class in use
+    ///@copydoc RungeKutta::RungeKutta()
+    ExplicitMultistep(){}
+    ///@copydoc construct()
+    ExplicitMultistep( std::string method, unsigned stages, const ContainerType& copyable): m_fem( method, stages, copyable){ }
+    ///@copydoc construct()
+    ExplicitMultistep( enum multistep_identifier method, unsigned stages, const ContainerType& copyable): m_fem( method, stages, copyable){}
+    ///@copydoc FilteredExplicitMultistep::construct()
+    void construct( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
+        m_fem.construct( method, stages, copyable);
+    }
+    ///@brief Return an object of same size as the object used for construction
+    ///@return A copyable object; what it contains is undefined, its size is important
+    const ContainerType& copyable()const{ return m_fem.copyable();}
+
+    /**
+     * @brief Initialize timestepper. Call before using the step function.
+     *
+     * This routine has to be called before the first timestep is made.
+     * @copydoc hide_rhs
+     * @param rhs The rhs functor
+     * @param t0 The intital time corresponding to u0
+     * @param u0 The initial value of the integration
+     * @param dt The timestep saved for later use
+     * @note the implementation is such that on output the last call to the explicit part \c ex is at \c (t0,u0). This might be interesting if the call to \c ex changes its state.
+     */
+    template< class RHS>
+    void init( RHS& rhs, value_type t0, const ContainerType& u0, value_type dt){
+        dg::IdentityFilter id;
+        m_fem.init( id, rhs, t0, u0, dt);
+    }
+
+    /**
+    * @brief Advance one timestep
+    *
+    * @copydoc hide_rhs
+    * @param rhs The rhs functor
+    * @param t (write-only), contains timestep corresponding to \c u on output
+    * @param u (write-only), contains next step of time-integration on output
+    * @note the implementation is such that on output the last call to the explicit part \c ex is at the new \c (t,u). This might be interesting if the call to \c ex changes its state.
+    * @attention The first few steps after the call to the init function are performed with a semi-implicit Runge-Kutta method to initialize the multistepper
+    */
+    template< class RHS>
+    void step( RHS& rhs, value_type& t, ContainerType& u){
+        dg::IdentityFilter id;
+        m_fem.step( id, rhs, t, u);
+    }
+
+  private:
+    FilteredExplicitMultistep<ContainerType> m_fem;
+};
+
 } //namespace dg
-- 
GitLab


From 184d93ded2d8ba821f046f98092aebe0badf36f3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 17 Dec 2020 17:14:40 +0100
Subject: [PATCH 418/540] Merge AdamsBashforth struct with ExplicitMultistep

If it becomes relevant at any point in time we can make the
storage reduction internally in the class
---
 inc/dg/multistep.h    | 168 ++++++++----------------------------------
 inc/dg/multistep_t.cu |   5 +-
 inc/dg/tableau.h      |   2 +-
 3 files changed, 35 insertions(+), 140 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 1d9e1ada4..f99b58be6 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -42,128 +42,6 @@ namespace dg{
 * @note a disadvantage of multistep is that timestep adaption is not easily done.
 */
 
-/**
-* @brief Adams-Bashforth explicit multistep time-integration
-* \f[ u^{n+1} = u^n + \Delta t\sum_{j=0}^{s-1} b_j f\left(t^n - j \Delta t, u^{n-j}\right) \f]
-*
-* with coefficients taken from https://en.wikipedia.org/wiki/Linear_multistep_method
-* @note This scheme has a smaller region of absolute stability than some of the \c ExplicitMultistep method
-* @copydoc hide_note_multistep
-* @copydoc hide_ContainerType
-* @ingroup time
-*/
-template<class ContainerType>
-struct AdamsBashforth
-{
-    using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
-    using container_type = ContainerType; //!< the type of the vector class in use
-    ///copydoc RungeKutta::RungeKutta()
-    AdamsBashforth(){}
-    ///@copydoc AdamsBashforth::construct()
-    AdamsBashforth( unsigned order, const ContainerType& copyable){
-        construct( order, copyable);
-    }
-    /**
-    * @brief Reserve internal workspace for the integration
-    *
-    * @param order (global) order (= number of steps in the multistep) of the method (Currently, one of 1, 2, 3, 4 or 5)
-    * @param copyable ContainerType of the size that is used in \c step
-    * @note it does not matter what values \c copyable contains, but its size is important
-    */
-    void construct(unsigned order, const ContainerType& copyable){
-        m_k = order;
-        m_f.assign( order, copyable);
-        m_u = copyable;
-        m_ab.resize( order);
-        switch (order){
-            case 1: m_ab = {1}; break;
-            case 2: m_ab = {1.5, -0.5}; break;
-            case 3: m_ab = { 23./12., -4./3., 5./12.}; break;
-            case 4: m_ab = {55./24., -59./24., 37./24., -3./8.}; break;
-            case 5: m_ab = { 1901./720., -1387./360., 109./30., -637./360., 251./720.}; break;
-            default: throw dg::Error(dg::Message()<<"Order not implemented in AdamsBashforth!");
-        }
-    }
-    ///@brief Return an object of same size as the object used for construction
-    ///@return A copyable object; what it contains is undefined, its size is important
-    const ContainerType& copyable()const{ return m_u;}
-
-    /**
-     * @brief Initialize timestepper. Call before using the step function.
-     *
-     * This routine has to be called before the first timestep is made.
-     * @copydoc hide_rhs
-     * @param rhs The rhs functor
-     * @param t0 The intital time corresponding to u0
-     * @param u0 The initial value of the integration
-     * @param dt The timestep
-     * @note the implementation is such that on output the last call to the rhs is at (t0,u0). This might be interesting if the call to the rhs changes its state.
-     */
-    template< class RHS>
-    void init( RHS& rhs, value_type t0, const ContainerType& u0, value_type dt);
-    /**
-    * @brief Advance u0 one timestep
-    *
-    * @copydoc hide_rhs
-    * @param rhs right hand side function or functor
-    * @param t (write-only) contains timestep corresponding to \c u on output
-    * @param u (write-only) contains next step of the integration on output
-    * @note the implementation is such that on output the last call to the rhs is at the new (t,u). This might be interesting if the call to the rhs changes its state.
-    * @attention The first few steps after the call to the init function are performed with an explicit Runge-Kutta method of the same order
-    */
-    template< class RHS>
-    void step( RHS& rhs, value_type& t, ContainerType& u);
-  private:
-    value_type m_tu, m_dt;
-    std::vector<ContainerType> m_f;
-    ContainerType m_u;
-    std::vector<value_type> m_ab;
-    unsigned m_k, m_counter;
-};
-
-///@cond
-template< class ContainerType>
-template< class RHS>
-void AdamsBashforth<ContainerType>::init( RHS& f, value_type t0, const ContainerType& u0, value_type dt)
-{
-    m_tu = t0, m_dt = dt;
-    f( t0, u0, m_f[m_k-1]); //f may not destroy u0
-    blas1::copy(  u0, m_u);
-    m_counter = 0;
-}
-
-template<class ContainerType>
-template< class RHS>
-void AdamsBashforth<ContainerType>::step( RHS& f, value_type& t, ContainerType& u)
-{
-    if( m_counter < m_k-1)
-    {
-        std::map<unsigned, enum tableau_identifier> order2method{
-            {1, EXPLICIT_EULER_1_1},
-            {2, MIDPOINT_2_2},
-            {3, KUTTA_3_3},
-            {4, CLASSIC_4_4},
-            {5, CASH_KARP_6_4_5}
-        };
-        dg::RungeKutta<ContainerType> rk( order2method.at(m_k), u);
-        rk.step( f, t, u, t, u, m_dt);
-        m_counter++;
-        m_tu = t;
-        blas1::copy(  u, m_u);
-        f( m_tu, m_u, m_f[m_k - 1 - m_counter]);
-        return;
-    }
-    for( unsigned i=0; i<m_k; i++)
-        blas1::axpby( m_dt*m_ab[i], m_f[i], 1., m_u);
-    //permute m_f[k-1]  to be the new m_f[0]
-    std::rotate( m_f.rbegin(), m_f.rbegin()+1, m_f.rend());
-    blas1::copy( m_u, u);
-    t = m_tu = m_tu + m_dt;
-    f( m_tu, m_u, m_f[0]); //evaluate f at new point
-}
-///@endcond
-
-
 /**
 * @brief Karniadakis semi-implicit multistep time-integration
 * \f[
@@ -574,21 +452,21 @@ struct BDF
     {
         m_bdf.construct( order, std::forward<SolverParams>(ps)...);
     }
-    ///@copydoc AdamsBashforth::copyable()
+    ///@copydoc RungeKutta::copyable()
     const ContainerType& copyable()const{ return m_bdf.copyable();}
     ///Write access to the internal solver for the implicit part
     SolverType& solver() { return m_bdf.solver();}
     ///Read access to the internal solver for the implicit part
     const SolverType& solver() const { return m_bdf.solver();}
 
-    ///@copydoc AdamsBashforth::init()
+    ///@copydoc ExplicitMultistep::init()
     template<class RHS>
     void init(RHS& rhs, value_type t0, const ContainerType& u0, value_type dt){
         dg::IdentityFilter id;
         m_bdf.init( id, rhs, t0, u0, dt);
     }
 
-    ///@copydoc AdamsBashforth::step()
+    ///@copydoc ExplicitMultistep::step()
     template<class RHS>
     void step(RHS& rhs, value_type& t, container_type& u){
         dg::IdentityFilter id;
@@ -601,12 +479,19 @@ struct BDF
 
 
 /**
- * @brief Identify coefficients for linear multistep methods
+ * @brief Identifiers for linear multistep methods
  * @sa ExplicitMultistep
 *  @ingroup time
  */
 enum multistep_identifier
 {
+    /** The family of schemes described in <a href = "https://en.wikipedia.org/wiki/Linear_multistep_method"> Linear multistep methods </a>
+     as **Adams-Bashforth**
+    * \f[ u^{n+1} = u^n + \Delta t\sum_{j=0}^{s-1} b_j f\left(t^n - j \Delta t, u^{n-j}\right) \f]
+     **Possible stages are 1, 2,..., 5**, the order of the method is the same as its stages
+    @note The Adams-Bashforth schemes implemented here need less storage but may have **a smaller region of absolute stability** than for example an extrapolated BDF method of the same order.
+    */
+    ADAMS_BASHFORTH,
     /** The family of schemes described in <a href =
      "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S.
      J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep
@@ -647,7 +532,7 @@ enum multistep_identifier
 * @brief General explicit linear multistep time-integration with Limiter / Filter
 * \f[
 * \begin{align}
-    \tilde v &= \sum_{j=0}^{s-1} \alpha_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}\beta_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right) \\
+    \tilde v &= \sum_{j=0}^{s-1} a_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}b_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right) \\
     v^{n+1} &= \Lambda\Pi \left( \tilde v\right)
     \end{align}
     \f]
@@ -659,12 +544,11 @@ enum multistep_identifier
     where \f$ f \f$ contains the equations.
     The coefficients for an order 3 "eBDF" scheme are given as an example:
     \f[
-    \alpha_0 = \frac{18}{11}\ \alpha_1 = -\frac{9}{11}\ \alpha_2 = \frac{2}{11} \\
-    \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11}
+    a_0 = \frac{18}{11}\ a_1 = -\frac{9}{11}\ a_2 = \frac{2}{11} \\
+    b_0 = \frac{18}{11}\ b_1 = -\frac{18}{11}\ b_2 = \frac{6}{11}
 \f]
 @sa multistep_identifier
 @note This scheme is the same as ExplicitMultistep with the additional option to use a filter
-@note The schemes implemented here need more storage but may have **a larger region of absolute stability** than an AdamsBashforth method of the same order.
 *
 * @copydoc hide_note_multistep
 * @copydoc hide_ContainerType
@@ -681,6 +565,7 @@ struct FilteredExplicitMultistep
     ///@copydoc construct()
     FilteredExplicitMultistep( std::string method, unsigned stages, const ContainerType& copyable){
         std::map < std::string, enum multistep_identifier> str2id{
+            {"Adams-Bashforth", ADAMS_BASHFORTH},
             {"eBDF", eBDF},
             {"TVB", TVB},
             {"SSP", SSP}
@@ -697,7 +582,7 @@ struct FilteredExplicitMultistep
     /**
      * @brief Reserve memory for the integration
      *
-     * Set the coefficients \f$ \alpha_i,\ \beta_i\f$
+     * Set the coefficients \f$ a_i,\ b_i\f$
      * @param method the name of the family of schemes to be used (a string can be converted to an enum with the same spelling) @sa multistep_identifier
      * @param stages (global) stages (= number of steps in the multistep) of the method (Currently possible values depend on the method), does not necessarily coincide with the order of the method
      * @param copyable ContainerType of the size that is used in \c step
@@ -750,6 +635,18 @@ struct FilteredExplicitMultistep
         m_a.resize( stages);
         m_b.resize( stages);
         switch( method){
+            case ADAMS_BASHFORTH:
+            m_a.assign(stages, 0);
+            m_a[0] = 1.;
+            switch (stages){
+                case 1: m_b = {1}; break;
+                case 2: m_b = {1.5, -0.5}; break;
+                case 3: m_b = { 23./12., -4./3., 5./12.}; break;
+                case 4: m_b = {55./24., -59./24., 37./24., -3./8.}; break;
+                case 5: m_b = { 1901./720., -1387./360., 109./30., -637./360., 251./720.}; break;
+                default: throw dg::Error(dg::Message()<<"Order not implemented in AdamsBashforth!");
+            }
+            break;
             case eBDF:
             switch (stages){
                 case 1: m_a = {1.};
@@ -881,7 +778,7 @@ void FilteredExplicitMultistep<ContainerType>::step(Limiter& l, RHS& f, value_ty
 * @brief General explicit linear multistep time-integration
 * \f[
 * \begin{align}
-    v^{n+1} = \sum_{j=0}^{s-1} \alpha_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}\beta_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right)
+    v^{n+1} = \sum_{j=0}^{s-1} a_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}b_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right)
     \end{align}
     \f]
 
@@ -892,11 +789,10 @@ void FilteredExplicitMultistep<ContainerType>::step(Limiter& l, RHS& f, value_ty
     where \f$ f \f$ contains the equations.
     The coefficients for an order 3 "eBDF" scheme are given as an example:
     \f[
-    \alpha_0 = \frac{18}{11}\ \alpha_1 = -\frac{9}{11}\ \alpha_2 = \frac{2}{11} \\
-    \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11}
+    a_0 = \frac{18}{11}\ a_1 = -\frac{9}{11}\ a_2 = \frac{2}{11} \\
+    b_0 = \frac{18}{11}\ b_1 = -\frac{18}{11}\ b_2 = \frac{6}{11}
 \f]
 @sa multistep_identifier
-@note The schemes implemented here need more storage but may have **a larger region of absolute stability** than an AdamsBashforth method of the same order.
 *
 * @copydoc hide_note_multistep
 * @copydoc hide_ContainerType
@@ -946,7 +842,7 @@ struct ExplicitMultistep
     * @param t (write-only), contains timestep corresponding to \c u on output
     * @param u (write-only), contains next step of time-integration on output
     * @note the implementation is such that on output the last call to the explicit part \c ex is at the new \c (t,u). This might be interesting if the call to \c ex changes its state.
-    * @attention The first few steps after the call to the init function are performed with a semi-implicit Runge-Kutta method to initialize the multistepper
+    * @attention The first few steps after the call to the init function are performed with a Runge-Kutta method (of the same order) to initialize the multistepper
     */
     template< class RHS>
     void step( RHS& rhs, value_type& t, ContainerType& u){
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 9e0a90fb6..f35851401 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -106,11 +106,11 @@ int main()
     double time = 0.;
     std::array<double,2> error( sol);
     exblas::udouble res;
-    std::cout << "### Test Adams Bashforth methods with "<<NT<<" steps\n";
+    std::cout << "### Test Explicit Multistep methods with "<<NT<<" steps\n";
     for( unsigned s=1; s<6; s++)
     {
         time = 0., y0 = init;
-        dg::AdamsBashforth< std::array<double,2> > ab( s, y0);
+        dg::ExplicitMultistep< std::array<double,2> > ab( "Adams-Bashforth", s, y0);
         ab.init( full, time, y0, dt);
         //main time loop
         for( unsigned k=0; k<NT; k++)
@@ -119,7 +119,6 @@ int main()
         res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << "Relative error AB "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
     }
-    std::cout << "### Test Explicit Multistep methods with "<<NT<<" steps\n";
     for( unsigned s=1; s<7; s++)
     {
         time = 0., y0 = init;
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index 53e82b462..3707cd5b3 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -992,7 +992,7 @@ ShuOsherTableau<real_type> ssprk_5_4()
 ///@endcond
 
 /**
-* @brief Identify the Butcher Tableaus provided by this library
+* @brief Identifiers for Butcher Tableaus
 *
 * We follow the naming convention of the ARKode library http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html (They also provide nice stability plots for their methods)
 * as NAME-S-P-Q or NAME-S-Q, where
-- 
GitLab


From 24d9e4b4993027f75d83844a3135549c0ca4dea1 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 18 Dec 2020 16:36:06 +0100
Subject: [PATCH 419/540] Change Filter function input to number

- swap RHS and Filter in timestepper input
- add filter.h to library header
---
 inc/dg/functors.h          | 17 +++++++------
 inc/dg/multistep.h         | 50 +++++++++++++++++++-------------------
 inc/dg/runge_kutta.h       |  4 +--
 inc/dg/runge_kutta_t.cu    |  2 +-
 inc/dg/topology/filter.h   | 20 +++++++--------
 inc/dg/topology/geometry.h |  1 +
 6 files changed, 49 insertions(+), 45 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index ef9dfab8d..db32ba8ef 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -1105,19 +1105,22 @@ struct DPolynomialHeaviside {
  * @brief \f$ \begin{cases}
     1 \text{ if } \eta < \eta_c \\
     \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
-    0 \text{ else}
+    0 \text{ else} \\
+    \eta=\frac{i}{1-n}
     \end{cases}\f$
 
-    This function is continuously differentiable
+    where n is the number of polynomial coefficients
+
+    This function is s times continuously differentiable everywhere
     @sa Its main use comes from the application in dg::ModalFilter
  */
 struct ExponentialFilter
 {
-    ExponentialFilter( double alpha, double eta_c, unsigned s): m_alpha(alpha),
-                                                                m_etac(eta_c),
-                                                                m_s(s) {}
-    double operator()( double eta) const
+    ExponentialFilter( double alpha, double eta_c, unsigned s, unsigned n):
+        m_alpha(alpha), m_etac(eta_c), m_s(s), m_n(n) {}
+    double operator()( unsigned i) const
     {
+        double eta = (double)i/(double)(m_n-1);
         if( eta < m_etac)
             return 1.;
         if( eta <= 1.)
@@ -1126,7 +1129,7 @@ struct ExponentialFilter
     }
     private:
     double m_alpha, m_etac;
-    unsigned m_s;
+    unsigned m_s, m_n;
 };
 
 /**
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index f99b58be6..c8085829d 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -319,12 +319,12 @@ struct FilteredBDF
     const SolverType& solver() const { return m_solver;}
 
     ///@copydoc FilteredExplicitMultistep::init()
-    template<class Limiter, class RHS>
-    void init(Limiter& limiter, RHS& rhs, value_type t0, const ContainerType& u0, value_type dt);
+    template<class RHS, class Limiter>
+    void init(RHS& rhs, Limiter& limiter, value_type t0, const ContainerType& u0, value_type dt);
 
     ///@copydoc FilteredExplicitMultistep::step()
-    template<class Limiter, class RHS>
-    void step(Limiter& limiter, RHS& rhs, value_type& t, container_type& u);
+    template<class RHS, class Limiter>
+    void step(RHS& rhs, Limiter& limiter, value_type& t, container_type& u);
     private:
     void init_coeffs(unsigned order){
         switch (order){
@@ -348,8 +348,8 @@ struct FilteredBDF
 
 ///@cond
 template< class ContainerType, class SolverType>
-template<class Limiter, class RHS>
-void FilteredBDF<ContainerType, SolverType>::init(Limiter& l, RHS& rhs, value_type t0,
+template<class RHS, class Limiter>
+void FilteredBDF<ContainerType, SolverType>::init(RHS& rhs, Limiter& l, value_type t0,
     const ContainerType& u0, value_type dt)
 {
     m_tu = t0, m_dt = dt;
@@ -359,8 +359,8 @@ void FilteredBDF<ContainerType, SolverType>::init(Limiter& l, RHS& rhs, value_ty
 }
 
 template< class ContainerType, class SolverType>
-template<class Limiter, class RHS>
-void FilteredBDF<ContainerType, SolverType>::step(Limiter& l, RHS& rhs, value_type& t, container_type& u)
+template<class RHS, class Limiter>
+void FilteredBDF<ContainerType, SolverType>::step(RHS& rhs, Limiter& l, value_type& t, container_type& u)
 {
     if( m_counter < m_k-1)
     {
@@ -463,14 +463,14 @@ struct BDF
     template<class RHS>
     void init(RHS& rhs, value_type t0, const ContainerType& u0, value_type dt){
         dg::IdentityFilter id;
-        m_bdf.init( id, rhs, t0, u0, dt);
+        m_bdf.init( rhs, id, t0, u0, dt);
     }
 
     ///@copydoc ExplicitMultistep::step()
     template<class RHS>
     void step(RHS& rhs, value_type& t, container_type& u){
         dg::IdentityFilter id;
-        m_bdf.step( id, rhs, t, u);
+        m_bdf.step( rhs, id, t, u);
     }
     private:
     FilteredBDF<ContainerType, SolverType> m_bdf;
@@ -603,32 +603,32 @@ struct FilteredExplicitMultistep
      * @brief Initialize timestepper. Call before using the step function.
      *
      * This routine has to be called before the first timestep is made.
-     * @copydoc hide_limiter
      * @copydoc hide_rhs
-     * @param limiter The limiter or filter to use
+     * @copydoc hide_limiter
      * @param rhs The rhs functor
+     * @param limiter The limiter or filter to use
      * @param t0 The intital time corresponding to u0
      * @param u0 The initial value of the integration
      * @param dt The timestep saved for later use
      * @note the implementation is such that on output the last call to the explicit part \c ex is at \c (t0,u0). This might be interesting if the call to \c ex changes its state.
      */
-    template< class Limiter, class RHS>
-    void init( Limiter& limiter, RHS& rhs, value_type t0, const ContainerType& u0, value_type dt);
+    template< class RHS, class Limiter>
+    void init( RHS& rhs, Limiter& limiter, value_type t0, const ContainerType& u0, value_type dt);
 
     /**
     * @brief Advance one timestep
     *
-    * @copydoc hide_limiter
-    * @param limiter The limiter or filter to use
     * @copydoc hide_rhs
+    * @copydoc hide_limiter
     * @param rhs The rhs functor
+    * @param limiter The limiter or filter to use
     * @param t (write-only), contains timestep corresponding to \c u on output
     * @param u (write-only), contains next step of time-integration on output
     * @note the implementation is such that on output the last call to the explicit part \c ex is at the new \c (t,u). This might be interesting if the call to \c ex changes its state.
     * @attention The first few steps after the call to the init function are performed with a Runge-Kutta method
     */
-    template< class Limiter, class RHS>
-    void step( Limiter& limiter, RHS& rhs, value_type& t, ContainerType& u);
+    template< class RHS, class Limiter>
+    void step( RHS& rhs, Limiter& limiter, value_type& t, ContainerType& u);
 
   private:
     void init_coeffs(enum multistep_identifier method, unsigned stages){
@@ -727,8 +727,8 @@ struct FilteredExplicitMultistep
 };
 ///@cond
 template< class ContainerType>
-template< class Limiter, class RHS>
-void FilteredExplicitMultistep<ContainerType>::init( Limiter& l, RHS& f, value_type t0, const ContainerType& u0, value_type dt)
+template< class RHS, class Limiter>
+void FilteredExplicitMultistep<ContainerType>::init( RHS& f, Limiter& l, value_type t0, const ContainerType& u0, value_type dt)
 {
     m_tu = t0, m_dt = dt;
     l.apply( u0, m_u[m_k-1]);
@@ -737,8 +737,8 @@ void FilteredExplicitMultistep<ContainerType>::init( Limiter& l, RHS& f, value_t
 }
 
 template<class ContainerType>
-template<class Limiter, class RHS>
-void FilteredExplicitMultistep<ContainerType>::step(Limiter& l, RHS& f, value_type& t, ContainerType& u)
+template<class RHS, class Limiter>
+void FilteredExplicitMultistep<ContainerType>::step(RHS& f, Limiter& l, value_type& t, ContainerType& u)
 {
     if( m_counter < m_k-1)
     {
@@ -751,7 +751,7 @@ void FilteredExplicitMultistep<ContainerType>::step(Limiter& l, RHS& f, value_ty
             {6, SSPRK_5_4}
         };
         ShuOsher<ContainerType> rk( order2method.at(m_k), u);
-        rk.step( l, f, t, u, t, u, m_dt);
+        rk.step( f, l, t, u, t, u, m_dt);
         m_counter++;
         m_tu = t;
         blas1::copy(  u, m_u[m_k-1-m_counter]);
@@ -831,7 +831,7 @@ struct ExplicitMultistep
     template< class RHS>
     void init( RHS& rhs, value_type t0, const ContainerType& u0, value_type dt){
         dg::IdentityFilter id;
-        m_fem.init( id, rhs, t0, u0, dt);
+        m_fem.init( rhs, id, t0, u0, dt);
     }
 
     /**
@@ -847,7 +847,7 @@ struct ExplicitMultistep
     template< class RHS>
     void step( RHS& rhs, value_type& t, ContainerType& u){
         dg::IdentityFilter id;
-        m_fem.step( id, rhs, t, u);
+        m_fem.step( rhs, id, t, u);
     }
 
   private:
diff --git a/inc/dg/runge_kutta.h b/inc/dg/runge_kutta.h
index 3e59c842e..f7fbc7401 100644
--- a/inc/dg/runge_kutta.h
+++ b/inc/dg/runge_kutta.h
@@ -652,8 +652,8 @@ struct ShuOsher
     * @param dt timestep
     * @note on return \c rhs(t1, u1) will be the last call to \c rhs (this is useful if \c RHS holds state, which is then updated to the current timestep)
     */
-    template<class Limiter, class RHS>
-    void step( Limiter& limiter, RHS& rhs, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
+    template<class RHS, class Limiter>
+    void step( RHS& rhs, Limiter& limiter, value_type t0, const ContainerType& u0, value_type& t1, ContainerType& u1, value_type dt){
         unsigned s = m_t.num_stages();
         std::vector<value_type> ts( m_t.num_stages()+1);
         ts[0] = t0;
diff --git a/inc/dg/runge_kutta_t.cu b/inc/dg/runge_kutta_t.cu
index 24a7cab34..dfdf1518b 100644
--- a/inc/dg/runge_kutta_t.cu
+++ b/inc/dg/runge_kutta_t.cu
@@ -109,7 +109,7 @@ int main()
         dg::blas1::copy( u, u1);
         double t0 = t_start;
         for( unsigned i=0; i<N; i++)
-            rk.step( id, functor, t0, u1, t0, u1, dt);
+            rk.step( functor, id, t0, u1, t0, u1, dt);
         dg::blas1::axpby( 1., sol , -1., u1);
         std::cout << "Norm of error in "<<std::setw(24) <<name<<"\t"<<sqrt(dg::blas1::dot( u1, u1))<<"\n";
     }
diff --git a/inc/dg/topology/filter.h b/inc/dg/topology/filter.h
index fd28c6512..f75dbc24f 100644
--- a/inc/dg/topology/filter.h
+++ b/inc/dg/topology/filter.h
@@ -1,4 +1,5 @@
 #pragma once
+#include "dg/functors.h"
 #include "fast_interpolation.h"
 
 /**@file
@@ -20,10 +21,7 @@ modal_filter( UnaryOp op, const RealGrid1d<real_type>& g )
     Operator<real_type> forward=g.dlt().forward();
     Operator<real_type> filter( g.n(), 0);
     for( unsigned i=0; i<g.n(); i++)
-    {
-        real_type eta = (real_type) i / (real_type)(g.n()-1);
-        filter(i,i) = op( eta);
-    }
+        filter(i,i) = op( i);
     filter = backward*filter*forward;
     //Assemble the matrix
     EllSparseBlockMat<real_type> A(g.N(), g.N(), 1, 1, g.n());
@@ -139,30 +137,32 @@ struct ModalFilter
      * @brief Create exponential filter \f$ \begin{cases}
     1 \text{ if } \eta < \eta_c \\
     \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
-    0 \text{ else}
+    0 \text{ else} \\
+    \eta := \frac{i}{n-1}
     \end{cases}\f$
      *
      * @tparam Topology Any grid
+     * @param alpha damping for the highest mode is \c exp( -alpha)
      * @param eta_c cutoff frequency (0<eta_c<1), 0.5 or 0 are good starting values
      * @param order 8 or 16 are good values
      * @param t The topology to apply the modal filter on
      * @sa dg::ExponentialFilter
      */
     template<class Topology>
-    ModalFilter( real_type eta_c, unsigned order, const Topology& t):
-        ModalFilter( ExponentialFilter( 36, eta_c, order), t)
+    ModalFilter( real_type alpha, real_type eta_c, unsigned order, const Topology& t):
+        ModalFilter( dg::ExponentialFilter( alpha, eta_c, order, t.n()), t)
     { }
     /**
      * @brief Create arbitrary filter
      *
      * @tparam Topology Any grid
-     * @tparam UnaryOp Model of Unary Function \c real_type \c f(real_type) The input will be the normalized modal number \f$ \eta := \frac{i}{n-1} \f$ where \f$ i=0,...,n-1\f$ and n is the number of polynomial coefficients in use
+     * @tparam UnaryOp Model of Unary Function \c real_type \c sigma(unsigned) The input will be the modal number \c i where \f$ i=0,...,n-1\f$ and \c n is the number of polynomial coefficients in use. The output is the filter strength for the given mode number
      * @param f The filter to evaluate on the normalized modal coefficients
      * @param t The topology to apply the modal filter on
      */
     template<class UnaryOp, class Topology>
-    ModalFilter( UnaryOp f, const Topology& t) : m_filter (
-            dg::create::modal_filter( f, t)) { }
+    ModalFilter( UnaryOp sigma, const Topology& t) : m_filter (
+            dg::create::modal_filter( sigma, t)) { }
 
     void apply( const ContainerType& x, ContainerType& y) const{ symv( 1., x, 0., y);}
     void symv( const ContainerType& x, ContainerType& y) const{ symv( 1., x,0,y);}
diff --git a/inc/dg/topology/geometry.h b/inc/dg/topology/geometry.h
index 932eaf62b..b80002313 100644
--- a/inc/dg/topology/geometry.h
+++ b/inc/dg/topology/geometry.h
@@ -4,6 +4,7 @@
 #include "thrust/host_vector.h"
 #include "evaluation.h"
 #include "weights.h"
+#include "filter.h"
 #ifdef MPI_VERSION
 #include "dg/backend/mpi_vector.h"
 #include "mpi_evaluation.h"
-- 
GitLab


From fc5a0336d01dda00e71e7644ffb4b533b595dc27 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 18 Dec 2020 16:38:52 +0100
Subject: [PATCH 420/540] Update filter test files

---
 inc/dg/topology/filter_mpit.cu | 4 ++--
 inc/dg/topology/filter_t.cu    | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/inc/dg/topology/filter_mpit.cu b/inc/dg/topology/filter_mpit.cu
index 7781a61bc..c79e4f3ab 100644
--- a/inc/dg/topology/filter_mpit.cu
+++ b/inc/dg/topology/filter_mpit.cu
@@ -40,7 +40,7 @@ int main(int argc, char* argv[])
     const dg::MDVec vec = dg::evaluate( function, g3);
     const dg::MDVec weights = dg::create::weights( g3);
     dg::MDVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
-    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( 0.5, 8, g3);
+    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( 36, 0.5, 8, g3);
     dg::MIDMatrix project = dg::create::projection( g2,g3);
     dg::MIDMatrix interpo = dg::create::interpolation( g3,g2);
 
@@ -71,7 +71,7 @@ int main(int argc, char* argv[])
     const dg::MDVec vec = dg::evaluate( function, g3);
     const dg::MDVec weights = dg::create::weights( g3);
     dg::MDVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
-    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( 0.5, 8, g3);
+    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( 36, 0.5, 8, g3);
     dg::MIDMatrix project = dg::create::projection( g2,g3);
     dg::MIDMatrix interpo = dg::create::interpolation( g3,g2);
 
diff --git a/inc/dg/topology/filter_t.cu b/inc/dg/topology/filter_t.cu
index b72a7ab0c..9735a3ed6 100644
--- a/inc/dg/topology/filter_t.cu
+++ b/inc/dg/topology/filter_t.cu
@@ -10,7 +10,7 @@
 double function( double x, double y){return sin(x)*sin(y);}
 double function( double x, double y, double z){return sin(x)*sin(y)*sin(z);}
 
-const unsigned n = 3, Nx = 8, Ny = 10, Nz = 6;
+const unsigned Nx = 8, Ny = 10, Nz = 6;
 
 int main()
 {
@@ -25,7 +25,7 @@ int main()
     const dg::DVec vec = dg::evaluate( function, g3);
     const dg::DVec weights = dg::create::weights( g3);
     dg::DVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
-    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 0.5, 8, g3);
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, g3);
     dg::IDMatrix project = dg::create::projection( g2,g3);
     dg::IDMatrix interpo = dg::create::interpolation( g3,g2);
 
@@ -49,7 +49,7 @@ int main()
     const dg::DVec vec = dg::evaluate( function, g3);
     const dg::DVec weights = dg::create::weights( g3);
     dg::DVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
-    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 0.5, 8, g3);
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, g3);
     dg::IDMatrix project = dg::create::projection( g2,g3);
     dg::IDMatrix interpo = dg::create::interpolation( g3,g2);
 
-- 
GitLab


From 54ab577617a7cb460ac297785f152ae900706fda Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 18 Dec 2020 16:39:42 +0100
Subject: [PATCH 421/540] Insert Modal Filter into shu code

first results are promising but detailed tests needed
---
 src/lamb_dipole/Makefile    | 16 +++----
 src/lamb_dipole/shu.cuh     | 44 +++++++++++-------
 src/lamb_dipole/shu2d_b.cu  | 90 ++++++++++++++++++-------------------
 src/lamb_dipole/shu_b.cu    | 61 +++++++++++++------------
 src/lamb_dipole/shu_hpc.cu  | 14 +++---
 src/lamb_dipole/shu_t.cu    | 64 +++++++++++++-------------
 src/lamb_dipole/shu_time.cu | 35 +++++++--------
 7 files changed, 165 insertions(+), 159 deletions(-)

diff --git a/src/lamb_dipole/Makefile b/src/lamb_dipole/Makefile
index 2a40ecc54..9788d456d 100644
--- a/src/lamb_dipole/Makefile
+++ b/src/lamb_dipole/Makefile
@@ -1,29 +1,29 @@
 device=gpu
 
-#configure machine 
+#configure machine
 include ../../config/default.mk
-include ../../config/*.mk 
+include ../../config/*.mk
 include ../../config/devices/devices.mk
 
 INCLUDE+= -I../         # other src libraries
 INCLUDE+= -I../../inc   # other project libraries
 
-all: shu_t shu_b shu2d_b shu_hpc
+all: shu_t shu_b shu2d_b shu_hpc shu_time
 
-shu_t: shu_t.cu shu.cuh 
+shu_t: shu_t.cu shu.cuh
 	$(CC) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) -DDG_DEBUG
 
-shu_b: shu_b.cu shu.cuh 
+shu_b: shu_b.cu shu.cuh
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB)
 
-shu2d_b: shu2d_b.cu shu.cuh 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE)  
+shu2d_b: shu2d_b.cu shu.cuh
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE)
 
 shu_hpc: shu_hpc.cu shu.cuh
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB)
 
 shu_time: shu_time.cu shu.cuh
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) 
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE)
 .PHONY: clean
 
 clean:
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index 23a2ad0d1..867740522 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -6,7 +6,7 @@
 
 #include "dg/algorithm.h"
 
-namespace dg
+namespace shu
 {
 template< class Matrix, class container>
 struct Diffusion
@@ -14,7 +14,7 @@ struct Diffusion
     Diffusion( const dg::Grid2d& g, double nu): nu_(nu),
         w2d( dg::create::weights( g) ), v2d( dg::create::inv_weights(g) ) ,
         LaplacianM( g, dg::normed)
-    { 
+    {
     }
     void operator()(double t, const container& x, container& y)
     {
@@ -31,15 +31,15 @@ struct Diffusion
 };
 
 template< class Matrix, class container >
-struct Shu 
+struct Shu
 {
     using value_type = dg::get_value_type<container>;
     typedef container Vector;
 
-    Shu( const Grid2d& grid, double eps);
+    Shu( const dg::Grid2d& grid, double eps);
 
-    const Elliptic<Matrix, container, container>& lap() const { return laplaceM;}
-    ArakawaX<CartesianGrid2d, Matrix, container>& arakawa() {return arakawa_;}
+    const dg::Elliptic<Matrix, container, container>& lap() const { return m_multi_laplaceM[0];}
+    dg::ArakawaX<dg::CartesianGrid2d, Matrix, container>& arakawa() {return m_arakawa;}
     /**
      * @brief Returns psi that belong to the last y in operator()
      *
@@ -50,28 +50,38 @@ struct Shu
     void operator()(double t, const Vector& y, Vector& yp);
   private:
     container psi, w2d, v2d;
-    Elliptic<CartesianGrid2d, Matrix, container> laplaceM;
-    ArakawaX<CartesianGrid2d, Matrix, container> arakawa_; 
-    Invert<container> invert;
+    std::vector<dg::Elliptic<dg::CartesianGrid2d, Matrix, container>> m_multi_laplaceM;
+    dg::ArakawaX<dg::CartesianGrid2d, Matrix, container> m_arakawa;
+    dg::Extrapolation<container> m_old_psi;
+    dg::MultigridCG2d<dg::CartesianGrid2d, Matrix, container> m_multigrid;
+    double m_eps;
 };
 
 template<class Matrix, class container>
-Shu< Matrix, container>::Shu( const Grid2d& g, double eps): 
+Shu< Matrix, container>::Shu( const dg::Grid2d& g, double eps):
     psi( g.size()),
-    w2d( create::weights( g)), v2d( create::inv_weights(g)),  
-    laplaceM( g, not_normed),
-    arakawa_( g), 
-    invert( psi, g.size(), eps)
+    w2d( dg::create::weights( g)), v2d( dg::create::inv_weights(g)),
+    m_arakawa( g),
+    m_old_psi( 2, w2d),
+    m_multigrid( g, 3),
+    m_eps(eps)
 {
+    m_multi_laplaceM.resize(3);
+    for( unsigned u=0; u<3; u++)
+        m_multi_laplaceM[u].construct( m_multigrid.grid(u), dg::not_normed, dg::centered, 1);
 }
 
 template< class Matrix, class container>
 void Shu<Matrix, container>::operator()(double t, const Vector& y, Vector& yp)
 {
-    invert( laplaceM, psi, y);
-    arakawa_( y, psi, yp); //A(y,psi)-> yp
+    m_old_psi.extrapolate( t, psi);
+    std::vector<unsigned> number = m_multigrid.direct_solve( m_multi_laplaceM, psi, y, m_eps);
+    m_old_psi.update( t, psi);
+    if( number[0] == m_multigrid.max_iter())
+        throw dg::Fail( m_eps);
+    m_arakawa( y, psi, yp); //A(y,psi)-> yp
 }
 
-}//namespace dg
+}//namespace shu
 
 #endif //_DG_SHU_CUH
diff --git a/src/lamb_dipole/shu2d_b.cu b/src/lamb_dipole/shu2d_b.cu
index 491854a79..55b5a26c8 100644
--- a/src/lamb_dipole/shu2d_b.cu
+++ b/src/lamb_dipole/shu2d_b.cu
@@ -7,9 +7,6 @@
 
 #include "shu.cuh"
 
-using namespace std;
-using namespace dg;
-
 const double lx = 2.*M_PI;//*50.;
 const double ly = 2.*M_PI;//*50.;
 
@@ -18,7 +15,6 @@ const double ly = 2.*M_PI;//*50.;
 const double T = 2.;
 ////const double eps = 1e-7; //CG method
 
-
 double D = 0.01;
 
 //const double kx = 2.*M_PI* (double)m/lx;
@@ -34,28 +30,28 @@ double solution_phi( double x, double y){ return sin(kx*x)*sin(ky*y)*exp(-ksqr*D
 //code for either lamb dipole or analytic sine function without graphics
 int main()
 {
-    Timer t;
+    dg::Timer t;
     ////////////////////////////////////////////////////////////
-    //cout << "Solve 2D incompressible NavierStokes with sin(x)sin(y) or Lamb dipole initial condition\n";
-    //cout << "Type n, N and eps\n";
-    //cin >> n >> Nx >>eps;
+    //std::cout << "Solve 2D incompressible NavierStokes with sin(x)sin(y) or Lamb dipole initial condition\n";
+    //std::cout << "Type n, N and eps\n";
+    //std::cin >> n >> Nx >>eps;
     //Ny = Nx;
-    //cout << "Type diffusion constant!\n";
-    //cin >> D;
-    //cout << "# of Legendre coefficients: " << n<<endl;
-    //cout << "# of grid cells:            " << Nx*Ny<<endl;
-    cout << "# grid NT dt eps eps_V eps_omega eps_E eps\n";
-    cout << "Diffusion " << D <<endl;
+    //std::cout << "Type diffusion constant!\n";
+    //std::cin >> D;
+    //std::cout << "# of Legendre coefficients: " << n<<std::endl;
+    //std::cout << "# of grid cells:            " << Nx*Ny<<std::endl;
+    std::cout << "# grid NT dt eps eps_V eps_omega eps_E eps\n";
+    std::cout << "Diffusion " << D <<std::endl;
 
     ////////////////////////////////////////////////////////////
     for( unsigned n=2; n<3; n++)
     {
-        cout << "P="<<n<<"\n";
+        std::cout << "P="<<n<<"\n";
         for(unsigned i=1; i<5; i++)
         {
             unsigned Nx = 8*pow(2,i), Ny = Nx;
-            Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
-            DVec w2d( create::weights(grid));
+            dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
+            dg::DVec w2d( dg::create::weights(grid));
 
             double dx = lx/(double)Nx;
             double eps = 1e-1/pow(10, n)*pow(dx,n);
@@ -63,58 +59,58 @@ int main()
             //if( D!= 0)
                 //NT = std::max((unsigned)(0.06*T*pow(4,n)/dx/dx), NT);
             const double dt = T/(double)NT;
-            //cout << "Runge Kutta stages          " << k <<endl;
-            //cout << "Timestep                    " << dt << endl;
-            //cout << "# of steps                  " << NT <<endl;
+            //cout << "Runge Kutta stages          " << k <<std::endl;
+            //cout << "Timestep                    " << dt << std::endl;
+            //cout << "# of steps                  " << NT <<std::endl;
             ////////////////////////////////////////////////////////////
 
-            DVec stencil = evaluate( one, grid);
-            DVec omega = evaluate( initial, grid );
-            const DVec sol = evaluate( solution, grid );
-            const DVec sol_phi = evaluate( solution_phi, grid );
+            dg::DVec stencil = dg::evaluate( dg::one, grid);
+            dg::DVec omega = dg::evaluate( initial, grid );
+            const dg::DVec sol = dg::evaluate( solution, grid );
+            const dg::DVec sol_phi = dg::evaluate( solution_phi, grid );
 
-            DVec y0( omega), y1( y0);
+            dg::DVec y0( omega), y1( y0);
             //make solver and stepper
-            Shu<DMatrix, DVec> shu( grid, eps);
-            Diffusion<DMatrix, DVec> diffusion( grid, D);
-            Karniadakis< DVec > karniadakis( y0, y0.size(), 1e-8);
+            shu::Shu<dg::DMatrix, dg::DVec> shu( grid, eps);
+            shu::Diffusion<dg::DMatrix, dg::DVec> diffusion( grid, D);
+            dg::Karniadakis< dg::DVec > karniadakis( y0, y0.size(), 1e-8);
 
             shu(0., y0, y1);
-            double vorticity = blas2::dot( stencil, w2d, sol);
-            double enstrophy = 0.5*blas2::dot( sol, w2d, sol);
-            double energy =    0.5*blas2::dot( sol, w2d, sol_phi) ;
+            double vorticity = dg::blas2::dot( stencil, w2d, sol);
+            double enstrophy = 0.5*dg::blas2::dot( sol, w2d, sol);
+            double energy =    0.5*dg::blas2::dot( sol, w2d, sol_phi) ;
 
             double time = 0;
             karniadakis.init( shu,diffusion, time, y0, dt);
             while( time < T)
             {
-                //step 
+                //step
 
                 t.tic();
                 karniadakis.step( shu, diffusion, time, y0);
                 t.toc();
                 time += dt;
                 //std::cout << "Time "<<time<< " "<<t.diff()<<"\n";
-                if( fabs(blas2::dot( w2d, y0)) > 1e16) 
+                if( fabs(dg::blas2::dot( w2d, y0)) > 1e16)
                 {
-                    cerr << "Sim unstable at time "<<time<<"!\n\n\n";
+                    std::cerr << "Sim unstable at time "<<time<<"!\n\n\n";
                     break;
                 }
             }
-            //cout << "Total simulation time:     "<<t.diff()<<"s\n";
-            //cout << "Average Time for one step: "<<t.diff()/(double)NT<<"s\n";
+            //std::cout << "Total simulation time:     "<<t.diff()<<"s\n";
+            //std::cout << "Average Time for one step: "<<t.diff()/(double)NT<<"s\n";
             ////////////////////////////////////////////////////////////////////
-            cout << Nx;
-            cout << " "<<NT;
-            cout << " "<<dt;
-            cout << " "<<eps;
-            cout << " "<<fabs(blas2::dot( stencil, w2d, y0)); 
-            cout << " "<<fabs(0.5*blas2::dot( w2d, y0) - enstrophy)/enstrophy;
-            cout << " "<<fabs(0.5*blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<" ";
-
-            blas1::axpby( 1., sol, -1., y0);
-            cout << " "<<sqrt( blas2::dot( w2d, y0))<< endl;
-            //cout << "Relative distance to solution "<<sqrt( blas2::dot( w2d, y0))/sqrt( blas2::dot( w2d, sol)) << endl;
+            std::cout << Nx;
+            std::cout << " "<<NT;
+            std::cout << " "<<dt;
+            std::cout << " "<<eps;
+            std::cout << " "<<fabs(dg::blas2::dot( stencil, w2d, y0));
+            std::cout << " "<<fabs(0.5*dg::blas2::dot( w2d, y0) - enstrophy)/enstrophy;
+            std::cout << " "<<fabs(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<" ";
+
+            dg::blas1::axpby( 1., sol, -1., y0);
+            std::cout << " "<<sqrt( dg::blas2::dot( w2d, y0))<< std::endl;
+            //std::cout << "Relative distance to solution "<<sqrt( dg::blas2::dot( w2d, y0))/sqrt( dg::blas2::dot( w2d, sol)) << std::endl;
 
         }
         std::cout << std::endl;
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 0f4b59b7b..0a154e61f 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -21,9 +21,6 @@ double shearLayer(double x, double y){
     return delta*cos(x) + 1./rho/cosh( (3.*M_PI/2.-y)/rho)/cosh( (3.*M_PI/2.-y)/rho);
 }
 
-using namespace std;
-using namespace dg;
-
 int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
@@ -40,8 +37,8 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////////////////////////////////////////////////////////////
-    Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
-    DVec w2d( create::weights(grid));
+    dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
+    dg::DVec w2d( dg::create::weights(grid));
     /////////////////////////////////////////////////////////////////
     std::stringstream title;
     GLFWwindow* w = draw::glfwInitAndCreateWindow(600, 600, "");
@@ -55,8 +52,8 @@ int main( int argc, char* argv[])
     else if ( p.initial == "shear")
         omega = dg::evaluate ( shearLayer, grid);
 
-    DVec stencil = evaluate( one, grid);
-    DVec y0( omega ), y1( y0);
+    dg::DVec stencil = evaluate( dg::one, grid);
+    dg::DVec y0( omega ), y1( y0);
     //subtract mean mass 
     if( p.bc_x == dg::PER && p.bc_y == dg::PER)
     {
@@ -64,18 +61,21 @@ int main( int argc, char* argv[])
         dg::blas1::axpby( -meanMass, stencil, 1., y0);
     }
     //make solver and stepper
-    Shu<DMatrix, DVec> shu( grid, p.eps);
-    Diffusion<DMatrix, DVec> diffusion( grid, p.D);
-    Karniadakis< DVec > karniadakis( y0, y0.size(), p.eps_time);
+    shu::Shu<dg::DMatrix, dg::DVec> shu( grid, p.eps);
+    //shu::Diffusion<dg::DMatrix, dg::DVec> diffusion( grid, p.D);
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, grid);
+    //dg::Karniadakis< dg::DVec > stepper( y0, y0.size(), p.eps_time);
+    dg::FilteredExplicitMultistep< dg::DVec > stepper( "eBDF", 3, y0);
+    //dg::ShuOsher<dg::DVec> stepper( "SSPRK-3-3", y0);
 
-    Timer t;
+    dg::Timer t;
     t.tic();
     shu( 0., y0, y1);
     t.toc();
-    cout << "Time for one rhs evaluation: "<<t.diff()<<"s\n";
-    double vorticity = blas2::dot( stencil , w2d, y0);
-    double enstrophy = 0.5*blas2::dot( y0, w2d, y0);
-    double energy =    0.5*blas2::dot( y0, w2d, shu.potential()) ;
+    std::cout << "Time for one rhs evaluation: "<<t.diff()<<"s\n";
+    double vorticity = dg::blas2::dot( stencil , w2d, y0);
+    double enstrophy = 0.5*dg::blas2::dot( y0, w2d, y0);
+    double energy =    0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
     
     std::cout << "Total energy:     "<<energy<<"\n";
     std::cout << "Total enstrophy:  "<<enstrophy<<"\n";
@@ -84,14 +84,15 @@ int main( int argc, char* argv[])
     double time = 0;
     ////////////////////////////////glfw//////////////////////////////
     //create visualisation vectors
-    DVec visual( grid.size());
-    HVec hvisual( grid.size());
+    dg::DVec visual( grid.size());
+    dg::HVec hvisual( grid.size());
     //transform vector to an equidistant grid
     dg::IDMatrix equidistant = dg::create::backscatter( grid );
     draw::ColorMapRedBlueExt colors( 1.);
-    karniadakis.init( shu, diffusion, time, y0, p.dt);
-    //cout << "Press any key to start!\n";
-    //double x; 
+    //stepper.init( shu, diffusion, time, y0, p.dt);
+    stepper.init( shu, filter, time, y0, p.dt);
+    //std::cout << "Press any key to start!\n";
+    //double x;
     //cin >> x;
     while (!glfwWindowShouldClose(w) && time < p.maxout*p.itstp*p.dt)
     {
@@ -109,23 +110,25 @@ int main( int argc, char* argv[])
         t.tic();
         for( unsigned i=0; i<p.itstp; i++)
         {
-            karniadakis.step( shu, diffusion, time, y0 );
+            //stepper.step( shu, diffusion, time, y0 );
+            stepper.step( shu, filter, time, y0 );
+            //stepper.step( shu, filter, time, y0, time, y0, p.dt );
         }
         t.toc();
-        //cout << "Timer for one step: "<<t.diff()/N<<"s\n";
+        //std::cout << "Timer for one step: "<<t.diff()/N<<"s\n";
         time += p.itstp*p.dt;
 
     }
     glfwTerminate();
     ////////////////////////////////////////////////////////////////////
-    cout << "Analytic formula enstrophy "<<lamb.enstrophy()<<endl;
-    cout << "Analytic formula energy    "<<lamb.energy()<<endl;
-    cout << "Total vorticity          is: "<<blas2::dot( stencil , w2d, y0) << "\n";
-    cout << "Relative enstrophy error is: "<<(0.5*blas2::dot( w2d, y0) - enstrophy)/enstrophy<<"\n";
-    cout << "Relative energy error    is: "<<(0.5*blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<"\n";
+    std::cout << "Analytic formula enstrophy "<<lamb.enstrophy()<<std::endl;
+    std::cout << "Analytic formula energy    "<<lamb.energy()<<std::endl;
+    std::cout << "Total vorticity          is: "<<dg::blas2::dot( stencil , w2d, y0) << "\n";
+    std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - enstrophy)/enstrophy<<"\n";
+    std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<"\n";
 
-    //blas1::axpby( 1., y0, -1, sol);
-    //cout << "Distance to solution: "<<sqrt(blas2::dot( w2d, sol ))<<endl;
+    //dg::blas1::axpby( 1., y0, -1, sol);
+    //cout << "Distance to solution: "<<sqrt(dg::blas2::dot( w2d, sol ))<<std::endl;
 
     //cout << "Press any key to quit!\n";
     //cin >> x;
diff --git a/src/lamb_dipole/shu_hpc.cu b/src/lamb_dipole/shu_hpc.cu
index dd8106378..ed2353c82 100644
--- a/src/lamb_dipole/shu_hpc.cu
+++ b/src/lamb_dipole/shu_hpc.cu
@@ -41,10 +41,13 @@ int main( int argc, char * argv[])
     dg::DVec y0( omega );
     const dg::DVec one = dg::evaluate( dg::one, grid);
     //make solver and stepper
-    dg::Shu<dg::DMatrix, dg::DVec> shu( grid, p.eps);
-    dg::Diffusion< dg::DMatrix, dg::DVec > diff( grid, p.D);
-    dg::Karniadakis< dg::DVec> karniadakis( y0, y0.size(), 1e-10);
-    karniadakis.init( shu, diff, 0., y0, p.dt);
+    shu::Shu<dg::DMatrix, dg::DVec> shu( grid, p.eps);
+    //shu::Diffusion< dg::DMatrix, dg::DVec > diff( grid, p.D);
+    //dg::Karniadakis< dg::DVec> karniadakis( y0, y0.size(), 1e-10);
+    //karniadakis.init( shu, diff, 0., y0, p.dt);
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, grid);
+    dg::FilteredExplicitMultistep<dg::DVec> stepper( "eBDF", 3, y0);
+    stepper.init( shu, filter, 0., y0, p.dt);
 
     dg::DVec varphi( grid.size()), potential;
     double vorticity = dg::blas2::dot( one , w2d, y0);
@@ -97,7 +100,8 @@ int main( int argc, char * argv[])
         ti.tic();
         for( unsigned j=0; j<p.itstp; j++)
         {
-            karniadakis.step( shu, diff, time, y0);//one step further
+            //karniadakis.step( shu, diff, time, y0);//one step further
+            stepper.step( shu, filter, time, y0);//one step further
             output1d[0] = vorticity = dg::blas2::dot( one , w2d, y0);
             output1d[1] = enstrophy = 0.5*dg::blas2::dot( y0, w2d, y0);
             potential = shu.potential();
diff --git a/src/lamb_dipole/shu_t.cu b/src/lamb_dipole/shu_t.cu
index 3896fa8bc..22aab0fb5 100644
--- a/src/lamb_dipole/shu_t.cu
+++ b/src/lamb_dipole/shu_t.cu
@@ -11,9 +11,6 @@
 #include "shu.cuh"
 
 
-using namespace std;
-using namespace dg;
-
 const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
 
@@ -30,12 +27,12 @@ int main()
 {
     unsigned n, Nx, Ny;
     double eps;
-    cout << "Type n, Nx, Ny and eps!\n";
-    cin >> n >> Nx >> Ny>>eps;
+    std::cout << "Type n, Nx, Ny and eps!\n";
+    std::cin >> n >> Nx >> Ny>>eps;
     const unsigned NT = (unsigned)(T*n*Nx/0.1/lx);
-    
-    Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
-    DVec w2d( create::weights( grid));
+
+    dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
+    dg::DVec w2d( dg::create::weights( grid));
     const double dt = T/(double)NT;
     /////////////////////////////////////////////////////////////////
     //create CUDA context that uses OpenGL textures in Glfw window
@@ -43,33 +40,36 @@ int main()
     GLFWwindow* w = draw::glfwInitAndCreateWindow(600, 600, "Navier Stokes");
     draw::RenderHostData render( 1,1);
     ////////////////////////////////////////////////////////////
-    cout << "# of Legendre coefficients: " << n<<endl;
-    cout << "# of grid cells:            " << Nx*Ny<<endl;
-    cout << "Timestep                    " << dt << endl;
-    //cout << "# of timesteps              " << NT << endl;
-    cout << "Diffusion                   " << D <<endl;
+    std::cout << "# of Legendre coefficients: " << n<<std::endl;
+    std::cout << "# of grid cells:            " << Nx*Ny<<std::endl;
+    std::cout << "Timestep                    " << dt << std::endl;
+    //std::cout << "# of timesteps              " << NT << std::endl;
+    std::cout << "Diffusion                   " << D <<std::endl;
     dg::Lamb lamb( 0.5*lx, 0.5*ly, 0.2*lx, 1);
-    HVec omega = evaluate ( lamb, grid);
-    DVec stencil = evaluate( one, grid);
-    DVec y0( omega);
-    Shu<dg::DMatrix, dg::DVec> test( grid, eps);
-    Diffusion<DMatrix, DVec> diffusion( grid, D);
-    Karniadakis< DVec > karniadakis( y0, y0.size(), 1e-8);
+    dg::HVec omega = evaluate ( lamb, grid);
+    dg::DVec stencil = evaluate( dg::one, grid);
+    dg::DVec y0( omega);
+    shu::Shu<dg::DMatrix, dg::DVec> test( grid, eps);
+    //shu::Diffusion<DMatrix, DVec> diffusion( grid, D);
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, grid);
+    //dg::Karniadakis< dg::DVec > stepper( y0, y0.size(), 1e-8);
+    dg::FilteredExplicitMultistep< dg::DVec > stepper( "TVB",3, y0);
 
     ////////////////////////////////glfw//////////////////////////////
     //create visualisation vectors
-    DVec visual( grid.size());
-    HVec hvisual( grid.size());
+    dg::DVec visual( grid.size());
+    dg::HVec hvisual( grid.size());
     //transform vector to an equidistant grid
     dg::IDMatrix equidistant = dg::create::backscatter( grid );
     draw::ColorMapRedBlueExt colors( 1.);
     double time = 0;
-    karniadakis.init( test, diffusion, time, y0, dt);
+    //stepper.init( test, diffusion, time, y0, dt);
+    stepper.init( test, filter, time, y0, dt);
     while (!glfwWindowShouldClose(w))
     {
         //transform field to an equidistant grid
-        cout << "Total vorticity is: "<<blas2::dot( stencil, w2d, y0) << "\n";
-        cout << "Total enstrophy is: "<<blas2::dot( w2d, y0)<<"\n";
+        std::cout << "Total vorticity is: "<<dg::blas2::dot( stencil, w2d, y0) << "\n";
+        std::cout << "Total enstrophy is: "<<dg::blas2::dot( w2d, y0)<<"\n";
         //compute the color scale
         dg::blas2::symv( equidistant, y0, visual );
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), -1., dg::AbsMax<float>() );
@@ -77,21 +77,19 @@ int main()
         //draw and swap buffers
         dg::blas1::transfer(visual, hvisual);
         render.renderQuad( hvisual, n*Nx, n*Ny, colors);
-        //step 
-        karniadakis.step( test,diffusion, time, y0 );
+        //step
+        //stepper.step( test,diffusion, time, y0 );
+        stepper.step( test,filter, time, y0 );
 
         glfwSwapBuffers(w);
         glfwWaitEvents();
     }
     glfwTerminate();
     ////////////////////////////////////////////////////////////////////
-    /*
-    cout << "Total vorticity is: "<< blas2::dot( stencil, w2d, y0) << "\n";
-    cout << "Total enstrophy  is "<<blas2::dot( y0, w2d, y0)<<"\n";
-    blas1::axpby( 1., sol.data(), -1., y0);
-    cudaThreadSynchronize();
-    cout << "Distance to solution "<<sqrt( blas2::dot( w2d, y0))<<endl; //don't forget sqrt when comuting errors
-    */
+    std::cout << "Total vorticity is: "<< dg::blas2::dot( stencil, w2d, y0) << "\n";
+    std::cout << "Total enstrophy  is "<<dg::blas2::dot( y0, w2d, y0)<<"\n";
+    //dg::blas1::axpby( 1., sol.data(), -1., y0);
+    //std::cout << "Distance to solution "<<sqrt( dg::blas2::dot( w2d, y0))<<std::endl; //don't forget sqrt when comuting errors
 
     return 0;
 
diff --git a/src/lamb_dipole/shu_time.cu b/src/lamb_dipole/shu_time.cu
index 5d51fbeda..ca79fd61a 100644
--- a/src/lamb_dipole/shu_time.cu
+++ b/src/lamb_dipole/shu_time.cu
@@ -6,10 +6,6 @@
 #include "shu.cuh"
 #include "parameters.h"
 
-
-
-using namespace std;
-using namespace dg;
 //const unsigned k=4;
 const double Tmax = 0.01;
 const double eps = 1e-14;
@@ -19,31 +15,30 @@ unsigned Nx = 100, Ny = Nx;
 int main( int argc, char * argv[])
 {
 
-
-    double dt0; 
+    double dt0;
     std::cout << "type dt0 (1e-3)!\n";
     std::cin >> dt0;
     std::cout << "k n dt Nx eps vort enstr energy\n";
-    Grid2d grid( 0, 1, 0, 1, n, Nx, Ny, dg::PER, dg::PER);
-    DVec w2d( create::weights(grid));
+    dg::Grid2d grid( 0, 1, 0, 1, n, Nx, Ny, dg::PER, dg::PER);
+    dg::DVec w2d( dg::create::weights(grid));
     dg::Lamb lamb( 0.5, 0.8, 0.1, 1.);
-    const HVec omega = evaluate ( lamb, grid);
-    Shu<dg::DMatrix, dg::DVec> shu( grid, eps);
-    const DVec stencil = evaluate( one, grid);
+    const dg::HVec omega = dg::evaluate ( lamb, grid);
+    shu::Shu<dg::DMatrix, dg::DVec> shu( grid, eps);
+    const dg::DVec stencil = dg::evaluate( dg::one, grid);
     for(unsigned i=0; i<6;i++)
     {
         double dt = dt0/pow(2,i);
         unsigned NT = (unsigned)(Tmax/dt);
         double time = 0;
-        //initiate solver 
-        DVec y0( omega ), y1( y0);
+        //initiate solver
+        dg::DVec y0( omega ), y1( y0);
         //make solver and stepper
-        AdamsBashforth< DVec > ab(3, y0);
+        dg::ExplicitMultistep< dg::DVec > ab("eBDF",3, y0);
         ab.init( shu, time, y0, dt);
 
-        double vorticity = blas2::dot( stencil, w2d, y1);
-        double enstrophy = 0.5*blas2::dot( y1, w2d, y1);
-        double energy =    0.5*blas2::dot( y1, w2d, shu.potential()) ;
+        double vorticity = dg::blas2::dot( stencil, w2d, y1);
+        double enstrophy = 0.5*dg::blas2::dot( y1, w2d, y1);
+        double energy =    0.5*dg::blas2::dot( y1, w2d, shu.potential()) ;
         /////////////////////////////////////////////////////////////////
         try{
         for( unsigned i=0; i<NT; i++)
@@ -56,9 +51,9 @@ int main( int argc, char * argv[])
             std::cerr << "Does Simulation respect CFL condition?\n";
         }
         std::cout << 2 <<" "<<n<<" "<<dt<<" "<<Nx<<" "<<eps<<" ";
-        std::cout << fabs(blas2::dot( stencil , w2d, y1));
-        std::cout << " "<<fabs(0.5*blas2::dot( y1, w2d, y1)-enstrophy)/enstrophy;
-        std::cout << " "<<fabs(0.5*blas2::dot( y1, w2d, shu.potential())-energy)/energy <<"\n";
+        std::cout << fabs(dg::blas2::dot( stencil , w2d, y1));
+        std::cout << " "<<fabs(0.5*dg::blas2::dot( y1, w2d, y1)-enstrophy)/enstrophy;
+        std::cout << " "<<fabs(0.5*dg::blas2::dot( y1, w2d, shu.potential())-energy)/energy <<"\n";
     }
     std::cout << std::endl;
     return 0;
-- 
GitLab


From 565e187f7fbafda07c7f1e67bd0774c4c15a1dde Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 11 Jan 2021 15:45:40 +0100
Subject: [PATCH 422/540] Introduce _t utility typedefs

the purpose is to make writing programs easier that compile for both mpi
and shared memory
---
 inc/dg/backend/typedefs.h        | 33 ++++++++++++++++++++++++++++++++
 inc/dg/topology/base_geometry.h  |  9 +++++++++
 inc/dg/topology/grid.h           |  7 +++++++
 inc/dg/topology/interpolation.h  |  8 ++++++++
 inc/dg/topology/mpi_base.h       |  6 ++++++
 inc/dg/topology/mpi_grid.h       |  4 ++++
 inc/dg/topology/mpi_projection.h |  6 ++++++
 inc/geometries/curvilinear.h     |  4 ++++
 inc/geometries/mpi_curvilinear.h |  2 ++
 src/feltor/feltor.cu             | 12 ++++++------
 src/feltor/feltor_hpc.cu         | 21 +++++++-------------
 11 files changed, 92 insertions(+), 20 deletions(-)

diff --git a/inc/dg/backend/typedefs.h b/inc/dg/backend/typedefs.h
index a22efcb9f..09af767d8 100644
--- a/inc/dg/backend/typedefs.h
+++ b/inc/dg/backend/typedefs.h
@@ -66,4 +66,37 @@ using fMDMatrix = dg::RowColDistMat<dg::fDMatrix, dg::fDCooMat, dg::fNNCD>; //!<
 }//namespace dg
 #endif //MPI_VERSION
 
+//MPI-independent definitions
+namespace dg{
+///@addtogroup typedefs
+///@{
+//vectors
+#ifdef MPI_VERSION
+using HVec_t  = MHVec;
+using fHVec_t = fMHVec;
+
+using DVec_t  = MDVec;
+using fDVec_t = fMDVec;
+
+//derivative matrices
+using HMatrix_t = MHMatrix;
+using fHMatrix_t = fMHMatrix;
+using DMatrix_t = MDMatrix;
+using fDMatrix_t = fMDMatrix;
+#else
+using HVec_t  = HVec;
+using fHVec_t = fHVec;
+
+using DVec_t  = DVec;
+using fDVec_t = fDVec;
+
+//derivative matrices
+using HMatrix_t = HMatrix;
+using fHMatrix_t = fHMatrix;
+using DMatrix_t = DMatrix;
+using fDMatrix_t = fDMatrix;
+#endif //MPI_VERSION
+///@}
+}//namespace dg
+
 #endif//_DG_TYPEDEFS_CUH_
diff --git a/inc/dg/topology/base_geometry.h b/inc/dg/topology/base_geometry.h
index 4c5db43a6..2862f1eca 100644
--- a/inc/dg/topology/base_geometry.h
+++ b/inc/dg/topology/base_geometry.h
@@ -284,6 +284,15 @@ using aProductGeometry3d    = dg::aRealProductGeometry3d<double>;
 using CartesianGrid2d       = dg::RealCartesianGrid2d<double>;
 using CartesianGrid3d       = dg::RealCartesianGrid3d<double>;
 using CylindricalGrid3d     = dg::RealCylindricalGrid3d<double>;
+#ifndef MPI_VERSION
+using aGeometry2d_t           = aGeometry2d           ;
+using aGeometry3d_t           = aGeometry3d           ;
+using aProductGeometry3d_t    = aProductGeometry3d    ;
+using CartesianGrid2d_t       = CartesianGrid2d       ;
+using CartesianGrid3d_t       = CartesianGrid3d       ;
+using CylindricalGrid3d_t     = CylindricalGrid3d     ;
+#endif //MPI_VERSION
+
 ///@}
 
 } //namespace dg
diff --git a/inc/dg/topology/grid.h b/inc/dg/topology/grid.h
index 38a269bd2..7fb465df4 100644
--- a/inc/dg/topology/grid.h
+++ b/inc/dg/topology/grid.h
@@ -833,6 +833,13 @@ using Grid2d        = dg::RealGrid2d<double>;
 using Grid3d        = dg::RealGrid3d<double>;
 using aTopology2d   = dg::aRealTopology2d<double>;
 using aTopology3d   = dg::aRealTopology3d<double>;
+#ifndef MPI_VERSION
+using Grid1d_t        = Grid1d      ;
+using Grid2d_t        = Grid2d      ;
+using Grid3d_t        = Grid3d      ;
+using aTopology2d_t   = aTopology2d ;
+using aTopology3d_t   = aTopology3d ;
+#endif
 ///@}
 
 }// namespace dg
diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index 6b2dd9ddf..30d130bda 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -25,6 +25,14 @@ using IHMatrix = tIHMatrix<double>;
 using IDMatrix = tIDMatrix<double>;
 //typedef cusp::csr_matrix<int, double, cusp::host_memory> IHMatrix; //!< CSR host Matrix
 //typedef cusp::csr_matrix<int, double, cusp::device_memory> IDMatrix; //!< CSR device Matrix
+#ifndef MPI_VERSION
+template<class real_type>
+using tIHMatrix_t = tIHMatrix<real_type>;
+template<class real_type>
+using tIDMatrix_t = tIDMatrix<real_type>;
+using IHMatrix_t = IHMatrix;
+using IDMatrix_t = IDMatrix;
+#endif //MPI_VERSION
 
 ///@}
 
diff --git a/inc/dg/topology/mpi_base.h b/inc/dg/topology/mpi_base.h
index f0eff86d4..6e659de74 100644
--- a/inc/dg/topology/mpi_base.h
+++ b/inc/dg/topology/mpi_base.h
@@ -262,6 +262,12 @@ using aProductMPIGeometry3d = dg::aRealProductMPIGeometry3d<double>;
 using CartesianMPIGrid2d    = dg::RealCartesianMPIGrid2d<double>;
 using CartesianMPIGrid3d    = dg::RealCartesianMPIGrid3d<double>;
 using CylindricalMPIGrid3d  = dg::RealCylindricalMPIGrid3d<double>;
+using aGeometry2d_t           = aMPIGeometry2d           ;
+using aGeometry3d_t           = aMPIGeometry3d           ;
+using aProductGeometry3d_t    = aProductMPIGeometry3d    ;
+using CartesianGrid2d_t       = CartesianMPIGrid2d       ;
+using CartesianGrid3d_t       = CartesianMPIGrid3d       ;
+using CylindricalGrid3d_t     = CylindricalMPIGrid3d     ;
 ///@}
 
 }//namespace dg
diff --git a/inc/dg/topology/mpi_grid.h b/inc/dg/topology/mpi_grid.h
index 5cb3de7fe..0812da19f 100644
--- a/inc/dg/topology/mpi_grid.h
+++ b/inc/dg/topology/mpi_grid.h
@@ -741,6 +741,10 @@ using MPIGrid2d         = dg::RealMPIGrid2d<double>;
 using MPIGrid3d         = dg::RealMPIGrid3d<double>;
 using aMPITopology2d    = dg::aRealMPITopology2d<double>;
 using aMPITopology3d    = dg::aRealMPITopology3d<double>;
+using Grid2d_t          = MPIGrid2d      ;
+using Grid3d_t          = MPIGrid3d      ;
+using aTopology2d_t     = aMPITopology2d ;
+using aTopology3d_t     = aMPITopology3d ;
 ///@}
 
 }//namespace dg
diff --git a/inc/dg/topology/mpi_projection.h b/inc/dg/topology/mpi_projection.h
index ea8452459..dbc1624cb 100644
--- a/inc/dg/topology/mpi_projection.h
+++ b/inc/dg/topology/mpi_projection.h
@@ -22,6 +22,12 @@ using MIHMatrix = tMIHMatrix<double>;
 using MIDMatrix = tMIDMatrix<double>;
 //typedef MPIDistMat< dg::IHMatrix, GeneralComm< dg::iHVec, dg::HVec > > MIHMatrix; //!< MPI distributed CSR host Matrix
 //typedef MPIDistMat< dg::IDMatrix, GeneralComm< dg::iDVec, dg::DVec > > MIDMatrix; //!< MPI distributed CSR device Matrix
+template<class real_type>
+using tIHMatrix_t = tMIHMatrix<real_type>;
+template<class real_type>
+using tIDMatrix_t = tMIDMatrix<real_type>;
+using IHMatrix_t = MIHMatrix;
+using IDMatrix_t = MIDMatrix;
 ///@}
 
 ///@cond
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 45d94f5f5..1fcabb9ee 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -204,6 +204,10 @@ struct RealCurvilinearProductGrid3d : public dg::aRealProductGeometry3d<real_typ
 
 using CurvilinearGrid2d         = dg::geo::RealCurvilinearGrid2d<double>;
 using CurvilinearProductGrid3d  = dg::geo::RealCurvilinearProductGrid3d<double>;
+#ifndef MPI_VERSION
+using CurvilinearGrid2d_t         = CurvilinearGrid2d        ;
+using CurvilinearProductGrid3d_t  = CurvilinearProductGrid3d ;
+#endif
 
 ///@}
 ///@cond
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index cce08129f..624c3320d 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -204,6 +204,8 @@ RealCurvilinearMPIGrid2d<real_type>::RealCurvilinearMPIGrid2d( const RealCurvili
 //
 using CurvilinearMPIGrid2d         = dg::geo::RealCurvilinearMPIGrid2d<double>;
 using CurvilinearProductMPIGrid3d  = dg::geo::RealCurvilinearProductMPIGrid3d<double>;
+using CurvilinearGrid2d_t         = CurvilinearMPIGrid2d        ;
+using CurvilinearProductGrid3d_t  = CurvilinearProductMPIGrid3d ;
 
 ///@}
 }//namespace geo
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 51841ac50..11281001d 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -10,12 +10,12 @@
 #include "feltor.h"
 #include "implicit.h"
 
-using HVec = dg::HVec;
-using DVec = dg::DVec;
-using DMatrix = dg::DMatrix;
-using IDMatrix = dg::IDMatrix;
-using IHMatrix = dg::IHMatrix;
-using Geometry = dg::CylindricalGrid3d;
+using HVec = dg::HVec_t;
+using DVec = dg::DVec_t;
+using DMatrix = dg::DMatrix_t;
+using IDMatrix = dg::IDMatrix_t;
+using IHMatrix = dg::IHMatrix_t;
+using Geometry = dg::CylindricalGrid3d_t;
 #define MPI_OUT
 
 #include "init.h"
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 3e3e0fbd9..6080ada0b 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -14,23 +14,16 @@
 #include "feltor.h"
 #include "implicit.h"
 
+using HVec = dg::HVec_t;
+using DVec = dg::DVec_t;
+using HMatrix = dg::HMatrix_t;
+using DMatrix = dg::DMatrix_t;
+using IDMatrix = dg::IDMatrix_t;
+using IHMatrix = dg::IHMatrix_t;
+using Geometry = dg::CylindricalGrid3d_t;
 #ifdef FELTOR_MPI
-using HVec = dg::MHVec;
-using DVec = dg::MDVec;
-using HMatrix = dg::MHMatrix;
-using DMatrix = dg::MDMatrix;
-using IDMatrix = dg::MIDMatrix;
-using IHMatrix = dg::MIHMatrix;
-using Geometry = dg::CylindricalMPIGrid3d;
 #define MPI_OUT if(rank==0)
 #else //FELTOR_MPI
-using HVec = dg::HVec;
-using DVec = dg::DVec;
-using HMatrix = dg::HMatrix;
-using DMatrix = dg::DMatrix;
-using IDMatrix = dg::IDMatrix;
-using IHMatrix = dg::IHMatrix;
-using Geometry = dg::CylindricalGrid3d;
 #define MPI_OUT
 #endif //FELTOR_MPI
 
-- 
GitLab


From 40914b52d6571dbcbde4299aba24d0f370056bee Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Jan 2021 13:01:24 +0100
Subject: [PATCH 423/540] Fix n=1 bug in Filter and add construct methods

for timesteppers
---
 inc/dg/adaptive.h  |  1 +
 inc/dg/arakawa.h   | 14 ++++++++++++
 inc/dg/functors.h  |  1 +
 inc/dg/multistep.h | 56 ++++++++++++++++++++++++++--------------------
 4 files changed, 48 insertions(+), 24 deletions(-)

diff --git a/inc/dg/adaptive.h b/inc/dg/adaptive.h
index 7e7ea1f64..11f8cfcd5 100644
--- a/inc/dg/adaptive.h
+++ b/inc/dg/adaptive.h
@@ -54,6 +54,7 @@ value_type i_control( value_type dt_old, value_type eps_0, value_type eps_1, val
 template<class value_type>
 struct PIDController
 {
+    //PID means proportional (present), integral (past), derivative (future)
     PIDController( ){}
     value_type operator()( value_type dt_old, value_type eps_n, value_type eps_n1, value_type eps_n2, unsigned embedded_order, unsigned order)const
     {
diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index ff4583ba8..bd00c911d 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -36,6 +36,7 @@ struct ArakawaX
     using matrix_type = Matrix;
     using container_type = Container;
     using value_type = get_value_type<Container>;
+    ArakawaX(){}
     /**
      * @brief Create Arakawa on a grid
      * @param g The grid
@@ -51,6 +52,19 @@ struct ArakawaX
      */
     ArakawaX( const Geometry& g, bc bcx, bc bcy);
 
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        //construct and swap
+        *this = ArakawaX( std::forward<Params>( ps)...);
+    }
+
     /**
      * @brief Compute Poisson bracket
      *
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index db32ba8ef..53e071f4c 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -1121,6 +1121,7 @@ struct ExponentialFilter
     double operator()( unsigned i) const
     {
         double eta = (double)i/(double)(m_n-1);
+        if( m_n == 1) eta = 0.;
         if( eta < m_etac)
             return 1.;
         if( eta <= 1.)
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index c8085829d..33b9cb460 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -119,7 +119,7 @@ struct Karniadakis
     */
     template<class ...SolverParams>
     void construct( SolverParams&& ...ps){
-        m_solver = Solver( std::forward<SolverParams>(ps)...);
+        m_solver = SolverType( std::forward<SolverParams>(ps)...);
         m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
         set_order(3);
         m_counter = 0;
@@ -488,7 +488,7 @@ enum multistep_identifier
     /** The family of schemes described in <a href = "https://en.wikipedia.org/wiki/Linear_multistep_method"> Linear multistep methods </a>
      as **Adams-Bashforth**
     * \f[ u^{n+1} = u^n + \Delta t\sum_{j=0}^{s-1} b_j f\left(t^n - j \Delta t, u^{n-j}\right) \f]
-     **Possible stages are 1, 2,..., 5**, the order of the method is the same as its stages
+    * **Possible stages are 1, 2,..., 5**, the order of the method is the same as its stages
     @note The Adams-Bashforth schemes implemented here need less storage but may have **a smaller region of absolute stability** than for example an extrapolated BDF method of the same order.
     */
     ADAMS_BASHFORTH,
@@ -504,12 +504,12 @@ enum multistep_identifier
      described in <a href =
      "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf">
      Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
-     * **Possible orders are 1, 2,..., 7**
+     * **Possible stages are 1, 2,..., 7** with the order the same as the number of stages
 
     */
     eBDF,
     /** The family of schemes described in <a href="https://doi.org/10.1016/j.jcp.2005.02.029">S.J. Ruuth and W. Hundsdorfer, High-order linear multistep methods with general monotonicity and boundedness properties, Journal of Computational Physics, Volume 209, Issue 1, 2005 </a> as Total variation Bound.
-     * These schemes have larger stable step sizes than the eBDF family, for
+     * These schemes have **larger stable step sizes than the eBDF family**, for
      * example TVB3 has 38% larger stepsize than eBDF3 and TVB4 has 109% larger
      * stepsize than eBDF4.  **Possible orders are 1,2,...,6**, The CFL
      * conditions in relation to forward Euler are (1-1: 1, 2-2: 0.5, 3-3: 0.54, 4-4: 0.46, 5-5:
@@ -521,8 +521,8 @@ enum multistep_identifier
      * We implement the lowest order schemes for each stage. The CFL conditions
      * in relation to forward Euler are (1-1: 1, 2-2: 0.5, 3-2: 0.5, 4-2: 0.66, 5-3:
      * 0.5, 6-3: 0.567).  We disregard the remaining
-     * schemes since their CFL condition is worse than a TVB scheme of the same
-     * order. These schemes are noteworthy because the coefficients b_i are all positive except for the 2-2 method.
+     * schemes since their CFL conditions are worse than the TVB scheme of the same
+     * order. These schemes are noteworthy because the coefficients b_i are all positive except for the 2-2 method and **the "4-2" and "6-3" methods allow slightly larger stepsize than eBDF** of same order (2 and 3)
      */
     SSP
 };
@@ -562,7 +562,15 @@ struct FilteredExplicitMultistep
     ///@copydoc RungeKutta::RungeKutta()
     FilteredExplicitMultistep(){}
 
-    ///@copydoc construct()
+    /**
+     * @brief Reserve memory for the integration
+     *
+     * Set the coefficients \f$ a_i,\ b_i\f$
+     * @param method the name of the family of schemes to be used (a string can be converted to an enum with the same spelling) @sa multistep_identifier
+     * @param stages (global) stages (= number of steps in the multistep) of the method (Currently possible values depend on the method), does not necessarily coincide with the order of the method
+     * @param copyable ContainerType of the size that is used in \c step
+     * @note it does not matter what values \c copyable contains, but its size is important
+     */
     FilteredExplicitMultistep( std::string method, unsigned stages, const ContainerType& copyable){
         std::map < std::string, enum multistep_identifier> str2id{
             {"Adams-Bashforth", ADAMS_BASHFORTH},
@@ -573,28 +581,28 @@ struct FilteredExplicitMultistep
         if( str2id.find(method) == str2id.end())
             throw dg::Error(dg::Message(_ping_)<<"Multistep coefficients for "<<method<<" not found!");
         else
-            construct( str2id[method], stages, copyable);
+            *this = FilteredExplicitMultistep( str2id[method], stages, copyable);
     }
-    ///@copydoc construct()
+    ///@copydoc FilteredExplicitMultistep(std::string,unsigned,const ContainerType&)
     FilteredExplicitMultistep( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
-        construct( method, stages, copyable);
-    }
-    /**
-     * @brief Reserve memory for the integration
-     *
-     * Set the coefficients \f$ a_i,\ b_i\f$
-     * @param method the name of the family of schemes to be used (a string can be converted to an enum with the same spelling) @sa multistep_identifier
-     * @param stages (global) stages (= number of steps in the multistep) of the method (Currently possible values depend on the method), does not necessarily coincide with the order of the method
-     * @param copyable ContainerType of the size that is used in \c step
-     * @note it does not matter what values \c copyable contains, but its size is important
-     */
-    void construct( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
         m_k = stages;
         m_f.assign( stages, copyable);
         m_u.assign( stages, copyable);
         init_coeffs(method, stages);
         m_counter = 0;
     }
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        //construct and swap
+        *this = FilteredExplicitMultistep( std::forward<Params>( ps)...);
+    }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_u[0];}
@@ -805,11 +813,11 @@ struct ExplicitMultistep
     using container_type = ContainerType; //!< the type of the vector class in use
     ///@copydoc RungeKutta::RungeKutta()
     ExplicitMultistep(){}
-    ///@copydoc construct()
+    ///@copydoc FilteredExplicitMultistep(std::string,unsigned,const ContainerType&)
     ExplicitMultistep( std::string method, unsigned stages, const ContainerType& copyable): m_fem( method, stages, copyable){ }
-    ///@copydoc construct()
+    ///@copydoc FilteredExplicitMultistep(std::string,unsigned,const ContainerType&)
     ExplicitMultistep( enum multistep_identifier method, unsigned stages, const ContainerType& copyable): m_fem( method, stages, copyable){}
-    ///@copydoc FilteredExplicitMultistep::construct()
+    ///@copydoc FilteredExplicitMultistep(std::string,unsigned,const ContainerType&)
     void construct( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
         m_fem.construct( method, stages, copyable);
     }
-- 
GitLab


From 40ff9d54ed01cdfc5cf0ac058bd6de44f8fe4514 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Jan 2021 13:02:24 +0100
Subject: [PATCH 424/540] Add construct method to Filter

---
 inc/dg/topology/filter.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/inc/dg/topology/filter.h b/inc/dg/topology/filter.h
index f75dbc24f..f242c5a52 100644
--- a/inc/dg/topology/filter.h
+++ b/inc/dg/topology/filter.h
@@ -164,6 +164,19 @@ struct ModalFilter
     ModalFilter( UnaryOp sigma, const Topology& t) : m_filter (
             dg::create::modal_filter( sigma, t)) { }
 
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        //construct and swap
+        *this = ModalFilter( std::forward<Params>( ps)...);
+    }
+
     void apply( const ContainerType& x, ContainerType& y) const{ symv( 1., x, 0., y);}
     void symv( const ContainerType& x, ContainerType& y) const{ symv( 1., x,0,y);}
     void symv(real_type alpha, const ContainerType& x, real_type beta, ContainerType& y) const
-- 
GitLab


From e492d4ce3157a7bb95b7d9db9fd79a560301e752 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Jan 2021 13:07:45 +0100
Subject: [PATCH 425/540] Rewrite Shu project into one file

the idea is to use this as a platform to test out various advection
schemes while still being able to reproduce the results from the
original paper
---
 src/lamb_dipole/Makefile           |  17 +-
 src/lamb_dipole/init.h             |  70 +++++++
 src/lamb_dipole/input/default.json | 120 ++++++++----
 src/lamb_dipole/parameters.h       |  76 --------
 src/lamb_dipole/shu.cuh            | 209 ++++++++++++++++-----
 src/lamb_dipole/shu2d_b.cu         | 131 -------------
 src/lamb_dipole/shu_b.cu           | 291 ++++++++++++++++++++---------
 src/lamb_dipole/shu_hpc.cu         | 138 --------------
 src/lamb_dipole/shu_t.cu           |  96 ----------
 src/lamb_dipole/shu_time.cu        |  61 ------
 10 files changed, 524 insertions(+), 685 deletions(-)
 create mode 100644 src/lamb_dipole/init.h
 delete mode 100644 src/lamb_dipole/parameters.h
 delete mode 100644 src/lamb_dipole/shu2d_b.cu
 delete mode 100644 src/lamb_dipole/shu_hpc.cu
 delete mode 100644 src/lamb_dipole/shu_t.cu
 delete mode 100644 src/lamb_dipole/shu_time.cu

diff --git a/src/lamb_dipole/Makefile b/src/lamb_dipole/Makefile
index 9788d456d..1f8053517 100644
--- a/src/lamb_dipole/Makefile
+++ b/src/lamb_dipole/Makefile
@@ -8,23 +8,12 @@ include ../../config/devices/devices.mk
 INCLUDE+= -I../         # other src libraries
 INCLUDE+= -I../../inc   # other project libraries
 
-all: shu_t shu_b shu2d_b shu_hpc shu_time
-
-shu_t: shu_t.cu shu.cuh
-	$(CC) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) -DDG_DEBUG
+all: shu_b
 
 shu_b: shu_b.cu shu.cuh
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB)
-
-shu2d_b: shu2d_b.cu shu.cuh
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE)
-
-shu_hpc: shu_hpc.cu shu.cuh
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB)
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(GLFLAGS) $(JSONLIB) -g
 
-shu_time: shu_time.cu shu.cuh
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE)
 .PHONY: clean
 
 clean:
-	rm -f shu_t shu_b shu2d_b shu_hpc shu_time
+	rm -f shu_b
diff --git a/src/lamb_dipole/init.h b/src/lamb_dipole/init.h
new file mode 100644
index 000000000..cd59e4b10
--- /dev/null
+++ b/src/lamb_dipole/init.h
@@ -0,0 +1,70 @@
+
+#include <map>
+#include <functional>
+#include "json/json.h"
+#include "dg/algorithm.h"
+#include "file/json_utilities.h"
+
+namespace shu{
+
+dg::CartesianGrid2d createGrid( Json::Value& js, enum file::error mode)
+{
+    unsigned n = file::get( mode, js, "grid", "n",3).asUInt();
+    unsigned Nx = file::get( mode, js, "grid", "Nx", 48).asUInt();
+    unsigned Ny = file::get( mode, js, "grid", "Ny", 48).asUInt();
+    double x0 = file::get_idx( mode, js, "grid", "x", 0u, 0.).asDouble();
+    double x1 = file::get_idx( mode, js, "grid", "x", 1u, 1.).asDouble();
+    double y0 = file::get_idx( mode, js, "grid", "y", 0u, 0.).asDouble();
+    double y1 = file::get_idx( mode, js, "grid", "y", 1u, 1.).asDouble();
+    dg::bc bcx = dg::str2bc ( file::get_idx( mode, js, "grid", "bc", 0u, "DIR").asString());
+    dg::bc bcy = dg::str2bc ( file::get_idx( mode, js, "grid", "bc", 1u, "PER").asString());
+    return dg::CartesianGrid2d( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy);
+}
+
+struct ShearLayer{
+
+    ShearLayer( double rho, double delta): m_rho(rho), m_delta(delta){}
+    DG_DEVICE
+    double operator()(double x, double y) const{
+    if( y<= M_PI)
+        return m_delta*cos(x) - 1./m_rho/cosh( (y-M_PI/2.)/m_rho)/cosh( (y-M_PI/2.)/m_rho);
+    return m_delta*cos(x) + 1./m_rho/cosh( (3.*M_PI/2.-y)/m_rho)/cosh( (3.*M_PI/2.-y)/m_rho);
+    }
+    private:
+    double m_rho;
+    double m_delta;
+};
+
+std::map<std::string, std::function< dg::HVec(
+    Json::Value& js, enum file::error mode,
+    const dg::CartesianGrid2d& grid) >
+> initial_conditions =
+{
+    {"lamb", [](
+        Json::Value& js, enum file::error mode,
+        const dg::CartesianGrid2d& grid)
+        {
+            dg::HVec omega;
+            double posX = file::get( mode, js, "init", "posX", 0.5).asDouble();
+            double posY = file::get( mode, js, "init", "posY", 0.8).asDouble();
+            double R = file::get( mode, js, "init", "sigma", 0.1).asDouble();
+            double U = file::get( mode, js, "init", "velocity", 1).asDouble();
+            dg::Lamb lamb( posX*grid.lx(), posY*grid.ly(), R, U);
+            omega = dg::evaluate ( lamb, grid);
+            return omega;
+        }
+    },
+    {"shear", [](
+        Json::Value& js, enum file::error mode,
+        const dg::CartesianGrid2d& grid)
+        {
+            dg::HVec omega;
+            double rho = file::get( mode, js, "init", "rho", M_PI/15.).asDouble();
+            double delta = file::get( mode, js, "init", "delta", 0.05).asDouble();
+            omega = dg::evaluate ( ShearLayer( rho, delta), grid);
+            return omega;
+        }
+    }
+};
+
+}//namespace shu
diff --git a/src/lamb_dipole/input/default.json b/src/lamb_dipole/input/default.json
index 3d8e91e8c..acf2a18b5 100644
--- a/src/lamb_dipole/input/default.json
+++ b/src/lamb_dipole/input/default.json
@@ -1,43 +1,85 @@
 
 //            * Input-File for "SHU" Code *
-//            -------------------------------------------
-
 {
-    //--------------------Space and Time discretization------------
-    
-    "n"  : 3,       //(# of polynomial coefficients)    =   1     (3)
-    "Nx" : 30,      //(grid points in x)                =   80  (192)
-    "Ny" : 30,      //(grid points in y)                =   80  (192)
-    "dt" : 0.21e-3, //(time step in units c_s/L_p)      =   0.21e-3    (0.01)
-     
-     //-----------------Algorithmic parameters-------------------
-     
-    "eps_pol" : 1e-6, // (stopping for conjugate gradient)
-    "eps_time" : 1e-9, // (stopping for implicit part)
-     
-     //-----------------Boundary conditions----------------------
-     
-    "lx" : 1, //6.2831853 (length of x )
-    "ly" : 1, //6.2831853 (length of y )
-    "bc_x" : "PER", //( periodic,  Dirichlet )
-    "bc_y" : "PER", //( periodic,  Dirichlet )
-     
-     //-----------------Physical parameters----------------------
-     
-    "nu_perp": 1e-4,  //(viscosity)  
-     
-     //-------------Initial dipole parameters---------------------
-     
-    "velocity" : 1,     // (blob speed)
-    "sigma"    : 0.1,   //(dipole radius in units of lx)
-    "posX"     : 0.5,   // ( in units of lx)
-    "posY"     : 0.8,   // ( in units of ly)
-     
-     //-----------------Miscellaneous----------------------------
-     
-    "itstp"   : 5,     // (steps between output)
-    "maxout"  : 200,   //# of outputs (excluding first)
-    "initial" : "lamb" // lamb or shear
-}
+    "grid":
+    {
+        "type": "Cartesian2d",
+        "n"  : 3,       //(# of polynomial coefficients)    =   1     (3)
+        "Nx" : 48,      //(grid points in x)                =   80  (192)
+        "Ny" : 48,      //(grid points in y)                =   80  (192)
+        "x": [0, 1], //6.283185307179586],
+        "y": [0, 1], //6.283185307179586],
+        "bc" : ["DIR", "PER"]
+    },
+    //"timestepper":
+    //{
+    //    "stepper": "Karniadakis",
+    //    "order": 3,
+    //    "dt" : 2e-3,
+    //    "eps_time" : 1e-9, // (stopping for implicit part)
+    //    "solver":{
+    //        "type": "default"
+    //    }
+    //},
+    //"timestepper":
+    //{
+    //    "stepper": "Shu-Osher",
+    //    "order": "SSPRK-3-3",
+    //    "dt" : 1e-3
+    //},
+    "timestepper":
+    {
+        "stepper": "FilteredMultistep",
+        "order": ["eBDF", 3],
+        "dt" : 5e-4
+    },
+    "regularization":
+    {
+        "type": "modal",
+        "alpha": 36,
+        "order": 8,
+        "eta_c": 0.5
+    },
+    //"regularization":
+    //{
+    //    "type" : "viscosity",
+    //    "order": 2,
+    //    "nu_perp": 1.0e-3  //(viscosity)  
+    //},
+    "output":
+    {
+        "type": "glfw",
+        //"type": "netcdf",
+        "itstp"   : 5,     // (steps between output)
+        "maxout"  : 100   //# of outputs (excluding first)
+    },
 
-//@ ------------------------------------------------------------
+    "elliptic":
+    {
+        "type" : "multigrid",
+        "stages": 3,
+        "eps_pol" : [1e-6,10,10] // (stopping on each steage )
+    },
+    "advection":
+    {
+        //"type": "arakawa",
+        "multiplication": "pointwise",
+        "type": "upwind"
+        //"multiplication": "projection"
+    },
+
+    "init":
+    {
+        "type" : "lamb",
+        "velocity" : 1,     // (blob speed)
+        "sigma"    : 0.1,   //(dipole radius in units of lx)
+        "posX"     : 0.5,   // ( in units of lx)
+        "posY"     : 0.8   // ( in units of ly)
+    }
+    //"init":
+    //{
+    //    "type" : "shear",
+    //    "rho":  0.20943951023931953, //pi/15
+    //    "delta": 0.05
+    //}
+}
diff --git a/src/lamb_dipole/parameters.h b/src/lamb_dipole/parameters.h
deleted file mode 100644
index 987187a96..000000000
--- a/src/lamb_dipole/parameters.h
+++ /dev/null
@@ -1,76 +0,0 @@
-#pragma once
-#include <string>
-#include "dg/enums.h"
-#include "json/json.h"
-
-/**
- * @brief Provide a mapping between input file and named parameters
- */
-struct Parameters
-{
-    unsigned n, Nx, Ny; 
-    double dt; 
-
-    double eps, eps_time;
-    double lx, ly; 
-    enum dg::bc bc_x, bc_y;
-
-    double D;
-    int global;
-
-    double U, R, posX, posY;
-
-    unsigned itstp; 
-    unsigned maxout;
-    std::string initial;
-
-    Parameters( const Json::Value& js) {
-        //names are chosen to be as consisten with other projects as possible
-        n  = js["n"].asUInt();
-        Nx = js["Nx"].asUInt();
-        Ny = js["Ny"].asUInt();
-        dt = js["dt"].asDouble();
-        eps = js["eps_pol"].asDouble();
-        eps_time = js["eps_time"].asDouble();
-        lx = js["lx"].asDouble();
-        ly = js["ly"].asDouble();
-        bc_x = dg::str2bc(js["bc_x"].asString());
-        bc_y = dg::str2bc(js["bc_y"].asString());
-        D = js["nu_perp"].asDouble();
-        R = js["sigma"].asDouble();
-        posX = js["posX"].asDouble();
-        posY = js["posY"].asDouble();
-        itstp = js["itstp"].asUInt();
-        maxout = js["maxout"].asUInt();
-        U = js.get("velocity", 1).asDouble();
-        initial = js.get("initial", "lamb").asString();
-    }
-
-    void display( std::ostream& os = std::cout ) const
-    {
-        os << "Physical parameters are: \n"
-            <<"    Viscosity:       = "<<D<<"\n";
-        os << "Boundary parameters are: \n"
-            <<"    lx = "<<lx<<"\n"
-            <<"    ly = "<<ly<<"\n"
-            <<"Boundary conditions in x are: \n"
-            <<"    "<<bc2str(bc_x)<<"\n"
-            <<"Boundary conditions in y are: \n"
-            <<"    "<<bc2str(bc_y)<<"\n";
-        os << "Algorithmic parameters are: \n"
-            <<"    n  = "<<n<<"\n"
-            <<"    Nx = "<<Nx<<"\n"
-            <<"    Ny = "<<Ny<<"\n"
-            <<"    dt = "<<dt<<"\n";
-        os  <<"Dipole parameters are: \n"
-            << "    radius:       "<<R<<"\n"
-            << "    velocity:     "<<U<<"\n"
-            << "    posX:         "<<posX<<"\n"
-            << "    posY:         "<<posY<<"\n";
-        os << "Stopping for CG:         "<<eps<<"\n"
-            <<"Steps between output:    "<<itstp<<"\n"
-            <<"Initial condition:       "<<initial<<"\n"
-            <<"Number of outputs:       "<<maxout<<std::endl;
-
-    }
-};
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index 867740522..e97788753 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -4,82 +4,203 @@
 #include <exception>
 #include <cusp/ell_matrix.h>
 
+#include "json/json.h"
+#include "file/json_utilities.h"
 #include "dg/algorithm.h"
 
 namespace shu
 {
-template< class Matrix, class container>
+
+struct Upwind{
+    DG_DEVICE
+    void operator()( double& result, double fw, double bw, double v)
+    {
+        if( v > 0)
+            result -= bw; // yp = - Div( F)
+        else
+            result -= fw;
+    }
+};
+template< class Geometry, class Matrix, class Container>
 struct Diffusion
 {
-    Diffusion( const dg::Grid2d& g, double nu): nu_(nu),
-        w2d( dg::create::weights( g) ), v2d( dg::create::inv_weights(g) ) ,
-        LaplacianM( g, dg::normed)
+    Diffusion( const Geometry& g, Json::Value& js, enum file::error mode)
     {
+        std::string regularization = file::get( mode, js, "regularization", "type", "modal").asString();
+        if( "viscosity" == regularization)
+        {
+            m_nu = file::get( mode, js, "regularization", "nu_perp", 1e-3).asDouble();
+            m_order = file::get( mode, js, "regularization", "order", 1).asUInt();
+            m_temp = dg::evaluate( dg::zero, g);
+            m_LaplacianM.construct( g, dg::normed);
+        }
     }
-    void operator()(double t, const container& x, container& y)
+    void operator()(double t, const Container& x, Container& y)
     {
-        dg::blas2::gemv( LaplacianM, x, y);
-        dg::blas1::scal( y, -nu_);
+        dg::blas1::copy( x, y);
+        for( unsigned p=0; p<m_order; p++)
+        {
+            using std::swap;
+            swap( m_temp, y);
+            dg::blas2::symv( m_nu, m_LaplacianM, m_temp, 0., y);
+        }
+        dg::blas1::scal( y, -1.);
     }
-    const container& weights(){return w2d;}
-    const container& inv_weights(){return v2d;}
-    const container& precond(){return v2d;}
+    const Container& weights(){ return m_LaplacianM.weights();}
+    const Container& inv_weights(){ return m_LaplacianM.inv_weights();}
+    const Container& precond(){ return m_LaplacianM.precond();}
   private:
-    double nu_;
-    const container w2d, v2d;
-    dg::Elliptic<dg::CartesianGrid2d, Matrix,container> LaplacianM;
+    double m_nu;
+    unsigned m_order;
+    Container m_temp;
+    dg::Elliptic<Geometry, Matrix,Container> m_LaplacianM;
 };
 
-template< class Matrix, class container >
+template< class Geometry, class IMatrix, class Matrix, class Container >
 struct Shu
 {
-    using value_type = dg::get_value_type<container>;
-    typedef container Vector;
+    using value_type = dg::get_value_type<Container>;
 
-    Shu( const dg::Grid2d& grid, double eps);
+    Shu( const Geometry& grid, Json::Value& js, enum file::error mode);
 
-    const dg::Elliptic<Matrix, container, container>& lap() const { return m_multi_laplaceM[0];}
-    dg::ArakawaX<dg::CartesianGrid2d, Matrix, container>& arakawa() {return m_arakawa;}
+    const dg::Elliptic<Matrix, Container, Container>& lap() const { return m_multi_laplaceM[0];}
+    dg::ArakawaX<Geometry, Matrix, Container>& arakawa() {return m_arakawa;}
     /**
      * @brief Returns psi that belong to the last y in operator()
      *
      * In a multistep scheme this belongs to the point HEAD-1
      * @return psi is the potential
      */
-    const container& potential( ) {return psi;}
-    void operator()(double t, const Vector& y, Vector& yp);
+    const Container& potential( ) {return m_psi;}
+    void operator()(double t, const Container& y, Container& yp);
   private:
-    container psi, w2d, v2d;
-    std::vector<dg::Elliptic<dg::CartesianGrid2d, Matrix, container>> m_multi_laplaceM;
-    dg::ArakawaX<dg::CartesianGrid2d, Matrix, container> m_arakawa;
-    dg::Extrapolation<container> m_old_psi;
-    dg::MultigridCG2d<dg::CartesianGrid2d, Matrix, container> m_multigrid;
-    double m_eps;
+    Container m_psi, m_v, m_temp[3], m_fine_psi, m_fine_v, m_fine_temp[3], m_fine_y, m_fine_yp;
+    std::vector<dg::Elliptic<Geometry, Matrix, Container>> m_multi_laplaceM;
+    dg::ArakawaX<Geometry, Matrix, Container> m_arakawa;
+    dg::Extrapolation<Container> m_old_psi;
+    dg::MultigridCG2d<Geometry, Matrix, Container> m_multigrid;
+    IMatrix m_inter, m_project;
+    Matrix m_forward[2], m_backward[2], m_centered[2];
+    std::vector<double> m_eps;
+    std::string m_advection, m_multiplication;
 };
 
-template<class Matrix, class container>
-Shu< Matrix, container>::Shu( const dg::Grid2d& g, double eps):
-    psi( g.size()),
-    w2d( dg::create::weights( g)), v2d( dg::create::inv_weights(g)),
-    m_arakawa( g),
-    m_old_psi( 2, w2d),
-    m_multigrid( g, 3),
-    m_eps(eps)
+template<class Geometry, class IMatrix, class Matrix, class Container>
+Shu< Geometry, IMatrix, Matrix, Container>::Shu(
+        const Geometry& g, Json::Value& js, enum file::error mode):
+    m_old_psi( 2, dg::evaluate( dg::zero, g)),
+    m_multigrid( g, 3)
 {
-    m_multi_laplaceM.resize(3);
-    for( unsigned u=0; u<3; u++)
+    m_advection = file::get( mode, js, "advection", "type", "arakawa").asString();
+    m_multiplication = file::get( mode, js, "advection", "multiplication", "pointwise").asString();
+
+    m_psi = dg::evaluate( dg::zero, g);
+    if( "projection" == m_multiplication)
+    {
+        Geometry fine_grid = g;
+        fine_grid.set( 2*g.n()-1, g.Nx(), g.Ny());
+        m_inter = dg::create::interpolation( fine_grid, g);
+        m_project = dg::create::projection( g, fine_grid);
+
+        m_centered[0] = dg::create::dx( fine_grid, g.bcx(), dg::centered);
+        m_centered[1] = dg::create::dy( fine_grid, g.bcy(), dg::centered);
+        m_forward[0] = dg::create::dx( fine_grid, dg::inverse( g.bcx()), dg::forward);
+        m_forward[1] = dg::create::dy( fine_grid, dg::inverse( g.bcy()), dg::forward);
+        m_backward[0] = dg::create::dx( fine_grid, dg::inverse( g.bcx()), dg::backward);
+        m_backward[1] = dg::create::dy( fine_grid, dg::inverse( g.bcy()), dg::backward);
+        m_fine_psi = dg::evaluate( dg::zero, fine_grid);
+        m_fine_y = dg::evaluate( dg::zero, fine_grid);
+        m_fine_yp = dg::evaluate( dg::zero, fine_grid);
+        m_fine_v = dg::evaluate( dg::zero, fine_grid);
+        m_fine_temp[0] = dg::evaluate( dg::zero, fine_grid);
+        m_fine_temp[1] = dg::evaluate( dg::zero, fine_grid);
+        m_fine_temp[2] = dg::evaluate( dg::zero, fine_grid);
+        m_arakawa.construct( fine_grid);
+    }
+    else
+    {
+        m_centered[0] = dg::create::dx( g, g.bcx(), dg::centered);
+        m_centered[1] = dg::create::dy( g, g.bcy(), dg::centered);
+        m_forward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::forward);
+        m_forward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::forward);
+        m_backward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::backward);
+        m_backward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::backward);
+        m_v = dg::evaluate( dg::zero, g);
+        m_temp[0] = dg::evaluate( dg::zero, g);
+        m_temp[1] = dg::evaluate( dg::zero, g);
+        m_temp[2] = dg::evaluate( dg::zero, g);
+        m_arakawa.construct( g);
+    }
+
+    unsigned stages = file::get( mode, js, "elliptic", "stages", 3).asUInt();
+    m_eps.resize(stages);
+    m_eps[0] = file::get_idx( mode, js, "elliptic", "eps_pol", 0, 1e-6).asDouble();
+    for( unsigned i=1;i<stages; i++)
+    {
+        m_eps[i] = file::get_idx( mode, js, "elliptic", "eps_pol", i, 1).asDouble();
+        m_eps[i]*= m_eps[0];
+    }
+    m_multi_laplaceM.resize(stages);
+    for( unsigned u=0; u<stages; u++)
         m_multi_laplaceM[u].construct( m_multigrid.grid(u), dg::not_normed, dg::centered, 1);
 }
 
-template< class Matrix, class container>
-void Shu<Matrix, container>::operator()(double t, const Vector& y, Vector& yp)
+template< class Geometry, class IMatrix, class Matrix, class Container>
+void Shu<Geometry, IMatrix, Matrix, Container>::operator()(double t, const Container& y, Container& yp)
 {
-    m_old_psi.extrapolate( t, psi);
-    std::vector<unsigned> number = m_multigrid.direct_solve( m_multi_laplaceM, psi, y, m_eps);
-    m_old_psi.update( t, psi);
+    //solve elliptic equation
+    m_old_psi.extrapolate( t, m_psi);
+    std::vector<unsigned> number = m_multigrid.direct_solve( m_multi_laplaceM, m_psi, y, m_eps);
+    m_old_psi.update( t, m_psi);
     if( number[0] == m_multigrid.max_iter())
-        throw dg::Fail( m_eps);
-    m_arakawa( y, psi, yp); //A(y,psi)-> yp
+        throw dg::Fail( m_eps[0]);
+    //now do advection with various schemes
+    if( "pointwise" == m_multiplication)
+    {
+        if( "arakawa" == m_advection)
+            m_arakawa( y, m_psi, yp); //A(y,psi)-> yp
+        else if( "upwind" == m_advection)
+        {
+            dg::blas1::copy( 0., yp);
+            //dx ( nv_x)
+            dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
+            dg::blas1::pointwiseDot( y, m_v, m_temp[0]); //f_x
+            dg::blas2::symv( m_forward[0], m_temp[0], m_temp[1]);
+            dg::blas2::symv( m_backward[0], m_temp[0], m_temp[2]);
+            dg::blas1::subroutine( shu::Upwind(), yp, m_temp[1], m_temp[2], m_v);
+            //dy ( nv_y)
+            dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_v); //v_y
+            dg::blas1::pointwiseDot( y, m_v, m_temp[0]); //f_y
+            dg::blas2::symv( m_forward[1], m_temp[0], m_temp[1]);
+            dg::blas2::symv( m_backward[1], m_temp[0], m_temp[2]);
+            dg::blas1::subroutine( shu::Upwind(), yp, m_temp[1], m_temp[2], m_v);
+        }
+    }
+    else // "projection " == multiplication
+    {
+        dg::blas2::symv( m_inter, y, m_fine_y);
+        dg::blas2::symv( m_inter, m_psi, m_fine_psi);
+        if( "arakawa" == m_advection)
+            m_arakawa( m_fine_y, m_fine_psi, m_fine_yp); //A(y,psi)-> yp
+        else if( "upwind" == m_advection)
+        {
+            dg::blas1::copy( 0., m_fine_yp);
+            //dx ( nv_x)
+            dg::blas2::symv( -1., m_centered[1], m_fine_psi, 0., m_fine_v); //v_x
+            dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_x
+            dg::blas2::symv( m_forward[0], m_fine_temp[0], m_fine_temp[1]);
+            dg::blas2::symv( m_backward[0], m_fine_temp[0], m_fine_temp[2]);
+            dg::blas1::subroutine( shu::Upwind(), m_fine_yp, m_fine_temp[1], m_fine_temp[2], m_fine_v);
+            //dy ( nv_y)
+            dg::blas2::symv( 1., m_centered[0], m_fine_psi, 0., m_fine_v); //v_y
+            dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_y
+            dg::blas2::symv( m_forward[1], m_fine_temp[0], m_fine_temp[1]);
+            dg::blas2::symv( m_backward[1], m_fine_temp[0], m_fine_temp[2]);
+            dg::blas1::subroutine( shu::Upwind(), m_fine_yp, m_fine_temp[1], m_fine_temp[2], m_fine_v);
+        }
+        dg::blas2::symv( m_project, m_fine_yp, yp);
+    }
+
 }
 
 }//namespace shu
diff --git a/src/lamb_dipole/shu2d_b.cu b/src/lamb_dipole/shu2d_b.cu
deleted file mode 100644
index 55b5a26c8..000000000
--- a/src/lamb_dipole/shu2d_b.cu
+++ /dev/null
@@ -1,131 +0,0 @@
-#include <iostream>
-#include <iomanip>
-#include <thrust/remove.h>
-#include <thrust/host_vector.h>
-
-#include "dg/algorithm.h"
-
-#include "shu.cuh"
-
-const double lx = 2.*M_PI;//*50.;
-const double ly = 2.*M_PI;//*50.;
-
-//const double U = 1.; //the dipole doesn't move with this velocity because box is not infinite
-//const double R = 0.2*lx;
-const double T = 2.;
-////const double eps = 1e-7; //CG method
-
-double D = 0.01;
-
-//const double kx = 2.*M_PI* (double)m/lx;
-//const double ky = 2.*M_PI* (double)m/ly;
-//const double ksqr = (kx*kx+ky*ky) ;//4.*M_PI*M_PI*(1./lx/lx + 1./ly/ly);
-const double kx = 1., ky = kx, ksqr = 2.;
-
-
-double initial( double x, double y){ return 2.*sin(kx*x)*sin(ky*y);}
-double solution( double x, double y){ return 2.*sin(kx*x)*sin(ky*y)*exp(-ksqr*D*T);}
-double solution_phi( double x, double y){ return sin(kx*x)*sin(ky*y)*exp(-ksqr*D*T);}
-
-//code for either lamb dipole or analytic sine function without graphics
-int main()
-{
-    dg::Timer t;
-    ////////////////////////////////////////////////////////////
-    //std::cout << "Solve 2D incompressible NavierStokes with sin(x)sin(y) or Lamb dipole initial condition\n";
-    //std::cout << "Type n, N and eps\n";
-    //std::cin >> n >> Nx >>eps;
-    //Ny = Nx;
-    //std::cout << "Type diffusion constant!\n";
-    //std::cin >> D;
-    //std::cout << "# of Legendre coefficients: " << n<<std::endl;
-    //std::cout << "# of grid cells:            " << Nx*Ny<<std::endl;
-    std::cout << "# grid NT dt eps eps_V eps_omega eps_E eps\n";
-    std::cout << "Diffusion " << D <<std::endl;
-
-    ////////////////////////////////////////////////////////////
-    for( unsigned n=2; n<3; n++)
-    {
-        std::cout << "P="<<n<<"\n";
-        for(unsigned i=1; i<5; i++)
-        {
-            unsigned Nx = 8*pow(2,i), Ny = Nx;
-            dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
-            dg::DVec w2d( dg::create::weights(grid));
-
-            double dx = lx/(double)Nx;
-            double eps = 1e-1/pow(10, n)*pow(dx,n);
-            unsigned NT = 4*(unsigned)(T*pow(2,n)/dx);
-            //if( D!= 0)
-                //NT = std::max((unsigned)(0.06*T*pow(4,n)/dx/dx), NT);
-            const double dt = T/(double)NT;
-            //cout << "Runge Kutta stages          " << k <<std::endl;
-            //cout << "Timestep                    " << dt << std::endl;
-            //cout << "# of steps                  " << NT <<std::endl;
-            ////////////////////////////////////////////////////////////
-
-            dg::DVec stencil = dg::evaluate( dg::one, grid);
-            dg::DVec omega = dg::evaluate( initial, grid );
-            const dg::DVec sol = dg::evaluate( solution, grid );
-            const dg::DVec sol_phi = dg::evaluate( solution_phi, grid );
-
-            dg::DVec y0( omega), y1( y0);
-            //make solver and stepper
-            shu::Shu<dg::DMatrix, dg::DVec> shu( grid, eps);
-            shu::Diffusion<dg::DMatrix, dg::DVec> diffusion( grid, D);
-            dg::Karniadakis< dg::DVec > karniadakis( y0, y0.size(), 1e-8);
-
-            shu(0., y0, y1);
-            double vorticity = dg::blas2::dot( stencil, w2d, sol);
-            double enstrophy = 0.5*dg::blas2::dot( sol, w2d, sol);
-            double energy =    0.5*dg::blas2::dot( sol, w2d, sol_phi) ;
-
-            double time = 0;
-            karniadakis.init( shu,diffusion, time, y0, dt);
-            while( time < T)
-            {
-                //step
-
-                t.tic();
-                karniadakis.step( shu, diffusion, time, y0);
-                t.toc();
-                time += dt;
-                //std::cout << "Time "<<time<< " "<<t.diff()<<"\n";
-                if( fabs(dg::blas2::dot( w2d, y0)) > 1e16)
-                {
-                    std::cerr << "Sim unstable at time "<<time<<"!\n\n\n";
-                    break;
-                }
-            }
-            //std::cout << "Total simulation time:     "<<t.diff()<<"s\n";
-            //std::cout << "Average Time for one step: "<<t.diff()/(double)NT<<"s\n";
-            ////////////////////////////////////////////////////////////////////
-            std::cout << Nx;
-            std::cout << " "<<NT;
-            std::cout << " "<<dt;
-            std::cout << " "<<eps;
-            std::cout << " "<<fabs(dg::blas2::dot( stencil, w2d, y0));
-            std::cout << " "<<fabs(0.5*dg::blas2::dot( w2d, y0) - enstrophy)/enstrophy;
-            std::cout << " "<<fabs(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<" ";
-
-            dg::blas1::axpby( 1., sol, -1., y0);
-            std::cout << " "<<sqrt( dg::blas2::dot( w2d, y0))<< std::endl;
-            //std::cout << "Relative distance to solution "<<sqrt( dg::blas2::dot( w2d, y0))/sqrt( dg::blas2::dot( w2d, sol)) << std::endl;
-
-        }
-        std::cout << std::endl;
-    }
-    //energy and enstrophy errrors are due to timestep only ( vorticity is exactly conserved)
-    // k = 2 | p = 3
-    // k = 3 | p = 4
-    // k = 4 | p = 5
-
-    //solution to sin(x)sin(y) 
-    // n = 1 
-    // n = 2 | p = 2
-    // n = 3 | p = 2.6
-    // n = 4 | p = 4
-
-    return 0;
-
-}
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 0a154e61f..ceff946c9 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -1,137 +1,256 @@
 #include <iostream>
 #include <iomanip>
 #include <sstream>
-#include <limits.h>  // UINT_MAX is needed in cusp (v0.5.1) but limits.h is not included
-#include <thrust/remove.h>
 #include <thrust/host_vector.h>
 
 #include "dg/algorithm.h"
 #include "dg/file/json_utilities.h"
 
 #include "draw/host_window.h"
+#include "dg/file/file.h"
 
+#include "init.h"
 #include "shu.cuh"
-#include "parameters.h"
-
-double delta =0.05;
-double rho =M_PI/15.;
-double shearLayer(double x, double y){
-    if( y<= M_PI)
-        return delta*cos(x) - 1./rho/cosh( (y-M_PI/2.)/rho)/cosh( (y-M_PI/2.)/rho);
-    return delta*cos(x) + 1./rho/cosh( (3.*M_PI/2.-y)/rho)/cosh( (3.*M_PI/2.-y)/rho);
-}
+
 
 int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
+    enum file::error mode = file::error::is_warning;
     if( argc == 1)
         file::file2Json( "input/default.json", js, file::comments::are_discarded);
-    else if( argc == 2)
-        file::file2Json( argv[1], js);
     else
-    {
-        std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
-        return -1;
-    }
-    const Parameters p( js);
-    p.display( std::cout);
+        file::file2Json( argv[1], js);
+    std::cout << js <<std::endl;
+
     /////////////////////////////////////////////////////////////////
-    dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
+    dg::CartesianGrid2d grid = shu::createGrid( js, mode);
     dg::DVec w2d( dg::create::weights(grid));
     /////////////////////////////////////////////////////////////////
-    std::stringstream title;
-    GLFWwindow* w = draw::glfwInitAndCreateWindow(600, 600, "");
-    draw::RenderHostData render( 1,1);
-    ////////////////////////////////////////////////////////////
-
-    dg::Lamb lamb( p.posX*p.lx, p.posY*p.ly, p.R, p.U);
-    dg::HVec omega;
-    if( p.initial == "lamb")
-        omega = dg::evaluate ( lamb, grid);
-    else if ( p.initial == "shear")
-        omega = dg::evaluate ( shearLayer, grid);
-
-    dg::DVec stencil = evaluate( dg::one, grid);
+
+    std::string initial = file::get( mode, js, "init", "type", "lamb").asString();
+    dg::HVec omega = shu::initial_conditions.at( initial)(js, mode, grid);
+
+
     dg::DVec y0( omega ), y1( y0);
-    //subtract mean mass 
-    if( p.bc_x == dg::PER && p.bc_y == dg::PER)
+    //subtract mean mass
+    if( grid.bcx() == dg::PER && grid.bcy() == dg::PER)
     {
-        double meanMass = dg::blas2::dot( y0, w2d, stencil)/(double)(p.lx*p.ly);
-        dg::blas1::axpby( -meanMass, stencil, 1., y0);
+        double meanMass = dg::blas1::dot( y0, w2d)/(double)(grid.lx()*grid.ly());
+        dg::blas1::axpby( -meanMass, 1., 1., y0);
     }
     //make solver and stepper
-    shu::Shu<dg::DMatrix, dg::DVec> shu( grid, p.eps);
-    //shu::Diffusion<dg::DMatrix, dg::DVec> diffusion( grid, p.D);
-    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, grid);
-    //dg::Karniadakis< dg::DVec > stepper( y0, y0.size(), p.eps_time);
-    dg::FilteredExplicitMultistep< dg::DVec > stepper( "eBDF", 3, y0);
-    //dg::ShuOsher<dg::DVec> stepper( "SSPRK-3-3", y0);
+    shu::Shu<dg::CartesianGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec>
+        shu( grid, js, mode);
+    shu::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, js, mode);
 
     dg::Timer t;
     t.tic();
     shu( 0., y0, y1);
     t.toc();
     std::cout << "Time for one rhs evaluation: "<<t.diff()<<"s\n";
-    double vorticity = dg::blas2::dot( stencil , w2d, y0);
+    double vorticity = dg::blas1::dot( w2d, y0);
     double enstrophy = 0.5*dg::blas2::dot( y0, w2d, y0);
     double energy =    0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
-    
+    dg::DVec varphi( grid.size());
+    shu.arakawa().variation( shu.potential(), varphi);
+    double variation = dg::blas1::dot( varphi, w2d);
+
     std::cout << "Total energy:     "<<energy<<"\n";
     std::cout << "Total enstrophy:  "<<enstrophy<<"\n";
     std::cout << "Total vorticity:  "<<vorticity<<"\n";
+    std::cout << "Total variation:  "<<variation<<"\n";
 
     double time = 0;
-    ////////////////////////////////glfw//////////////////////////////
-    //create visualisation vectors
-    dg::DVec visual( grid.size());
-    dg::HVec hvisual( grid.size());
-    //transform vector to an equidistant grid
-    dg::IDMatrix equidistant = dg::create::backscatter( grid );
-    draw::ColorMapRedBlueExt colors( 1.);
-    //stepper.init( shu, diffusion, time, y0, p.dt);
-    stepper.init( shu, filter, time, y0, p.dt);
-    //std::cout << "Press any key to start!\n";
-    //double x;
-    //cin >> x;
-    while (!glfwWindowShouldClose(w) && time < p.maxout*p.itstp*p.dt)
+    std::string stepper = file::get( mode, js, "timestepper", "stepper", "FilteredMultistep").asString();
+    std::string regularization = file::get( mode, js, "regularization", "type", "moddal").asString();
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter;
+    dg::Karniadakis<dg::DVec> karniadakis;
+    dg::ShuOsher<dg::DVec> shu_osher;
+    dg::FilteredExplicitMultistep<dg::DVec> multistep;
+    if( regularization == "modal")
     {
-        dg::blas2::symv( equidistant, y0, visual);
-        colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), -1., dg::AbsMax<double>() );
-        //draw and swap buffers
-        dg::blas1::transfer( visual, hvisual);
-        render.renderQuad( hvisual, p.n*p.Nx, p.n*p.Ny, colors);
-        title << "Time "<<time<< " \ttook "<<t.diff()/(double)p.itstp<<"\t per step";
-        glfwSetWindowTitle(w, title.str().c_str());
-        title.str("");
-        glfwPollEvents();
-        glfwSwapBuffers(w);
-        //step 
-        t.tic();
-        for( unsigned i=0; i<p.itstp; i++)
+        double alpha = file::get( mode, js, "regularization", "alpha", 36).asDouble();
+        double order = file::get( mode, js, "regularization", "order", 8).asDouble();
+        double eta_c = file::get( mode, js, "regularization", "eta_c", 0.5).asDouble();
+        filter.construct( alpha, eta_c, order, grid);
+    }
+    double dt = file::get( mode, js, "timestepper", "dt", 2e-3).asDouble();
+    if( "Karniadakis" == stepper)
+    {
+        if( regularization != "viscosity")
         {
-            //stepper.step( shu, diffusion, time, y0 );
-            stepper.step( shu, filter, time, y0 );
-            //stepper.step( shu, filter, time, y0, time, y0, p.dt );
+            throw dg::Error(dg::Message(_ping_)<<"Warning! Karniadakis only works with viscosity regularization! Exit now!");
+
+            return -1;
         }
-        t.toc();
-        //std::cout << "Timer for one step: "<<t.diff()/N<<"s\n";
-        time += p.itstp*p.dt;
+        double eps_time = file::get( mode, js, "timestepper", "eps_time", 1e-10).asDouble();
+        karniadakis.construct( y0, y0.size(), eps_time);
+        karniadakis.init( shu, diffusion, time, y0, dt);
+    }
+    else if( "Shu-Osher" == stepper)
+    {
+        shu_osher.construct( "SSPRK-3-3", y0);
+    }
+    else if( "FilteredMultistep" == stepper)
+    {
+        multistep.construct( "eBDF", 3, y0);
+        multistep.init( shu, filter, time, y0, dt);
+    }
+    else
+    {
+        throw dg::Error(dg::Message(_ping_)<<"Error! Timestepper not recognized!\n");
+
+        return -1;
+    }
+    unsigned maxout = file::get( mode, js, "output", "maxout", 100).asUInt();
+    unsigned itstp = file::get( mode, js, "output", "itstp", 5).asUInt();
+    std::string output = file::get( mode, js, "output", "type", "glfw").asString();
+    if( "glfw" == output)
+    {
+        ////////////////////////////////glfw//////////////////////////////
+        //create visualisation vectors
+        dg::DVec visual( grid.size());
+        dg::HVec hvisual( grid.size());
+        //transform vector to an equidistant grid
+        std::stringstream title;
+        GLFWwindow* w = draw::glfwInitAndCreateWindow(600, 600, "");
+        draw::RenderHostData render( 1,1);
+        draw::ColorMapRedBlueExt colors( 1.);
+        dg::IDMatrix equidistant = dg::create::backscatter( grid );
+        while (!glfwWindowShouldClose(w) && time < maxout*itstp*dt)
+        {
+            dg::blas2::symv( equidistant, y0, visual);
+            colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), -1., dg::AbsMax<double>() );
+            //draw and swap buffers
+            dg::blas1::transfer( visual, hvisual);
+            render.renderQuad( hvisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
+            title << "Time "<<time<< " \ttook "<<t.diff()/(double)itstp<<"\t per step";
+            glfwSetWindowTitle(w, title.str().c_str());
+            title.str("");
+            glfwPollEvents();
+            glfwSwapBuffers(w);
+            //step
+            t.tic();
+            if( "Karniadakis" == stepper)
+                for( unsigned i=0; i<itstp; i++)
+                    karniadakis.step( shu, diffusion, time, y0);
+            else if ( "FilteredMultistep" == stepper)
+                for( unsigned i=0; i<itstp; i++)
+                    multistep.step( shu, filter, time, y0);
+            else if ( "Shu-Osher" == stepper)
+                for( unsigned i=0; i<itstp; i++)
+                    shu_osher.step( shu, filter, time, y0, time, y0, dt);
+            t.toc();
+            //std::cout << "Timer for one step: "<<t.diff()/N<<"s\n";
+        }
+        glfwTerminate();
+    }
+    else
+    {
+        std::string outputfile;
+        if( argc == 1 || argc == 2)
+            outputfile = "shu.nc";
+        else
+            outputfile = argv[2];
+        ////////////////////////////set up netcdf/////////////////////////////////////
+        file::NC_Error_Handle err;
+        int ncid;
+        err = nc_create( outputfile.c_str(),NC_NETCDF4|NC_CLOBBER, &ncid);
+        std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
+        err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
+        int dim_ids[3], tvarID;
+        int EtimeID, EtimevarID;
+        err = file::define_dimensions( ncid, dim_ids, &tvarID, grid);
+        err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+        //field IDs
+        std::string names3d[2] = {"vorticity_field", "potential"};
+        std::string names1d[4] = {"vorticity", "enstrophy", "energy", "variation"};
+        int dataIDs[2], variableIDs[4];
+        for( unsigned i=0; i<2; i++){
+            err = nc_def_var( ncid, names3d[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);}
+        for( unsigned i=0; i<4; i++){
+            err = nc_def_var( ncid, names1d[i].data(), NC_DOUBLE, 1, &EtimeID, &variableIDs[i]);}
+        err = nc_enddef(ncid);
+        size_t start[3] = {0, 0, 0};
+        size_t count[3] = {1, grid.n()*grid.Ny(), grid.n()*grid.Nx()};
+        size_t Estart[] = {0};
+        size_t Ecount[] = {1};
+        ///////////////////////////////////first output/////////////////////////
+        std::vector<dg::HVec> transferH(2);
+        dg::blas1::transfer( y0, transferH[0]);
+        dg::blas1::transfer( shu.potential(), transferH[1]);
+        for( int k=0;k<2; k++)
+            err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH[k].data() );
+        err = nc_put_vara_double( ncid, tvarID, start, count, &time);
+        double output1d[4] = {vorticity, enstrophy, energy, variation};
+        for( int k=0;k<4; k++)
+            err = nc_put_vara_double( ncid, variableIDs[k], Estart, Ecount, &output1d[k] );
+        err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
+        ///////////////////////////////////timeloop/////////////////////////
+        unsigned step=0;
+        try{
+        for( unsigned i=1; i<=maxout; i++)
+        {
+
+            dg::Timer ti;
+            ti.tic();
+            for( unsigned j=0; j<itstp; j++)
+            {
+                if( "Karniadakis" == stepper)
+                    karniadakis.step( shu, diffusion, time, y0);
+                else if ( "FilteredMultistep" == stepper)
+                    multistep.step( shu, filter, time, y0);
+                else if ( "Shu-Osher" == stepper)
+                    shu_osher.step( shu, filter, time, y0, time, y0, dt);
 
+                output1d[0] = dg::blas1::dot( w2d, y0);
+                output1d[1] = 0.5*dg::blas2::dot( y0, w2d, y0);
+                output1d[2] = 0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
+                shu.arakawa().variation(shu.potential(), varphi);
+                output1d[3] = dg::blas1::dot( varphi, w2d);
+                Estart[0] += 1;
+                for( int k=0;k<4; k++)
+                    err = nc_put_vara_double( ncid, variableIDs[k], Estart, Ecount, &output1d[k] );
+                err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
+                if( energy>1e6)
+                    throw dg::Error(dg::Message(_ping_)<<"Energy exploded! Exit\n");
+            }
+            step+=itstp;
+            //output all fields
+            dg::blas1::transfer( y0, transferH[0]);
+            dg::blas1::transfer( shu.potential(), transferH[1]);
+            start[0] = i;
+            for( int k=0;k<2; k++)
+                err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH[k].data() );
+            err = nc_put_vara_double( ncid, tvarID, start, count, &time);
+            ti.toc();
+            std::cout << "\n\t Step "<<step <<" of "<<itstp*maxout <<" at time "<<time;
+            std::cout << "\n\t Average time for one step: "<<ti.diff()/(double)itstp<<"s\n\n"<<std::flush;
+        }
+        }
+        catch( dg::Fail& fail) {
+            std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
+            std::cerr << "Does Simulation respect CFL condition?\n";
+        }
+        err = nc_close(ncid);
     }
-    glfwTerminate();
     ////////////////////////////////////////////////////////////////////
-    std::cout << "Analytic formula enstrophy "<<lamb.enstrophy()<<std::endl;
-    std::cout << "Analytic formula energy    "<<lamb.energy()<<std::endl;
-    std::cout << "Total vorticity          is: "<<dg::blas2::dot( stencil , w2d, y0) << "\n";
+    std::cout << "Time "<<time<<std::endl;
+    //dg::Lamb solution( p.posX*p.lx, p.posY*p.ly - p.U*time, p.R, p.U);
+    //dg::DVec sol = dg::evaluate( solution, grid);
+    //dg::blas1::axpby( 1., y0, -1., sol);
+    //double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
+    //std::cout << "Analytic error to solution "<<error<<std::endl;
+    //std::cout << "Analytic formula enstrophy "<<lamb.enstrophy()<<std::endl;
+    //std::cout << "Analytic formula energy    "<<lamb.energy()<<std::endl;
+    std::cout << "Absolute vorticity error is: "<<dg::blas1::dot( w2d, y0) - vorticity << "\n";
     std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - enstrophy)/enstrophy<<"\n";
     std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<"\n";
+    shu.arakawa().variation(shu.potential(), varphi);
+    std::cout << "Variation relative to 0 is:  "<<variation - dg::blas1::dot( varphi, w2d) << std::endl;
 
-    //dg::blas1::axpby( 1., y0, -1, sol);
-    //cout << "Distance to solution: "<<sqrt(dg::blas2::dot( w2d, sol ))<<std::endl;
-
-    //cout << "Press any key to quit!\n";
-    //cin >> x;
     return 0;
 
 }
diff --git a/src/lamb_dipole/shu_hpc.cu b/src/lamb_dipole/shu_hpc.cu
deleted file mode 100644
index ed2353c82..000000000
--- a/src/lamb_dipole/shu_hpc.cu
+++ /dev/null
@@ -1,138 +0,0 @@
-#include <iostream>
-#include <iomanip>
-
-#include "dg/file/file.h"
-#include "shu.cuh"
-#include "parameters.h"
-
-double delta =0.05;
-double rho =M_PI/15.;
-double shearLayer(double x, double y){
-    if( y<= M_PI)
-        return delta*cos(x) - 1./rho/cosh( (y-M_PI/2.)/rho)/cosh( (y-M_PI/2.)/rho);
-    return delta*cos(x) + 1./rho/cosh( (3.*M_PI/2.-y)/rho)/cosh( (3.*M_PI/2.-y)/rho);
-}
-
-int main( int argc, char * argv[])
-{
-    dg::Timer t;
-    ////////////////////////Parameter initialisation//////////////////////////
-    Json::Value js;
-    if( argc != 3)
-    {
-        std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile] [outputfile]\n";
-        return -1;
-    }
-    else
-        file::file2Json( argv[1], js);
-    std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
-    const Parameters p( js);
-    p.display( std::cout);
-
-    //////////////initiate solver/////////////////////////////////////////
-    dg::Grid2d grid( 0, p.lx, 0, p.ly, p.n, p.Nx, p.Ny, p.bc_x, p.bc_y);
-    dg::DVec w2d( dg::create::weights(grid));
-    dg::Lamb lamb( p.posX*p.lx, p.posY*p.ly, p.R, p.U);
-    dg::HVec omega;
-    if( p.initial == "lamb")
-        omega = dg::evaluate ( lamb, grid);
-    else if ( p.initial == "shear")
-        omega = dg::evaluate ( shearLayer, grid);
-    dg::DVec y0( omega );
-    const dg::DVec one = dg::evaluate( dg::one, grid);
-    //make solver and stepper
-    shu::Shu<dg::DMatrix, dg::DVec> shu( grid, p.eps);
-    //shu::Diffusion< dg::DMatrix, dg::DVec > diff( grid, p.D);
-    //dg::Karniadakis< dg::DVec> karniadakis( y0, y0.size(), 1e-10);
-    //karniadakis.init( shu, diff, 0., y0, p.dt);
-    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, grid);
-    dg::FilteredExplicitMultistep<dg::DVec> stepper( "eBDF", 3, y0);
-    stepper.init( shu, filter, 0., y0, p.dt);
-
-    dg::DVec varphi( grid.size()), potential;
-    double vorticity = dg::blas2::dot( one , w2d, y0);
-    double enstrophy = 0.5*dg::blas2::dot( y0, w2d, y0);
-    double energy =    0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
-    potential = shu.potential();
-    shu.arakawa().variation( potential, varphi);
-    double variation = dg::blas2::dot( varphi, w2d, one );
-    double time = 0;
-    /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
-    int ncid;
-    err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
-    err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
-    int dim_ids[3], tvarID;
-    int EtimeID, EtimevarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid);
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
-    //field IDs
-    std::string names3d[2] = {"vorticity_field", "potential"}; 
-    std::string names1d[4] = {"vorticity", "enstrophy", "energy", "variation"}; 
-    int dataIDs[2], variableIDs[4]; 
-    for( unsigned i=0; i<2; i++){
-        err = nc_def_var( ncid, names3d[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);}
-    for( unsigned i=0; i<4; i++){
-        err = nc_def_var( ncid, names1d[i].data(), NC_DOUBLE, 1, &EtimeID, &variableIDs[i]);}
-    err = nc_enddef(ncid);
-    size_t start[3] = {0, 0, 0};
-    size_t count[3] = {1, grid.n()*grid.Ny(), grid.n()*grid.Nx()};
-    size_t Estart[] = {0};
-    size_t Ecount[] = {1};
-    ///////////////////////////////////first output/////////////////////////
-    std::vector<dg::HVec> output(2);
-    dg::blas1::transfer( y0, output[0]);
-    dg::blas1::transfer( shu.potential(), output[1]);
-    for( int k=0;k<2; k++)
-        err = nc_put_vara_double( ncid, dataIDs[k], start, count, output[k].data() );
-    err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-    double output1d[4] = {vorticity, enstrophy, energy, variation};
-    for( int k=0;k<4; k++)
-        err = nc_put_vara_double( ncid, variableIDs[k], Estart, Ecount, &output1d[k] );
-    err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
-    ///////////////////////////////////timeloop/////////////////////////
-    unsigned step=0;
-    try{
-    for( unsigned i=0; i<p.maxout; i++)
-    {
-
-        dg::Timer ti;
-        ti.tic();
-        for( unsigned j=0; j<p.itstp; j++)
-        {
-            //karniadakis.step( shu, diff, time, y0);//one step further
-            stepper.step( shu, filter, time, y0);//one step further
-            output1d[0] = vorticity = dg::blas2::dot( one , w2d, y0);
-            output1d[1] = enstrophy = 0.5*dg::blas2::dot( y0, w2d, y0);
-            potential = shu.potential();
-            output1d[2] = energy    = 0.5*dg::blas2::dot( y0, w2d, potential) ;
-            shu.arakawa().variation(potential, varphi);
-            output1d[3] = variation = dg::blas2::dot( varphi, w2d, one );
-            Estart[0] += 1;
-            for( int k=0;k<4; k++)
-                err = nc_put_vara_double( ncid, variableIDs[k], Estart, Ecount, &output1d[k] );
-            err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
-            if( energy>1e6) throw dg::Fail(p.eps);
-        }
-        step+=p.itstp;
-        //output all fields
-        dg::blas1::transfer( y0, output[0]);
-        dg::blas1::transfer( shu.potential(), output[1]);
-        start[0] = i;
-        for( int k=0;k<2; k++)
-            err = nc_put_vara_double( ncid, dataIDs[k], start, count, output[k].data() );
-        err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-        ti.toc();
-        std::cout << "\n\t Step "<<step <<" of "<<p.itstp*p.maxout <<" at time "<<time;
-        std::cout << "\n\t Average time for one step: "<<ti.diff()/(double)p.itstp<<"s\n\n"<<std::flush;
-    }
-    }
-    catch( dg::Fail& fail) { 
-        std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-        std::cerr << "Does Simulation respect CFL condition?\n";
-    }
-    err = nc_close(ncid);
-
-    return 0;
-
-}
diff --git a/src/lamb_dipole/shu_t.cu b/src/lamb_dipole/shu_t.cu
deleted file mode 100644
index 22aab0fb5..000000000
--- a/src/lamb_dipole/shu_t.cu
+++ /dev/null
@@ -1,96 +0,0 @@
-#include <iostream>
-#include <iomanip>
-#include <sstream>
-#include <thrust/remove.h>
-#include <thrust/host_vector.h>
-
-#include "draw/host_window.h"
-
-#include "dg/algorithm.h"
-
-#include "shu.cuh"
-
-
-const double lx = 2.*M_PI;
-const double ly = 2.*M_PI;
-
-// const unsigned k = 2;
-const double D = 0.01;
-const double T = 1.;
-
-
-double initial( double x, double y){return 2.*sin(x)*sin(y);}
-double solution( double x, double y) {return 2.*sin(x)*sin(y)*exp( -2.*T*D);}
-
-
-int main()
-{
-    unsigned n, Nx, Ny;
-    double eps;
-    std::cout << "Type n, Nx, Ny and eps!\n";
-    std::cin >> n >> Nx >> Ny>>eps;
-    const unsigned NT = (unsigned)(T*n*Nx/0.1/lx);
-
-    dg::Grid2d grid( 0, lx, 0, ly, n, Nx, Ny, dg::PER, dg::PER);
-    dg::DVec w2d( dg::create::weights( grid));
-    const double dt = T/(double)NT;
-    /////////////////////////////////////////////////////////////////
-    //create CUDA context that uses OpenGL textures in Glfw window
-    std::stringstream title;
-    GLFWwindow* w = draw::glfwInitAndCreateWindow(600, 600, "Navier Stokes");
-    draw::RenderHostData render( 1,1);
-    ////////////////////////////////////////////////////////////
-    std::cout << "# of Legendre coefficients: " << n<<std::endl;
-    std::cout << "# of grid cells:            " << Nx*Ny<<std::endl;
-    std::cout << "Timestep                    " << dt << std::endl;
-    //std::cout << "# of timesteps              " << NT << std::endl;
-    std::cout << "Diffusion                   " << D <<std::endl;
-    dg::Lamb lamb( 0.5*lx, 0.5*ly, 0.2*lx, 1);
-    dg::HVec omega = evaluate ( lamb, grid);
-    dg::DVec stencil = evaluate( dg::one, grid);
-    dg::DVec y0( omega);
-    shu::Shu<dg::DMatrix, dg::DVec> test( grid, eps);
-    //shu::Diffusion<DMatrix, DVec> diffusion( grid, D);
-    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, grid);
-    //dg::Karniadakis< dg::DVec > stepper( y0, y0.size(), 1e-8);
-    dg::FilteredExplicitMultistep< dg::DVec > stepper( "TVB",3, y0);
-
-    ////////////////////////////////glfw//////////////////////////////
-    //create visualisation vectors
-    dg::DVec visual( grid.size());
-    dg::HVec hvisual( grid.size());
-    //transform vector to an equidistant grid
-    dg::IDMatrix equidistant = dg::create::backscatter( grid );
-    draw::ColorMapRedBlueExt colors( 1.);
-    double time = 0;
-    //stepper.init( test, diffusion, time, y0, dt);
-    stepper.init( test, filter, time, y0, dt);
-    while (!glfwWindowShouldClose(w))
-    {
-        //transform field to an equidistant grid
-        std::cout << "Total vorticity is: "<<dg::blas2::dot( stencil, w2d, y0) << "\n";
-        std::cout << "Total enstrophy is: "<<dg::blas2::dot( w2d, y0)<<"\n";
-        //compute the color scale
-        dg::blas2::symv( equidistant, y0, visual );
-        colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), -1., dg::AbsMax<float>() );
-        std::cout << "Color scale " << colors.scale() <<"\n";
-        //draw and swap buffers
-        dg::blas1::transfer(visual, hvisual);
-        render.renderQuad( hvisual, n*Nx, n*Ny, colors);
-        //step
-        //stepper.step( test,diffusion, time, y0 );
-        stepper.step( test,filter, time, y0 );
-
-        glfwSwapBuffers(w);
-        glfwWaitEvents();
-    }
-    glfwTerminate();
-    ////////////////////////////////////////////////////////////////////
-    std::cout << "Total vorticity is: "<< dg::blas2::dot( stencil, w2d, y0) << "\n";
-    std::cout << "Total enstrophy  is "<<dg::blas2::dot( y0, w2d, y0)<<"\n";
-    //dg::blas1::axpby( 1., sol.data(), -1., y0);
-    //std::cout << "Distance to solution "<<sqrt( dg::blas2::dot( w2d, y0))<<std::endl; //don't forget sqrt when comuting errors
-
-    return 0;
-
-}
diff --git a/src/lamb_dipole/shu_time.cu b/src/lamb_dipole/shu_time.cu
deleted file mode 100644
index ca79fd61a..000000000
--- a/src/lamb_dipole/shu_time.cu
+++ /dev/null
@@ -1,61 +0,0 @@
-#include <iostream>
-#include <iomanip>
-
-#include "dg/algorithm.h"
-
-#include "shu.cuh"
-#include "parameters.h"
-
-//const unsigned k=4;
-const double Tmax = 0.01;
-const double eps = 1e-14;
-const unsigned n=1; //make error in space small
-unsigned Nx = 100, Ny = Nx;
-
-int main( int argc, char * argv[])
-{
-
-    double dt0;
-    std::cout << "type dt0 (1e-3)!\n";
-    std::cin >> dt0;
-    std::cout << "k n dt Nx eps vort enstr energy\n";
-    dg::Grid2d grid( 0, 1, 0, 1, n, Nx, Ny, dg::PER, dg::PER);
-    dg::DVec w2d( dg::create::weights(grid));
-    dg::Lamb lamb( 0.5, 0.8, 0.1, 1.);
-    const dg::HVec omega = dg::evaluate ( lamb, grid);
-    shu::Shu<dg::DMatrix, dg::DVec> shu( grid, eps);
-    const dg::DVec stencil = dg::evaluate( dg::one, grid);
-    for(unsigned i=0; i<6;i++)
-    {
-        double dt = dt0/pow(2,i);
-        unsigned NT = (unsigned)(Tmax/dt);
-        double time = 0;
-        //initiate solver
-        dg::DVec y0( omega ), y1( y0);
-        //make solver and stepper
-        dg::ExplicitMultistep< dg::DVec > ab("eBDF",3, y0);
-        ab.init( shu, time, y0, dt);
-
-        double vorticity = dg::blas2::dot( stencil, w2d, y1);
-        double enstrophy = 0.5*dg::blas2::dot( y1, w2d, y1);
-        double energy =    0.5*dg::blas2::dot( y1, w2d, shu.potential()) ;
-        /////////////////////////////////////////////////////////////////
-        try{
-        for( unsigned i=0; i<NT; i++)
-        {
-            ab.step( shu, time, y1);
-        }
-        }
-        catch( dg::Fail& fail) { 
-            std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
-            std::cerr << "Does Simulation respect CFL condition?\n";
-        }
-        std::cout << 2 <<" "<<n<<" "<<dt<<" "<<Nx<<" "<<eps<<" ";
-        std::cout << fabs(dg::blas2::dot( stencil , w2d, y1));
-        std::cout << " "<<fabs(0.5*dg::blas2::dot( y1, w2d, y1)-enstrophy)/enstrophy;
-        std::cout << " "<<fabs(0.5*dg::blas2::dot( y1, w2d, shu.potential())-energy)/energy <<"\n";
-    }
-    std::cout << std::endl;
-    return 0;
-
-}
-- 
GitLab


From c38a0338e6faed856a345d2ebed90367324365ed Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Jan 2021 13:09:09 +0100
Subject: [PATCH 426/540] Rename HVec_t to x::HVec and similar

the philosophy is that x is more symbolic for the dual behaviour
of this typedef working for both shared and distributed memory
---
 inc/dg/backend/typedefs.h        | 34 +++++++++++++++++---------------
 inc/dg/topology/base_geometry.h  | 14 +++++++------
 inc/dg/topology/grid.h           | 12 ++++++-----
 inc/dg/topology/interpolation.h  | 10 ++++++----
 inc/dg/topology/mpi_base.h       | 14 +++++++------
 inc/dg/topology/mpi_grid.h       | 10 ++++++----
 inc/dg/topology/mpi_projection.h | 10 ++++++----
 inc/geometries/curvilinear.h     |  6 ++++--
 inc/geometries/mpi_curvilinear.h |  6 ++++--
 src/feltor/feltor.cu             | 12 +++++------
 src/feltor/feltor_hpc.cu         | 14 ++++++-------
 11 files changed, 80 insertions(+), 62 deletions(-)

diff --git a/inc/dg/backend/typedefs.h b/inc/dg/backend/typedefs.h
index 09af767d8..1c08dd4eb 100644
--- a/inc/dg/backend/typedefs.h
+++ b/inc/dg/backend/typedefs.h
@@ -71,31 +71,33 @@ namespace dg{
 ///@addtogroup typedefs
 ///@{
 //vectors
+namespace x{
 #ifdef MPI_VERSION
-using HVec_t  = MHVec;
-using fHVec_t = fMHVec;
+using HVec  = MHVec;
+using fHVec = fMHVec;
 
-using DVec_t  = MDVec;
-using fDVec_t = fMDVec;
+using DVec  = MDVec;
+using fDVec = fMDVec;
 
 //derivative matrices
-using HMatrix_t = MHMatrix;
-using fHMatrix_t = fMHMatrix;
-using DMatrix_t = MDMatrix;
-using fDMatrix_t = fMDMatrix;
+using HMatrix = MHMatrix;
+using fHMatrix = fMHMatrix;
+using DMatrix = MDMatrix;
+using fDMatrix = fMDMatrix;
 #else
-using HVec_t  = HVec;
-using fHVec_t = fHVec;
+using HVec  = HVec;
+using fHVec = fHVec;
 
-using DVec_t  = DVec;
-using fDVec_t = fDVec;
+using DVec  = DVec;
+using fDVec = fDVec;
 
 //derivative matrices
-using HMatrix_t = HMatrix;
-using fHMatrix_t = fHMatrix;
-using DMatrix_t = DMatrix;
-using fDMatrix_t = fDMatrix;
+using HMatrix = HMatrix;
+using fHMatrix = fHMatrix;
+using DMatrix = DMatrix;
+using fDMatrix = fDMatrix;
 #endif //MPI_VERSION
+}//namespace x
 ///@}
 }//namespace dg
 
diff --git a/inc/dg/topology/base_geometry.h b/inc/dg/topology/base_geometry.h
index 2862f1eca..fa1d41a6f 100644
--- a/inc/dg/topology/base_geometry.h
+++ b/inc/dg/topology/base_geometry.h
@@ -285,12 +285,14 @@ using CartesianGrid2d       = dg::RealCartesianGrid2d<double>;
 using CartesianGrid3d       = dg::RealCartesianGrid3d<double>;
 using CylindricalGrid3d     = dg::RealCylindricalGrid3d<double>;
 #ifndef MPI_VERSION
-using aGeometry2d_t           = aGeometry2d           ;
-using aGeometry3d_t           = aGeometry3d           ;
-using aProductGeometry3d_t    = aProductGeometry3d    ;
-using CartesianGrid2d_t       = CartesianGrid2d       ;
-using CartesianGrid3d_t       = CartesianGrid3d       ;
-using CylindricalGrid3d_t     = CylindricalGrid3d     ;
+namespace x{
+using aGeometry2d           = aGeometry2d           ;
+using aGeometry3d           = aGeometry3d           ;
+using aProductGeometry3d    = aProductGeometry3d    ;
+using CartesianGrid2d       = CartesianGrid2d       ;
+using CartesianGrid3d       = CartesianGrid3d       ;
+using CylindricalGrid3d     = CylindricalGrid3d     ;
+}//namespace x
 #endif //MPI_VERSION
 
 ///@}
diff --git a/inc/dg/topology/grid.h b/inc/dg/topology/grid.h
index 7fb465df4..7ba7facda 100644
--- a/inc/dg/topology/grid.h
+++ b/inc/dg/topology/grid.h
@@ -834,11 +834,13 @@ using Grid3d        = dg::RealGrid3d<double>;
 using aTopology2d   = dg::aRealTopology2d<double>;
 using aTopology3d   = dg::aRealTopology3d<double>;
 #ifndef MPI_VERSION
-using Grid1d_t        = Grid1d      ;
-using Grid2d_t        = Grid2d      ;
-using Grid3d_t        = Grid3d      ;
-using aTopology2d_t   = aTopology2d ;
-using aTopology3d_t   = aTopology3d ;
+namespace x {
+using Grid1d        = Grid1d      ;
+using Grid2d        = Grid2d      ;
+using Grid3d        = Grid3d      ;
+using aTopology2d   = aTopology2d ;
+using aTopology3d   = aTopology3d ;
+} //namespace x
 #endif
 ///@}
 
diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index 30d130bda..859c76685 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -26,12 +26,14 @@ using IDMatrix = tIDMatrix<double>;
 //typedef cusp::csr_matrix<int, double, cusp::host_memory> IHMatrix; //!< CSR host Matrix
 //typedef cusp::csr_matrix<int, double, cusp::device_memory> IDMatrix; //!< CSR device Matrix
 #ifndef MPI_VERSION
+namespace x{
 template<class real_type>
-using tIHMatrix_t = tIHMatrix<real_type>;
+using tIHMatrix = tIHMatrix<real_type>;
 template<class real_type>
-using tIDMatrix_t = tIDMatrix<real_type>;
-using IHMatrix_t = IHMatrix;
-using IDMatrix_t = IDMatrix;
+using tIDMatrix = tIDMatrix<real_type>;
+using IHMatrix = IHMatrix;
+using IDMatrix = IDMatrix;
+} //namespace x
 #endif //MPI_VERSION
 
 ///@}
diff --git a/inc/dg/topology/mpi_base.h b/inc/dg/topology/mpi_base.h
index 6e659de74..1c1ab91d7 100644
--- a/inc/dg/topology/mpi_base.h
+++ b/inc/dg/topology/mpi_base.h
@@ -262,12 +262,14 @@ using aProductMPIGeometry3d = dg::aRealProductMPIGeometry3d<double>;
 using CartesianMPIGrid2d    = dg::RealCartesianMPIGrid2d<double>;
 using CartesianMPIGrid3d    = dg::RealCartesianMPIGrid3d<double>;
 using CylindricalMPIGrid3d  = dg::RealCylindricalMPIGrid3d<double>;
-using aGeometry2d_t           = aMPIGeometry2d           ;
-using aGeometry3d_t           = aMPIGeometry3d           ;
-using aProductGeometry3d_t    = aProductMPIGeometry3d    ;
-using CartesianGrid2d_t       = CartesianMPIGrid2d       ;
-using CartesianGrid3d_t       = CartesianMPIGrid3d       ;
-using CylindricalGrid3d_t     = CylindricalMPIGrid3d     ;
+namespace x{
+using aGeometry2d           = aMPIGeometry2d           ;
+using aGeometry3d           = aMPIGeometry3d           ;
+using aProductGeometry3d    = aProductMPIGeometry3d    ;
+using CartesianGrid2d       = CartesianMPIGrid2d       ;
+using CartesianGrid3d       = CartesianMPIGrid3d       ;
+using CylindricalGrid3d     = CylindricalMPIGrid3d     ;
+}//namespace x
 ///@}
 
 }//namespace dg
diff --git a/inc/dg/topology/mpi_grid.h b/inc/dg/topology/mpi_grid.h
index 0812da19f..04017f180 100644
--- a/inc/dg/topology/mpi_grid.h
+++ b/inc/dg/topology/mpi_grid.h
@@ -741,10 +741,12 @@ using MPIGrid2d         = dg::RealMPIGrid2d<double>;
 using MPIGrid3d         = dg::RealMPIGrid3d<double>;
 using aMPITopology2d    = dg::aRealMPITopology2d<double>;
 using aMPITopology3d    = dg::aRealMPITopology3d<double>;
-using Grid2d_t          = MPIGrid2d      ;
-using Grid3d_t          = MPIGrid3d      ;
-using aTopology2d_t     = aMPITopology2d ;
-using aTopology3d_t     = aMPITopology3d ;
+namespace x{
+using Grid2d          = MPIGrid2d      ;
+using Grid3d          = MPIGrid3d      ;
+using aTopology2d     = aMPITopology2d ;
+using aTopology3d     = aMPITopology3d ;
+}//namespace x
 ///@}
 
 }//namespace dg
diff --git a/inc/dg/topology/mpi_projection.h b/inc/dg/topology/mpi_projection.h
index dbc1624cb..6b1b52b39 100644
--- a/inc/dg/topology/mpi_projection.h
+++ b/inc/dg/topology/mpi_projection.h
@@ -22,12 +22,14 @@ using MIHMatrix = tMIHMatrix<double>;
 using MIDMatrix = tMIDMatrix<double>;
 //typedef MPIDistMat< dg::IHMatrix, GeneralComm< dg::iHVec, dg::HVec > > MIHMatrix; //!< MPI distributed CSR host Matrix
 //typedef MPIDistMat< dg::IDMatrix, GeneralComm< dg::iDVec, dg::DVec > > MIDMatrix; //!< MPI distributed CSR device Matrix
+namespace x{
 template<class real_type>
-using tIHMatrix_t = tMIHMatrix<real_type>;
+using tIHMatrix = tMIHMatrix<real_type>;
 template<class real_type>
-using tIDMatrix_t = tMIDMatrix<real_type>;
-using IHMatrix_t = MIHMatrix;
-using IDMatrix_t = MIDMatrix;
+using tIDMatrix = tMIDMatrix<real_type>;
+using IHMatrix = MIHMatrix;
+using IDMatrix = MIDMatrix;
+} //namespace x
 ///@}
 
 ///@cond
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 1fcabb9ee..79b55d503 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -205,8 +205,10 @@ struct RealCurvilinearProductGrid3d : public dg::aRealProductGeometry3d<real_typ
 using CurvilinearGrid2d         = dg::geo::RealCurvilinearGrid2d<double>;
 using CurvilinearProductGrid3d  = dg::geo::RealCurvilinearProductGrid3d<double>;
 #ifndef MPI_VERSION
-using CurvilinearGrid2d_t         = CurvilinearGrid2d        ;
-using CurvilinearProductGrid3d_t  = CurvilinearProductGrid3d ;
+namespace x{
+using CurvilinearGrid2d         = CurvilinearGrid2d        ;
+using CurvilinearProductGrid3d  = CurvilinearProductGrid3d ;
+}
 #endif
 
 ///@}
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 624c3320d..93e1f2d14 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -204,8 +204,10 @@ RealCurvilinearMPIGrid2d<real_type>::RealCurvilinearMPIGrid2d( const RealCurvili
 //
 using CurvilinearMPIGrid2d         = dg::geo::RealCurvilinearMPIGrid2d<double>;
 using CurvilinearProductMPIGrid3d  = dg::geo::RealCurvilinearProductMPIGrid3d<double>;
-using CurvilinearGrid2d_t         = CurvilinearMPIGrid2d        ;
-using CurvilinearProductGrid3d_t  = CurvilinearProductMPIGrid3d ;
+namespace x{
+using CurvilinearGrid2d         = CurvilinearMPIGrid2d        ;
+using CurvilinearProductGrid3d  = CurvilinearProductMPIGrid3d ;
+}//namespace x
 
 ///@}
 }//namespace geo
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 11281001d..15fe0e998 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -10,12 +10,12 @@
 #include "feltor.h"
 #include "implicit.h"
 
-using HVec = dg::HVec_t;
-using DVec = dg::DVec_t;
-using DMatrix = dg::DMatrix_t;
-using IDMatrix = dg::IDMatrix_t;
-using IHMatrix = dg::IHMatrix_t;
-using Geometry = dg::CylindricalGrid3d_t;
+using HVec = dg::x::HVec;
+using DVec = dg::x::DVec;
+using DMatrix = dg::x::DMatrix;
+using IDMatrix = dg::x::IDMatrix;
+using IHMatrix = dg::x::IHMatrix;
+using Geometry = dg::x::CylindricalGrid3d;
 #define MPI_OUT
 
 #include "init.h"
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 6080ada0b..ca65b1088 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -14,13 +14,13 @@
 #include "feltor.h"
 #include "implicit.h"
 
-using HVec = dg::HVec_t;
-using DVec = dg::DVec_t;
-using HMatrix = dg::HMatrix_t;
-using DMatrix = dg::DMatrix_t;
-using IDMatrix = dg::IDMatrix_t;
-using IHMatrix = dg::IHMatrix_t;
-using Geometry = dg::CylindricalGrid3d_t;
+using HVec = dg::x::HVec;
+using DVec = dg::x::DVec;
+using HMatrix = dg::x::HMatrix;
+using DMatrix = dg::x::DMatrix;
+using IDMatrix = dg::x::IDMatrix;
+using IHMatrix = dg::x::IHMatrix;
+using Geometry = dg::x::CylindricalGrid3d;
 #ifdef FELTOR_MPI
 #define MPI_OUT if(rank==0)
 #else //FELTOR_MPI
-- 
GitLab


From 28bf635daabf327bda076870192b6c7cf3a2e71b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Jan 2021 15:23:40 +0100
Subject: [PATCH 427/540] Fix volume bug in Arakawa and Poisson

- and remove variation function from class
- volume form is still not really tested
---
 inc/dg/arakawa.h    | 29 ++++-------------------------
 inc/dg/poisson.h    | 28 ++++------------------------
 inc/dg/poisson_t.cu |  9 ---------
 3 files changed, 8 insertions(+), 58 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index bd00c911d..6111a24c8 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -86,7 +86,7 @@ struct ArakawaX
      */
     template<class ContainerType0>
     void set_chi( const ContainerType0& new_chi) {
-        dg::blas1::pointwiseDot( new_chi, m_inv_perp_vol, m_chi);
+        dg::blas1::pointwiseDivide( new_chi, m_perp_vol, m_chi);
     }
 
     /**
@@ -108,29 +108,10 @@ struct ArakawaX
         return m_bdyf;
     }
 
-    /**
-     * @brief Compute the total variation integrand
-     *
-     * Computes \f[ (\nabla\phi)^2 = \partial_i \phi g^{ij}\partial_j \phi \f]
-     * in the plane of a 2x1 product space
-     * @param phi function
-     * @param varphi may equal phi, contains result on output
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerType0, class ContainerType1>
-    void variation( const ContainerType0& phi, ContainerType1& varphi)
-    {
-        blas2::symv( m_bdxf, phi, m_dxrhs);
-        blas2::symv( m_bdyf, phi, m_dyrhs);
-        tensor::multiply2d( m_metric, m_dxrhs, m_dyrhs, varphi, m_helper);
-        blas1::pointwiseDot( 1., varphi, m_dxrhs, 1., m_helper, m_dyrhs, 0., varphi);
-    }
-
   private:
     Container m_dxlhs, m_dxrhs, m_dylhs, m_dyrhs, m_helper;
     Matrix m_bdxf, m_bdyf;
-    SparseTensor<Container> m_metric;
-    Container m_chi, m_inv_perp_vol;
+    Container m_chi, m_perp_vol;
 };
 ///@cond
 template<class Geometry, class Matrix, class Container>
@@ -143,10 +124,8 @@ ArakawaX<Geometry, Matrix, Container>::ArakawaX( const Geometry& g, bc bcx, bc b
     m_bdxf(dg::create::dx( g, bcx, dg::centered)),
     m_bdyf(dg::create::dy( g, bcy, dg::centered))
 {
-    m_metric=g.metric();
-    m_chi = dg::tensor::determinant2d(m_metric);
-    dg::blas1::transform(m_chi, m_chi, dg::SQRT<get_value_type<Container>>());
-    m_inv_perp_vol = m_chi;
+    m_chi = m_perp_vol = dg::tensor::volume2d(g.metric());
+    dg::blas1::pointwiseDivide( 1., m_perp_vol, m_chi);
 }
 
 template<class T>
diff --git a/inc/dg/poisson.h b/inc/dg/poisson.h
index 65be11c21..7960b976e 100644
--- a/inc/dg/poisson.h
+++ b/inc/dg/poisson.h
@@ -79,7 +79,7 @@ struct Poisson
      */
     template<class ContainerType0>
     void set_chi( const ContainerType0& new_chi) {
-        dg::blas1::pointwiseDot( new_chi, m_inv_perp_vol, m_chi);
+        dg::blas1::pointwiseDivide( new_chi, m_perp_vol, m_chi);
     }
 
     /**
@@ -117,29 +117,11 @@ struct Poisson
     const Matrix& dyrhs() {
         return m_dyrhs;
     }
-    /**
-     * @brief Compute the total variation integrand, uses bc of rhs of poisson bracket
-     *
-     * Computes \f[ (\nabla\phi)^2 = \partial_i \phi g^{ij}\partial_j \phi \f]
-     * in the plane of a 2x1 product space
-     * @param phi function
-     * @param varphi may equal phi, contains result on output
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerType0, class ContainerType1>
-    void variationRHS( const ContainerType0& phi, ContainerType1& varphi)
-    {
-        blas2::symv( m_dxrhs, phi, m_dxrhsrhs);
-        blas2::symv( m_dyrhs, phi, m_dyrhsrhs);
-        tensor::multiply2d( m_metric, m_dxrhsrhs, m_dyrhsrhs, varphi, m_helper);
-        blas1::pointwiseDot( 1., varphi, m_dxrhsrhs, 1., m_helper, m_dyrhsrhs, 0., varphi);
-    }
 
   private:
     Container m_dxlhslhs, m_dxrhsrhs, m_dylhslhs, m_dyrhsrhs, m_helper;
     Matrix m_dxlhs, m_dylhs, m_dxrhs, m_dyrhs;
-    SparseTensor<Container> m_metric;
-    Container m_chi, m_inv_perp_vol;
+    Container m_chi, m_perp_vol;
 };
 
 ///@cond
@@ -161,10 +143,8 @@ Poisson<Geometry, Matrix, Container>::Poisson(  const Geometry& g, bc bcxlhs, bc
     m_dxrhs(dg::create::dx( g, bcxrhs, dg::centered)),
     m_dyrhs(dg::create::dy( g, bcyrhs, dg::centered))
 {
-    m_metric=g.metric();
-    m_chi = dg::tensor::determinant2d(m_metric);
-    dg::blas1::transform(m_chi, m_chi, dg::SQRT<get_value_type<Container>>());
-    m_inv_perp_vol = m_chi;
+    m_chi = m_perp_vol = dg::tensor::volume2d(g.metric());
+    dg::blas1::pointwiseDivide( 1., m_perp_vol, m_chi);
 }
 
 template< class Geometry, class Matrix, class Container>
diff --git a/inc/dg/poisson_t.cu b/inc/dg/poisson_t.cu
index b333dbabc..7440c9b8e 100644
--- a/inc/dg/poisson_t.cu
+++ b/inc/dg/poisson_t.cu
@@ -37,10 +37,6 @@ double jacobian( double x, double y)
 {
     return cos(x)*cos(y)*cos(x)*cos(y) - sin(x)*sin(y)*sin(x)*sin(y);
 }
-double variationRHS( double x, double y)
-{
-    return cos(x)*cos(y)*cos(x)*cos(y) + sin(x)*sin(y)*sin(x)*sin(y);
-}
 
 int main()
 {
@@ -61,7 +57,6 @@ int main()
     const dg::DVec w2d = dg::create::weights( grid);
     const dg::DVec eins = dg::evaluate( dg::one, grid);
     const dg::DVec sol = dg::evaluate ( jacobian, grid);
-    const dg::DVec variation = dg::evaluate ( variationRHS, grid);
     exblas::udouble res;
     std::cout << std::scientific;
     res.d = dg::blas2::dot( eins, w2d, jac);
@@ -73,9 +68,5 @@ int main()
     dg::blas1::axpby( 1., sol, -1., jac);
     res.d = sqrt(dg::blas2::dot( w2d, jac));
     std::cout << "Distance to solution "<<res.d<<"\t"<<res.i<<std::endl;
-    poisson.variationRHS( rhs, jac);
-    dg::blas1::axpby( 1., variation, -1., jac);
-    res.d = sqrt( dg::blas2::dot( w2d, jac));
-    std::cout << "Variation distance to solution "<<res.d<<"\t"<<res.i<<std::endl;
     return 0;
 }
-- 
GitLab


From 9fde94f44f07e6acf99bd4c22c70d13ee6726eb0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Jan 2021 15:25:42 +0100
Subject: [PATCH 428/540] Remove variation test from arakawa_t

---
 inc/dg/arakawa_t.cu | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/inc/dg/arakawa_t.cu b/inc/dg/arakawa_t.cu
index 0a09085df..1a83a3321 100644
--- a/inc/dg/arakawa_t.cu
+++ b/inc/dg/arakawa_t.cu
@@ -40,9 +40,6 @@ double jacobian( double x, double y) {
     return cos(x)*cos(y)*cos(x)*cos(y) - sin(x)*sin(y)*sin(x)*sin(y);
 }
 //![function]
-double variationRHS( double x, double y) {
-    return cos(x)*cos(y)*cos(x)*cos(y) + sin(x)*sin(y)*sin(x)*sin(y);
-}
 /*
 ////These are for comparing to FD arakawa results
 //double left( double x, double y) {return sin(2.*M_PI*(x-hx/2.));}
@@ -106,11 +103,6 @@ int main()
     //n = 5 -> p = 5    |
     // quantities are all conserved to 1e-15 for periodic bc
     // for dirichlet bc these are not better conserved than normal jacobian
-    const dg::DVec variation = dg::evaluate ( variationRHS, grid);
-    arakawa.variation( rhs, jac);
-    dg::blas1::axpby( 1., variation, -1., jac);
-    res.d = sqrt( dg::blas2::dot( w2d, jac));
-    std::cout << "Variation distance   "<<res.d<<"\t"<<res.i-binary[4]<<std::endl; //don't forget sqrt when comuting errors
     std::cout << "\nContinue with topology/average_t.cu !\n\n";
     return 0;
 }
-- 
GitLab


From 81c0ee290d70492374d949eca10688f1ece5d12e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Jan 2021 16:27:17 +0100
Subject: [PATCH 429/540] Move variation to shu.cuh object

- and insert DG_BENCHMARK in Makefile
- there is a strange bug on the GPU version, it randomly crashes, does
this only happen on GTX card?
---
 src/lamb_dipole/Makefile |  2 +-
 src/lamb_dipole/shu.cuh  | 28 ++++++++++++++++++----------
 src/lamb_dipole/shu_b.cu | 15 ++++++++++++---
 3 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/src/lamb_dipole/Makefile b/src/lamb_dipole/Makefile
index 1f8053517..caeafc9a8 100644
--- a/src/lamb_dipole/Makefile
+++ b/src/lamb_dipole/Makefile
@@ -11,7 +11,7 @@ INCLUDE+= -I../../inc   # other project libraries
 all: shu_b
 
 shu_b: shu_b.cu shu.cuh
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(GLFLAGS) $(JSONLIB) -g
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
 .PHONY: clean
 
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index e97788753..e896f32f7 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -65,14 +65,17 @@ struct Shu
 
     const dg::Elliptic<Matrix, Container, Container>& lap() const { return m_multi_laplaceM[0];}
     dg::ArakawaX<Geometry, Matrix, Container>& arakawa() {return m_arakawa;}
-    /**
-     * @brief Returns psi that belong to the last y in operator()
-     *
-     * In a multistep scheme this belongs to the point HEAD-1
-     * @return psi is the potential
-     */
+
     const Container& potential( ) {return m_psi;}
+
     void operator()(double t, const Container& y, Container& yp);
+
+    void variation( const Container& phi, Container& variation_phi){
+        dg::blas2::symv( m_centered_phi[0], phi,  m_temp[0]);
+        dg::blas2::symv( m_centered_phi[1], phi,  m_temp[1]);
+        dg::tensor::multiply2d( m_metric, m_temp[0], m_temp[1], variation_phi, m_temp[2]);
+        dg::blas1::pointwiseDot( 1., m_temp[0], variation_phi, 1., m_temp[1], m_temp[2], 0., variation_phi);
+    }
   private:
     Container m_psi, m_v, m_temp[3], m_fine_psi, m_fine_v, m_fine_temp[3], m_fine_y, m_fine_yp;
     std::vector<dg::Elliptic<Geometry, Matrix, Container>> m_multi_laplaceM;
@@ -81,8 +84,10 @@ struct Shu
     dg::MultigridCG2d<Geometry, Matrix, Container> m_multigrid;
     IMatrix m_inter, m_project;
     Matrix m_forward[2], m_backward[2], m_centered[2];
+    Matrix m_centered_phi[2]; // for variation
     std::vector<double> m_eps;
     std::string m_advection, m_multiplication;
+    dg::SparseTensor<Container> m_metric;
 };
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
@@ -93,8 +98,15 @@ Shu< Geometry, IMatrix, Matrix, Container>::Shu(
 {
     m_advection = file::get( mode, js, "advection", "type", "arakawa").asString();
     m_multiplication = file::get( mode, js, "advection", "multiplication", "pointwise").asString();
+    m_metric = g.metric();
 
     m_psi = dg::evaluate( dg::zero, g);
+    m_centered_phi[0] = dg::create::dx( g, g.bcx(), dg::centered);
+    m_centered_phi[1] = dg::create::dy( g, g.bcy(), dg::centered);
+    m_v = dg::evaluate( dg::zero, g);
+    m_temp[0] = dg::evaluate( dg::zero, g);
+    m_temp[1] = dg::evaluate( dg::zero, g);
+    m_temp[2] = dg::evaluate( dg::zero, g);
     if( "projection" == m_multiplication)
     {
         Geometry fine_grid = g;
@@ -125,10 +137,6 @@ Shu< Geometry, IMatrix, Matrix, Container>::Shu(
         m_forward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::forward);
         m_backward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::backward);
         m_backward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::backward);
-        m_v = dg::evaluate( dg::zero, g);
-        m_temp[0] = dg::evaluate( dg::zero, g);
-        m_temp[1] = dg::evaluate( dg::zero, g);
-        m_temp[2] = dg::evaluate( dg::zero, g);
         m_arakawa.construct( g);
     }
 
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index ceff946c9..9ce7d598c 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -54,7 +54,7 @@ int main( int argc, char* argv[])
     double enstrophy = 0.5*dg::blas2::dot( y0, w2d, y0);
     double energy =    0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
     dg::DVec varphi( grid.size());
-    shu.arakawa().variation( shu.potential(), varphi);
+    shu.variation( shu.potential(), varphi);
     double variation = dg::blas1::dot( varphi, w2d);
 
     std::cout << "Total energy:     "<<energy<<"\n";
@@ -133,6 +133,7 @@ int main( int argc, char* argv[])
             glfwSwapBuffers(w);
             //step
             t.tic();
+            try{
             if( "Karniadakis" == stepper)
                 for( unsigned i=0; i<itstp; i++)
                     karniadakis.step( shu, diffusion, time, y0);
@@ -142,6 +143,12 @@ int main( int argc, char* argv[])
             else if ( "Shu-Osher" == stepper)
                 for( unsigned i=0; i<itstp; i++)
                     shu_osher.step( shu, filter, time, y0, time, y0, dt);
+            }
+            catch( dg::Fail& fail) {
+                std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
+                std::cerr << "Does Simulation respect CFL condition?\n";
+                return -1;
+            }
             t.toc();
             //std::cout << "Timer for one step: "<<t.diff()/N<<"s\n";
         }
@@ -208,7 +215,7 @@ int main( int argc, char* argv[])
                 output1d[0] = dg::blas1::dot( w2d, y0);
                 output1d[1] = 0.5*dg::blas2::dot( y0, w2d, y0);
                 output1d[2] = 0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
-                shu.arakawa().variation(shu.potential(), varphi);
+                shu.variation(shu.potential(), varphi);
                 output1d[3] = dg::blas1::dot( varphi, w2d);
                 Estart[0] += 1;
                 for( int k=0;k<4; k++)
@@ -233,6 +240,8 @@ int main( int argc, char* argv[])
         catch( dg::Fail& fail) {
             std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
             std::cerr << "Does Simulation respect CFL condition?\n";
+            err = nc_close(ncid);
+            return -1;
         }
         err = nc_close(ncid);
     }
@@ -248,7 +257,7 @@ int main( int argc, char* argv[])
     std::cout << "Absolute vorticity error is: "<<dg::blas1::dot( w2d, y0) - vorticity << "\n";
     std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - enstrophy)/enstrophy<<"\n";
     std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<"\n";
-    shu.arakawa().variation(shu.potential(), varphi);
+    shu.variation(shu.potential(), varphi);
     std::cout << "Variation relative to 0 is:  "<<variation - dg::blas1::dot( varphi, w2d) << std::endl;
 
     return 0;
-- 
GitLab


From 00a10f76c9c126128febe508734ffcd5512c3e10 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 13 Jan 2021 16:37:29 +0100
Subject: [PATCH 430/540] Add WITHOUT_GLFW feature to compile on cluster

---
 src/lamb_dipole/Makefile | 3 +++
 src/lamb_dipole/shu_b.cu | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/src/lamb_dipole/Makefile b/src/lamb_dipole/Makefile
index caeafc9a8..ae39b1ed6 100644
--- a/src/lamb_dipole/Makefile
+++ b/src/lamb_dipole/Makefile
@@ -13,6 +13,9 @@ all: shu_b
 shu_b: shu_b.cu shu.cuh
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
 
+shu_hpc: shu_b.cu shu.cuh
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g -DDG_BENCHMARK -DWITHOUT_GLFW
+
 .PHONY: clean
 
 clean:
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 9ce7d598c..3261c1219 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -6,7 +6,9 @@
 #include "dg/algorithm.h"
 #include "dg/file/json_utilities.h"
 
+#ifndef WITHOUT_GLFW
 #include "draw/host_window.h"
+#endif
 #include "dg/file/file.h"
 
 #include "init.h"
@@ -107,6 +109,7 @@ int main( int argc, char* argv[])
     unsigned maxout = file::get( mode, js, "output", "maxout", 100).asUInt();
     unsigned itstp = file::get( mode, js, "output", "itstp", 5).asUInt();
     std::string output = file::get( mode, js, "output", "type", "glfw").asString();
+#ifndef WITHOUT_GLFW
     if( "glfw" == output)
     {
         ////////////////////////////////glfw//////////////////////////////
@@ -155,6 +158,7 @@ int main( int argc, char* argv[])
         glfwTerminate();
     }
     else
+#endif //WITHOUT_GLFW
     {
         std::string outputfile;
         if( argc == 1 || argc == 2)
-- 
GitLab


From 132e67ee706dca194f36e26f058d21dc2d2088c8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 14 Jan 2021 17:06:05 +0100
Subject: [PATCH 431/540] Add centered differences for advection

- also note that variation is the same as energy
---
 src/lamb_dipole/Makefile |  4 ++--
 src/lamb_dipole/shu.cuh  | 23 +++++++++++++++++++++++
 src/lamb_dipole/shu_b.cu | 27 +++++++++++++++++----------
 3 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/src/lamb_dipole/Makefile b/src/lamb_dipole/Makefile
index ae39b1ed6..f6c7e6c1f 100644
--- a/src/lamb_dipole/Makefile
+++ b/src/lamb_dipole/Makefile
@@ -1,4 +1,4 @@
-device=gpu
+device=omp
 
 #configure machine
 include ../../config/default.mk
@@ -11,7 +11,7 @@ INCLUDE+= -I../../inc   # other project libraries
 all: shu_b
 
 shu_b: shu_b.cu shu.cuh
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(GLFLAGS) $(JSONLIB) -g -DDG_BENCHMARK
+	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(GLFLAGS) $(JSONLIB) -g
 
 shu_hpc: shu_b.cu shu.cuh
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g -DDG_BENCHMARK -DWITHOUT_GLFW
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index e896f32f7..9f668639d 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -183,6 +183,17 @@ void Shu<Geometry, IMatrix, Matrix, Container>::operator()(double t, const Conta
             dg::blas2::symv( m_backward[1], m_temp[0], m_temp[2]);
             dg::blas1::subroutine( shu::Upwind(), yp, m_temp[1], m_temp[2], m_v);
         }
+        else if( "centered" == m_advection)
+        {
+            //dx ( nv_x)
+            dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
+            dg::blas1::pointwiseDot( y, m_v, m_temp[0]); //f_x
+            dg::blas2::symv( -1., m_centered[0], m_temp[0], 0., yp);
+            //dy ( nv_y)
+            dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_v); //v_y
+            dg::blas1::pointwiseDot( y, m_v, m_temp[0]); //f_y
+            dg::blas2::symv( -1., m_centered[1], m_temp[0], 1., yp);
+        }
     }
     else // "projection " == multiplication
     {
@@ -206,6 +217,18 @@ void Shu<Geometry, IMatrix, Matrix, Container>::operator()(double t, const Conta
             dg::blas2::symv( m_backward[1], m_fine_temp[0], m_fine_temp[2]);
             dg::blas1::subroutine( shu::Upwind(), m_fine_yp, m_fine_temp[1], m_fine_temp[2], m_fine_v);
         }
+        else if( "centered" == m_advection)
+        {
+            //dx ( nv_x)
+            dg::blas2::symv( -1., m_centered[1], m_fine_psi, 0., m_fine_v); //v_x
+            dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_x
+            dg::blas2::symv( -1., m_centered[0], m_fine_temp[0], 0., m_fine_yp);
+            //dy ( nv_y)
+            dg::blas2::symv( 1., m_centered[0], m_fine_psi, 0., m_fine_v); //v_y
+            dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_y
+            dg::blas2::symv( -1., m_centered[1], m_fine_temp[0], 1., m_fine_yp);
+        }
+
         dg::blas2::symv( m_project, m_fine_yp, yp);
     }
 
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 3261c1219..a72873dc7 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -57,7 +57,7 @@ int main( int argc, char* argv[])
     double energy =    0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
     dg::DVec varphi( grid.size());
     shu.variation( shu.potential(), varphi);
-    double variation = dg::blas1::dot( varphi, w2d);
+    double variation = 0.5*dg::blas1::dot( varphi, w2d);
 
     std::cout << "Total energy:     "<<energy<<"\n";
     std::cout << "Total enstrophy:  "<<enstrophy<<"\n";
@@ -220,7 +220,7 @@ int main( int argc, char* argv[])
                 output1d[1] = 0.5*dg::blas2::dot( y0, w2d, y0);
                 output1d[2] = 0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
                 shu.variation(shu.potential(), varphi);
-                output1d[3] = dg::blas1::dot( varphi, w2d);
+                output1d[3] = 0.5*dg::blas1::dot( varphi, w2d);
                 Estart[0] += 1;
                 for( int k=0;k<4; k++)
                     err = nc_put_vara_double( ncid, variableIDs[k], Estart, Ecount, &output1d[k] );
@@ -251,18 +251,25 @@ int main( int argc, char* argv[])
     }
     ////////////////////////////////////////////////////////////////////
     std::cout << "Time "<<time<<std::endl;
-    //dg::Lamb solution( p.posX*p.lx, p.posY*p.ly - p.U*time, p.R, p.U);
-    //dg::DVec sol = dg::evaluate( solution, grid);
-    //dg::blas1::axpby( 1., y0, -1., sol);
-    //double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
-    //std::cout << "Analytic error to solution "<<error<<std::endl;
-    //std::cout << "Analytic formula enstrophy "<<lamb.enstrophy()<<std::endl;
-    //std::cout << "Analytic formula energy    "<<lamb.energy()<<std::endl;
+    if( "lamb" == initial)
+    {
+        double posX = file::get( mode, js, "init", "posX", 0.5).asDouble();
+        double posY = file::get( mode, js, "init", "posY", 0.8).asDouble();
+        double R = file::get( mode, js, "init", "sigma", 0.1).asDouble();
+        double U = file::get( mode, js, "init", "velocity", 1).asDouble();
+        dg::Lamb lamb( posX*grid.lx(), posY*grid.ly() - U*time, R, U);
+        dg::DVec sol = dg::evaluate( lamb, grid);
+        dg::blas1::axpby( 1., y0, -1., sol);
+        double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
+        std::cout << "Analytic error to solution "<<error<<std::endl;
+        std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - lamb.enstrophy())/lamb.enstrophy()<<"\n";
+        std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - lamb.energy())/lamb.energy()<<"\n";
+    }
     std::cout << "Absolute vorticity error is: "<<dg::blas1::dot( w2d, y0) - vorticity << "\n";
     std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - enstrophy)/enstrophy<<"\n";
     std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<"\n";
     shu.variation(shu.potential(), varphi);
-    std::cout << "Variation relative to 0 is:  "<<variation - dg::blas1::dot( varphi, w2d) << std::endl;
+    std::cout << "Relative variation error is: "<<(0.5*dg::blas1::dot( varphi, w2d)-variation)/variation << std::endl;
 
     return 0;
 
-- 
GitLab


From 4dbd619c3de7d132a008295d186c65ba522babf0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 14 Jan 2021 18:27:38 +0100
Subject: [PATCH 432/540] Add documentation and manufactured solution

- documentation should be expanded
---
 doc/related_pages/references.bib   |  20 ++++
 src/lamb_dipole/Makefile           |   4 +-
 src/lamb_dipole/init.h             |  46 ++++++++
 src/lamb_dipole/input/default.json |   6 ++
 src/lamb_dipole/shu.cuh            |  16 ++-
 src/lamb_dipole/shu.tex            | 164 +++++++++++++++++++++++++++++
 src/lamb_dipole/shu_b.cu           |  18 +++-
 7 files changed, 269 insertions(+), 5 deletions(-)
 create mode 100644 src/lamb_dipole/shu.tex

diff --git a/doc/related_pages/references.bib b/doc/related_pages/references.bib
index d4121fa7d..e5adcfc35 100644
--- a/doc/related_pages/references.bib
+++ b/doc/related_pages/references.bib
@@ -53,6 +53,16 @@
   Doi           = {10.1137/S0036142997316712},
 }
 
+@Article{Liu2000,
+  Title         = {A High-Order Discontinuous Galerkin Method for 2D Incompressible Flows},
+  Author        = {Liu, J.G. and Shu, C.W.},
+  Journal       = {J. Comput. Phys.},
+  Year          = {2000},
+  Pages         = {557-596},
+  Volume        = {160},
+  Doi           = {10.1006/jcph.2000.6475},
+}
+
 @Article{Cockburn2001runge,
   Title         = {{Runge--Kutta discontinuous Galerkin methods for convection-dominated problems}},
   Author        = {Cockburn, B. and Shu, C. W.},
@@ -116,6 +126,16 @@
   Doi                      = {10.1016/j.cpc.2014.07.007},
 }
 
+@Article{Nielsen1997,
+  Title                    = {Formation and temporal evolution of the Lamb-dipole},
+  Author                   = {Nielsen, A.H. and Rasmussen, J.Juul},
+  Journal                  = {Phys. Fluids},
+  Year                     = {1997},
+  Pages                    = {982--991},
+  Volume                   = {9},
+  Doi                      = {10.1063/1.869193},
+}
+
 @Book{Frankel,
   Title                    = {{The geometry of physics: an introduction}},
   Author                   = {Frankel, Theodore},
diff --git a/src/lamb_dipole/Makefile b/src/lamb_dipole/Makefile
index f6c7e6c1f..5a43bc1bf 100644
--- a/src/lamb_dipole/Makefile
+++ b/src/lamb_dipole/Makefile
@@ -10,10 +10,10 @@ INCLUDE+= -I../../inc   # other project libraries
 
 all: shu_b
 
-shu_b: shu_b.cu shu.cuh
+shu_b: shu_b.cu shu.cuh init.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(GLFLAGS) $(JSONLIB) -g
 
-shu_hpc: shu_b.cu shu.cuh
+shu_hpc: shu_b.cu shu.cuh init.h
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -g -DDG_BENCHMARK -DWITHOUT_GLFW
 
 .PHONY: clean
diff --git a/src/lamb_dipole/init.h b/src/lamb_dipole/init.h
index cd59e4b10..3d73eb6bb 100644
--- a/src/lamb_dipole/init.h
+++ b/src/lamb_dipole/init.h
@@ -1,3 +1,4 @@
+#pragma once
 
 #include <map>
 #include <functional>
@@ -35,6 +36,39 @@ struct ShearLayer{
     double m_delta;
 };
 
+struct MMSVorticity{
+    MMSVorticity( double radius, double velocity, double time): m_s(radius), m_v(velocity), m_t(time) {}
+    DG_DEVICE
+    double operator()( double x, double y)const{
+        return -4.*x*exp( - (x*x + (m_t*m_v + y)*(m_t*m_v+y))/m_s/m_s)*
+            (-2*m_s*m_s + x*x+ (m_t*m_v + y)*(m_t*m_v+y))/m_s/m_s/m_s/m_s;
+    }
+    private:
+    double m_s, m_v, m_t;
+};
+
+struct MMSPotential{
+    MMSPotential( double radius, double velocity, double time): m_s(radius), m_v(velocity), m_t(time) {}
+    DG_DEVICE
+    double operator()( double x, double y, double t)const{
+        return exp( - (x*x + (m_t*m_v + y)*(m_t*m_v+y))/m_s/m_s) * x;
+    }
+    private:
+    double m_s, m_v, m_t;
+};
+struct MMSSource{
+    MMSSource( ): m_s(1), m_v(0) {}
+    MMSSource( double radius, double velocity): m_s(radius), m_v(velocity) {}
+    DG_DEVICE
+    double operator()( double x, double y, double t)const{
+        double ss = m_s*m_s;
+        double ad = y+m_v*t;
+        return 8.*x/ss/ss/ss*ad*exp( - 2.*(x*x + ad*ad)/ss)*( -ss +exp( (x*x+ad*ad)/ss)*m_v*(-3.*ss + x*x + ad*ad));
+    }
+    private:
+    double m_s, m_v;
+};
+
 std::map<std::string, std::function< dg::HVec(
     Json::Value& js, enum file::error mode,
     const dg::CartesianGrid2d& grid) >
@@ -64,6 +98,18 @@ std::map<std::string, std::function< dg::HVec(
             omega = dg::evaluate ( ShearLayer( rho, delta), grid);
             return omega;
         }
+    },
+    {"mms", [](
+        Json::Value& js, enum file::error mode,
+        const dg::CartesianGrid2d& grid)
+        {
+            dg::HVec omega;
+            double sigma = file::get( mode, js, "init", "sigma", 0.2).asDouble();
+            double velocity = file::get( mode, js, "init", "velocity", 0.1).asDouble();
+            std::cout << "Sigma "<<sigma<<" "<<velocity<<std::endl;;
+            omega = dg::evaluate ( MMSVorticity( sigma, velocity, 0.), grid);
+            return omega;
+        }
     }
 };
 
diff --git a/src/lamb_dipole/input/default.json b/src/lamb_dipole/input/default.json
index acf2a18b5..9e594aad8 100644
--- a/src/lamb_dipole/input/default.json
+++ b/src/lamb_dipole/input/default.json
@@ -82,4 +82,10 @@
     //    "rho":  0.20943951023931953, //pi/15
     //    "delta": 0.05
     //}
+    //"init": //use [-1,1]x[-1,1]
+    //{
+    //    "type" : "mms",
+    //    "sigma":  0.2,
+    //    "velocity": 1
+    //}
 }
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index 9f668639d..f902db8f9 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -7,6 +7,7 @@
 #include "json/json.h"
 #include "file/json_utilities.h"
 #include "dg/algorithm.h"
+#include "init.h"
 
 namespace shu
 {
@@ -76,6 +77,10 @@ struct Shu
         dg::tensor::multiply2d( m_metric, m_temp[0], m_temp[1], variation_phi, m_temp[2]);
         dg::blas1::pointwiseDot( 1., m_temp[0], variation_phi, 1., m_temp[1], m_temp[2], 0., variation_phi);
     }
+    void set_mms_source( double sigma, double velocity) {
+        m_mms = shu::MMSSource( sigma, velocity);
+        m_add_mms = true;
+    }
   private:
     Container m_psi, m_v, m_temp[3], m_fine_psi, m_fine_v, m_fine_temp[3], m_fine_y, m_fine_yp;
     std::vector<dg::Elliptic<Geometry, Matrix, Container>> m_multi_laplaceM;
@@ -88,13 +93,19 @@ struct Shu
     std::vector<double> m_eps;
     std::string m_advection, m_multiplication;
     dg::SparseTensor<Container> m_metric;
+
+    shu::MMSSource m_mms;
+    bool m_add_mms = false;
+    Container m_x, m_y;
 };
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
 Shu< Geometry, IMatrix, Matrix, Container>::Shu(
         const Geometry& g, Json::Value& js, enum file::error mode):
     m_old_psi( 2, dg::evaluate( dg::zero, g)),
-    m_multigrid( g, 3)
+    m_multigrid( g, 3),
+    m_x( dg::evaluate( dg::cooX2d, g)),
+    m_y( dg::evaluate( dg::cooY2d, g))
 {
     m_advection = file::get( mode, js, "advection", "type", "arakawa").asString();
     m_multiplication = file::get( mode, js, "advection", "multiplication", "pointwise").asString();
@@ -228,9 +239,10 @@ void Shu<Geometry, IMatrix, Matrix, Container>::operator()(double t, const Conta
             dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_y
             dg::blas2::symv( -1., m_centered[1], m_fine_temp[0], 1., m_fine_yp);
         }
-
         dg::blas2::symv( m_project, m_fine_yp, yp);
     }
+    if( m_add_mms) //for the manufactured solution we need to add a source term
+        dg::blas1::evaluate( yp, dg::plus_equals(), m_mms, m_x, m_y, t);
 
 }
 
diff --git a/src/lamb_dipole/shu.tex b/src/lamb_dipole/shu.tex
new file mode 100644
index 000000000..bf541ed73
--- /dev/null
+++ b/src/lamb_dipole/shu.tex
@@ -0,0 +1,164 @@
+%%%%%%%%%%%%%%%%%%%%%definitions%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+
+\input{../../doc/related_pages/header.tex}
+\input{../../doc/related_pages/newcommands.tex}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%DOCUMENT%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{document}
+
+\title{Testing Advection Schemes}
+\author{ M.~Wiesenberger}
+\maketitle
+
+\begin{abstract}
+  This is a program to test various advection schemes on the 2d incompressible Euler
+  equation used in Reference~\cite{Einkemmer2014}.
+\end{abstract}
+
+\section{Equations}
+We implement the 2d incompressible Euler equation
+\begin{subequations}
+\begin{align}
+ \frac{\partial \omega}{\partial t} + \{ \phi, \omega\} = 0 \\
+ -\Delta \phi = \omega
+\end{align}
+\label{eq:euler_poisson}
+\end{subequations}
+with vorticity $\omega$ and stream-function $\phi$.
+The Poisson bracket is given by $\{ \phi, \omega\} := \phi_x \omega_y - \phi_y \omega_x$.
+Eq.~\eqref{eq:euler_poisson} is a reformulation of the standard conservative form
+\begin{subequations}
+\begin{align}
+    \frac{\partial \omega}{\partial t} + \nabla\cdot({\vec v \omega}) = 0 \\
+\nabla\cdot\vec v = 0 \quad \omega = -(\nabla\times \vec v)\cdot \zhat
+\end{align}
+\label{eq:euler_conservative}
+\end{subequations}
+with $v_x = - \phi_y$ and $v_y = \phi_x$.
+
+Eqs.~\eqref{eq:euler_poisson} have an infinite amount of conserved quantities
+among them the total vorticity $V$, the kinetic energy $E$ and the enstrophy $\Omega$
+ \begin{align}
+     V := \int_D \omega \dA\quad
+     E :=\frac{1}{2} \int_D \left( \nabla \phi\right)^2 \dA \quad
+     \Omega:= \frac{1}{2} \int_D \omega^2 \dA
+ \end{align}
+
+
+\section{Initialization}
+We will consider several different initial conditions in order to test
+our numerical methods
+\subsection{Lamb Dipole}
+The Lamb dipole is a stationary solution to the Euler equations~\cite{Nielsen1997} with infinite
+boundary conditions
+\begin{align}
+    \omega(x,y,0) = \begin{cases}
+        \frac{2\lambda U}{J_0(\lambda R)} J_1(\lambda R) \cos \theta,\ r < R,\\
+        0, \text{ else}
+    \end{cases}
+\end{align}
+Unfortunately, for a finite box this is not an exact solution any more.
+on the domain $[0,1]\times [0,1]$.
+\subsection{Manufactured Solution}
+We manufacture a solution via
+\begin{align}
+    \phi(x,y,t) &=
+    x \exp\left( - \frac{ x^2 + (y+vt)^2}{\sigma^2}\right) \\
+    \omega(x,y,t) &= -\Delta \phi = -\sigma^{-4} \left[ 4\phi(x,y,t) ( x^2-2\sigma^2  + (y+tv)^2)\right]
+\end{align}
+which is solution to the modified equations
+\begin{subequations}
+\begin{align}
+    \frac{\partial \omega}{\partial t} + \{ \phi, \omega\} = S(x,y,t) \\
+ -\Delta \phi = \omega
+\end{align}
+\label{eq:euler_poisson}
+\end{subequations}
+with the source
+\begin{align}
+    S(x,y,t) = 8 x \sigma^{-6}(y+vt)\exp\left( - 2\frac{ x^2 + (y+vt)^2}{\sigma^2} \right)\left(-\sigma^2  + \exp\left( \frac{ x^2 + (y+vt)^2}{\sigma^2} \right) v( -3\sigma^2 + x^2 + (y+vt)^2) \right)
+\end{align}
+on the domain $[-1,1]\times [-1,1]$.
+
+
+\subsection{ Double Shear layer}
+Here, we follow~\cite{Liu2000} and test the scheme on a double shear layer problem.
+\begin{align}
+    \omega(x,y,0) = \begin{cases}
+        \delta \cos(x) - \frac{1}{\rho} \text{sech}^2 \left(\frac{y-\pi/2}{\rho}\right),\ y \leq \pi \\
+        \delta \cos(x) + \frac{1}{\rho} \text{sech}^2 \left(\frac{3\pi/2-y}{\rho}\right),\ y > \pi \\
+    \end{cases}
+\end{align}
+where $\rho = \pi/15$ and $\delta =0.05$ on the domain $[0,2\pi]\times [0,2\pi]$.
+This solution will quickly roll-up and generate smaller and smaller scales.
+A thin shear layer corresponds to $\rho = \pi/50$ or smaller.
+
+\section{Numerical methods}
+Our goal is to try out various time integration and advection discretization techniques\ldots
+\subsection{Arakawa scheme and centered flux}
+Reference~\cite{Liu2000} reports that the centered flux does not have any numerical
+diffusion while the upwind flux does.
+We also know from Godunov's theorem
+that any linear advection scheme of order 2 or higher is prone to oscillations.
+They also prove that upwind and centered fluxes do not dissipate energy.
+From finite differences we know that centered differences for the advection term
+are unstable (or at least produce a lot of oscillations).
+\subsection{Modal filters}
+\ldots
+\subsection{Artificial viscosity}
+\ldots
+
+\section{Compilation and useage}
+The program shu\_b.cu compiles with
+\begin{verbatim}
+make <shu_b> device = <omp gpu>
+make <shu_hpc> device = <omp gpu>
+\end{verbatim}
+and depends on both GLFW3 and NETCDF. If GLFW3 is not available then compile shu\_hpc which avoids this dependency.
+Run with
+\begin{verbatim}
+path/to/feltor/src/shu/shu_b input.json
+\end{verbatim}
+
+\subsection{Input file structure}
+Input file format: json
+
+%%This is a booktabs table
+\begin{longtable}{llll>{\RaggedRight}p{7cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
+\bottomrule
+\end{longtable}
+
+The default value is taken if the value name is not found in the input file. If there is no default and
+the value is not found,
+the program exits with an error message.
+
+\subsection{Structure of output file}
+Output file format: netcdf-4/hdf5
+%
+%Name | Type | Dimensionality | Description
+%---|---|---|---|
+\begin{longtable}{lll>{\RaggedRight}p{7cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
+inputfile  &             text attribute & 1 & verbose input file as a string \\
+energy\_time             & Coord. Var. & 1 (energy\_time) & timesteps at which 1d variables are written \\
+time                     & Coord. Var. & 1 (time) & time at which fields are written \\
+x                        & Coord. Var. & 1 (x) & x-coordinate  \\
+y                        & Coord. Var. & 1 (y) & y-coordinate \\
+vorticity\_field         & Dataset & 3 (time, y, x) & electon density $n$ \\
+potential                & Dataset & 3 (time, y, x) & electric potential $\phi$  \\
+vorticity                & Dataset & 1 (energy\_time) & Vorticity $V$  \\
+enstrophy                & Dataset & 1 (energy\_time) & Enstropy $\Omega$  \\
+energy                   & Dataset & 1 (energy\_time) & Total energy integral computed using $E = \int_D \phi\omega \dA$ \\
+variation                & Dataset & 1 (energy\_time) & Total energy integral computed directly $\int_D (\nabla \phi)^2$  \\
+\bottomrule
+\end{longtable}
+
+%..................................................................
+\bibliography{../../doc/related_pages/references}
+%..................................................................
+
+
+\end{document}
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index a72873dc7..f0c463a13 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -34,7 +34,6 @@ int main( int argc, char* argv[])
     std::string initial = file::get( mode, js, "init", "type", "lamb").asString();
     dg::HVec omega = shu::initial_conditions.at( initial)(js, mode, grid);
 
-
     dg::DVec y0( omega ), y1( y0);
     //subtract mean mass
     if( grid.bcx() == dg::PER && grid.bcy() == dg::PER)
@@ -46,6 +45,13 @@ int main( int argc, char* argv[])
     shu::Shu<dg::CartesianGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec>
         shu( grid, js, mode);
     shu::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, js, mode);
+    if( "mms" == initial)
+    {
+        double sigma = file::get( mode, js, "init", "sigma", 0.2).asDouble();
+        double velocity = file::get( mode, js, "init", "velocity", 0.1).asDouble();
+        shu.set_mms_source( sigma, velocity);
+    }
+
 
     dg::Timer t;
     t.tic();
@@ -265,6 +271,16 @@ int main( int argc, char* argv[])
         std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - lamb.enstrophy())/lamb.enstrophy()<<"\n";
         std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - lamb.energy())/lamb.energy()<<"\n";
     }
+    if( "mms" == initial)
+    {
+        double R = file::get( mode, js, "init", "sigma", 0.1).asDouble();
+        double U = file::get( mode, js, "init", "velocity", 1).asDouble();
+        shu::MMSVorticity vortex( R, U, time);
+        dg::DVec sol = dg::evaluate( vortex, grid);
+        dg::blas1::axpby( 1., y0, -1., sol);
+        double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
+        std::cout << "Analytic error to solution "<<error<<std::endl;
+    }
     std::cout << "Absolute vorticity error is: "<<dg::blas1::dot( w2d, y0) - vorticity << "\n";
     std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - enstrophy)/enstrophy<<"\n";
     std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<"\n";
-- 
GitLab


From d142aa39fbb90a5926bde451304f4f2adebc10cc Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 15 Jan 2021 16:58:46 +0100
Subject: [PATCH 433/540] Fix periodicity in manufactured solution

---
 doc/related_pages/references.bib |  8 +++++
 src/lamb_dipole/init.h           | 52 +++++++++++++++++++++++---------
 src/lamb_dipole/shu.cuh          |  4 +--
 src/lamb_dipole/shu.tex          |  9 ++++--
 src/lamb_dipole/shu_b.cu         | 18 ++---------
 5 files changed, 56 insertions(+), 35 deletions(-)

diff --git a/doc/related_pages/references.bib b/doc/related_pages/references.bib
index e5adcfc35..f72dc2eae 100644
--- a/doc/related_pages/references.bib
+++ b/doc/related_pages/references.bib
@@ -193,6 +193,14 @@
   Series                   = {Texts in Applied Mathematics},
 }
 
+@Book{LeVeque,
+  Title                    = {Finite Volume Methods for Hyperbolic Problems},
+  Author                   = {LeVeque, R.J.},
+  Publisher                = {Cambridge University Press},
+  Year                     = {2002},
+  Series                   = {Cambridge Texts in Applied Mathematics},
+}
+
 @Article{Stegmeir2017,
   Title                    = {Advances in the flux-coordinate independent approach },
   Author                   = {Andreas Stegmeir and Omar Maj and David Coster and Karl Lackner and Markus Held and Matthias Wiesenberger},
diff --git a/src/lamb_dipole/init.h b/src/lamb_dipole/init.h
index 3d73eb6bb..b0a202b11 100644
--- a/src/lamb_dipole/init.h
+++ b/src/lamb_dipole/init.h
@@ -37,36 +37,60 @@ struct ShearLayer{
 };
 
 struct MMSVorticity{
-    MMSVorticity( double radius, double velocity, double time): m_s(radius), m_v(velocity), m_t(time) {}
+    MMSVorticity( double radius, double velocity, double ly, double time): m_s(radius), m_v(velocity), m_ly( ly), m_t(time) {}
     DG_DEVICE
     double operator()( double x, double y)const{
-        return -4.*x*exp( - (x*x + (m_t*m_v + y)*(m_t*m_v+y))/m_s/m_s)*
-            (-2*m_s*m_s + x*x+ (m_t*m_v + y)*(m_t*m_v+y))/m_s/m_s/m_s/m_s;
+        return evaluate( x, y, 0) + evaluate( x,y,m_ly) + evaluate( x,y,-m_ly)
+               + evaluate( x,y,2*m_ly) + evaluate( x,y,-2*m_ly);
     }
     private:
-    double m_s, m_v, m_t;
+    DG_DEVICE
+    double evaluate( double x, double y, double y0) const
+    {
+        double ad = y - y0 +m_v*m_t;
+        if( fabs(ad ) > m_ly) return 0;
+        return -4.*x*exp( - (x*x + ad*ad)/m_s/m_s)*
+            (-2*m_s*m_s + x*x+ ad*ad)/m_s/m_s/m_s/m_s;
+    }
+    double m_s, m_v, m_ly, m_t;
 };
 
 struct MMSPotential{
-    MMSPotential( double radius, double velocity, double time): m_s(radius), m_v(velocity), m_t(time) {}
+    MMSPotential( double radius, double velocity, double ly, double time): m_s(radius), m_v(velocity), m_ly( ly), m_t(time) {}
     DG_DEVICE
-    double operator()( double x, double y, double t)const{
-        return exp( - (x*x + (m_t*m_v + y)*(m_t*m_v+y))/m_s/m_s) * x;
+    double operator()( double x, double y)const{
+        return evaluate( x, y, 0) + evaluate( x,y,m_ly) + evaluate( x,y,-m_ly)
+               + evaluate( x,y,2*m_ly) + evaluate( x,y,-2*m_ly);
     }
     private:
-    double m_s, m_v, m_t;
+    DG_DEVICE
+    double evaluate( double x, double y, double y0) const
+    {
+        double ad = y - y0 +m_v*m_t;
+        if( fabs(ad ) > m_ly) return 0;
+        return exp( - (x*x + ad*ad)/m_s/m_s) * x;
+    }
+    double m_s, m_v, m_ly, m_t;
 };
+
 struct MMSSource{
     MMSSource( ): m_s(1), m_v(0) {}
-    MMSSource( double radius, double velocity): m_s(radius), m_v(velocity) {}
+    MMSSource( double radius, double velocity, double ly): m_s(radius), m_v(velocity), m_ly( ly){}
     DG_DEVICE
     double operator()( double x, double y, double t)const{
+        return evaluate( x, y,t, 0) + evaluate( x,y,t,m_ly) + evaluate( x,y,t,-m_ly)
+               + evaluate( x,y,t,2*m_ly) + evaluate( x,y,t,-2*m_ly);
+    }
+    private:
+    DG_DEVICE
+    double evaluate( double x, double y, double t, double y0) const
+    {
         double ss = m_s*m_s;
-        double ad = y+m_v*t;
+        double ad = y - y0 +m_v*t;
+        if( fabs(ad ) > m_ly) return 0;
         return 8.*x/ss/ss/ss*ad*exp( - 2.*(x*x + ad*ad)/ss)*( -ss +exp( (x*x+ad*ad)/ss)*m_v*(-3.*ss + x*x + ad*ad));
     }
-    private:
-    double m_s, m_v;
+    double m_s, m_v, m_ly;
 };
 
 std::map<std::string, std::function< dg::HVec(
@@ -106,8 +130,8 @@ std::map<std::string, std::function< dg::HVec(
             dg::HVec omega;
             double sigma = file::get( mode, js, "init", "sigma", 0.2).asDouble();
             double velocity = file::get( mode, js, "init", "velocity", 0.1).asDouble();
-            std::cout << "Sigma "<<sigma<<" "<<velocity<<std::endl;;
-            omega = dg::evaluate ( MMSVorticity( sigma, velocity, 0.), grid);
+            std::cout << "Sigma "<<sigma<<" "<<velocity<<std::endl;
+            omega = dg::evaluate ( MMSVorticity( sigma, velocity,grid.ly(), 0.), grid);
             return omega;
         }
     }
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index f902db8f9..08c7bd8d8 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -77,8 +77,8 @@ struct Shu
         dg::tensor::multiply2d( m_metric, m_temp[0], m_temp[1], variation_phi, m_temp[2]);
         dg::blas1::pointwiseDot( 1., m_temp[0], variation_phi, 1., m_temp[1], m_temp[2], 0., variation_phi);
     }
-    void set_mms_source( double sigma, double velocity) {
-        m_mms = shu::MMSSource( sigma, velocity);
+    void set_mms_source( double sigma, double velocity, double ly) {
+        m_mms = shu::MMSSource( sigma, velocity, ly);
         m_add_mms = true;
     }
   private:
diff --git a/src/lamb_dipole/shu.tex b/src/lamb_dipole/shu.tex
index bf541ed73..ab0de6de1 100644
--- a/src/lamb_dipole/shu.tex
+++ b/src/lamb_dipole/shu.tex
@@ -94,12 +94,15 @@ This solution will quickly roll-up and generate smaller and smaller scales.
 A thin shear layer corresponds to $\rho = \pi/50$ or smaller.
 
 \section{Numerical methods}
-Our goal is to try out various time integration and advection discretization techniques\ldots
+Our goal is to try out various time integration and advection discretization techniques.
+We know from Godunov's theorem
+that any linear advection scheme of order 2 or higher is prone to oscillations.
+\subsection{Forward time and centered space}
+It is well known that the forward in time, centered in space method for solving
+hyperbolic systems is unconditionally unstable~\cite{LeVeque}.
 \subsection{Arakawa scheme and centered flux}
 Reference~\cite{Liu2000} reports that the centered flux does not have any numerical
 diffusion while the upwind flux does.
-We also know from Godunov's theorem
-that any linear advection scheme of order 2 or higher is prone to oscillations.
 They also prove that upwind and centered fluxes do not dissipate energy.
 From finite differences we know that centered differences for the advection term
 are unstable (or at least produce a lot of oscillations).
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index f0c463a13..ad8d8bf88 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -49,7 +49,7 @@ int main( int argc, char* argv[])
     {
         double sigma = file::get( mode, js, "init", "sigma", 0.2).asDouble();
         double velocity = file::get( mode, js, "init", "velocity", 0.1).asDouble();
-        shu.set_mms_source( sigma, velocity);
+        shu.set_mms_source( sigma, velocity, grid.ly());
     }
 
 
@@ -257,25 +257,11 @@ int main( int argc, char* argv[])
     }
     ////////////////////////////////////////////////////////////////////
     std::cout << "Time "<<time<<std::endl;
-    if( "lamb" == initial)
-    {
-        double posX = file::get( mode, js, "init", "posX", 0.5).asDouble();
-        double posY = file::get( mode, js, "init", "posY", 0.8).asDouble();
-        double R = file::get( mode, js, "init", "sigma", 0.1).asDouble();
-        double U = file::get( mode, js, "init", "velocity", 1).asDouble();
-        dg::Lamb lamb( posX*grid.lx(), posY*grid.ly() - U*time, R, U);
-        dg::DVec sol = dg::evaluate( lamb, grid);
-        dg::blas1::axpby( 1., y0, -1., sol);
-        double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
-        std::cout << "Analytic error to solution "<<error<<std::endl;
-        std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - lamb.enstrophy())/lamb.enstrophy()<<"\n";
-        std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - lamb.energy())/lamb.energy()<<"\n";
-    }
     if( "mms" == initial)
     {
         double R = file::get( mode, js, "init", "sigma", 0.1).asDouble();
         double U = file::get( mode, js, "init", "velocity", 1).asDouble();
-        shu::MMSVorticity vortex( R, U, time);
+        shu::MMSVorticity vortex( R, U, grid.ly(), time);
         dg::DVec sol = dg::evaluate( vortex, grid);
         dg::blas1::axpby( 1., y0, -1., sol);
         double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
-- 
GitLab


From 7cd004dd176f08446269bf9817472a2e2bb6cabb Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 18 Jan 2021 15:16:34 +0100
Subject: [PATCH 434/540] Add input parameters documentation to lamb dipole

---
 src/lamb_dipole/shu.tex | 201 ++++++++++++++++++++++++++++++++++------
 1 file changed, 174 insertions(+), 27 deletions(-)

diff --git a/src/lamb_dipole/shu.tex b/src/lamb_dipole/shu.tex
index ab0de6de1..b01a9e7cd 100644
--- a/src/lamb_dipole/shu.tex
+++ b/src/lamb_dipole/shu.tex
@@ -20,7 +20,7 @@ We implement the 2d incompressible Euler equation
 \begin{subequations}
 \begin{align}
  \frac{\partial \omega}{\partial t} + \{ \phi, \omega\} = 0 \\
- -\Delta \phi = \omega
+ -\Delta \phi = \omega \label{eq:euler_poisson_elliptic}
 \end{align}
 \label{eq:euler_poisson}
 \end{subequations}
@@ -59,6 +59,23 @@ boundary conditions
 \end{align}
 Unfortunately, for a finite box this is not an exact solution any more.
 on the domain $[0,1]\times [0,1]$.
+The Lamb dipole is chosen with the following parameters in the input file
+%%This is a booktabs table
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+grid & dict &  & Grid parameters \\
+\qquad x & float[2]& [0,1] & Choose x box boundaries appropriately \\
+\qquad y & float[2]& [0,1] & Choose y box boundaries appropriately \\
+\qquad bc & string[2] & [PER, PER] & Choose boundary conditions [x,y] appropriately \\
+init &  dict &   & Parameters for initialization \\
+\qquad type      & string & lamb & This choice necessitates the following parameters \\
+\qquad velocity  & float &  1    &  blob speed \\
+\qquad sigma     & float &  0.1  & dipole radius in units of lx \\
+\qquad posX      & float &  0.5  & in units of lx \\
+\qquad posY      & float &  0.8  & in units of ly \\
+\bottomrule
+\end{longtable}
 \subsection{Manufactured Solution}
 We manufacture a solution via
 \begin{align}
@@ -70,16 +87,31 @@ which is solution to the modified equations
 \begin{subequations}
 \begin{align}
     \frac{\partial \omega}{\partial t} + \{ \phi, \omega\} = S(x,y,t) \\
- -\Delta \phi = \omega
+    -\Delta \phi = \omega
 \end{align}
-\label{eq:euler_poisson}
+\label{eq:euler_poisson_modified}
 \end{subequations}
 with the source
 \begin{align}
-    S(x,y,t) = 8 x \sigma^{-6}(y+vt)\exp\left( - 2\frac{ x^2 + (y+vt)^2}{\sigma^2} \right)\left(-\sigma^2  + \exp\left( \frac{ x^2 + (y+vt)^2}{\sigma^2} \right) v( -3\sigma^2 + x^2 + (y+vt)^2) \right)
+    S(x,y,t) =& 8 x \sigma^{-6}(y+vt)\exp\left( - 2\frac{ x^2 + (y+vt)^2}{\sigma^2} \right) \nonumber\\
+    &\left(-\sigma^2  + \exp\left( \frac{ x^2 + (y+vt)^2}{\sigma^2} \right) v( -3\sigma^2 + x^2 + (y+vt)^2) \right)
 \end{align}
 on the domain $[-1,1]\times [-1,1]$.
 
+The manufactured solution is chosen with the following parameters in the input file
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+grid & dict &  & Grid parameters \\
+\qquad x & float[2]& [-1,1] & Choose x box boundaries appropriately \\
+\qquad y & float[2]& [-1,1] & Choose y box boundaries appropriately \\
+\qquad bc & string[2] & [DIR, PER] & Choose boundary conditions appropriately \\
+init &  dict &   & Parameters for initialization \\
+\qquad type      & string & mms & This choice necessitates the following parameters \\
+\qquad sigma      & float & 0.2 & The width $\sigma$ \\
+\qquad velocity   & float & 1 & The velocity $v$ \\
+\bottomrule
+\end{longtable}
 
 \subsection{ Double Shear layer}
 Here, we follow~\cite{Liu2000} and test the scheme on a double shear layer problem.
@@ -92,24 +124,138 @@ Here, we follow~\cite{Liu2000} and test the scheme on a double shear layer probl
 where $\rho = \pi/15$ and $\delta =0.05$ on the domain $[0,2\pi]\times [0,2\pi]$.
 This solution will quickly roll-up and generate smaller and smaller scales.
 A thin shear layer corresponds to $\rho = \pi/50$ or smaller.
+The double shear layer initialization is chosen with the following parameters in the input file
+%%This is a booktabs table
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+grid & dict &  & Grid parameters \\
+\qquad x & float[2]& [0, 6.283185307179586] & Choose x box boundaries appropriately \\
+\qquad y & float[2]& [0, 6.283185307179586] & Choose y box boundaries appropriately \\
+\qquad bc & string[2] & [PER, PER] & Choose boundary conditions appropriately \\
+init &  dict &   & Parameters for initialization \\
+\qquad type      & string & shear & This choice necessitates the following parameters \\
+\qquad rho    & float & 0.20943951023931953 & The width $\rho = \pi/15$ \\
+\qquad delta  & float & 0.05 & The velocity $v$ \\
+\bottomrule
+\end{longtable}
 
 \section{Numerical methods}
 Our goal is to try out various time integration and advection discretization techniques.
 We know from Godunov's theorem
 that any linear advection scheme of order 2 or higher is prone to oscillations.
-\subsection{Forward time and centered space}
-It is well known that the forward in time, centered in space method for solving
-hyperbolic systems is unconditionally unstable~\cite{LeVeque}.
-\subsection{Arakawa scheme and centered flux}
-Reference~\cite{Liu2000} reports that the centered flux does not have any numerical
-diffusion while the upwind flux does.
-They also prove that upwind and centered fluxes do not dissipate energy.
-From finite differences we know that centered differences for the advection term
-are unstable (or at least produce a lot of oscillations).
-\subsection{Modal filters}
-\ldots
-\subsection{Artificial viscosity}
-\ldots
+\subsection{Spatial grid}
+The spatial grid is a two-dimensional Cartesian product-grid adaptable with the following parameters
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+grid & dict & & \\
+\qquad n  & integer & 3  & The number of polynomial coefficients \\
+\qquad Nx & integer & 48 & Number of cells in x \\
+\qquad Ny & integer & 48 & Number of cells in y \\
+\qquad  x & float[2]& [0,1] & Boundaries in x \\
+\qquad  y & float[2]& [0,1] & Boundaries in y \\
+\qquad bc & string[2] & [DIR, PER] & Boundary conditions in [x,y] \\
+\bottomrule
+\end{longtable}
+\subsection{Time steppers}
+Possible time-steppers are
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+timestepper & dict & & \\
+\qquad stepper  & string& Karniadakis & The \textit{dg::Karniadakis} class \\
+\qquad order    & float & 3 & Order of Karniadakis (1,2 or 3)\\
+\qquad dt       & float & 2e-3 & Fixed timestep \\
+\qquad eps\_time & float & 1e-9 & Accuracy requirement for implicit solver \\
+timestepper & dict & & \\
+\qquad stepper & string & Shu-Osher & The \textit{dg::ShuOsher} class \\
+\qquad order   & string & SSPRK-3-3 & Check dg documentation for possible values \\
+\qquad dt      & float & 1e-3 & Fixed Time-step \\
+timestepper & dict & & \\
+\qquad stepper & string & FilteredMultistep & The \textit{dg::FilteredExplicitMultistep} class \\
+\qquad order   & [string, integer] & [eBDF,3] & Check dg documentation for possible values \\
+\qquad dt      & float & 2e-3 & Fixed timestep \\
+\bottomrule
+\end{longtable}
+\subsection{Regularization technique}
+We support both artificial viscosity and modal filtering choosable by the following
+parameters in the input file
+For artificial viscosity Eqs.~\eqref{eq:euler_poisson} are modified to
+\begin{subequations}
+\begin{align}
+    \frac{\partial \omega}{\partial t} + \{ \phi, \omega\} = -(-\nu_\perp \Delta)^s \omega\\
+ -\Delta \phi = \omega
+\end{align}
+\label{eq:euler_poisson_viscous}
+\end{subequations}
+where $\nu_\perp$ is the viscosity coefficient and $s=1,2,3,\cdots$ is the order
+\begin{longtable}{lllp{7.5cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+regularization & dict & & \\
+\qquad type  & string& viscosity & Choosable only for \textbf{Karniadakis} timestepper (because it is solved implicitly) \\
+\qquad order    & integer & 2 & Order: 1 is normal diffusion, 2 is hyperdiffusion, can be arbitrarily high, but higher orders might take longer to solve \\
+\qquad nu\_perp    & float & 1e-3 & Viscosity coefficient \\
+\bottomrule
+\end{longtable}
+The other regularization method is the modal filter that applies an exponential filter
+\begin{align}
+    \begin{cases}
+    1 \text{ if } \eta < \eta_c \\
+    \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
+    0 \text{ else} \\
+    \eta := \frac{i}{n-1}
+    \end{cases}
+\end{align}
+and is choosable with the following parameters
+\begin{longtable}{lllp{7.5cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+regularization & dict & & \\
+\qquad type  & string& modal & Not choosable for \textbf{Karniadakis} timestepper\\
+\qquad order & integer & 8 & Order: normally 8 or 16 \\
+\qquad eta\_c & float & 0.5 & cutoff wavelength below which no damping is applied \\
+\qquad alpha & float & 36 & damping coefficient determining damping for highest wavenumber \\
+\bottomrule
+\end{longtable}
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\subsection{Elliptic solver}
+Currently, the multigrid solver is the only one choosable
+to solve Eq.~\eqref{eq:euler_poisson_elliptic}
+\begin{longtable}{lllp{7.5cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+elliptic & dict & & \\
+\qquad type  & string& multigrid & Actually a nested iterations class \\
+\qquad stages    & integer & 3 & Number of stages (3 is best in virtually all cases) \\
+\qquad eps\_pol    & float[stages] & [1e-6,10,10] & Accuracy requirement on each stage of the multigrid scheme. $\eps_0 = \eps_{pol,0}$, $\eps_i = \eps_{pol,i} \eps_{pol,0}$  for $i>1$. \\
+\bottomrule
+\end{longtable}
+\subsection{Advection schemes}
+The following parameters control the advection scheme in the code
+\begin{longtable}{lllp{7.5cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+advection & dict & & \\
+\qquad type  & string& arakawa & Discretize Eqs.~\eqref{eq:euler_poisson} using Arakawa's scheme \cite{Einkemmer2014} \\
+\qquad type  & string& centered & Discretize Eqs.~\eqref{eq:euler_conservative} using the centered flux \\
+\qquad type  & string& upwind & Discretize Eqs.~\eqref{eq:euler_conservative} using the upwind flux \\
+\qquad multiplication    & string & pointwise & The multiplications in the scheme
+are done pointwise in nodal space\\
+\qquad multiplication    & string & projection & The multiplications in the scheme
+are done by first interpolating to a higher polynomial grid with $n_{fine} = 2n -1$, and projecting the result back to the coarse grid\\
+\bottomrule
+\end{longtable}
+%\subsection{Forward time and centered space}
+%It is well known that the forward in time, centered in space method for solving
+%hyperbolic systems is unconditionally unstable~\cite{LeVeque}.
+%\subsection{Arakawa scheme and centered flux}
+%Reference~\cite{Liu2000} reports that the centered flux does not have any numerical
+%diffusion while the upwind flux does.
+%They also prove that upwind and centered fluxes do not dissipate energy.
+%From finite differences we know that centered differences for the advection term
+%are unstable (or at least produce a lot of oscillations).
 
 \section{Compilation and useage}
 The program shu\_b.cu compiles with
@@ -123,20 +269,21 @@ Run with
 path/to/feltor/src/shu/shu_b input.json
 \end{verbatim}
 
-\subsection{Input file structure}
+\subsection{Output structure}
 Input file format: json
-
-%%This is a booktabs table
-\begin{longtable}{llll>{\RaggedRight}p{7cm}}
+We can either display the results in real-time to screen using the glfw3 library or
+write the results to a file in netcdf-4 format.
+This is regulated by the output paramters in the input file
+\begin{longtable}{lllp{7cm}}
 \toprule
-\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Example} & \textbf{Default} & \textbf{Description}  \\ \midrule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+output & dict & & \\
+\qquad type  & string& glfw & Use glfw to display results in a window while computing (requires to compile with the glfw3 library) \\
+\qquad type  & string& netcdf & Use netcdf to write results into a file (see next section for information about what is written in there) \\
+\qquad itstp  & integer& 4 & The number of steps between outputs of 2d fields \\
+\qquad maxout  & integer& 500 & The total number of field outputs. The endtime is T=itstp*maxout*dt \\
 \bottomrule
 \end{longtable}
-
-The default value is taken if the value name is not found in the input file. If there is no default and
-the value is not found,
-the program exits with an error message.
-
 \subsection{Structure of output file}
 Output file format: netcdf-4/hdf5
 %
-- 
GitLab


From 2248b638bfe67946b9d8d95c936f989eae84e1f6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 18 Jan 2021 16:04:29 +0100
Subject: [PATCH 435/540] Add Sine initialization to shu

---
 src/lamb_dipole/init.h   |  9 +++++++++
 src/lamb_dipole/shu.tex  | 24 ++++++++++++++++++++++++
 src/lamb_dipole/shu_b.cu | 14 ++++++++++++++
 3 files changed, 47 insertions(+)

diff --git a/src/lamb_dipole/init.h b/src/lamb_dipole/init.h
index b0a202b11..d888ff1c2 100644
--- a/src/lamb_dipole/init.h
+++ b/src/lamb_dipole/init.h
@@ -123,6 +123,15 @@ std::map<std::string, std::function< dg::HVec(
             return omega;
         }
     },
+    {"sine", [](
+        Json::Value& js, enum file::error mode,
+        const dg::CartesianGrid2d& grid)
+        {
+            dg::HVec omega;
+            omega = dg::evaluate ( [](double x, double y) { return 2*sin(x)*sin(y);}, grid);
+            return omega;
+        }
+    },
     {"mms", [](
         Json::Value& js, enum file::error mode,
         const dg::CartesianGrid2d& grid)
diff --git a/src/lamb_dipole/shu.tex b/src/lamb_dipole/shu.tex
index b01a9e7cd..5bd3d82f1 100644
--- a/src/lamb_dipole/shu.tex
+++ b/src/lamb_dipole/shu.tex
@@ -112,6 +112,30 @@ init &  dict &   & Parameters for initialization \\
 \qquad velocity   & float & 1 & The velocity $v$ \\
 \bottomrule
 \end{longtable}
+\subsection{Simple sine function}
+A simple sine function is given by
+\begin{align}
+    \omega(x,y,0) = 2 \sin(x)\sin(y)
+\end{align}
+which has an analytical solution
+\begin{align}
+\omega(x,y,t) = 2 \sin(x)\sin(y)\exp( -(2\nu)^s t)
+\end{align}
+if there is artificial viscosity and is invariant else.
+on the domain $[0,2\pi]\times [0,2\pi]$.
+
+This solution is chosen with the following parameters in the input file
+\begin{longtable}{llll}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+grid & dict &  & Grid parameters \\
+\qquad x & float[2]& [0, 6.283185307179586] & Choose x box boundaries appropriately \\
+\qquad y & float[2]& [0, 6.283185307179586] & Choose y box boundaries appropriately \\
+\qquad bc & string[2] & [DIR, PER] & Choose boundary conditions appropriately \\
+init &  dict &   & Parameters for initialization \\
+\qquad type      & string & sine & No other parameters are necessary \\
+\bottomrule
+\end{longtable}
 
 \subsection{ Double Shear layer}
 Here, we follow~\cite{Liu2000} and test the scheme on a double shear layer problem.
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index ad8d8bf88..36ea77ff4 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -267,6 +267,20 @@ int main( int argc, char* argv[])
         double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
         std::cout << "Analytic error to solution "<<error<<std::endl;
     }
+    if( "sine" == initial)
+    {
+        double nu = 0.;
+        unsigned order = 1;
+        if( "viscosity" == regularization)
+        {
+            nu = file::get( mode, js, "regularization", "nu_perp", 1e-3).asDouble();
+            order = file::get( mode, js, "regularization", "order", 1).asUInt();
+        }
+        dg::DVec sol = dg::evaluate( [time,nu,order](double x, double y) {return 2*sin(x)*sin(y)*exp( -pow(2.*nu,order)*time);}, grid);
+        dg::blas1::axpby( 1., y0, -1., sol);
+        double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
+        std::cout << "Analytic error to solution "<<error<<std::endl;
+    }
     std::cout << "Absolute vorticity error is: "<<dg::blas1::dot( w2d, y0) - vorticity << "\n";
     std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - enstrophy)/enstrophy<<"\n";
     std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<"\n";
-- 
GitLab


From 110618ef55db9ce78c88af09e5543f5681bde864 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 18 Jan 2021 17:54:40 +0100
Subject: [PATCH 436/540] Correct path in feltor_hpc file output

---
 src/feltor/feltor_hpc.cu |   4 +-
 src/lamb_dipole/diag.h   | 111 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+), 2 deletions(-)
 create mode 100644 src/lamb_dipole/diag.h

diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index ca65b1088..3c81ff908 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -293,7 +293,7 @@ int main( int argc, char* argv[])
     }
     /// Set global attributes
     std::map<std::string, std::string> att;
-    att["title"] = "Output file of feltor/src/feltor_hpc.cu";
+    att["title"] = "Output file of feltor/src/feltor/feltor_hpc.cu";
     att["Conventions"] = "CF-1.7";
     ///Get local time and begin file history
     auto ttt = std::time(nullptr);
@@ -304,7 +304,7 @@ int main( int argc, char* argv[])
     oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
     for( int i=0; i<argc; i++) oss << " "<<argv[i];
     att["history"] = oss.str();
-    att["comment"] = "Find more info in feltor/src/feltor.tex";
+    att["comment"] = "Find more info in feltor/src/feltor/feltor.tex";
     att["source"] = "FELTOR";
     att["references"] = "https://github.com/feltor-dev/feltor";
     att["inputfile"] = inputfile;
diff --git a/src/lamb_dipole/diag.h b/src/lamb_dipole/diag.h
new file mode 100644
index 000000000..f48aa8db2
--- /dev/null
+++ b/src/lamb_dipole/diag.h
@@ -0,0 +1,111 @@
+#pragma once
+
+#include "dg/algorithm.h"
+#include "dg/file/file.h"
+#include "shu.cuh"
+
+namespace shu{
+
+struct Variables{
+    shu::Shu<dg::CartesianGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec>& shu;
+    const dg::CartesianGrid2d& grid;
+    const dg::DVec& y0;
+    const double& time;
+    const dg::DVec& weights;
+    double duration;
+    enum file::error mode;
+    Json::Value& js;
+};
+
+struct Record1d{
+    std::string name;
+    std::string long_name;
+    std::function<double( Variables&)> function;
+};
+
+struct Record{
+    std::string name;
+    std::string long_name;
+    std::function<void( dg::DVec&, Variables&)> function;
+};
+
+std::vector<Record> diagnostics2d_list = {
+    {"vorticity_field", "Vorticity in 2d",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.y0, result);
+        }
+    },
+    {"potential", "stream function",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas1::copy(v.shu.potential(), result);
+        }
+    },
+    {"vx", "Velocity in x",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas2::symv( -1., v.shu.dy(), v.shu.potential(), 0., result); //vx
+        }
+    },
+    {"vy", "Velocity in y",
+        []( dg::DVec& result, Variables& v ) {
+             dg::blas2::symv( +1., v.shu.dx(), v.shu.potential(), 0., result); //vy
+        }
+    }
+};
+
+std::vector<Record1d> diagnostics1d_list = {
+    {"vorticity", "Integrated Vorticity",
+        []( Variables& v ) {
+            return dg::blas1::dot(v.y0, v.weights);
+        }
+    },
+    {"enstrophy", "Integrated enstrophy",
+        []( Variables& v ) {
+            return 0.5*dg::blas2::dot( v.y0, v.weights, v.y0);
+        }
+    },
+    {"energy", "Integrated energy",
+        []( Variables& v ) {
+            return 0.5*dg::blas2::dot( v.y0, v.weights, v.shu.potential()) ;
+        }
+    },
+    {"time_per_step", "Computation time per step",
+        []( Variables& v ) {
+            return v.duration;
+        }
+    },
+    {"error", "Relative error to analytical solution (not available for every intitial condition)",
+        []( Variables& v ) {
+            std::string initial = file::get( v.mode, v.js, "init", "type", "lamb").asString();
+            if( "mms" == initial)
+            {
+                double R = file::get( v.mode, v.js, "init", "sigma", 0.1).asDouble();
+                double U = file::get( v.mode, v.js, "init", "velocity", 1).asDouble();
+                shu::MMSVorticity vortex( R, U, v.grid.ly(), v.time);
+                dg::DVec sol = dg::evaluate( vortex, v.grid);
+                dg::blas1::axpby( 1., v.y0, -1., sol);
+                return sqrt( dg::blas2::dot( sol, v.weights, sol)/dg::blas2::dot( v.y0 , v.weights, v.y0));
+            }
+            else if( "sine" == initial)
+            {
+                double nu = 0.;
+                unsigned order = 1;
+                std::string regularization = file::get( v.mode, v.js, "regularization", "type", "moddal").asString();
+                if( "viscosity" == regularization)
+                {
+                    nu = file::get( v.mode, v.js, "regularization", "nu_perp", 1e-3).asDouble();
+                    order = file::get( v.mode, v.js, "regularization", "order", 1).asUInt();
+                }
+                double time = v.time;
+                dg::DVec sol = dg::evaluate( [time,nu,order](double x, double y) {
+                    return 2.*sin(x)*sin(y)*exp( -pow(2.*nu,order)*time); },
+                    v.grid);
+                dg::blas1::axpby( 1., v.y0, -1., sol);
+                return sqrt( dg::blas2::dot( sol, v.weights, sol)/
+                             dg::blas2::dot( v.y0 , v.weights, v.y0));
+            }
+            return 0.;
+        }
+    }
+};
+
+}//namespace shu
-- 
GitLab


From e1d2e09e096a6ce5147cb4ab71508ae7997366dd Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 18 Jan 2021 17:55:47 +0100
Subject: [PATCH 437/540] Restructure file output in shu

---
 src/lamb_dipole/diag.h   |  18 +---
 src/lamb_dipole/shu.cuh  |   2 +
 src/lamb_dipole/shu.tex  |  14 +--
 src/lamb_dipole/shu_b.cu | 206 ++++++++++++++++++++-------------------
 4 files changed, 122 insertions(+), 118 deletions(-)

diff --git a/src/lamb_dipole/diag.h b/src/lamb_dipole/diag.h
index f48aa8db2..cef1f5757 100644
--- a/src/lamb_dipole/diag.h
+++ b/src/lamb_dipole/diag.h
@@ -30,7 +30,7 @@ struct Record{
 };
 
 std::vector<Record> diagnostics2d_list = {
-    {"vorticity_field", "Vorticity in 2d",
+    {"vorticity", "Vorticity in 2d",
         []( dg::DVec& result, Variables& v ) {
              dg::blas1::copy(v.y0, result);
         }
@@ -40,30 +40,20 @@ std::vector<Record> diagnostics2d_list = {
              dg::blas1::copy(v.shu.potential(), result);
         }
     },
-    {"vx", "Velocity in x",
-        []( dg::DVec& result, Variables& v ) {
-             dg::blas2::symv( -1., v.shu.dy(), v.shu.potential(), 0., result); //vx
-        }
-    },
-    {"vy", "Velocity in y",
-        []( dg::DVec& result, Variables& v ) {
-             dg::blas2::symv( +1., v.shu.dx(), v.shu.potential(), 0., result); //vy
-        }
-    }
 };
 
 std::vector<Record1d> diagnostics1d_list = {
-    {"vorticity", "Integrated Vorticity",
+    {"vorticity_1d", "Integrated Vorticity",
         []( Variables& v ) {
             return dg::blas1::dot(v.y0, v.weights);
         }
     },
-    {"enstrophy", "Integrated enstrophy",
+    {"enstrophy_1d", "Integrated enstrophy",
         []( Variables& v ) {
             return 0.5*dg::blas2::dot( v.y0, v.weights, v.y0);
         }
     },
-    {"energy", "Integrated energy",
+    {"energy_1d", "Integrated energy",
         []( Variables& v ) {
             return 0.5*dg::blas2::dot( v.y0, v.weights, v.shu.potential()) ;
         }
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index 08c7bd8d8..d5cedd239 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -68,6 +68,8 @@ struct Shu
     dg::ArakawaX<Geometry, Matrix, Container>& arakawa() {return m_arakawa;}
 
     const Container& potential( ) {return m_psi;}
+    Matrix& dx() {return m_centered[0];}
+    Matrix& dy() {return m_centered[1];}
 
     void operator()(double t, const Container& y, Container& yp);
 
diff --git a/src/lamb_dipole/shu.tex b/src/lamb_dipole/shu.tex
index 5bd3d82f1..d43880104 100644
--- a/src/lamb_dipole/shu.tex
+++ b/src/lamb_dipole/shu.tex
@@ -295,6 +295,7 @@ path/to/feltor/src/shu/shu_b input.json
 
 \subsection{Output structure}
 Input file format: json
+
 We can either display the results in real-time to screen using the glfw3 library or
 write the results to a file in netcdf-4 format.
 This is regulated by the output paramters in the input file
@@ -317,18 +318,19 @@ Output file format: netcdf-4/hdf5
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Dimension} & \textbf{Description}  \\ \midrule
 inputfile  &             text attribute & 1 & verbose input file as a string \\
-energy\_time             & Coord. Var. & 1 (energy\_time) & timesteps at which 1d variables are written \\
 time                     & Coord. Var. & 1 (time) & time at which fields are written \\
 x                        & Coord. Var. & 1 (x) & x-coordinate  \\
 y                        & Coord. Var. & 1 (y) & y-coordinate \\
-vorticity\_field         & Dataset & 3 (time, y, x) & electon density $n$ \\
+vorticity                & Dataset & 3 (time, y, x) & electon density $n$ \\
 potential                & Dataset & 3 (time, y, x) & electric potential $\phi$  \\
-vorticity                & Dataset & 1 (energy\_time) & Vorticity $V$  \\
-enstrophy                & Dataset & 1 (energy\_time) & Enstropy $\Omega$  \\
-energy                   & Dataset & 1 (energy\_time) & Total energy integral computed using $E = \int_D \phi\omega \dA$ \\
-variation                & Dataset & 1 (energy\_time) & Total energy integral computed directly $\int_D (\nabla \phi)^2$  \\
+vorticity\_1d            & Dataset & 1 (time) & Vorticity integral $V$  \\
+enstrophy\_1d            & Dataset & 1 (time) & Enstropy integral $\Omega$  \\
+energy\_1d               & Dataset & 1 (time) & Total energy integral computed using $E = \int_D \phi\omega \dA$ \\
+time\_per\_step          & Dataset & 1 (time) & Average time for one output \\
+error                    & Dataset & 1 (time) & Relative error to analytical solution if available, 0 else \\
 \bottomrule
 \end{longtable}
+The output fields are determined in the file \texttt{feltor/src/lamb\_dipole/diag.h}.
 
 %..................................................................
 \bibliography{../../doc/related_pages/references}
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 36ea77ff4..b3b13c8eb 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -12,6 +12,7 @@
 #include "dg/file/file.h"
 
 #include "init.h"
+#include "diag.h"
 #include "shu.cuh"
 
 
@@ -52,25 +53,20 @@ int main( int argc, char* argv[])
         shu.set_mms_source( sigma, velocity, grid.ly());
     }
 
-
+    double time = 0;
+    shu::Variables var = {shu, grid, y0, time, w2d, 0., mode, js};
     dg::Timer t;
     t.tic();
     shu( 0., y0, y1);
     t.toc();
-    std::cout << "Time for one rhs evaluation: "<<t.diff()<<"s\n";
-    double vorticity = dg::blas1::dot( w2d, y0);
-    double enstrophy = 0.5*dg::blas2::dot( y0, w2d, y0);
-    double energy =    0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
-    dg::DVec varphi( grid.size());
-    shu.variation( shu.potential(), varphi);
-    double variation = 0.5*dg::blas1::dot( varphi, w2d);
-
-    std::cout << "Total energy:     "<<energy<<"\n";
-    std::cout << "Total enstrophy:  "<<enstrophy<<"\n";
-    std::cout << "Total vorticity:  "<<vorticity<<"\n";
-    std::cout << "Total variation:  "<<variation<<"\n";
+    var.duration = t.diff();
+    for( auto record : shu::diagnostics1d_list)
+    {
+        double result = record.function( var);
+        std::cout  << "Diagnostics "<<record.name<<" "<<result<<"\n";
+    }
 
-    double time = 0;
+    /// ////////////// Initialize timestepper ///////////////////////
     std::string stepper = file::get( mode, js, "timestepper", "stepper", "FilteredMultistep").asString();
     std::string regularization = file::get( mode, js, "regularization", "type", "moddal").asString();
     dg::ModalFilter<dg::DMatrix, dg::DVec> filter;
@@ -143,68 +139,110 @@ int main( int argc, char* argv[])
             //step
             t.tic();
             try{
-            if( "Karniadakis" == stepper)
-                for( unsigned i=0; i<itstp; i++)
-                    karniadakis.step( shu, diffusion, time, y0);
-            else if ( "FilteredMultistep" == stepper)
-                for( unsigned i=0; i<itstp; i++)
-                    multistep.step( shu, filter, time, y0);
-            else if ( "Shu-Osher" == stepper)
-                for( unsigned i=0; i<itstp; i++)
-                    shu_osher.step( shu, filter, time, y0, time, y0, dt);
-            }
-            catch( dg::Fail& fail) {
+                for( unsigned j=0; j<itstp; j++)
+                {
+                    if( "Karniadakis" == stepper)
+                        karniadakis.step( shu, diffusion, time, y0);
+                    else if ( "FilteredMultistep" == stepper)
+                        multistep.step( shu, filter, time, y0);
+                    else if ( "Shu-Osher" == stepper)
+                        shu_osher.step( shu, filter, time, y0, time, y0, dt);
+                }
+            } catch( dg::Fail& fail) {
                 std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
                 std::cerr << "Does Simulation respect CFL condition?\n";
                 return -1;
             }
             t.toc();
-            //std::cout << "Timer for one step: "<<t.diff()/N<<"s\n";
+            var.duration = t.diff()/(double)itstp;
         }
         glfwTerminate();
     }
     else
 #endif //WITHOUT_GLFW
     {
+        std::string inputfile = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
         std::string outputfile;
         if( argc == 1 || argc == 2)
             outputfile = "shu.nc";
         else
             outputfile = argv[2];
-        ////////////////////////////set up netcdf/////////////////////////////////////
+        /// //////////////////////set up netcdf/////////////////////////////////////
         file::NC_Error_Handle err;
-        int ncid;
-        err = nc_create( outputfile.c_str(),NC_NETCDF4|NC_CLOBBER, &ncid);
-        std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
-        err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
+        int ncid=-1;
+        try{
+            err = nc_create( outputfile.c_str(),NC_NETCDF4|NC_CLOBBER, &ncid);
+        }catch( std::exception& e)
+        {
+            std::cerr << "ERROR creating file "<<outputfile<<std::endl;
+            std::cerr << e.what()<<std::endl;
+           return -1;
+        }
+        /// Set global attributes
+        std::map<std::string, std::string> att;
+        att["title"] = "Output file of feltor/src/lamb_dipole/shu_b.cu";
+        att["Conventions"] = "CF-1.7";
+        ///Get local time and begin file history
+        auto ttt = std::time(nullptr);
+        auto tm = *std::localtime(&ttt);
+
+        std::ostringstream oss;
+        ///time string  + program-name + args
+        oss << std::put_time(&tm, "%Y-%m-%d %H:%M:%S");
+        for( int i=0; i<argc; i++) oss << " "<<argv[i];
+        att["history"] = oss.str();
+        att["comment"] = "Find more info in feltor/src/lamb_dipole/shu.tex";
+        att["source"] = "FELTOR";
+        att["references"] = "https://github.com/feltor-dev/feltor";
+        att["inputfile"] = inputfile;
+        for( auto pair : att)
+            err = nc_put_att_text( ncid, NC_GLOBAL,
+                pair.first.data(), pair.second.size(), pair.second.data());
+
         int dim_ids[3], tvarID;
-        int EtimeID, EtimevarID;
-        err = file::define_dimensions( ncid, dim_ids, &tvarID, grid);
-        err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
-        //field IDs
-        std::string names3d[2] = {"vorticity_field", "potential"};
-        std::string names1d[4] = {"vorticity", "enstrophy", "energy", "variation"};
-        int dataIDs[2], variableIDs[4];
-        for( unsigned i=0; i<2; i++){
-            err = nc_def_var( ncid, names3d[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);}
-        for( unsigned i=0; i<4; i++){
-            err = nc_def_var( ncid, names1d[i].data(), NC_DOUBLE, 1, &EtimeID, &variableIDs[i]);}
+        std::map<std::string, int> id1d, id3d;
+        err = file::define_dimensions( ncid, dim_ids, &tvarID, grid,
+                {"time", "y", "x"});
+
+        //Create field IDs
+        for( auto& record : shu::diagnostics2d_list)
+        {
+            std::string name = record.name;
+            std::string long_name = record.long_name;
+            id3d[name] = 0;
+            err = nc_def_var( ncid, name.data(), NC_DOUBLE, 3, dim_ids,
+                    &id3d.at(name));
+            err = nc_put_att_text( ncid, id3d.at(name), "long_name", long_name.size(),
+                long_name.data());
+        }
+        for( auto& record : shu::diagnostics1d_list)
+        {
+            std::string name = record.name;
+            std::string long_name = record.long_name;
+            id1d[name] = 0;
+            err = nc_def_var( ncid, name.data(), NC_DOUBLE, 1, &dim_ids[0],
+                &id1d.at(name));
+            err = nc_put_att_text( ncid, id1d.at(name), "long_name", long_name.size(),
+                long_name.data());
+        }
         err = nc_enddef(ncid);
         size_t start[3] = {0, 0, 0};
         size_t count[3] = {1, grid.n()*grid.Ny(), grid.n()*grid.Nx()};
-        size_t Estart[] = {0};
-        size_t Ecount[] = {1};
         ///////////////////////////////////first output/////////////////////////
-        std::vector<dg::HVec> transferH(2);
-        dg::blas1::transfer( y0, transferH[0]);
-        dg::blas1::transfer( shu.potential(), transferH[1]);
-        for( int k=0;k<2; k++)
-            err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH[k].data() );
+        dg::DVec resultD = dg::evaluate( dg::zero, grid);
+        dg::HVec resultH( resultD);
+        for( auto& record : shu::diagnostics2d_list)
+        {
+            record.function( resultD, var);
+            dg::assign( resultD, resultH);
+            file::put_vara_double( ncid, id3d.at(record.name), start[0], grid, resultH);
+        }
+        for( auto& record : shu::diagnostics1d_list)
+        {
+            double result = record.function( var);
+            nc_put_vara_double( ncid, id1d.at(record.name), start, count, &result);
+        }
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-        double output1d[4] = {vorticity, enstrophy, energy, variation};
-        for( int k=0;k<4; k++)
-            err = nc_put_vara_double( ncid, variableIDs[k], Estart, Ecount, &output1d[k] );
-        err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
         ///////////////////////////////////timeloop/////////////////////////
         unsigned step=0;
         try{
@@ -221,30 +259,26 @@ int main( int argc, char* argv[])
                     multistep.step( shu, filter, time, y0);
                 else if ( "Shu-Osher" == stepper)
                     shu_osher.step( shu, filter, time, y0, time, y0, dt);
-
-                output1d[0] = dg::blas1::dot( w2d, y0);
-                output1d[1] = 0.5*dg::blas2::dot( y0, w2d, y0);
-                output1d[2] = 0.5*dg::blas2::dot( y0, w2d, shu.potential()) ;
-                shu.variation(shu.potential(), varphi);
-                output1d[3] = 0.5*dg::blas1::dot( varphi, w2d);
-                Estart[0] += 1;
-                for( int k=0;k<4; k++)
-                    err = nc_put_vara_double( ncid, variableIDs[k], Estart, Ecount, &output1d[k] );
-                err = nc_put_vara_double( ncid, EtimevarID, Estart, Ecount, &time);
-                if( energy>1e6)
-                    throw dg::Error(dg::Message(_ping_)<<"Energy exploded! Exit\n");
             }
             step+=itstp;
+            ti.toc();
+            var.duration = ti.diff() / (double) itstp;
             //output all fields
-            dg::blas1::transfer( y0, transferH[0]);
-            dg::blas1::transfer( shu.potential(), transferH[1]);
             start[0] = i;
-            for( int k=0;k<2; k++)
-                err = nc_put_vara_double( ncid, dataIDs[k], start, count, transferH[k].data() );
+            for( auto& record : shu::diagnostics2d_list)
+            {
+                record.function( resultD, var);
+                dg::assign( resultD, resultH);
+                file::put_vara_double( ncid, id3d.at(record.name), start[0], grid, resultH);
+            }
+            for( auto& record : shu::diagnostics1d_list)
+            {
+                double result = record.function( var);
+                nc_put_vara_double( ncid, id1d.at(record.name), start, count, &result);
+            }
             err = nc_put_vara_double( ncid, tvarID, start, count, &time);
-            ti.toc();
             std::cout << "\n\t Step "<<step <<" of "<<itstp*maxout <<" at time "<<time;
-            std::cout << "\n\t Average time for one step: "<<ti.diff()/(double)itstp<<"s\n\n"<<std::flush;
+            std::cout << "\n\t Average time for one step: "<<var.duration<<"s\n\n"<<std::flush;
         }
         }
         catch( dg::Fail& fail) {
@@ -257,35 +291,11 @@ int main( int argc, char* argv[])
     }
     ////////////////////////////////////////////////////////////////////
     std::cout << "Time "<<time<<std::endl;
-    if( "mms" == initial)
+    for( auto record : shu::diagnostics1d_list)
     {
-        double R = file::get( mode, js, "init", "sigma", 0.1).asDouble();
-        double U = file::get( mode, js, "init", "velocity", 1).asDouble();
-        shu::MMSVorticity vortex( R, U, grid.ly(), time);
-        dg::DVec sol = dg::evaluate( vortex, grid);
-        dg::blas1::axpby( 1., y0, -1., sol);
-        double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
-        std::cout << "Analytic error to solution "<<error<<std::endl;
-    }
-    if( "sine" == initial)
-    {
-        double nu = 0.;
-        unsigned order = 1;
-        if( "viscosity" == regularization)
-        {
-            nu = file::get( mode, js, "regularization", "nu_perp", 1e-3).asDouble();
-            order = file::get( mode, js, "regularization", "order", 1).asUInt();
-        }
-        dg::DVec sol = dg::evaluate( [time,nu,order](double x, double y) {return 2*sin(x)*sin(y)*exp( -pow(2.*nu,order)*time);}, grid);
-        dg::blas1::axpby( 1., y0, -1., sol);
-        double error = dg::blas2::dot( sol, w2d, sol)/dg::blas2::dot( y0 , w2d, y0);
-        std::cout << "Analytic error to solution "<<error<<std::endl;
+        double result = record.function( var);
+        std::cout  << "Diagnostics "<<record.name<<" "<<result<<"\n";
     }
-    std::cout << "Absolute vorticity error is: "<<dg::blas1::dot( w2d, y0) - vorticity << "\n";
-    std::cout << "Relative enstrophy error is: "<<(0.5*dg::blas2::dot( w2d, y0) - enstrophy)/enstrophy<<"\n";
-    std::cout << "Relative energy error    is: "<<(0.5*dg::blas2::dot( shu.potential(), w2d, y0) - energy)/energy<<"\n";
-    shu.variation(shu.potential(), varphi);
-    std::cout << "Relative variation error is: "<<(0.5*dg::blas1::dot( varphi, w2d)-variation)/variation << std::endl;
 
     return 0;
 
-- 
GitLab


From 0450f7158d265656f8594a646bd631c185796eee Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 Jan 2021 13:11:30 +0100
Subject: [PATCH 438/540] Modify filter construction to allow recursive vecs

---
 inc/dg/functors.h                    | 17 +++++++++++++++--
 inc/dg/topology/fast_interpolation.h | 19 ++++++-------------
 inc/dg/topology/filter.h             | 26 +++-----------------------
 inc/dg/topology/filter_t.cu          | 18 ++++++++++++++++--
 src/lamb_dipole/shu_b.cu             |  2 +-
 5 files changed, 41 insertions(+), 41 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 53e071f4c..ddb8c43ff 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -1116,8 +1116,21 @@ struct DPolynomialHeaviside {
  */
 struct ExponentialFilter
 {
-    ExponentialFilter( double alpha, double eta_c, unsigned s, unsigned n):
-        m_alpha(alpha), m_etac(eta_c), m_s(s), m_n(n) {}
+    /**
+     * @brief Create exponential filter \f$ \begin{cases}
+    1 \text{ if } \eta < \eta_c \\
+    \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
+    0 \text{ else} \\
+    \eta := \frac{i}{n-1}
+    \end{cases}\f$
+     *
+     * @param alpha damping for the highest mode is \c exp( -alpha)
+     * @param eta_c cutoff frequency (0<eta_c<1), 0.5 or 0 are good starting values
+     * @param order 8 or 16 are good values
+     * @param n The number of polynomial coefficients
+     */
+    ExponentialFilter( double alpha, double eta_c, unsigned order, unsigned n):
+        m_alpha(alpha), m_etac(eta_c), m_s(order), m_n(n) {}
     double operator()( unsigned i) const
     {
         double eta = (double)i/(double)(m_n-1);
diff --git a/inc/dg/topology/fast_interpolation.h b/inc/dg/topology/fast_interpolation.h
index 7a15e9aec..40f2ca1c9 100644
--- a/inc/dg/topology/fast_interpolation.h
+++ b/inc/dg/topology/fast_interpolation.h
@@ -41,8 +41,8 @@ struct MultiMatrix
     */
     MultiMatrix( int dimension): inter_(dimension), temp_(dimension-1 > 0 ? dimension-1 : 0 ){}
 
-    template<class OtherMatrix, class OtherContainer>
-    MultiMatrix( const MultiMatrix<OtherMatrix, OtherContainer>& src){
+    template<class OtherMatrix, class OtherContainer, class ... Params>
+    MultiMatrix( const MultiMatrix<OtherMatrix, OtherContainer>& src, Params&& ... ps){
         unsigned dimsM = src.get_matrices().size();
         unsigned dimsT = src.get_temp().size();
         inter_.resize( dimsM);
@@ -50,19 +50,12 @@ struct MultiMatrix
         for( unsigned i=0; i<dimsM; i++)
             inter_[i] = src.get_matrices()[i];
         for( unsigned i=0; i<dimsT; i++)
-            dg::assign( src.get_temp()[i].data(), temp_[i].data());
+            dg::assign( src.get_temp()[i].data(), temp_[i].data(), std::forward<Params>(ps)...);
 
     }
-    template<class OtherMatrix, class OtherContainer, class ...Params>
-    void construct( const MultiMatrix<OtherMatrix, OtherContainer>& src, Params&& ...ps){
-        unsigned dimsM = src.get_matrices().size();
-        unsigned dimsT = src.get_temp().size();
-        inter_.resize( dimsM);
-        temp_.resize(  dimsT);
-        for( unsigned i=0; i<dimsM; i++)
-            inter_[i] = src.get_matrices()[i];
-        for( unsigned i=0; i<dimsT; i++)
-            dg::assign( src.get_temp()[i].data(), temp_[i].data(), std::forward<Params>(ps)...);
+    template<class ...Params>
+    void construct( Params&& ...ps){
+        *this = MultiMatrix( std::forward<Params>(ps)...);
     }
 
 
diff --git a/inc/dg/topology/filter.h b/inc/dg/topology/filter.h
index f242c5a52..008d7dfb8 100644
--- a/inc/dg/topology/filter.h
+++ b/inc/dg/topology/filter.h
@@ -132,26 +132,6 @@ struct ModalFilter
 {
     using real_type = get_value_type<ContainerType>;
     ModalFilter(){}
-
-    /**
-     * @brief Create exponential filter \f$ \begin{cases}
-    1 \text{ if } \eta < \eta_c \\
-    \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
-    0 \text{ else} \\
-    \eta := \frac{i}{n-1}
-    \end{cases}\f$
-     *
-     * @tparam Topology Any grid
-     * @param alpha damping for the highest mode is \c exp( -alpha)
-     * @param eta_c cutoff frequency (0<eta_c<1), 0.5 or 0 are good starting values
-     * @param order 8 or 16 are good values
-     * @param t The topology to apply the modal filter on
-     * @sa dg::ExponentialFilter
-     */
-    template<class Topology>
-    ModalFilter( real_type alpha, real_type eta_c, unsigned order, const Topology& t):
-        ModalFilter( dg::ExponentialFilter( alpha, eta_c, order, t.n()), t)
-    { }
     /**
      * @brief Create arbitrary filter
      *
@@ -160,9 +140,9 @@ struct ModalFilter
      * @param f The filter to evaluate on the normalized modal coefficients
      * @param t The topology to apply the modal filter on
      */
-    template<class UnaryOp, class Topology>
-    ModalFilter( UnaryOp sigma, const Topology& t) : m_filter (
-            dg::create::modal_filter( sigma, t)) { }
+    template<class UnaryOp, class Topology, class ...Params>
+    ModalFilter( UnaryOp sigma, const Topology& t, Params&& ...ps) : m_filter (
+            dg::create::modal_filter( sigma, t), std::forward<Params>(ps)...) { }
 
     /**
     * @brief Perfect forward parameters to one of the constructors
diff --git a/inc/dg/topology/filter_t.cu b/inc/dg/topology/filter_t.cu
index 9735a3ed6..f42de67fc 100644
--- a/inc/dg/topology/filter_t.cu
+++ b/inc/dg/topology/filter_t.cu
@@ -25,7 +25,7 @@ int main()
     const dg::DVec vec = dg::evaluate( function, g3);
     const dg::DVec weights = dg::create::weights( g3);
     dg::DVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
-    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, g3);
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( dg::ExponentialFilter(36, 0.5, 8, g3.n()), g3);
     dg::IDMatrix project = dg::create::projection( g2,g3);
     dg::IDMatrix interpo = dg::create::interpolation( g3,g2);
 
@@ -49,7 +49,7 @@ int main()
     const dg::DVec vec = dg::evaluate( function, g3);
     const dg::DVec weights = dg::create::weights( g3);
     dg::DVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
-    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( 36, 0.5, 8, g3);
+    dg::ModalFilter<dg::DMatrix, dg::DVec> filter( dg::ExponentialFilter(36, 0.5, 8, g3.n()), g3);
     dg::IDMatrix project = dg::create::projection( g2,g3);
     dg::IDMatrix interpo = dg::create::interpolation( g3,g2);
 
@@ -64,6 +64,20 @@ int main()
         std::cout << "3D TEST FAILED!\n";
     else
         std::cout << "3D TEST PASSED!\n";
+    //Test recursive filter
+    dg::ModalFilter<dg::DMatrix, std::vector<dg::DVec>> vec_filter( dg::ExponentialFilter(36, 0.5, 8, g3.n()), g3, 3);
+    const std::vector<dg::DVec> vec_vec ( 3, vec);
+    std::vector<dg::DVec> filtered_vec_vec ( vec_vec);
+    vec_filter.apply( vec_vec, filtered_vec_vec);
+    dg::blas1::axpby( 1., filtered_vec, -1., filtered_vec_vec[2]);
+    error = sqrt(dg::blas2::dot( filtered_vec_vec[2], weights, filtered_vec_vec[2])/ dg::blas2::dot( filtered_vec, weights, filtered_vec));
+    std::cout << "Error by filtering: "<<error<<std::endl;
+
+    if( error > 1e-14)
+        std::cout << "Vector TEST FAILED!\n";
+    else
+        std::cout << "Vector TEST PASSED!\n";
     }
+
     return 0;
 }
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index b3b13c8eb..1cd152f10 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -78,7 +78,7 @@ int main( int argc, char* argv[])
         double alpha = file::get( mode, js, "regularization", "alpha", 36).asDouble();
         double order = file::get( mode, js, "regularization", "order", 8).asDouble();
         double eta_c = file::get( mode, js, "regularization", "eta_c", 0.5).asDouble();
-        filter.construct( alpha, eta_c, order, grid);
+        filter.construct( dg::ExponentialFilter(alpha, eta_c, order, grid.n()), grid);
     }
     double dt = file::get( mode, js, "timestepper", "dt", 2e-3).asDouble();
     if( "Karniadakis" == stepper)
-- 
GitLab


From d7341b2623a6204b9899c53fd2ff1068b3734ed4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 19 Jan 2021 18:09:34 +0100
Subject: [PATCH 439/540] Support none as regularization input

---
 src/lamb_dipole/shu.tex  | 14 ++++++++++++--
 src/lamb_dipole/shu_b.cu | 39 +++++++++++++++++++++++++++++++++------
 2 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/src/lamb_dipole/shu.tex b/src/lamb_dipole/shu.tex
index d43880104..878829a0c 100644
--- a/src/lamb_dipole/shu.tex
+++ b/src/lamb_dipole/shu.tex
@@ -203,8 +203,18 @@ timestepper & dict & & \\
 \bottomrule
 \end{longtable}
 \subsection{Regularization technique}
-We support both artificial viscosity and modal filtering choosable by the following
-parameters in the input file
+Choose either no regularization or artificial viscosity or modal filtering by the following
+parameters in the input file.
+
+For no regularization choose
+\begin{longtable}{lllp{7.5cm}}
+\toprule
+\rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
+regularization & dict & & \\
+\qquad type  & string& none & No regularization\\
+\bottomrule
+\end{longtable}
+
 For artificial viscosity Eqs.~\eqref{eq:euler_poisson} are modified to
 \begin{subequations}
 \begin{align}
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 1cd152f10..92f77948b 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -70,6 +70,8 @@ int main( int argc, char* argv[])
     std::string stepper = file::get( mode, js, "timestepper", "stepper", "FilteredMultistep").asString();
     std::string regularization = file::get( mode, js, "regularization", "type", "moddal").asString();
     dg::ModalFilter<dg::DMatrix, dg::DVec> filter;
+    dg::IdentityFilter id;
+    bool apply_filter = true;
     dg::Karniadakis<dg::DVec> karniadakis;
     dg::ShuOsher<dg::DVec> shu_osher;
     dg::FilteredExplicitMultistep<dg::DVec> multistep;
@@ -80,12 +82,14 @@ int main( int argc, char* argv[])
         double eta_c = file::get( mode, js, "regularization", "eta_c", 0.5).asDouble();
         filter.construct( dg::ExponentialFilter(alpha, eta_c, order, grid.n()), grid);
     }
+    if( regularization == "none")
+        apply_filter = false;
     double dt = file::get( mode, js, "timestepper", "dt", 2e-3).asDouble();
     if( "Karniadakis" == stepper)
     {
         if( regularization != "viscosity")
         {
-            throw dg::Error(dg::Message(_ping_)<<"Warning! Karniadakis only works with viscosity regularization! Exit now!");
+            throw dg::Error(dg::Message(_ping_)<<"Error: Karniadakis only works with viscosity regularization! Exit now!");
 
             return -1;
         }
@@ -100,7 +104,10 @@ int main( int argc, char* argv[])
     else if( "FilteredMultistep" == stepper)
     {
         multistep.construct( "eBDF", 3, y0);
-        multistep.init( shu, filter, time, y0, dt);
+        if( apply_filter)
+            multistep.init( shu, filter, time, y0, dt);
+        else
+            multistep.init( shu, id, time, y0, dt);
     }
     else
     {
@@ -144,9 +151,19 @@ int main( int argc, char* argv[])
                     if( "Karniadakis" == stepper)
                         karniadakis.step( shu, diffusion, time, y0);
                     else if ( "FilteredMultistep" == stepper)
-                        multistep.step( shu, filter, time, y0);
+                    {
+                        if( apply_filter)
+                            multistep.step( shu, filter, time, y0);
+                        else
+                            multistep.step( shu, id, time, y0);
+                    }
                     else if ( "Shu-Osher" == stepper)
-                        shu_osher.step( shu, filter, time, y0, time, y0, dt);
+                    {
+                        if( apply_filter)
+                            shu_osher.step( shu, filter, time, y0, time, y0, dt);
+                        else
+                            shu_osher.step( shu, id, time, y0, time, y0, dt);
+                    }
                 }
             } catch( dg::Fail& fail) {
                 std::cerr << "CG failed to converge to "<<fail.epsilon()<<"\n";
@@ -256,9 +273,19 @@ int main( int argc, char* argv[])
                 if( "Karniadakis" == stepper)
                     karniadakis.step( shu, diffusion, time, y0);
                 else if ( "FilteredMultistep" == stepper)
-                    multistep.step( shu, filter, time, y0);
+                {
+                    if( apply_filter)
+                        multistep.step( shu, filter, time, y0);
+                    else
+                        multistep.step( shu, id, time, y0);
+                }
                 else if ( "Shu-Osher" == stepper)
-                    shu_osher.step( shu, filter, time, y0, time, y0, dt);
+                {
+                    if( apply_filter)
+                        shu_osher.step( shu, filter, time, y0, time, y0, dt);
+                    else
+                        shu_osher.step( shu, id, time, y0, time, y0, dt);
+                }
             }
             step+=itstp;
             ti.toc();
-- 
GitLab


From 9ccd2c280f189d533321188acc2f3567675a9089 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 21 Jan 2021 21:13:47 +0100
Subject: [PATCH 440/540] Add static output to shu

in particular xc and yc in order to easily plot
---
 src/lamb_dipole/diag.h   | 13 +++++++++++++
 src/lamb_dipole/shu_b.cu | 20 ++++++++++++++++++--
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/src/lamb_dipole/diag.h b/src/lamb_dipole/diag.h
index cef1f5757..007ec9637 100644
--- a/src/lamb_dipole/diag.h
+++ b/src/lamb_dipole/diag.h
@@ -42,6 +42,19 @@ std::vector<Record> diagnostics2d_list = {
     },
 };
 
+std::vector<Record> diagnostics2d_static_list = {
+    { "xc", "x-coordinate in Cartesian coordinate system",
+        []( dg::DVec& result, Variables& v ) {
+            result = dg::evaluate( dg::cooX2d, v.grid);
+        }
+    },
+    { "yc", "y-coordinate in Cartesian coordinate system",
+        []( dg::DVec& result, Variables& v ) {
+            result = dg::evaluate( dg::cooY2d, v.grid);
+        }
+    }
+};
+
 std::vector<Record1d> diagnostics1d_list = {
     {"vorticity_1d", "Integrated Vorticity",
         []( Variables& v ) {
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 92f77948b..6c09e6897 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -242,12 +242,28 @@ int main( int argc, char* argv[])
             err = nc_put_att_text( ncid, id1d.at(name), "long_name", long_name.size(),
                 long_name.data());
         }
+        // Output static vars
+        dg::DVec resultD = dg::evaluate( dg::zero, grid);
+        dg::HVec resultH( resultD);
+        for( auto& record : shu::diagnostics2d_static_list)
+        {
+            std::string name = record.name;
+            std::string long_name = record.long_name;
+            int staticID = 0;
+            err = nc_def_var( ncid, name.data(), NC_DOUBLE, 2, &dim_ids[1],
+                &staticID);
+            err = nc_put_att_text( ncid, staticID, "long_name", long_name.size(),
+                long_name.data());
+            err = nc_enddef(ncid);
+            record.function( resultD, var);
+            dg::assign( resultD, resultH);
+            file::put_var_double( ncid, staticID, grid, resultH);
+            err = nc_redef(ncid);
+        }
         err = nc_enddef(ncid);
         size_t start[3] = {0, 0, 0};
         size_t count[3] = {1, grid.n()*grid.Ny(), grid.n()*grid.Nx()};
         ///////////////////////////////////first output/////////////////////////
-        dg::DVec resultD = dg::evaluate( dg::zero, grid);
-        dg::HVec resultH( resultD);
         for( auto& record : shu::diagnostics2d_list)
         {
             record.function( resultD, var);
-- 
GitLab


From 952c5a2ed4f4667428108ee187d9ebe46c2dd6fa Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 21 Jan 2021 22:08:25 +0100
Subject: [PATCH 441/540] Add better error checks for output type

---
 src/lamb_dipole/shu_b.cu | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 6c09e6897..f5d182b2e 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -20,7 +20,7 @@ int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
-    enum file::error mode = file::error::is_warning;
+    enum file::error mode = file::error::is_throw;
     if( argc == 1)
         file::file2Json( "input/default.json", js, file::comments::are_discarded);
     else
@@ -175,7 +175,7 @@ int main( int argc, char* argv[])
         }
         glfwTerminate();
     }
-    else
+    else if( "netcdf" == output)
 #endif //WITHOUT_GLFW
     {
         std::string inputfile = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
@@ -332,6 +332,12 @@ int main( int argc, char* argv[])
         }
         err = nc_close(ncid);
     }
+    else
+    {
+        throw dg::Error(dg::Message(_ping_)<<"Error: Wrong value for output type "<<output<<" Must be glfw or netcdf! Exit now!");
+
+        return -1;
+    }
     ////////////////////////////////////////////////////////////////////
     std::cout << "Time "<<time<<std::endl;
     for( auto record : shu::diagnostics1d_list)
-- 
GitLab


From d9107781509d437a6e530c36ed3231fa0176cb33 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 28 Jan 2021 10:11:49 +0100
Subject: [PATCH 442/540] Make shu throw if viscosity and not Karniadakis

---
 src/lamb_dipole/shu_b.cu | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index f5d182b2e..ae2864a32 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -82,8 +82,11 @@ int main( int argc, char* argv[])
         double eta_c = file::get( mode, js, "regularization", "eta_c", 0.5).asDouble();
         filter.construct( dg::ExponentialFilter(alpha, eta_c, order, grid.n()), grid);
     }
-    if( regularization == "none")
+    else
         apply_filter = false;
+    if( regularization == "viscosity" && stepper != "Karniadakis")
+        throw dg::Error(dg::Message(_ping_)<<"Error: Viscosity only works with Karniadakis! Exit now!");
+
     double dt = file::get( mode, js, "timestepper", "dt", 2e-3).asDouble();
     if( "Karniadakis" == stepper)
     {
-- 
GitLab


From 87532bdbf6dc2894785a38eda10e90b32da6b3df Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 28 Jan 2021 11:11:16 +0100
Subject: [PATCH 443/540] Add Tsitouras RK 5(4) methods

The default method for ODE integration in Julia
---
 inc/dg/runge_kutta_t.cu |  5 ++-
 inc/dg/tableau.h        | 80 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+), 1 deletion(-)

diff --git a/inc/dg/runge_kutta_t.cu b/inc/dg/runge_kutta_t.cu
index dfdf1518b..d566f0610 100644
--- a/inc/dg/runge_kutta_t.cu
+++ b/inc/dg/runge_kutta_t.cu
@@ -78,6 +78,8 @@ int main()
         "Cash-Karp-6-4-5",
         "Fehlberg-6-4-5",
         "Dormand-Prince-7-4-5",
+        "Tsitouras09-7-4-5",
+        "Tsitouras11-7-4-5",
         "ARK-8-4-5 (explicit)",
         "Verner-8-5-6",
         "Fehlberg-13-7-8",
@@ -89,7 +91,8 @@ int main()
         std::array<double, 2> u1(u), sol = solution(t_end, damping, omega_0, omega_drive);
         dg::stepperRK(name, functor, t_start, u, t_end, u1, N);
         dg::blas1::axpby( 1., sol , -1., u1);
-        std::cout << "Norm of error in "<<std::setw(24) <<name<<"\t"<<sqrt(dg::blas1::dot( u1, u1))<<"\n";
+        auto b = dg::create::tableau<double>(name);
+        std::cout << "Norm of error in "<<std::setw(24) <<name<<"\t"<<sqrt(dg::blas1::dot( u1, u1))<<(b.isFsal()?" (fsal)" : "") <<"\n";
     }
     std::cout << "Shu-Osher Methods with "<<N<<" steps:\n";
     names = std::vector<std::string> {
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index 3707cd5b3..32643e7a0 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -465,6 +465,76 @@ ButcherTableau<real_type> dormand_prince_7_4_5()
     };
     return ButcherTableau<real_type>(7,data);
 }
+template<class real_type>
+ButcherTableau<real_type> tsitouras09_7_4_5()
+{
+    real_type b[7] = {
+        0.091937670648056,1.156529958312496,-0.781330409541651,
+        0.197624776163019,0.271639883438847,0.063598120979232,0
+    };
+    real_type bt[7] = {
+        0.092167469090589,1.131750860603267,-0.759749304413104,
+        0.205573577541223,0.264767065074229,0.040490332103796,1./40.
+    };
+    real_type c[7] = {
+        0., 0.231572163526079, 0.212252555252816,0.596693497318054,
+        0.797009955708112,1.,1.
+    };
+    real_type a[7*7] = {
+        0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,
+        0,-0.059103796886580,0,0,0,0,0,
+        0,4.560080615554683,-4.006458683473722,0,0,0,0,
+        0,-2.443935658802774,2.631461258707441,0.524706566208284,0,0,0,
+        0,9.516251378071800,-8.467630087008555,-0.987888827522473,0.867009765724064,0,0,
+        0,0,0,0,0,0,0
+    };
+    for( unsigned i=0; i<6; i++)
+    {
+        real_type tmp=0;
+        for( unsigned j=0; j<7; j++)
+            tmp += a[i*7+j];
+        a[i*7] = c[i] - tmp;
+    }
+    for( unsigned j=0; j<7; j++)
+        a[6*7+j] = b[j];
+    return ButcherTableau<real_type>(7,4,5,a,b,bt,c);
+}
+
+template<class real_type>
+ButcherTableau<real_type> tsitouras11_7_4_5()
+{
+    real_type b[7] = {
+        0.09646076681806523,0.01,0.4798896504144996,
+        1.379008574103742,-3.290069515436081,2.324710524099774,0.,
+    };
+    real_type bt[7] = {
+        0.001780011052226,0.000816434459657,-0.007880878010262,0.144711007173263,-0.582357165452555,0.458082105929187,1./66.
+    };
+    real_type c[7] = {
+        0.,0.161,0.327,0.9,0.9800255409045097,1.,1.
+    };
+    real_type a[7*7] = {
+        0,0,0,0,0,0,0,
+        0,0,0,0,0,0,0,
+        0,0.3354806554923570,0,0,0,0,0,
+        0,-6.359448489975075,4.362295432869581,0,0,0,0,
+        0,-11.74888356406283,7.495539342889836,-0.09249506636175525,0,0,0,
+        0,-12.92096931784711,8.159367898576159,-0.07158497328140100,-0.02826905039406838,0,0,
+        0,0,0,0,0,0,0
+    };
+    for( unsigned i=0; i<6; i++)
+    {
+        real_type tmp=0;
+        for( unsigned j=0; j<7; j++)
+            tmp += a[i*7+j];
+        a[i*7] = c[i] - tmp;
+    }
+    for( unsigned j=0; j<7; j++)
+        a[6*7+j] = b[j];
+    return ButcherTableau<real_type>(7,4,5,a,b,bt,c);
+}
+
 template<class real_type>
 ButcherTableau<real_type> ark548l2sa_erk_8_4_5()
 {
@@ -1019,6 +1089,8 @@ enum tableau_identifier{
     CASH_KARP_6_4_5,//!< <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Cash-Karp-6-4-5</a>
     FEHLBERG_6_4_5,//!< <a href="https://en.wikipedia.org/wiki/Runge%E2%80%93Kutta%E2%80%93Fehlberg_method">Fehlberg-6-4-5</a>
     DORMAND_PRINCE_7_4_5,//!< <a href="https://en.wikipedia.org/wiki/Dormand%E2%80%93Prince_method">Dormand-Prince-7-4-5</a>
+    TSITOURAS09_7_4_5,//!< <a href="https://doi.org/10.1063/1.3241561">Tsitouras 5(4) method from 2009</a> (fsal), The default method in Julia
+    TSITOURAS11_7_4_5,//!< <a href="https://doi.org/10.1016/j.camwa.2011.06.002">Tsitouras 5(4) method from 2011</a> (fsal), Further improves Tsitouras09
     ARK548L2SA_ERK_8_4_5,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-4-2-3 (explicit)</a>
     VERNER_8_5_6,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Verner-8-5-6</a>
     FEHLBERG_13_7_8,//!< <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Fehlberg-13-7-8</a>
@@ -1067,6 +1139,8 @@ static std::unordered_map<std::string, enum tableau_identifier> str2id{
     {"Cash-Karp-6-4-5", CASH_KARP_6_4_5},
     {"Fehlberg-6-4-5", FEHLBERG_6_4_5},
     {"Dormand-Prince-7-4-5", DORMAND_PRINCE_7_4_5},
+    {"Tsitouras09-7-4-5", TSITOURAS09_7_4_5},
+    {"Tsitouras11-7-4-5", TSITOURAS11_7_4_5},
     {"ARK-8-4-5 (explicit)", ARK548L2SA_ERK_8_4_5},
     {"Verner-8-5-6", VERNER_8_5_6},
     {"Fehlberg-13-7-8", FEHLBERG_13_7_8},
@@ -1161,6 +1235,10 @@ ButcherTableau<real_type> tableau( enum tableau_identifier id)
             return dg::tableau::fehlberg_6_4_5<real_type>();
         case DORMAND_PRINCE_7_4_5:
             return dg::tableau::dormand_prince_7_4_5<real_type>();
+        case TSITOURAS09_7_4_5:
+            return dg::tableau::tsitouras09_7_4_5<real_type>();
+        case TSITOURAS11_7_4_5:
+            return dg::tableau::tsitouras11_7_4_5<real_type>();
         case ARK548L2SA_ERK_8_4_5:
             return dg::tableau::ark548l2sa_erk_8_4_5<real_type>();
         case VERNER_8_5_6:
@@ -1242,6 +1320,8 @@ ButcherTableau<real_type> tableau( std::string name)
  *   Cash_Karp-6-4-5        | dg::CASH_KARP_6_4_5        | <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Cash-Karp</a>
  *   Fehlberg-6-4-5         | dg::FEHLBERG_6_4_5         | <a href="https://en.wikipedia.org/wiki/Runge%E2%80%93Kutta%E2%80%93Fehlberg_method">Runge-Kutta-Fehlberg</a>
  *   Dormand-Prince-7-4-5   | dg::DORMAND_PRINCE_7_4_5   | <a href="https://en.wikipedia.org/wiki/Dormand%E2%80%93Prince_method">Dormand-Prince method</a> (fsal)
+ *   Tsitouras09-7-4-5   | dg::TSITOURAS09_7_4_5   | <a href="https://doi.org/10.1063/1.3241561">Tsitouras 5(4) method from 2009</a> (fsal), The default method in Julia
+ *   Tsitouras11-7-4-5   | dg::TSITOURAS11_7_4_5   | <a href="https://doi.org/10.1016/j.camwa.2011.06.002">Tsitouras 5(4) method from 2011</a> (fsal) Further improves Tsitouras09
  *   ARK-8-4-5 (explicit)   | dg::ARK548L2SA_ERK_8_4_5   | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">ARK-4-2-3 (explicit)</a>
  *   Verner-8-5-6           | dg::VERNER_8_5_6           | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Verner-8-5-6</a>
  *   Fehlberg-13-7-8        | dg::FEHLBERG_13_7_8        | <a href="http://runge.math.smu.edu/arkode_dev/doc/guide/build/html/Butcher.html">Fehlberg-13-7-8</a>
-- 
GitLab


From 946bdfd29cd0e443288e14b4040b8a7d38c29b8d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Feb 2021 13:33:17 +0100
Subject: [PATCH 444/540] Add some file types to gitignore

---
 .gitignore | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 5eb941ffc..54914461d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,4 +19,9 @@ src/feltor/doc/
 *.dat
 *.eps
 *.swp
-*.txt
\ No newline at end of file
+*.txt
+*.pdf
+*.aux
+*.log
+*.png
+*.pvsm
-- 
GitLab


From 5f3e3d12b645f7db5a2fa24e668988ff127fe3d8 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Feb 2021 13:34:46 +0100
Subject: [PATCH 445/540] Remove whitespaces in dg_introduction.tex

---
 .../dg_introduction/dg_introduction.tex            | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/related_pages/dg_introduction/dg_introduction.tex b/doc/related_pages/dg_introduction/dg_introduction.tex
index 22ce46ca6..bec264fc1 100644
--- a/doc/related_pages/dg_introduction/dg_introduction.tex
+++ b/doc/related_pages/dg_introduction/dg_introduction.tex
@@ -109,8 +109,8 @@ $f_{nj} := f(x^a_{nj})$ and note that
 \end{subequations}
 where $f_{nj}$ are the elements of $\vec f$,
  $\Eins\in\mathbb{R}^{N\times N}$ is the identity matrix and $F,B\in\mathbb{R}^{P\times P}$. Furthermore, we use
-$\otimes$ to denote the Kronecker product which is bilinear and associative. 
-The discontinuous Galerkin expansion $f_h$ of a function $f$ in the interval $[a,b]$ can 
+$\otimes$ to denote the Kronecker product which is bilinear and associative.
+The discontinuous Galerkin expansion $f_h$ of a function $f$ in the interval $[a,b]$ can
 then readily be given as
 \begin{align}
     f_h(x) = \sum_{n=1}^N \sum_{k=0}^{P-1} \bar f^{nk} p_{nk}(x),
@@ -124,7 +124,7 @@ where
     \end{cases}
     \label{}
 \end{align}
-As an example, we plot Eq.~\eqref{eq:dgexpansion} for $f(x)=\sin(2x)$ in Fig.~\ref{fig:discretization}. 
+As an example, we plot Eq.~\eqref{eq:dgexpansion} for $f(x)=\sin(2x)$ in Fig.~\ref{fig:discretization}.
 \begin{figure}[htpb]
     \includegraphics[width= 0.9\textwidth]{discretization.pdf}
     \caption{ 
@@ -133,9 +133,9 @@ As an example, we plot Eq.~\eqref{eq:dgexpansion} for $f(x)=\sin(2x)$ in Fig.~\r
     }
     \label{fig:discretization}
 \end{figure}
-Already with a very low resolution of $P=3$ and $N=5$ 
+Already with a very low resolution of $P=3$ and $N=5$
 we get an acceptable function approximation. We clearly see the discontinuities
-at the cell boundaries. 
+at the cell boundaries.
 
 The use of Legendre polynomials yields a natural approximation of the integrals of $f$
 via Gauss--Legendre quadrature
@@ -153,7 +153,7 @@ With these formulas we have a simple, accurate, and fast
 method to evaluate integrals on the entire domain. This is applied, for example, to compute
 errors in the $L_2$-norm.
 
-We now define some useful quantities that simplify our notation (note that $i,j=0,\dots,P-1$) 
+We now define some useful quantities that simplify our notation (note that $i,j=0,\dots,P-1$)
 \begin{subequations}
     \begin{align}
         S_{ij} &:= \int_{-h/2}^{h/2} p_i\left(\frac{2}{h} x\right)p_j\left(\frac{2}{h} x\right) \dx = \frac{h}{2i+1}\delta_{ij} \\%=: s_i \delta_{ij}\\ 
@@ -679,7 +679,7 @@ or in a time-discreized way
     n_k^{n+1} = (1-\nu \Delta t (k(k+1))^s ) n_k^n
 \end{align}
 which is entirely analogous to the derivation above.
-In the same way we can read this as applying a filter functions to the Legendre modes.
+In the same way we can read this as applying a filter function to the Legendre modes.
 \begin{align}
     n_k^{n+1} = \sigma(k) n_k^n
 \end{align}
-- 
GitLab


From 2c9efd19312cc498c94b393b42ea7a59c8279243 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Feb 2021 13:51:42 +0100
Subject: [PATCH 446/540] ATTENTION: change file namespace to dg::file

this is to make it consistent with the geometries library and the header
name of dg/file/file.h
---
 doc/header.html           | 2 +-
 inc/file/Doxyfile         | 2 +-
 inc/file/easy_output.h    | 3 +++
 inc/file/json_utilities.h | 3 +++
 inc/file/nc_utilities.h   | 3 +++
 inc/file/netcdf_mpit.cpp  | 4 ++--
 inc/file/netcdf_t.cpp     | 4 ++--
 7 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/doc/header.html b/doc/header.html
index 135c86fee..f6c96f72a 100644
--- a/doc/header.html
+++ b/doc/header.html
@@ -26,7 +26,7 @@ $extrastylesheet
       <!-- <li><a href="../../../index.html">/</a><li>-->
     <li><a href="../../dg/html/modules.html">dG</a></li>
     <li><a href="../../geometries/html/modules.html">geometries</a></li>
-    <li><a href="../../file/html/namespacefile.html">file</a></li>
+    <li><a href="../../file/html/namespacedg_1_1file.html">file</a></li>
     <li><a href="../../exblas/html/namespaceexblas.html">exblas</a></li>
   </ul>
   <!--End My own title area-->
diff --git a/inc/file/Doxyfile b/inc/file/Doxyfile
index 7cc781f81..db44d5d2a 100644
--- a/inc/file/Doxyfile
+++ b/inc/file/Doxyfile
@@ -44,7 +44,7 @@ PROJECT_NUMBER         =
 # for a project that appears at the top of each page and should give viewer a
 # quick idea about the purpose of the project. Keep the description short.
 
-PROJECT_BRIEF          = "Facilitate the handling of Json and NetCDF files in \"dg/file/file.h\""
+PROJECT_BRIEF          = "Facilitate the handling of Json and NetCDF files in \"dg/file/file.h\", which consists of \"dg/file/nc_utilities.h\" (-lnetcdf -lhdf5 -lhdf5_hl) and \"dg/file/json_utilities.h\" (-ljsoncpp) "
 
 # With the PROJECT_LOGO tag one can specify a logo or an icon that is included
 # in the documentation. The maximum height of the logo should not exceed 55
diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
index 3205bf8eb..b0d42ed87 100644
--- a/inc/file/easy_output.h
+++ b/inc/file/easy_output.h
@@ -13,6 +13,8 @@
  * Contains Error handling class
  */
 
+namespace dg
+{
 namespace file
 {
 
@@ -399,3 +401,4 @@ void put_vara_double(int ncid, int varid, unsigned slice,
 #endif //MPI_VERSION
 
 }//namespace file
+}//namespace dg
diff --git a/inc/file/json_utilities.h b/inc/file/json_utilities.h
index b5e8389a8..d5b22c514 100644
--- a/inc/file/json_utilities.h
+++ b/inc/file/json_utilities.h
@@ -13,6 +13,8 @@
 
 //Note that the json utilities are separate from netcdf utilities because
 //of the different library dependencies that they incur
+namespace dg
+{
 namespace file
 {
 
@@ -308,3 +310,4 @@ static inline void string2Json(std::string input, Json::Value& js, enum comments
 }
 
 }//namespace file
+}//namespace dg
diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index d9ae4d094..f91b83855 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -20,6 +20,8 @@
  */
 
 
+namespace dg
+{
 /**
 * @brief Namespace for netcdf output related classes and functions following the
  <a href="http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html">CF-conventions</a>
@@ -309,3 +311,4 @@ inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRe
 #endif //MPI_VERSION
 
 } //namespace file
+} //namespace dg
diff --git a/inc/file/netcdf_mpit.cpp b/inc/file/netcdf_mpit.cpp
index 43e9764e7..620675a15 100644
--- a/inc/file/netcdf_mpit.cpp
+++ b/inc/file/netcdf_mpit.cpp
@@ -30,13 +30,13 @@ int main(int argc, char* argv[])
 
     //create NetCDF File
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     MPI_Info info = MPI_INFO_NULL;
     err = nc_create_par( "testmpi.nc", NC_NETCDF4|NC_MPIIO|NC_CLOBBER, MPI_COMM_WORLD, info, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "input", hello.size(), hello.data());
 
     int dimids[4], tvarID;
-    err = file::define_dimensions( ncid, dimids, &tvarID, g);
+    err = dg::file::define_dimensions( ncid, dimids, &tvarID, g);
     int dataID;
     err = nc_def_var( ncid, "data", NC_DOUBLE, 4, dimids, &dataID);
 
diff --git a/inc/file/netcdf_t.cpp b/inc/file/netcdf_t.cpp
index 18eddb000..4327351b8 100644
--- a/inc/file/netcdf_t.cpp
+++ b/inc/file/netcdf_t.cpp
@@ -26,13 +26,13 @@ int main()
     std::string hello = "Hello world\n";
     thrust::host_vector<double> data = dg::evaluate( function, g);
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     err = nc_create( "test.nc", NC_NETCDF4|NC_CLOBBER, &ncid); //for netcdf4
     //err = nc_create( "test.nc", NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "input", hello.size(), hello.data());
 
     int dim_ids[4], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, g);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, g);
 
     int dataID, scalarID, vectorID[3];
     err = nc_def_var( ncid, "data", NC_DOUBLE, 1, dim_ids, &dataID);
-- 
GitLab


From 146b2cdc48e0267a81cadb6036470d642320c43d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Feb 2021 14:29:06 +0100
Subject: [PATCH 447/540] Create Multistep tableaus

add new multistep methods
and make Multistep classes consistent with Runge Kutta classes
- ImExMultistep <-> ARKStep
- ImplicitMultistep <-> DIRKStep
- ExplicitMultistep <-> ERKStep
---
 inc/dg/adaptive.h          |   2 +-
 inc/dg/dg_doc.h            |   3 +
 inc/dg/multistep.h         | 694 +++++++++++++++++--------------------
 inc/dg/multistep_t.cu      | 107 ++----
 inc/dg/multistep_tableau.h | 657 +++++++++++++++++++++++++++++++++++
 inc/dg/tableau.h           |  13 +-
 6 files changed, 1017 insertions(+), 459 deletions(-)
 create mode 100644 inc/dg/multistep_tableau.h

diff --git a/inc/dg/adaptive.h b/inc/dg/adaptive.h
index 11f8cfcd5..62600af8a 100644
--- a/inc/dg/adaptive.h
+++ b/inc/dg/adaptive.h
@@ -6,7 +6,7 @@
 namespace dg
 {
 
-///@addtogroup time
+///@addtogroup time_utils
 ///@{
 /*! @brief Compute \f[ \sqrt{\sum_i x_i^2}\f] using \c dg::blas1::dot
  *
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 77e7eae3b..050bdfb70 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -51,6 +51,9 @@
  *      Algorithms that make use only of blas level 1 and 2 functions
  * @{
  *     @defgroup time Time integrators
+ *     @{
+ *          @defgroup time_utils Utilities for time integration
+ *     @}
  *     @defgroup invert Linear and nonlinear solvers
  *     @defgroup root Root finding
  * @}
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 33b9cb460..6512ee843 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -3,21 +3,22 @@
 #include <map>
 #include "implicit.h"
 #include "runge_kutta.h"
+#include "multistep_tableau.h"
 
+//MW: if ever we want to change the SolverType at runtime (with an input parameter e.g.) make it a new parameter in the solve method (either abstract type or template like RHS)
 
 /*! @file
   @brief contains multistep explicit& implicit time-integrators
   */
 namespace dg{
 
-//MW: if ever we want to change the SolverType at runtime (with an input parameter e.g.) make it a new parameter in the solve method (either abstract type or template like RHS)
 
 /*! @class hide_explicit_implicit
- * @tparam Explicit The explicit part of the right hand side
-        is a functor type with no return value (subroutine)
-        of signature <tt> void operator()(value_type, const ContainerType&, ContainerType&)</tt>
-        The first argument is the time, the second is the input vector, which the functor may \b not override, and the third is the output,
-        i.e. y' = f(t, y) translates to f(t, y, y').
+* @tparam Explicit The explicit part of the right hand side
+    is a functor type with no return value (subroutine)
+    of signature <tt> void operator()(value_type, const ContainerType&, ContainerType&)</tt>
+    The first argument is the time, the second is the input vector, which the functor may \b not override, and the third is the output,
+    i.e. y' = f(t, y) translates to f(t, y, y').
         The two ContainerType arguments never alias each other in calls to the functor.
  * @tparam Implicit The implicit part of the right hand side
         is a functor type with no return value (subroutine)
@@ -35,95 +36,97 @@ namespace dg{
 * @note Uses only \c blas1::axpby routines to integrate one step.
 * @note The difference between a multistep and a single step method like RungeKutta
 * is that the multistep only takes one right-hand-side evaluation per step.
-* This might be advantageous if the right hand side is expensive to evaluate like in
-* partial differential equations. However, it might also happen that the stability
-* region of the one-step method is larger so that a larger timestep can be taken there
-* and on average they take just the same rhs evaluations.
+* This is advantageous if the right hand side is expensive to evaluate.
+* Even though Runge Kutta methods can have a larger absolute timestep, if
+* the effective timestep per rhs evaluation is compared, multistep methods
+* generally win.
 * @note a disadvantage of multistep is that timestep adaption is not easily done.
 */
 
 /**
-* @brief Karniadakis semi-implicit multistep time-integration
-* \f[
-* \begin{align}
-    v^{n+1} = \sum_{q=0}^2 \alpha_q v^{n-q} + \Delta t\left[\left(\sum_{q=0}^2\beta_q  \hat E(t^{n}-q\Delta t, v^{n-q})\right) + \gamma_0\hat I(t^{n}+\Delta t, v^{n+1})\right]
-    \end{align}
-    \f]
-
-    which discretizes
-    \f[
-    \frac{\partial v}{\partial t} = \hat E(t,v) + \hat I(t,v)
-    \f]
+ * @brief Semi-implicit multistep time-integration
+ * \f[
+ * \begin{align}
+     v^{n+1} = \sum_{q=0}^{s-1} a_q v^{n-q} + \Delta t\left[\left(\sum_{q=0}^{s-1}b_q  \hat E(t^{n}-q\Delta t, v^{n-q}) + \sum_{q=1}^{s} c_q \hat I( t^n - q\Delta t, v^{n-q})\right) + c_0\hat I(t^{n}+\Delta t, v^{n+1})\right]
+     \end{align}
+     \f]
+     which discretizes
+     \f[
+     \frac{\partial v}{\partial t} = \hat E(t,v) + \hat I(t,v)
+     \f]
     where \f$ \hat E \f$ contains the explicit and \f$ \hat I \f$ the implicit part of the equations.
-    The coefficients are
-    \f[
-    \alpha_0 = \frac{18}{11}\ \alpha_1 = -\frac{9}{11}\ \alpha_2 = \frac{2}{11} \\
-    \beta_0 = \frac{18}{11}\ \beta_1 = -\frac{18}{11}\ \beta_2 = \frac{6}{11} \\
-    \gamma_0 = \frac{6}{11}
-\f]
-    for the default third order method and
-    \f[
-    \alpha_0 = \frac{4}{3}\ \alpha_1 = -\frac{1}{3}\ \alpha_2 = 0 \\
-    \beta_0  = \frac{4}{3}\ \beta_1 = -\frac{2}{3}\ \beta_2 = 0 \\
-    \gamma_0 = \frac{2}{3}
-\f]
-    for a second order method and
+    As an example, the coefficients for the 3-step, 3rd order "Karniadakis" scheme are
     \f[
-    \alpha_0 = 1\ \alpha_1 = 0\ \alpha_2 = 0\\
-    \beta_0  = 1\ \beta_1 = 0\ \beta_2 = 0\\
-    \gamma_0 = 1
+    a_0 = \frac{18}{11}\ a_1 = -\frac{9}{11}\  a_2 = \frac{2}{11} \\
+    b_0 = \frac{18}{11}\ b_1 = -\frac{18}{11}\ b_2 = \frac{6}{11} \\
+    c_0 = \frac{6}{11}\quad c_1 = c_2 = c_3 = 0 \\
 \f]
-for a semi-implicit (first order) Euler method
-*
-* The necessary Inversion in the imlicit part is provided by the \c SolverType class.
-* Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$).
-* @note This scheme implements <a href = "https://dx.doi.org/10.1016/0021-9991(91)90007-8"> Karniadakis, et al. J. Comput. Phys. 97 (1991)</a>
-* @note The implicit part equals a third order backward differentiation formula (BDF) https://en.wikipedia.org/wiki/Backward_differentiation_formula
-* while the explicit part equals the Minimal Projecting method by
-<a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
-* or **extrapolated BDF** in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S. J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a>
-*
-The following code example demonstrates how to implement the method of manufactured solutions on a 2d partial differential equation with the dg library:
-* @snippet multistep_t.cu function
-* In the main function:
-* @snippet multistep_t.cu karniadakis
-* @note In our experience the implicit treatment of diffusive or hyperdiffusive
+    You can use your own coefficients defined as a \c dg::MultistepTableau
+    or use one of the predefined coefficients in
+    @copydoc hide_imex_multistep_tableaus
+ *
+ * The necessary Inversion in the implicit part is provided by the \c SolverType class.
+ * Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$).
+ *
+ * The following code example demonstrates how to implement the method of manufactured solutions on a 2d partial differential equation with the dg library:
+ * @snippet multistep_t.cu function
+ * In the main function:
+ * @snippet multistep_t.cu karniadakis
+ * @note In our experience the implicit treatment of diffusive or hyperdiffusive
 terms may significantly reduce the required number of time steps. This
 outweighs the increased computational cost of the additional matrix inversions.
-However, each PDE is different and general statements like this one should be treated with care.
+However, each PDE is different and general statements like this one should be
+treated with care.
 * @copydoc hide_note_multistep
 * @copydoc hide_SolverType
 * @copydoc hide_ContainerType
 * @ingroup time
 */
 template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
-struct Karniadakis
+struct ImExMultistep
 {
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
     ///@copydoc RungeKutta::RungeKutta()
-    Karniadakis(){}
+    ImExMultistep(){}
 
-    ///@copydoc construct()
-    template<class ...SolverParams>
-    Karniadakis( SolverParams&& ...ps):m_solver( std::forward<SolverParams>(ps)...){
-        m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
-        set_order(3);
-        m_counter = 0;
-    }
-    /**
-     * @brief Reserve memory for the integration
+    /*! @brief Reserve memory for integration and construct Solver
      *
+     * @param tableau Tableau, name or identifier that \c ConvertsToMultistepTableau
      * @param ps Parameters that are forwarded to the constructor of \c SolverType
      * @tparam SolverParams Type of parameters (deduced by the compiler)
-    */
+     */
     template<class ...SolverParams>
-    void construct( SolverParams&& ...ps){
-        m_solver = SolverType( std::forward<SolverParams>(ps)...);
-        m_f.fill(m_solver.copyable()), m_u.fill(m_solver.copyable());
-        set_order(3);
+    ImExMultistep( ConvertsToMultistepTableau<value_type> tableau,
+            SolverParams&& ...ps):
+        m_t(tableau),
+        m_solver( std::forward<SolverParams>(ps)...)
+    {
+        //only store implicit part if needed
+        unsigned size_f = 0;
+        for( unsigned i=0; i<m_t.steps(); i++ )
+        {
+            if( m_t.im( i+1) != 0 )
+                size_f = i+1;
+        }
+        m_im.assign( size_f, m_solver.copyable());
+
+        m_u.assign( m_t.steps(), m_solver.copyable());
+        m_ex.assign( m_t.steps(), m_solver.copyable());
         m_counter = 0;
     }
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        //construct and swap
+        *this = ImExMultistep( std::forward<Params>( ps)...);
+    }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_u[0];}
@@ -158,102 +161,127 @@ struct Karniadakis
     template< class Explicit, class Implicit>
     void step( Explicit& ex, Implicit& im, value_type& t, ContainerType& u);
 
-    /**
-     * @brief Set the order of the method
-     *
-     * Change the coefficients \f$ \alpha_i,\ \beta_i,\ \gamma_0\f$ to a first,
-     * second or third order set.
-     * @param order 1 (Euler), 2 or 3 (default)
-     */
-    void set_order(unsigned order){
-        switch( order){
-            case 1:
-                a[0] = 1.;  b[0] = 1.;
-                a[1] = 0.;  b[1] = 0.;
-                a[2] = 0.;  b[2] = 0.;   //Euler
-                g0 = 1.;
-                break;
-            case 2:
-                a[0] =  4./3.;  b[0] = 4./3.;
-                a[1] = -1./3.;  b[1] = -2./3.;
-                a[2] = 0.;      b[2] = 0.;   //2nd Karniadakis
-                g0 = 2./3.;
-                break;
-            default:
-                a[0] =  18./11.;    b[0] =  18./11.;
-                a[1] = -9./11.;     b[1] = -18./11.;
-                a[2] = 2./11.;      b[2] = 6./11.;   //Karniadakis
-                g0 = 6./11.;
-                break;
-        }
-    }
   private:
+    dg::MultistepTableau<value_type> m_t;
     SolverType m_solver;
-    std::array<ContainerType,3> m_u, m_f;
-    value_type m_t, m_dt;
-    value_type a[3];
-    value_type b[3], g0 = 6./11.;
+    std::vector<ContainerType> m_u, m_ex, m_im;
+    ContainerType m_tmp;
+    value_type m_tu, m_dt;
     unsigned m_counter; //counts how often step has been called after init
 };
 
+
+/** @brief Deprecated  (use ImExMultistep and select "Karniadakis" from the multistep tableaus)
+* @ingroup time
+* @sa dg::ImExMultistep
+*/
+template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
+struct Karniadakis
+{
+    using value_type = get_value_type<ContainerType>;
+    using container_type = ContainerType;
+    Karniadakis(){}
+    template<class ...SolverParams>
+    Karniadakis( SolverParams&& ...ps): m_imex( "Karniadakis", std::forward<SolverParams> (ps)...) { }
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        *this = Karniadakis( std::forward<Params>( ps)...);
+    }
+    const ContainerType& copyable()const{ return m_imex.copyable();}
+    SolverType& solver() { return m_imex.solver();}
+    const SolverType& solver() const { return m_imex.solver();}
+    template< class Explicit, class Implicit>
+    void init( Explicit& ex, Implicit& im, value_type t0, const ContainerType& u0, value_type dt){
+        m_imex.init( ex, im, t0, u0, dt);
+    }
+    template< class Explicit, class Implicit>
+    void step( Explicit& ex, Implicit& im, value_type& t, ContainerType& u){
+        m_imex.step( ex, im, t, u);
+    }
+  private:
+    ImExMultistep<ContainerType, SolverType> m_imex;
+};
+
 ///@cond
 template< class ContainerType, class SolverType>
 template< class RHS, class Diffusion>
-void Karniadakis<ContainerType, SolverType>::init( RHS& f, Diffusion& diff, value_type t0, const ContainerType& u0, value_type dt)
+void ImExMultistep<ContainerType, SolverType>::init( RHS& f, Diffusion& diff, value_type t0, const ContainerType& u0, value_type dt)
 {
-    m_t = t0, m_dt = dt;
-    blas1::copy(  u0, m_u[2]);
-    f( t0, u0, m_f[2]); //f may not destroy u0
+    m_tu = t0, m_dt = dt;
+    unsigned s = m_t.steps();
+    blas1::copy(  u0, m_u[s-1]);
+    f( t0, u0, m_ex[s-1]); //f may not destroy u0
     m_counter = 0;
+    if( s-1-m_counter < m_im.size())
+        diff( m_tu, m_u[s-1-m_counter], m_im[s-1-m_counter]);
 }
 
 template<class ContainerType, class SolverType>
 template< class RHS, class Diffusion>
-void Karniadakis<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, value_type& t, ContainerType& u)
+void ImExMultistep<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, value_type& t, ContainerType& u)
 {
-    if( m_counter < 2)
+    unsigned s = m_t.steps();
+    if( m_counter < s - 1)
     {
-        ARKStep<ContainerType, SolverType> ark( "ARK-4-2-3", m_solver);
+        std::map<unsigned, std::string> order2method{
+            {1, "ARK-4-2-3"},
+            {2, "ARK-4-2-3"},
+            {3, "ARK-4-2-3"},
+            {4, "ARK-6-3-4"},
+            {5, "ARK-8-4-5"},
+            {6, "ARK-8-4-5"},
+            {7, "ARK-8-4-5"}
+        };
+        ARKStep<ContainerType, SolverType> ark( order2method.at( m_t.order()), m_solver);
         ContainerType tmp ( u);
         ark.step( f, diff, t, u, t, u, m_dt, tmp);
         m_counter++;
-        m_t = t;
-        if( m_counter == 1)
-        {
-            blas1::copy(  u, m_u[1]);
-            f( m_t, m_u[1], m_f[1]);
-        }
-        else
-        {
-            blas1::copy(  u, m_u[0]);
-            f( m_t, m_u[0], m_f[0]);
-        }
-        m_solver = ark.solver();
+        m_tu = t;
+        dg::blas1::copy( u, m_u[s-1-m_counter]);
+        f( m_tu, m_u[s-1-m_counter], m_ex[s-1-m_counter]);
+        //only assign to f if we actually need to store it
+        if( s-1-m_counter < m_im.size())
+            diff( m_tu, m_u[s-1-m_counter], m_im[s-1-m_counter]);
+        m_solver = ark.solver(); // store the state of the solver
         return;
     }
+    //compute right hand side of inversion equation
+    dg::blas1::axpbypgz( m_t.a(0), m_u[0], m_dt*m_t.ex(0), m_ex[0], 0., m_tmp);
+    for (unsigned i = 1; i < s; i++)
+        dg::blas1::axpbypgz( m_t.a(i), m_u[i], m_dt*m_t.ex(i), m_ex[i], 1., m_tmp);
+    for (unsigned i = 0; i < m_im.size(); i++)
+        dg::blas1::axpby( m_dt*m_t.im(i+1), m_im[i], 1., m_tmp);
+    t = m_tu = m_tu + m_dt;
 
-    blas1::axpbypgz( m_dt*b[0], m_f[0], m_dt*b[1], m_f[1], m_dt*b[2], m_f[2]);
-    blas1::axpbypgz( a[0], m_u[0], a[1], m_u[1], a[2], m_u[2]);
-    //permute m_f[2], m_u[2]  to be the new m_f[0], m_u[0]
-    std::rotate( m_f.rbegin(), m_f.rbegin()+1, m_f.rend());
-    std::rotate( m_u.rbegin(), m_u.rbegin()+1, m_u.rend());
-    blas1::axpby( 1., m_f[0], 1., m_u[0]);
-    //compute implicit part
     value_type alpha[2] = {2., -1.};
     //value_type alpha[2] = {1., 0.};
-    blas1::axpby( alpha[0], m_u[1], alpha[1],  m_u[2], u); //extrapolate previous solutions
-    t = m_t = m_t + m_dt;
-    m_solver.solve( -m_dt*g0, diff, t, u, m_u[0]);
+    if( s > 1 ) //everything higher than Euler
+        dg::blas1::axpby( alpha[0], m_u[0], alpha[1],  m_u[1], u);
+    else
+        dg::blas1::copy( m_u[0], u);
+
+    //Rotate 1 to the right (note the reverse iterator here!)
+    std::rotate( m_u.rbegin(), m_u.rbegin() + 1, m_u.rend());
+    std::rotate( m_ex.rbegin(), m_ex.rbegin() + 1, m_ex.rend());
+    if( !m_im.empty())
+        std::rotate( m_im.rbegin(), m_im.rbegin() + 1, m_im.rend());
+    //compute implicit part
+    m_solver.solve( -m_dt*m_t.im(0), diff, t, u, m_tmp);
+
     blas1::copy( u, m_u[0]); //store result
-    f(m_t, m_u[0], m_f[0]); //call f on new point
+    f(m_tu, m_u[0], m_ex[0]); //call f on new point
+    if( 0 < m_im.size())
+        diff( m_tu, m_u[0], m_im[0]); //call diff on new point
+
 }
 ///@endcond
 
 /**
-* @brief Backward differentiation formula implicit multistep time-integration with Limiter/Filter
+* @brief Implicit multistep time-integration with Limiter/Filter
 * \f[
 * \begin{align}
-    \tilde v &= \sum_{q=0}^{s-1} \alpha_q v^{n-q} + \Delta t\beta\hat I(t^{n}+\Delta t, v^{n+1}) \\
+    \tilde v &= \sum_{i=0}^{s-1} a_i v^{n-i} + \Delta t \sum_{i=1}^{s} c_i\hat I(t^{n+1-i}, v^{n+1-i}) + \Delta t c_{0} \hat I (t + \Delta t, \tilde v) \\
     v^{n+1} &= \Lambda\Pi\left(\tilde v\right)
     \end{align}
     \f]
@@ -263,53 +291,73 @@ void Karniadakis<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, valu
     \frac{\partial v}{\partial t} = \hat I(t,v)
     \f]
     where \f$ \hat I \f$ represents the right hand side of the equations.
-    The coefficients for up to order 6 can be found at
-    https://en.wikipedia.org/wiki/Backward_differentiation_formula
+    You can use your own coefficients defined as a \c dg::MultistepTableau
+    or use one of the predefined coefficients in
+    @copydoc hide_implicit_multistep_tableaus
+    and
+    @copydoc hide_imex_multistep_tableaus
 *
-* The necessary Inversion in the imlicit part is provided by the \c SolverType class.
+* The necessary Inversion in the implicit part is provided by the \c SolverType class.
 * Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$). For nonlinear right hand side we recommend the AndersonSolver
 *
 * @note In our experience the implicit treatment of diffusive or hyperdiffusive
 terms can significantly reduce the required number of time steps. This
 outweighs the increased computational cost of the additional inversions.
+However, each PDE is different and general statements like this one should be
+treated with care.
 * @copydoc hide_note_multistep
 * @copydoc hide_SolverType
 * @copydoc hide_ContainerType
 * @ingroup time
+* @attention The filter function inside the Implicit Multistep method is a
+* somewhat experimental feature, so use this class over
+* \c dg::ImplicitMultistep at your own risk
 */
 template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
-struct FilteredBDF
+struct FilteredImplicitMultistep
 {
 
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
     ///@copydoc RungeKutta::RungeKutta()
-    FilteredBDF(){}
-
-    ///@copydoc construct()
-    template<class ...SolverParams>
-    FilteredBDF( unsigned order, SolverParams&& ...ps):m_solver( std::forward<SolverParams>(ps)...), m_u( order, m_solver.copyable()), m_f(m_solver.copyable()) {
-        init_coeffs(order);
-        m_k = order;
-        m_counter = 0;
-    }
+    FilteredImplicitMultistep(){}
 
     /*! @brief Reserve memory for integration and construct Solver
      *
-     * @param order Order of the BDF formula (1 <= order <= 6)
+     * @param tableau Tableau, name or identifier that \c ConvertsToMultistepTableau
      * @param ps Parameters that are forwarded to the constructor of \c SolverType
      * @tparam SolverParams Type of parameters (deduced by the compiler)
      */
     template<class ...SolverParams>
-    void construct(unsigned order, SolverParams&& ...ps)
+    FilteredImplicitMultistep( ConvertsToMultistepTableau<value_type> tableau,
+        SolverParams&& ...ps):
+        m_t( tableau),
+        m_solver( std::forward<SolverParams>(ps)...)
     {
-        m_solver = SolverType( std::forward<SolverParams>(ps)...);
-        init_coeffs(order);
-        m_k = order;
-        m_u.assign( order, m_solver.copyable());
-        m_f = m_solver.copyable();
+        unsigned size_f = 0;
+        for( unsigned i=0; i<m_t.steps(); i++ )
+        {
+            if( m_t.im( i+1) != 0 )
+                size_f = i+1;
+        }
+        m_f.assign( size_f, m_solver.copyable());
+        m_u.assign( m_t.steps(), m_solver.copyable());
+        m_tmp = m_solver.copyable();
         m_counter = 0;
     }
+
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct(Params&& ...ps)
+    {
+        //construct and swap
+        *this = FilteredImplicitMultistep(  std::forward<Params>(ps)...);
+    }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
     const ContainerType& copyable()const{ return m_u[0];}
@@ -326,43 +374,37 @@ struct FilteredBDF
     template<class RHS, class Limiter>
     void step(RHS& rhs, Limiter& limiter, value_type& t, container_type& u);
     private:
-    void init_coeffs(unsigned order){
-        switch (order){
-            case 1: m_bdf = {1.}; m_beta = 1.; break;
-            case 2: m_bdf = {4./3., -1./3.}; m_beta = 2./3.; break;
-            case 3: m_bdf = { 18./11., -9./11., 2./11.}; m_beta = 6./11.; break;
-            case 4: m_bdf = {48./25., -36./25., 16./25., -3./25.}; m_beta = 12./25.; break;
-            case 5: m_bdf = { 300./137., -300./137., 200./137., -75./137., 12./137.}; m_beta = 60/137.; break;
-            case 6: m_bdf = { 360./147., -450./147., 400./147., -225./147., 72./147., -10./147.}; m_beta = 60/147.; break;
-            default: throw dg::Error(dg::Message()<<"Order not implemented in BDF!");
-        }
-    }
+    dg::MultistepTableau<value_type> m_t;
     SolverType m_solver;
     value_type m_tu, m_dt;
     std::vector<ContainerType> m_u;
-    ContainerType m_f;
-    std::vector<value_type> m_bdf;
-    value_type m_beta;
-    unsigned m_k, m_counter = 0; //counts how often step has been called after init
+    std::vector<ContainerType> m_f;
+    ContainerType m_tmp;
+    unsigned m_counter = 0; //counts how often step has been called after init
 };
 
 ///@cond
 template< class ContainerType, class SolverType>
 template<class RHS, class Limiter>
-void FilteredBDF<ContainerType, SolverType>::init(RHS& rhs, Limiter& l, value_type t0,
+void FilteredImplicitMultistep<ContainerType, SolverType>::init(RHS& rhs, Limiter& l, value_type t0,
     const ContainerType& u0, value_type dt)
 {
     m_tu = t0, m_dt = dt;
-    l.apply( u0, m_u[m_k-1]);
-    //rhs(m_tu, m_u[m_k-1], m_f); //call f on new point
+    l.apply( u0, m_u[m_u.size()-1]);
     m_counter = 0;
+    //only assign to f if we actually need to store it
+    unsigned s = m_t.steps();
+    if( s-1-m_counter < m_f.size())
+        rhs( m_tu, m_u[s-1-m_counter], m_f[s-1-m_counter]);
 }
 
 template< class ContainerType, class SolverType>
 template<class RHS, class Limiter>
-void FilteredBDF<ContainerType, SolverType>::step(RHS& rhs, Limiter& l, value_type& t, container_type& u)
+void FilteredImplicitMultistep<ContainerType, SolverType>::step(RHS& rhs, Limiter& l, value_type& t, container_type& u)
 {
-    if( m_counter < m_k-1)
+    unsigned s = m_t.steps();
+        //max( m_u.size(), m_f.size())
+    if( m_counter < s - 1)
     {
         std::map<unsigned, enum tableau_identifier> order2method{
             {1, IMPLICIT_EULER_1_1},
@@ -370,41 +412,55 @@ void FilteredBDF<ContainerType, SolverType>::step(RHS& rhs, Limiter& l, value_ty
             {3, KVAERNO_4_2_3},
             {4, SDIRK_5_3_4},
             {5, KVAERNO_7_4_5},
-            {6, KVAERNO_7_4_5}
+            {6, KVAERNO_7_4_5},
+            {7, KVAERNO_7_4_5}
         };
-        ImplicitRungeKutta<ContainerType, SolverType> dirk( order2method.at(m_k), m_solver);
+        ImplicitRungeKutta<ContainerType, SolverType> dirk(
+                order2method.at(m_t.order()), m_solver);
         dirk.step( rhs, t, u, t, u, m_dt);
         m_counter++;
         m_tu = t;
-        l.apply( u, m_u[m_k-1-m_counter]);
-        dg::blas1::copy(  m_u[m_k-1-m_counter], u);
-        //f( m_tu, m_u[m_k-1-m_counter], m_f);
+        l.apply( u, m_u[s-1-m_counter]);
+        dg::blas1::copy(  m_u[s-1-m_counter], u);
+        //only assign to f if we actually need to store it
+        if( s-1-m_counter < m_f.size())
+            rhs( m_tu, m_u[s-1-m_counter], m_f[s-1-m_counter]);
+        m_solver = dirk.solver(); // store the state of the solver
         return;
     }
     //compute right hand side of inversion equation
-    dg::blas1::axpby( m_bdf[0], m_u[0], 0., m_f);
-    for (unsigned i = 1; i < m_k; i++){
-        dg::blas1::axpby( m_bdf[i], m_u[i], 1., m_f);
-    }
+    dg::blas1::axpby( m_t.a(0), m_u[0], 0., m_tmp);
+    for (unsigned i = 1; i < s; i++)
+        dg::blas1::axpby( m_t.a(i), m_u[i], 1., m_tmp);
+    for (unsigned i = 0; i < m_f.size(); i++)
+        dg::blas1::axpby( m_dt*m_t.im(i+1), m_f[i], 1., m_tmp);
     t = m_tu = m_tu + m_dt;
 
     value_type alpha[2] = {2., -1.};
-    if( m_k > 1 ) //everything higher than Euler
+    //value_type alpha[2] = {1., 0.};
+    if( s > 1 ) //everything higher than Euler
         dg::blas1::axpby( alpha[0], m_u[0], alpha[1],  m_u[1], u);
     else
         dg::blas1::copy( m_u[0], u);
-    std::rotate(m_u.rbegin(), m_u.rbegin() + 1, m_u.rend()); //Rotate 1 to the right (note the reverse iterator here!)
-    m_solver.solve( -m_dt*m_beta, rhs, t, u, m_f);
+
+    //Rotate 1 to the right (note the reverse iterator here!)
+    std::rotate(m_u.rbegin(), m_u.rbegin() + 1, m_u.rend());
+    if( !m_f.empty())
+        std::rotate(m_f.rbegin(), m_f.rbegin() + 1, m_f.rend());
+    m_solver.solve( -m_dt*m_t.im(0), rhs, t, u, m_tmp);
+
     l.apply( u, m_u[0]);
+    if( 0 < m_f.size())
+        rhs( m_tu, m_u[0], m_f[0]);
     dg::blas1::copy(  m_u[0], u);
 }
 ///@endcond
 
 /**
-* @brief Backward differentiation formula implicit multistep time-integration
+* @brief Implicit multistep time-integration
 * \f[
 * \begin{align}
-    v^{n+1} = \sum_{q=0}^{s-1} \alpha_q v^{n-q} + \Delta t\beta\hat I(t^{n}+\Delta t, v^{n+1})
+    v^{n+1} &= \sum_{i=0}^{s-1} a_i v^{n-i} + \Delta t \sum_{i=1}^{s} c_i\hat I(t^{n+1-i}, v^{n+1-i}) + \Delta t c_{0} \hat I (t + \Delta t, v^{n+1}) \\
     \end{align}
     \f]
 
@@ -413,44 +469,55 @@ void FilteredBDF<ContainerType, SolverType>::step(RHS& rhs, Limiter& l, value_ty
     \frac{\partial v}{\partial t} = \hat I(t,v)
     \f]
     where \f$ \hat I \f$ represents the right hand side of the equations.
-    The coefficients for up to order 6 can be found at
-    https://en.wikipedia.org/wiki/Backward_differentiation_formula
+    You can use your own coefficients defined as a \c dg::MultistepTableau
+    or use one of the predefined coefficients in
+    @copydoc hide_implicit_multistep_tableaus
+    and
+    @copydoc hide_imex_multistep_tableaus
 *
-* The necessary Inversion in the imlicit part is provided by the \c SolverType class.
+* The necessary Inversion in the implicit part is provided by the \c SolverType class.
 * Per Default, a conjugate gradient method is used (therefore \f$ \hat I(t,v)\f$ must be linear in \f$ v\f$). For nonlinear right hand side we recommend the AndersonSolver
 *
 * @note In our experience the implicit treatment of diffusive or hyperdiffusive
 terms can significantly reduce the required number of time steps. This
 outweighs the increased computational cost of the additional inversions.
+However, each PDE is different and general statements like this one should be
+treated with care.
 * @copydoc hide_note_multistep
 * @copydoc hide_SolverType
 * @copydoc hide_ContainerType
 * @ingroup time
 */
 template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
-struct BDF
+struct ImplicitMultistep
 {
 
     using value_type = get_value_type<ContainerType>;//!< the value type of the time variable (float or double)
     using container_type = ContainerType; //!< the type of the vector class in use
     ///@copydoc RungeKutta::RungeKutta()
-    BDF(){}
-
-    ///@copydoc construct()
-    template<class ...SolverParams>
-    BDF( unsigned order, SolverParams&& ...ps): m_bdf( order,
-            std::forward<SolverParams>(ps)...) {}
+    ImplicitMultistep(){}
 
     /*! @brief Reserve memory for integration and construct Solver
      *
-     * @param order Order of the BDF formula (1 <= order <= 6)
+     * @param tableau Tableau, name or identifier that \c ConvertsToMultistepTableau
      * @param ps Parameters that are forwarded to the constructor of \c SolverType
      * @tparam SolverParams Type of parameters (deduced by the compiler)
      */
     template<class ...SolverParams>
-    void construct(unsigned order, SolverParams&& ...ps)
+    ImplicitMultistep( ConvertsToMultistepTableau<value_type> tableau, SolverParams&& ...ps): m_bdf( tableau,
+            std::forward<SolverParams>(ps)...) {}
+
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct(Params&& ...ps)
     {
-        m_bdf.construct( order, std::forward<SolverParams>(ps)...);
+        //construct and swap
+        *this = ImplicitMultistep(  std::forward<Params>(ps)...);
     }
     ///@copydoc RungeKutta::copyable()
     const ContainerType& copyable()const{ return m_bdf.copyable();}
@@ -473,61 +540,11 @@ struct BDF
         m_bdf.step( rhs, id, t, u);
     }
     private:
-    FilteredBDF<ContainerType, SolverType> m_bdf;
+    FilteredImplicitMultistep<ContainerType, SolverType> m_bdf;
 };
 
 
 
-/**
- * @brief Identifiers for linear multistep methods
- * @sa ExplicitMultistep
-*  @ingroup time
- */
-enum multistep_identifier
-{
-    /** The family of schemes described in <a href = "https://en.wikipedia.org/wiki/Linear_multistep_method"> Linear multistep methods </a>
-     as **Adams-Bashforth**
-    * \f[ u^{n+1} = u^n + \Delta t\sum_{j=0}^{s-1} b_j f\left(t^n - j \Delta t, u^{n-j}\right) \f]
-    * **Possible stages are 1, 2,..., 5**, the order of the method is the same as its stages
-    @note The Adams-Bashforth schemes implemented here need less storage but may have **a smaller region of absolute stability** than for example an extrapolated BDF method of the same order.
-    */
-    ADAMS_BASHFORTH,
-    /** The family of schemes described in <a href =
-     "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S.
-     J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep
-     methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a>
-     as **extrapolated BDF**  where it is found to be TVB (**total variation
-     bound**), i.e. \f$ || v^n|| \leq M ||v^0||\f$  where the norm signifies
-     the total variation semi-norm. (Note that total variation diminishing
-     (TVD) means M=1, and strong stability preserving (SSP) is the same as TVD, TVB schemes converge to the correct entropy solutions of hyperbolic conservation laws)
-     is the same as the **Minimal Projecting** scheme
-     described in <a href =
-     "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf">
-     Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a>
-     * **Possible stages are 1, 2,..., 7** with the order the same as the number of stages
-
-    */
-    eBDF,
-    /** The family of schemes described in <a href="https://doi.org/10.1016/j.jcp.2005.02.029">S.J. Ruuth and W. Hundsdorfer, High-order linear multistep methods with general monotonicity and boundedness properties, Journal of Computational Physics, Volume 209, Issue 1, 2005 </a> as Total variation Bound.
-     * These schemes have **larger stable step sizes than the eBDF family**, for
-     * example TVB3 has 38% larger stepsize than eBDF3 and TVB4 has 109% larger
-     * stepsize than eBDF4.  **Possible orders are 1,2,...,6**, The CFL
-     * conditions in relation to forward Euler are (1-1: 1, 2-2: 0.5, 3-3: 0.54, 4-4: 0.46, 5-5:
-     * 0.38, 6-6: 0.33), we disregard the remaining schemes since their
-     * CFL conditions are worse
-     */
-    TVB,
-    /** The family of schemes described in <a href="https://doi.org/10.1007/BF02728985">Gottlieb, S. On high order strong stability preserving runge-kutta and multi step time discretizations. J Sci Comput 25, 105–128 (2005)</a> as Strong Stability preserving.
-     * We implement the lowest order schemes for each stage. The CFL conditions
-     * in relation to forward Euler are (1-1: 1, 2-2: 0.5, 3-2: 0.5, 4-2: 0.66, 5-3:
-     * 0.5, 6-3: 0.567).  We disregard the remaining
-     * schemes since their CFL conditions are worse than the TVB scheme of the same
-     * order. These schemes are noteworthy because the coefficients b_i are all positive except for the 2-2 method and **the "4-2" and "6-3" methods allow slightly larger stepsize than eBDF** of same order (2 and 3)
-     */
-    SSP
-};
-
-
 /**
 * @brief General explicit linear multistep time-integration with Limiter / Filter
 * \f[
@@ -547,12 +564,18 @@ enum multistep_identifier
     a_0 = \frac{18}{11}\ a_1 = -\frac{9}{11}\ a_2 = \frac{2}{11} \\
     b_0 = \frac{18}{11}\ b_1 = -\frac{18}{11}\ b_2 = \frac{6}{11}
 \f]
-@sa multistep_identifier
+    You can use your own coefficients defined as a \c dg::MultistepTableau
+    or use one of the predefined coefficients in
+    @copydoc hide_explicit_multistep_tableaus
+
 @note This scheme is the same as ExplicitMultistep with the additional option to use a filter
 *
 * @copydoc hide_note_multistep
 * @copydoc hide_ContainerType
 * @ingroup time
+* @attention The filter function inside the Explicit Multistep method is a
+* somewhat experimental feature, so use this class over
+* \c dg::ExplicitMultistep at your own risk
 */
 template<class ContainerType>
 struct FilteredExplicitMultistep
@@ -571,24 +594,11 @@ struct FilteredExplicitMultistep
      * @param copyable ContainerType of the size that is used in \c step
      * @note it does not matter what values \c copyable contains, but its size is important
      */
-    FilteredExplicitMultistep( std::string method, unsigned stages, const ContainerType& copyable){
-        std::map < std::string, enum multistep_identifier> str2id{
-            {"Adams-Bashforth", ADAMS_BASHFORTH},
-            {"eBDF", eBDF},
-            {"TVB", TVB},
-            {"SSP", SSP}
-        };
-        if( str2id.find(method) == str2id.end())
-            throw dg::Error(dg::Message(_ping_)<<"Multistep coefficients for "<<method<<" not found!");
-        else
-            *this = FilteredExplicitMultistep( str2id[method], stages, copyable);
-    }
-    ///@copydoc FilteredExplicitMultistep(std::string,unsigned,const ContainerType&)
-    FilteredExplicitMultistep( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
-        m_k = stages;
-        m_f.assign( stages, copyable);
-        m_u.assign( stages, copyable);
-        init_coeffs(method, stages);
+    FilteredExplicitMultistep( ConvertsToMultistepTableau<value_type> tableau,
+            const ContainerType& copyable): m_t(tableau)
+    {
+        m_f.assign( m_t.steps(), copyable);
+        m_u.assign( m_t.steps(), copyable);
         m_counter = 0;
     }
     /**
@@ -639,99 +649,10 @@ struct FilteredExplicitMultistep
     void step( RHS& rhs, Limiter& limiter, value_type& t, ContainerType& u);
 
   private:
-    void init_coeffs(enum multistep_identifier method, unsigned stages){
-        m_a.resize( stages);
-        m_b.resize( stages);
-        switch( method){
-            case ADAMS_BASHFORTH:
-            m_a.assign(stages, 0);
-            m_a[0] = 1.;
-            switch (stages){
-                case 1: m_b = {1}; break;
-                case 2: m_b = {1.5, -0.5}; break;
-                case 3: m_b = { 23./12., -4./3., 5./12.}; break;
-                case 4: m_b = {55./24., -59./24., 37./24., -3./8.}; break;
-                case 5: m_b = { 1901./720., -1387./360., 109./30., -637./360., 251./720.}; break;
-                default: throw dg::Error(dg::Message()<<"Order not implemented in AdamsBashforth!");
-            }
-            break;
-            case eBDF:
-            switch (stages){
-                case 1: m_a = {1.};
-                        m_b = {1.}; break;
-                case 2: m_a = {4./3., -1./3.};
-                        m_b = {4./3., -2./3.}; break;
-                case 3: m_a = { 18./11., -9./11., 2./11.};
-                        m_b = { 18./11., -18./11., 6./11.}; break; //CLM = 0.38... (% of Euler step size)
-                case 4: m_a = {48./25., -36./25., 16./25., -3./25.};
-                        m_b = {48./25.,-72./25.,48./25.,-12./25.}; break; //CLM = 0.21...
-                case 5: m_a = { 300./137., -300./137., 200./137., -75./137., 12./137.};
-                        m_b = {300./137.,-600./137.,600./137.,-300./137.,60./137.}; break;
-                case 6: m_a = { 360./147., -450./147., 400./147., -225./147., 72./147., -10./147.};
-                        m_b = {360./147.,-900./147.,1200./147.,-900./147.,360./147.,-60./147.}; break;
-                case 7: m_a = { 2940./1089.,-4410./1089.,4900./1089.,-3675./1089.,1764./1089.,-490./1089.,60./1089.};
-                        m_b = { 2940./1089.,-8820./1089.,14700./1089.,-14700./1089.,8820./1089.,-2940./1089.,420./1089.}; break;
-                default: throw dg::Error(dg::Message()<<"Order not implemented in Minimal Projecting!");
-            }
-            break;
-            case TVB:
-            switch(stages){
-                case 1: m_a = {1.};
-                        m_b = {1.}; break;
-                case 2: m_a = {4./3., -1./3.};
-                        m_b = {4./3., -2./3.}; break; //CLM = 0.5
-                case 3: //CLM = 0.54...
-                    m_a[0] =  1.908535476882378;     m_b[0] =  1.502575553858997;
-                    m_a[1] = -1.334951446162515;     m_b[1] = -1.654746338401493;
-                    m_a[2] = 0.426415969280137;      m_b[2] = 0.670051276940255;
-                    break;
-                case 4: //CLM = 0.45...
-                    m_a[0] = 2.628241000683208;     m_b[0] = 1.618795874276609;
-                    m_a[1] = -2.777506277494861;     m_b[1] = -3.052866947601049;
-                    m_a[2] = 1.494730011212510;     m_b[2] = 2.229909318681302;
-                    m_a[3] = -0.345464734400857;     m_b[3] = -0.620278703629274;
-                    break;
-                case 5: //CLM = 0.37...
-                    m_a[0] = 3.308891758551210;     m_b[0] = 1.747442076919292;
-                    m_a[1] = -4.653490937946655;     m_b[1] = -4.630745565661800;
-                    m_a[2] = 3.571762873789854;     m_b[2] = 5.086056171401077;
-                    m_a[3] = -1.504199914126327;     m_b[3] = -2.691494591660196;
-                    m_a[4] = 0.277036219731918;     m_b[4] = 0.574321855183372;
-                    break;
-                case 6: //CLM = 0.32...
-                    m_a[0] = 4.113382628475685;     m_b[0] = 1.825457674048542;
-                    m_a[1] = -7.345730559324184;     m_b[1] = -6.414174588309508;
-                    m_a[2] = 7.393648314992094;     m_b[2] = 9.591671249204753;
-                    m_a[3] = -4.455158576186636;     m_b[3] = -7.583521888026967;
-                    m_a[4] = 1.523638279938299;     m_b[4] = 3.147082225022105;
-                    m_a[5] = -0.229780087895259;     m_b[5] = -0.544771649561925;
-                    break;
-                default: throw dg::Error(dg::Message()<<"Order not implemented in TVB scheme!");
-            }
-            break;
-            case SSP:
-            switch(stages){
-                case 1: m_a = {1.};
-                        m_b = {1.}; break;
-                case 2: m_a = {4./5., 1./5.};
-                        m_b = {8./5., -2./5.}; break; //CLM = 0.5 ... ,order 2
-                case 3: m_a = { 3./4., 0., 1./4.};
-                        m_b = { 3./2., 0., 0. }; break; //CLM = 0.5..., order 2
-                case 4: m_a = {8./9., 0., 0., 1./9.};
-                        m_b = {4./3., 0., 0., 0.}; break; //CLM = 0.66..., order 2
-                case 5: m_a = {25./32., 0., 0., 0., 7./32.};
-                        m_b = {25./16.,0.,0.,0.,5./16.}; break; //CLM 0.5, order 3
-                case 6: m_a = {108./125.,0.,0.,0.,0.,17./125.};
-                        m_b = {36./25.,0.,0.,0.,0.,6./25.}; break; //CLM 0.567, order 3
-                default: throw dg::Error(dg::Message()<<"Stage not implemented in SSP scheme!");
-            }
-            break;
-        }
-    }
+    dg::MultistepTableau<value_type> m_t;
     std::vector<ContainerType> m_u, m_f;
     value_type m_tu, m_dt;
-    std::vector<value_type> m_a, m_b;
-    unsigned m_k, m_counter; //counts how often step has been called after init
+    unsigned m_counter; //counts how often step has been called after init
 };
 ///@cond
 template< class ContainerType>
@@ -739,8 +660,9 @@ template< class RHS, class Limiter>
 void FilteredExplicitMultistep<ContainerType>::init( RHS& f, Limiter& l, value_type t0, const ContainerType& u0, value_type dt)
 {
     m_tu = t0, m_dt = dt;
-    l.apply( u0, m_u[m_k-1]);
-    f(m_tu, m_u[m_k-1], m_f[m_k-1]); //call f on new point
+    unsigned s = m_t.steps();
+    l.apply( u0, m_u[s-1]);
+    f(m_tu, m_u[s-1], m_f[s-1]); //call f on new point
     m_counter = 0;
 }
 
@@ -748,7 +670,8 @@ template<class ContainerType>
 template<class RHS, class Limiter>
 void FilteredExplicitMultistep<ContainerType>::step(RHS& f, Limiter& l, value_type& t, ContainerType& u)
 {
-    if( m_counter < m_k-1)
+    unsigned s = m_t.steps();
+    if( m_counter < s-1)
     {
         std::map<unsigned, enum tableau_identifier> order2method{
             {1, SSPRK_2_2},
@@ -756,23 +679,24 @@ void FilteredExplicitMultistep<ContainerType>::step(RHS& f, Limiter& l, value_ty
             {3, SSPRK_3_3},
             {4, SSPRK_5_4},
             {5, SSPRK_5_4},
-            {6, SSPRK_5_4}
+            {6, SSPRK_5_4},
+            {7, SSPRK_5_4}
         };
-        ShuOsher<ContainerType> rk( order2method.at(m_k), u);
+        ShuOsher<ContainerType> rk( order2method.at(m_t.order()), u);
         rk.step( f, l, t, u, t, u, m_dt);
         m_counter++;
         m_tu = t;
-        blas1::copy(  u, m_u[m_k-1-m_counter]);
-        f( m_tu, m_u[m_k-1-m_counter], m_f[m_k-1-m_counter]);
+        blas1::copy(  u, m_u[s-1-m_counter]);
+        f( m_tu, m_u[s-1-m_counter], m_f[s-1-m_counter]);
         return;
     }
     //compute new t,u
     t = m_tu = m_tu + m_dt;
-    dg::blas1::axpby( m_a[0], m_u[0], m_dt*m_b[0], m_f[0], u);
-    for (unsigned i = 1; i < m_k; i++){
-        dg::blas1::axpbypgz( m_a[i], m_u[i], m_dt*m_b[i], m_f[i], 1., u);
+    dg::blas1::axpby( m_t.a(0), m_u[0], m_dt*m_t.ex(0), m_f[0], u);
+    for (unsigned i = 1; i < s; i++){
+        dg::blas1::axpbypgz( m_t.a(i), m_u[i], m_dt*m_t.ex(i), m_f[i], 1., u);
     }
-    //permute m_f[m_k-1], m_u[m_k-1]  to be the new m_f[0], m_u[0]
+    //permute m_f[s-1], m_u[s-1]  to be the new m_f[0], m_u[0]
     std::rotate( m_f.rbegin(), m_f.rbegin()+1, m_f.rend());
     std::rotate( m_u.rbegin(), m_u.rbegin()+1, m_u.rend());
     //apply limiter
@@ -800,7 +724,9 @@ void FilteredExplicitMultistep<ContainerType>::step(RHS& f, Limiter& l, value_ty
     a_0 = \frac{18}{11}\ a_1 = -\frac{9}{11}\ a_2 = \frac{2}{11} \\
     b_0 = \frac{18}{11}\ b_1 = -\frac{18}{11}\ b_2 = \frac{6}{11}
 \f]
-@sa multistep_identifier
+    You can use your own coefficients defined as a \c dg::MultistepTableau
+    or use one of the predefined coefficients in
+    @copydoc hide_explicit_multistep_tableaus
 *
 * @copydoc hide_note_multistep
 * @copydoc hide_ContainerType
@@ -813,13 +739,19 @@ struct ExplicitMultistep
     using container_type = ContainerType; //!< the type of the vector class in use
     ///@copydoc RungeKutta::RungeKutta()
     ExplicitMultistep(){}
-    ///@copydoc FilteredExplicitMultistep(std::string,unsigned,const ContainerType&)
-    ExplicitMultistep( std::string method, unsigned stages, const ContainerType& copyable): m_fem( method, stages, copyable){ }
-    ///@copydoc FilteredExplicitMultistep(std::string,unsigned,const ContainerType&)
-    ExplicitMultistep( enum multistep_identifier method, unsigned stages, const ContainerType& copyable): m_fem( method, stages, copyable){}
-    ///@copydoc FilteredExplicitMultistep(std::string,unsigned,const ContainerType&)
-    void construct( enum multistep_identifier method, unsigned stages, const ContainerType& copyable){
-        m_fem.construct( method, stages, copyable);
+    ///@copydoc FilteredExplicitMultistep::FilteredExplicitMultistep(ConvertsToMultistepTableau<value_type>,const ContainerType&)
+    ExplicitMultistep( ConvertsToMultistepTableau<value_type> tableau, const ContainerType& copyable): m_fem( tableau, copyable){ }
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct(Params&& ...ps)
+    {
+        //construct and swap
+        *this = ExplicitMultistep(  std::forward<Params>(ps)...);
     }
     ///@brief Return an object of same size as the object used for construction
     ///@return A copyable object; what it contains is undefined, its size is important
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index f35851401..bab30129a 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -7,13 +7,12 @@
 #include "adaptive.h"
 #include "elliptic.h"
 
-//![function]
 //method of manufactured solution
 std::array<double,2> solution( double t, double nu) {
     return {exp( -nu*t) + cos(t), exp( -nu*t) + sin(t)};
 }
 
-//the explicit part contains the source Tp = S(x,y,t)
+//![function]
 struct Explicit
 {
     Explicit( double nu): m_nu( nu) {}
@@ -39,6 +38,7 @@ struct Implicit
   private:
     double m_nu;
 };
+// solve y + alpha I(t,y)  = rhs
 struct ImplicitSolver
 {
     ImplicitSolver( double nu): m_nu(nu) { }
@@ -52,6 +52,8 @@ struct ImplicitSolver
   private:
     double m_nu;
 };
+
+//![function]
 struct FullSolver
 {
     FullSolver( double nu): m_nu(nu) { }
@@ -66,7 +68,6 @@ struct FullSolver
     double m_nu;
 };
 
-//![function]
 
 struct Full
 {
@@ -107,96 +108,62 @@ int main()
     std::array<double,2> error( sol);
     exblas::udouble res;
     std::cout << "### Test Explicit Multistep methods with "<<NT<<" steps\n";
-    for( unsigned s=1; s<6; s++)
-    {
-        time = 0., y0 = init;
-        dg::ExplicitMultistep< std::array<double,2> > ab( "Adams-Bashforth", s, y0);
-        ab.init( full, time, y0, dt);
-        //main time loop
-        for( unsigned k=0; k<NT; k++)
-            ab.step( full, time, y0);
-        dg::blas1::axpby( -1., sol, 1., y0);
-        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
-        std::cout << "Relative error AB "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
-    }
-    for( unsigned s=1; s<7; s++)
-    {
-        time = 0., y0 = init;
-        dg::ExplicitMultistep< std::array<double,2> > ab( "eBDF", s, y0);
-        ab.init( full, time, y0, dt);
-        //main time loop
-        for( unsigned k=0; k<NT; k++)
-            ab.step( full, time, y0);
-        dg::blas1::axpby( -1., sol, 1., y0);
-        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
-        std::cout << "Relative error eBDF "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
-    }
-    for( unsigned s=1; s<6; s++)
-    {
-        time = 0., y0 = init;
-        dg::ExplicitMultistep< std::array<double,2> > ab( "TVB", s, y0);
-        ab.init( full, time, y0, dt);
-        //main time loop
-        for( unsigned k=0; k<NT; k++)
-            ab.step( full, time, y0);
-        dg::blas1::axpby( -1., sol, 1., y0);
-        res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
-        std::cout << "Relative error TVB  "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
-    }
-    for( unsigned s=1; s<6; s++)
+    std::vector<std::string> ex_names{
+    "AB-1-1", "AB-2-2", "AB-3-3", "AB-4-4", "AB-5-5",
+    "eBDF-1-1", "eBDF-2-2", "eBDF-3-3", "eBDF-4-4", "eBDF-5-5", "eBDF-6-6", "eBDF-7-7",
+    "TVB-1-1", "TVB-2-2", "TVB-3-3", "TVB-4-4", "TVB-5-5", "TVB-6-6",
+    "SSP-1-1", "SSP-2-2", "SSP-3-2", "SSP-4-2", "SSP-5-3", "SSP-6-3",
+    };
+    for( auto name : ex_names)
     {
         time = 0., y0 = init;
-        dg::ExplicitMultistep< std::array<double,2> > ab( "SSP", s, y0);
+        dg::ExplicitMultistep< std::array<double,2> > ab( name, y0);
         ab.init( full, time, y0, dt);
         //main time loop
         for( unsigned k=0; k<NT; k++)
             ab.step( full, time, y0);
         dg::blas1::axpby( -1., sol, 1., y0);
         res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
-        std::cout << "Relative error SSP  "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
+        std::cout << "Relative error: "<<std::setw(20) <<name<<"\t"<< res.d<<"\t"<<res.i<<std::endl;
     }
     std::cout << "### Test implicit multistep methods with "<<NT<<" steps\n";
-    for( unsigned s=1; s<7; s++)
+    std::vector<std::string> imex_names{
+    "Euler","ImEx-Adams-2-2", "ImEx-Adams-3-3", "ImEx-BDF-2-2",
+    "ImEx-BDF-3-3", "ImEx-BDF-4-4", "ImEx-BDF-5-5", "ImEx-BDF-6-6", "ImEx-BDF-7-7",
+    "ImEx-TVB-3-3", "ImEx-TVB-4-4", "ImEx-TVB-5-5",
+    };
+    for( auto name : imex_names)
     {
         time = 0., y0 = init;
-        dg::BDF< std::array<double,2>, FullSolver > bdf( s, nu);
+        dg::ImplicitMultistep< std::array<double,2>, FullSolver > bdf( name, nu);
         bdf.init( full, time, y0, dt);
         //main time loop
         for( unsigned k=0; k<NT; k++)
             bdf.step( full, time, y0);
         dg::blas1::axpby( -1., sol, 1., y0);
         res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
-        std::cout << "Relative error BDF "<<s<<"        is "<< res.d<<"\t"<<res.i<<std::endl;
+        std::cout << "Relative error: "<<std::setw(20) <<name<<"\t"<< res.d<<"\t"<<res.i<<std::endl;
     }
     Explicit ex( nu);
     Implicit im( nu);
-    std::cout << "### Test semi-implicit Karniadakis methods with "<<NT<<" steps\n";
-    //![karniadakis]
-    //construct time stepper
-    dg::Karniadakis< std::array<double,2>, ImplicitSolver > karniadakis( nu);
-    time = 0., y0 = init; //y0 and init are of type std::array<double,2> and contain the initial condition
-    //initialize the timestepper (ex and im are objects of type Explicit and Implicit defined above)
-    karniadakis.init( ex, im, time, y0, dt);
-    //main time loop (NT = 20)
-    for( unsigned i=0; i<NT; i++)
-        karniadakis.step( ex, im, time, y0); //inplace step
-    //![karniadakis]
-    dg::blas1::axpby( -1., sol, 1., y0);
-    res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
-    std::cout << "Relative error Karniadakis 3 is "<< res.d<<"\t"<<res.i<<std::endl;
-    for( unsigned s=2; s>0; s--)
+    // Test Semi-Implicit methods
+    for( auto name : imex_names)
     {
-        time = 0., y0 = init;
-        karniadakis.set_order(s);
-        karniadakis.init( ex, im, time, y0, dt);
-        for( unsigned i=0; i<NT; i++)
-            karniadakis.step( ex, im, time, y0);
+        //![karniadakis]
+        //construct time stepper
+        dg::ImExMultistep< std::array<double,2>, ImplicitSolver > imex( name, nu);
+        time = 0., y0 = init; //y0 and init are of type std::array<double,2> and contain the initial condition
+        //initialize the timestepper (ex and im are objects of type Explicit and Implicit defined above)
+        imex.init( ex, im, time, y0, dt);
+        //main time loop (NT = 20)
+        for( unsigned k=0; k<NT; k++)
+            imex.step( ex, im, time, y0); //inplace step
+        //![karniadakis]
         dg::blas1::axpby( -1., sol, 1., y0);
         res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
-        std::cout << "Relative error Karniadakis "<<s<<" is "<< res.d<<"\t"<<res.i<<std::endl;
+        std::cout << "Relative error: "<<std::setw(20) <<name<<"\t"<< res.d<<"\t"<<res.i<<std::endl;
     }
 
-
     std::cout << "### Test semi-implicit ARK methods\n";
     std::vector<std::string> names{"ARK-4-2-3", "ARK-6-3-4", "ARK-8-4-5"};
     double rtol = 1e-7, atol = 1e-10;
@@ -223,7 +190,7 @@ int main()
     }
     std::cout << "### Test Strang operator splitting\n";
 
-    std::vector<std::string> ex_names{
+    std::vector<std::string> rk_names{
         "Heun-Euler-2-1-2",
         "Bogacki-Shampine-4-2-3",
         "ARK-4-2-3 (explicit)",
@@ -235,7 +202,7 @@ int main()
         "Dormand-Prince-7-4-5",
         "ARK-8-4-5 (explicit)"
     };
-    for( auto name : ex_names)
+    for( auto name : rk_names)
     {
         time = 0., y0 = init;
         dg::Adaptive<dg::ERKStep<std::array<double,2>>> adapt( name, y0);
@@ -258,7 +225,7 @@ int main()
         dg::blas1::axpby( -1., sol, 1., y0);
         res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << std::setw(4)<<counter <<" steps! ";
-        std::cout << "Relative error "<<std::setw(24) <<name<<"\t"<<res.d<<"\n";
+        std::cout << "Relative error: "<<std::setw(24) <<name<<"\t"<<res.d<<"\n";
     }
     return 0;
 }
diff --git a/inc/dg/multistep_tableau.h b/inc/dg/multistep_tableau.h
new file mode 100644
index 000000000..f07c36339
--- /dev/null
+++ b/inc/dg/multistep_tableau.h
@@ -0,0 +1,657 @@
+#pragma once
+
+#include <vector>
+#include <string>
+#include <unordered_map>
+
+namespace dg{
+/**
+ * @brief Manage coefficients of Multistep methods
+ *
+ * The general s-step multistep method has the form
+ * \f[ y^{n+1} = \sum_{i=0}^{s-1} a_i y^{n-i} + h \sum_{i=0}^{s-1} b_i E( t_{n-i}, y_{n-i}) + h \sum_{i=0}^s c_i I( t_{n+1-i}, y^{n+1-i})
+ * \f]
+ * where E is the explicit and I is implicit part. A purely implicit method is
+ * one where all \f$ b_i\f$ are zero, while an explicit one is one where all
+ * \f$ c_i\f$ are zero.
+ * A tableau thus consists of the three arrays a, b and c the number of steps
+ * and the order of the method.
+ * Currently available methods are:
+ *
+ * ImEx methods
+ * @copydoc hide_imex_multistep_tableaus
+ * @note ImEx multistep tableaus can be used in ExplicitMultistep, ImplicitMultistep and ImExMultistep
+ *
+ * Explicit methods
+ * @copydoc hide_explicit_multistep_tableaus
+ *
+ * @tparam real_type type of the coefficients
+ * @sa ExplicitMultistep, ImplicitMultistep, ImExMultistep
+ * @ingroup time_utils
+ */
+template<class real_type>
+struct MultistepTableau
+{
+    using value_type = real_type;
+    ///No memory allocation
+    MultistepTableau(){}
+    /*! @brief Construct a tableau
+     *
+     * @param steps number of stages
+     * @param order (global) order of the resulting method
+     * @param a_v s real numbers
+     * @param b_v s real numbers (can be empty, which then sets them all to 0 constructing an implicit method)
+     * @param c_v s+1 real numbers (can be empty, which constructs an explicit method by assigning all c_i 0)
+     */
+    MultistepTableau( unsigned steps, unsigned order, const
+            std::vector<real_type>& a_v, const std::vector<real_type>& b_v,
+            const std::vector<real_type>& c_v): m_steps(steps), m_order(order),
+            m_a(a_v), m_b(b_v), m_c(c_v){
+                if( m_c.empty())
+                    m_c.assign( steps+1, 0);
+                if( m_b.empty())
+                    m_b.assign( steps, 0);
+    }
+
+    /**
+    * @brief Read the a_i coefficients
+    * @param i idx number 0<=i<s, i>=s results in undefined behaviour
+    * @return a_i
+    */
+    real_type a( unsigned i){ return m_a[i];}
+    /**
+    * @brief Read the explicit (b_i) coefficients
+    * @param i idx number 0<=i<s, i>=s results in undefined behaviour
+    * @return b_i
+    */
+    real_type ex( unsigned i){ return m_b[i];}
+    /**
+    * @brief Read the implicit (c_i) coefficients
+    * @param i idx number 0<=i<s+1, i>=s+1 results in undefined behaviour
+    * @return c_i
+    */
+    real_type im( unsigned i){ return m_c[i];}
+    ///The number of stages s
+    unsigned steps() const  {
+        return m_steps;
+    }
+    ///global order of accuracy for the method
+    unsigned order() const {
+        return m_order;
+    }
+    ///True if any of the explicit coefficients b_i are non-zero
+    bool isExplicit() const{
+        for( unsigned i=0; i<m_steps; i++)
+            if( m_b[i]!=0)
+                return true;
+        return false;
+    }
+    ///True if any of the implicit coefficients c_i are non-zero
+    bool isImplicit() const{
+        for( unsigned i=0; i<m_steps+1; i++)
+            if( m_c[i]!=0)
+                return true;
+        return false;
+    }
+    private:
+    unsigned m_steps, m_order;
+    std::vector<real_type> m_a, m_b, m_c;
+};
+
+///@cond
+namespace tableau
+{
+template<class real_type>
+MultistepTableau<real_type> imex_euler_1_1()
+{
+    unsigned steps = 1, order = 1;
+    std::vector<real_type> a(steps,0), b(steps, 0), c(steps+1,0);
+    a[0] = b[0] = c[0] = 1;
+    return MultistepTableau<real_type>( steps, order, a, b, c);
+}
+template<class real_type>
+MultistepTableau<real_type> imex_adams_2_2()
+{
+    // 2nd order AB method extended to  ImEx
+    // C = 4/9 ~ 0.444  D = 0.33
+    unsigned steps = 2, order = 2;
+    std::vector<real_type> a(steps,0), b(steps, 0), c(steps+1,0);
+    a[0] = 1.;
+    b[0] =  3./2.;
+    b[1] = -1./2.;
+    c[0] = 9./16.;
+    c[1] = -3./8.;
+    c[2] = 1./16.;
+    return MultistepTableau<real_type>( steps, order, a, b, c);
+}
+template<class real_type>
+MultistepTableau<real_type> imex_adams_3_3()
+{
+    // 3rd order AB method extended to ImEx
+    // C ~ 0.16, D = 0.67
+    unsigned steps = 3, order = 3;
+    std::vector<real_type> a(steps,0), b(steps, 0), c(steps+1,0);
+    a[0] = 1.;
+    b[0] =  23./12.;
+    b[1] = -4./3.;
+    b[2] = 5./12.;
+    c[0] =   4661./10000.;
+    c[1] = -15551./30000.;
+    c[2] =     1949/30000;
+    c[3] = -1483./30000.;
+    return MultistepTableau<real_type>( steps, order, a, b, c);
+}
+template<class real_type>
+MultistepTableau<real_type> imex_bdf(unsigned steps)
+{
+    unsigned order = steps;
+    std::vector<real_type> a(steps,0), b(steps, 0), c(steps+1,0);
+    switch( steps)
+    {
+        case( 2):
+        // C = 5/8 ~ 0.625  D = 0
+        a[0] =  4./3.;  b[0] = 4./3.;
+        a[1] = -1./3.;  b[1] = -2./3.;
+        c[0] = 2./3.;
+        break;
+        case(3):
+        //The Karniadakis method
+        // C = 7/18 ~ 0.39  D = 0
+        a[0] =  18./11.;    b[0] =  18./11.;
+        a[1] = -9./11.;     b[1] = -18./11.;
+        a[2] = 2./11.;      b[2] = 6./11.;
+        c[0] = 6./11.;
+        break;
+        case(4):
+        // C = 7/32 ~ 0.22 , D = 0
+        a[0] =  48./25.;    b[0] =  48./25.;
+        a[1] = -36./25.;    b[1] = -72./25.;
+        a[2] =  16./25.;    b[2] =  48./25.;
+        a[3] = - 3./25.;    b[3] = -12./25.;
+        c[0] = 12./25.;
+        break;
+        case(5):
+        // C = 0.0867 , D = 0
+        a[0] = 300./137.;    b[0] = 300./137.;
+        a[1] = -300./137.;   b[1] = -600./137.;
+        a[2] = 200./137.;    b[2] = 600./137.;
+        a[3] = -75./137.;    b[3] = -300./137.;
+        a[4] = 12./137.;     b[4] = 60./137.;
+        c[0] = 60./137.;
+        break;
+        case (6):
+        a = {360./147.,-450./147.,400./147.,-225./147.,72./147.,-10./147.};
+        b = {360./147.,-900./147.,1200./147.,-900./147.,360./147.,-60./147.};
+        c[0] = 60./147.;
+        break;
+        case (7):
+        a = { 2940./1089.,-4410./1089.,4900./1089.,-3675./1089.,1764./1089.,-490./1089.,60./1089.};
+        b = { 2940./1089.,-8820./1089.,14700./1089.,-14700./1089.,8820./1089.,-2940./1089.,420./1089.};
+        c[0] = 420./1089.;
+        break;
+    }
+    return MultistepTableau<real_type>( steps, order, a, b, c);
+}
+
+template<class real_type>
+MultistepTableau<real_type> imex_tvb(unsigned steps)
+{
+    unsigned order = steps;
+    std::vector<real_type> a(steps,0), b(steps, 0), c(steps+1,0);
+    switch( steps)
+    {
+        case(3):
+        // C = 0.536 D = 0.639
+        a[0] =  3909./2048.;     b[0] =  18463./12288.;
+        a[1] = -1367./1024.;     b[1] = -1271./768.;
+        a[2] =  873./2048.;      b[2] = 8233./12288.;
+        c[0] =  1089./2048.;
+        c[1] = -1139./12288.;
+        c[2] = -367./6144.;
+        c[3] =  1699./12288.;
+        break;
+        case(4):
+        // C = 0.458 , D = 0.685
+        a[0] =  21531./8192.;     b[0] =  13261./8192.;
+        a[1] = -22753./8192.;     b[1] = -75029./24576.;
+        a[2] =  12245./8192.;     b[2] =  54799./24576.;
+        a[3] = -2831./8192. ;     b[3] = -15245./24576.;
+        c[0] =  4207./8192.;
+        c[1] = -3567./8192.;
+        c[2] =  697./24576.;
+        c[3] = 4315./24576.;
+        c[4] = -41./384.;
+        break;
+        case(5):
+        // C = 0.376 , D = 0.709
+        a[0] =  13553./4096.;     b[0] = 10306951./5898240.;
+        a[1] = -38121./8192.;     b[1] = -13656497./2949120.;
+        a[2] =  7315./2048.;      b[2] = 1249949./245760.;
+        a[3] = -6161/4096. ;      b[3] = -7937687./2949120.;
+        a[4] = 2269./8192.;       b[4] = 3387361./5898240.;
+        c[0] =  4007./8192.;
+        c[1] =  -4118249./5898240.;
+        c[2] =  768703./2949120.;
+        c[3] = 47849./245760.;
+        c[4] = -725087./2949120.;
+        c[5] = 502321./5898240.;
+        break;
+    }
+    return MultistepTableau<real_type>( steps, order, a, b, c);
+}
+
+template<class real_type>
+MultistepTableau<real_type> ab(unsigned order)
+{
+    unsigned steps = order;
+    std::vector<real_type> a(steps,0), b(steps, 0), c(steps+1,0);
+    a[0]= 1.;
+    switch (order){
+        case 1: b = {1}; break;
+        case 2: b = {1.5, -0.5}; break;
+        case 3: b = { 23./12., -4./3., 5./12.}; break;
+        case 4: b = {55./24., -59./24., 37./24., -3./8.}; break;
+        case 5: b = { 1901./720., -1387./360., 109./30., -637./360., 251./720.}; break;
+        default: throw dg::Error(dg::Message()<<"Order "<<order<<" not implemented in AdamsBashforth!");
+    }
+    return MultistepTableau<real_type>( steps, order, a, b, c);
+}
+
+template<class real_type>
+MultistepTableau<real_type> tvb(unsigned steps)
+{
+    unsigned order = steps;
+    std::vector<real_type> a(steps,0), b(steps, 0), c(steps+1,0);
+    switch (steps){
+        case 1:
+            a = {1.};
+                b = {1.}; break;
+        case 2:
+            a = {4./3., -1./3.};
+            b = {4./3., -2./3.}; break; //CLM = 0.5
+        case 3: //CLM = 0.54...
+            a[0] =  1.908535476882378;  b[0] =  1.502575553858997;
+            a[1] = -1.334951446162515;  b[1] = -1.654746338401493;
+            a[2] = 0.426415969280137;   b[2] = 0.670051276940255;
+            break;
+        case 4: //CLM = 0.45...
+            a[0] = 2.628241000683208;   b[0] = 1.618795874276609;
+            a[1] = -2.777506277494861;  b[1] = -3.052866947601049;
+            a[2] = 1.494730011212510;   b[2] = 2.229909318681302;
+            a[3] = -0.345464734400857;  b[3] = -0.620278703629274;
+            break;
+        case 5: //CLM = 0.37...
+            a[0] = 3.308891758551210;   b[0] = 1.747442076919292;
+            a[1] = -4.653490937946655;  b[1] = -4.630745565661800;
+            a[2] = 3.571762873789854;   b[2] = 5.086056171401077;
+            a[3] = -1.504199914126327;  b[3] = -2.691494591660196;
+            a[4] = 0.277036219731918;   b[4] = 0.574321855183372;
+            break;
+        case 6: //CLM = 0.32...
+            a[0] = 4.113382628475685;   b[0] = 1.825457674048542;
+            a[1] = -7.345730559324184;  b[1] = -6.414174588309508;
+            a[2] = 7.393648314992094;   b[2] = 9.591671249204753;
+            a[3] = -4.455158576186636;  b[3] = -7.583521888026967;
+            a[4] = 1.523638279938299;   b[4] = 3.147082225022105;
+            a[5] = -0.229780087895259;  b[5] = -0.544771649561925;
+            break;
+        default: throw dg::Error(dg::Message()<<"Order "<<steps<<" not implemented in TVB scheme!");
+    }
+    return MultistepTableau<real_type>( steps, order, a, b, c);
+}
+template<class real_type>
+MultistepTableau<real_type> ssp(unsigned steps)
+{
+    std::vector<real_type> a(steps,0), b(steps, 0), c(steps+1,0);
+    unsigned order = 0;
+    switch (steps){
+        case 1: order = 1;
+                a = {1.};
+                b = {1.}; break;
+        case 2: order = 2;
+                a = {4./5., 1./5.};
+                b = {8./5., -2./5.}; break; //CLM = 0.5 ... ,order 2
+        case 3: order = 2;
+                a = { 3./4., 0., 1./4.};
+                b = { 3./2., 0., 0. }; break; //CLM = 0.5..., order 2
+        case 4: order = 2;
+                a = {8./9., 0., 0., 1./9.};
+                b = {4./3., 0., 0., 0.}; break; //CLM = 0.66..., order 2
+        case 5: order = 3;
+                a = {25./32., 0., 0., 0., 7./32.};
+                b = {25./16.,0.,0.,0.,5./16.}; break; //CLM 0.5, order 3
+        case 6: order = 3;
+                a = {108./125.,0.,0.,0.,0.,17./125.};
+                b = {36./25.,0.,0.,0.,0.,6./25.}; break; //CLM 0.567, order 3
+        default: throw dg::Error(dg::Message()<<"Stage "<<steps<<" not implemented in SSP scheme!");
+    }
+    return MultistepTableau<real_type>( steps, order, a, b, c);
+}
+
+}//namespace tableau
+///@endcond
+/**
+* @brief Identifiers for Multistep Tableaus
+*
+* We follow the naming convention
+* as NAME_S_Q
+*  - NAME is the author or name of the method
+*  - S is the number of steps in the method
+*  - Q is the global order of the method
+*
+*  @ingroup time_utils
+*  @sa ExplicitMultistep, ImplicitMultistep, ImExMultistep
+*/
+enum multistep_identifier{
+    //IMEX methods
+    IMEX_EULER_1_1,
+    IMEX_ADAMS_2_2,
+    IMEX_ADAMS_3_3,
+    IMEX_BDF_2_2,
+    IMEX_BDF_3_3,
+    IMEX_BDF_4_4,
+    IMEX_BDF_5_5,
+    IMEX_BDF_6_6,
+    IMEX_BDF_7_7,
+    IMEX_TVB_3_3,
+    IMEX_TVB_4_4,
+    IMEX_TVB_5_5,
+    // Explicit methods
+    AB_1_1,
+    AB_2_2,
+    AB_3_3,
+    AB_4_4,
+    AB_5_5,
+    eBDF_1_1,
+    eBDF_2_2,
+    eBDF_3_3,
+    eBDF_4_4,
+    eBDF_5_5,
+    eBDF_6_6,
+    eBDF_7_7,
+    TVB_1_1,
+    TVB_2_2,
+    TVB_3_3,
+    TVB_4_4,
+    TVB_5_5,
+    TVB_6_6,
+    SSP_1_1,
+    SSP_2_2,
+    SSP_3_2,
+    SSP_4_2,
+    SSP_5_3,
+    SSP_6_3,
+    // implicit methods
+    BDF_1_1, //!<
+    BDF_2_2, //!<
+    BDF_3_3, //!<
+    BDF_4_4, //!<
+    BDF_5_5, //!<
+    BDF_6_6, //!<
+    BDF_7_7, //!<
+
+};
+
+///@cond
+namespace create{
+
+static std::unordered_map<std::string, enum multistep_identifier> str2lmsid{
+    //Implicit-Explicit methods
+    {"Euler", IMEX_EULER_1_1},
+    {"Euler-1-1", IMEX_EULER_1_1},
+    {"ImEx-Adams-2-2", IMEX_ADAMS_2_2},
+    {"ImEx-Adams-3-3", IMEX_ADAMS_3_3},
+    {"ImEx-BDF-2-2", IMEX_BDF_2_2},
+    {"ImEx-BDF-3-3", IMEX_BDF_3_3},
+    {"Karniadakis",  IMEX_BDF_3_3},
+    {"ImEx-BDF-4-4", IMEX_BDF_4_4},
+    {"ImEx-BDF-5-5", IMEX_BDF_5_5},
+    {"ImEx-BDF-6-6", IMEX_BDF_6_6},
+    {"ImEx-BDF-7-7", IMEX_BDF_7_7},
+    {"ImEx-TVB-3-3", IMEX_TVB_3_3},
+    {"ImEx-TVB-4-4", IMEX_TVB_4_4},
+    {"ImEx-TVB-5-5", IMEX_TVB_5_5},
+    //Explicit methods
+    {"AB-1-1", AB_1_1},
+    {"AB-2-2", AB_2_2},
+    {"AB-3-3", AB_3_3},
+    {"AB-4-4", AB_4_4},
+    {"AB-5-5", AB_5_5},
+    {"eBDF-1-1", eBDF_1_1},
+    {"eBDF-2-2", eBDF_2_2},
+    {"eBDF-3-3", eBDF_3_3},
+    {"eBDF-4-4", eBDF_4_4},
+    {"eBDF-5-5", eBDF_5_5},
+    {"eBDF-6-6", eBDF_6_6},
+    {"eBDF-7-7", eBDF_7_7},
+    {"TVB-1-1", TVB_1_1},
+    {"TVB-2-2", TVB_2_2},
+    {"TVB-3-3", TVB_3_3},
+    {"TVB-4-4", TVB_4_4},
+    {"TVB-5-5", TVB_5_5},
+    {"TVB-6-6", TVB_6_6},
+    {"SSP-1-1", SSP_1_1},
+    {"SSP-2-2", SSP_2_2},
+    {"SSP-3-2", SSP_3_2},
+    {"SSP-4-2", SSP_4_2},
+    {"SSP-5-3", SSP_5_3},
+    {"SSP-6-3", SSP_6_3},
+    // implicit methods
+    {"BDF-1-1", BDF_1_1},
+    {"BDF-2-2", BDF_2_2},
+    {"BDF-3-3", BDF_3_3},
+    {"BDF-4-4", BDF_4_4},
+    {"BDF-5-5", BDF_5_5},
+    {"BDF-6-6", BDF_6_6},
+    {"BDF-7-7", BDF_7_7}
+};
+enum multistep_identifier str2lmstableau( std::string name)
+{
+    if( str2lmsid.find(name) == str2lmsid.end())
+        throw dg::Error(dg::Message(_ping_)<<"Multistep coefficients for "<<name<<" not found!");
+    else
+        return str2lmsid[name];
+}
+std::string lmstableau2str( enum multistep_identifier id)
+{
+    for( auto name: str2lmsid)
+    {
+        if( name.second == id)
+            return name.first;
+    }
+    throw dg::Error(dg::Message(_ping_)<<"Tableau conversion failed!");
+}
+
+template<class real_type>
+MultistepTableau<real_type> lmstableau( enum multistep_identifier id)
+{
+    switch(id){
+        case IMEX_EULER_1_1:
+            return dg::tableau::imex_euler_1_1<real_type>();
+        case IMEX_ADAMS_2_2:
+            return dg::tableau::imex_adams_2_2<real_type>();
+        case IMEX_ADAMS_3_3:
+            return dg::tableau::imex_adams_3_3<real_type>();
+        case IMEX_BDF_2_2:
+            return dg::tableau::imex_bdf<real_type>(2);
+        case IMEX_BDF_3_3:
+            return dg::tableau::imex_bdf<real_type>(3);
+        case IMEX_BDF_4_4:
+            return dg::tableau::imex_bdf<real_type>(4);
+        case IMEX_BDF_5_5:
+            return dg::tableau::imex_bdf<real_type>(5);
+        case IMEX_BDF_6_6:
+            return dg::tableau::imex_bdf<real_type>(6);
+        case IMEX_BDF_7_7:
+            return dg::tableau::imex_bdf<real_type>(7);
+        case IMEX_TVB_3_3:
+            return dg::tableau::imex_tvb<real_type>(3);
+        case IMEX_TVB_4_4:
+            return dg::tableau::imex_tvb<real_type>(4);
+        case IMEX_TVB_5_5:
+            return dg::tableau::imex_tvb<real_type>(5);
+        case AB_1_1:
+            return dg::tableau::ab<real_type>(1);
+        case AB_2_2:
+            return dg::tableau::ab<real_type>(2);
+        case AB_3_3:
+            return dg::tableau::ab<real_type>(3);
+        case AB_4_4:
+            return dg::tableau::ab<real_type>(4);
+        case AB_5_5:
+            return dg::tableau::ab<real_type>(5);
+        case eBDF_1_1:
+            return dg::tableau::imex_bdf<real_type>(1);
+        case eBDF_2_2:
+            return dg::tableau::imex_bdf<real_type>(2);
+        case eBDF_3_3:
+            return dg::tableau::imex_bdf<real_type>(3);
+        case eBDF_4_4:
+            return dg::tableau::imex_bdf<real_type>(4);
+        case eBDF_5_5:
+            return dg::tableau::imex_bdf<real_type>(5);
+        case eBDF_6_6:
+            return dg::tableau::imex_bdf<real_type>(6);
+        case eBDF_7_7:
+            return dg::tableau::imex_bdf<real_type>(7);
+        case TVB_1_1:
+            return dg::tableau::tvb<real_type>(1);
+        case TVB_2_2:
+            return dg::tableau::tvb<real_type>(2);
+        case TVB_3_3:
+            return dg::tableau::tvb<real_type>(3);
+        case TVB_4_4:
+            return dg::tableau::tvb<real_type>(4);
+        case TVB_5_5:
+            return dg::tableau::tvb<real_type>(5);
+        case TVB_6_6:
+            return dg::tableau::tvb<real_type>(6);
+        case SSP_1_1:
+            return dg::tableau::ssp<real_type>(1);
+        case SSP_2_2:
+            return dg::tableau::ssp<real_type>(2);
+        case SSP_3_2:
+            return dg::tableau::ssp<real_type>(3);
+        case SSP_4_2:
+            return dg::tableau::ssp<real_type>(4);
+        case SSP_5_3:
+            return dg::tableau::ssp<real_type>(5);
+        case SSP_6_3:
+            return dg::tableau::ssp<real_type>(6);
+        case BDF_1_1:
+            return dg::tableau::imex_euler_1_1<real_type>();
+        case BDF_2_2:
+            return dg::tableau::imex_bdf<real_type>(2);
+        case BDF_3_3:
+            return dg::tableau::imex_bdf<real_type>(3);
+        case BDF_4_4:
+            return dg::tableau::imex_bdf<real_type>(4);
+        case BDF_5_5:
+            return dg::tableau::imex_bdf<real_type>(5);
+        case BDF_6_6:
+            return dg::tableau::imex_bdf<real_type>(6);
+        case BDF_7_7:
+            return dg::tableau::imex_bdf<real_type>(7);
+    }
+    return MultistepTableau<real_type>(); //avoid compiler warning
+}
+
+
+template<class real_type>
+MultistepTableau<real_type> lmstableau( std::string name)
+{
+    return lmstableau<real_type>( str2lmstableau(name));
+}
+
+}//namespace create
+///@endcond
+/*! @class hide_imex_multistep_tableaus
+ *
+ *    Name  | Identifier | Description
+ *   -------|------------| -----------
+ *   ImEx-Euler-1-1         | dg::IMEX_EULER_1_1 | Explicit Euler combined with Implicit Euler
+    ImEx-Adams-X-X | dg::IMEX_ADAMS_X_X | <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> @note **X can be 2 (C=0.44) or 3 (C=0.16)**
+    ImEx-BDF_X_X | dg::IMEX_BDF_X_X | The family of schems described in <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> <br>The implicit part is a normal BDF scheme https://en.wikipedia.org/wiki/Backward_differentiation_formula while the explicit part equals the Minimal Projecting method by <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a> or **extrapolated BDF** in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S. J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> <br> @note Possible values for **X: 1 ( C=1), 2 (C=0.63), 3 (C=0.39), 4 (C=0.22), 5( C=0.09), * 6, 7** <br> Note that X=3 is identical to the "Karniadakis" scheme
+    * Karniadakis | dg::IMEX_BDF_3_3 | The ImEx-BDF-3-3 scheme is identical to the widely used "Karniadakis" scheme <a href = "https://dx.doi.org/10.1016/0021-9991(91)90007-8"> Karniadakis, et al. J. Comput. Phys. 97 (1991)</a>
+    ImEx-TVB-X-X | dg::IMEX_TVB_X_X | The family of schems described in < <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> <br> The explicit part is a TVB scheme while the implicit part is optimized to maximize damping of high wavelength <br> @note Possible values for **X: 3 (C=0.54), 4 (C=0.458), 5 (C=0.376)**
+    *
+ @note the CFL coefficient C is given relative to the forward Euler method: \f$ \Delta t < C \Delta_{FE}\f$.
+ */
+
+/*! @class hide_explicit_multistep_tableaus
+
+ *    Name  | Identifier | Description
+ *   -------|------------| -----------
+ *   AB-X-X | dg::AB_X_X | The family of schemes described in <a href = "https://en.wikipedia.org/wiki/Linear_multistep_method"> Linear multistep methods </a> as **Adams-Bashforth** \f[ u^{n+1} = u^n + \Delta t\sum_{j=0}^{s-1} b_j f\left(t^n - j \Delta t, u^{n-j}\right) \f] @note **Possible stages are X: 1, 2,..., 5**, the order of the method is the same as its stages @note The Adams-Bashforth schemes implemented here need less storage but may have **a smaller region of absolute stability** than for example an extrapolated BDF method of the same order.
+ * eBDF-X-X | dg::eBDF_X_X | The family of schemes described in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S.  J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> as **extrapolated BDF**  where it is found to be TVB (**total variation bound**). The schemes also appear as **Minimal Projecting** scheme described in <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a> * <br> @note **Possible stages are X: 1 (C=1), 2, 3 (C=0.39), 4, 5, 6, 7** with the order the same as the number of stages
+ * TVB-X-X | dg::TVB_X_X | The family of schemes described in <a href="https://doi.org/10.1016/j.jcp.2005.02.029">S.J. Ruuth and W. Hundsdorfer, High-order linear multistep methods with general monotonicity and boundedness properties, Journal of Computational Physics, Volume 209, Issue 1, 2005 </a> as Total variation Bound. These schemes have **larger stable step sizes than the eBDF family**, <br> @note **Possible values for X are 1 (C=1),2 (C=0.5), 3 (C=0.54), 4 (C=0.46), 5 (C=0.38) 6 (C=0.33)**. We highlight that TVB-3-3 has 38% larger stepsize than eBDF-3-3 and TVB-4-4 has 109% larger stepsize than eBDF-4-4.
+ * SSP-X-Y | dg::SSP_X_Y | The family of schemes described in <a href="https://doi.org/10.1007/BF02728985">Gottlieb, S. On high order strong stability preserving runge-kutta and multi step time discretizations. J Sci Comput 25, 105–128 (2005)</a> as Strong Stability preserving. We implement the lowest order schemes for each stage and disregard the remaining schemes in the paper since their CFL conditions are worse than the TVB scheme of the same order.  @note Possible values for **X-Y : 1-1 (C=1), 2-2 (C=0.5), 3-2 (C=0.5), 4-2 (C=0.66), 5-3: (C=0.5), 6-3 (C=0.567)**.@note These schemes are noteworthy because the coefficients b_i are all positive except for the 2-2 method and **the "4-2" and "6-3" methods allow slightly larger stepsize but increased storage requirements than TVB** of same order (2 and 3).
+
+ *@note Total variation bound (TVB) means \f$ || v^n|| \leq M ||v^0||\f$  where the norm signifies the total variation semi-norm. Total variation diminishing
+     (TVD) means M=1, and strong stability preserving (SSP) is the same as TVD, TVB schemes converge to the correct entropy solutions of hyperbolic conservation laws)
+ */
+
+/*! @class hide_implicit_multistep_tableaus
+ *
+ *    Name  | Identifier | Description
+ *   -------|------------| -----------
+ *   BDF-X-X | dg::BDF_X_X | The coefficients for backward differences can be found at https://en.wikipedia.org/wiki/Backward_differentiation_formula <br> @note A BDF scheme is simply constructed by discretizing the time derivative with a n-th order backward difference formula and evaluating the right hand side at the new timestep
+ *
+*/
+
+/*! @brief Convert identifiers to their corresponding \c dg::MultistepTableau
+ *
+ * This is a helper class to simplify the interfaces of our timestepper functions and classes.
+ * The sole purpose is to implicitly convert either a MultistepTableau or one of
+ * the following identifiers to an instance of a MultistepTableau.
+ *
+ * Explicit methods
+ * @copydoc hide_explicit_multistep_tableaus
+ * Implicit methods
+ * @copydoc hide_implicit_multistep_tableaus
+ * Implicit-Explicit methods
+ * @copydoc hide_imex_multistep_tableaus
+ * @param real_type The type of the coefficients in the MultistepTableau
+ * @ingroup time_utils
+ */
+template<class real_type>
+struct ConvertsToMultistepTableau
+{
+    using value_type = real_type;
+    ///Of course a MultistepTableau converts to a MultistepTableau
+    ///Useful if you constructed your very own coefficients
+    ConvertsToMultistepTableau( MultistepTableau<real_type> tableau): m_t(tableau){}
+
+    /*! @brief Create MultistepTableau from \c dg::tableau_identifier
+    *
+    * The use of this constructor might be a bit awkward because you'll have to write all caps.
+    * @param id the identifier, for example \c dg::eBDF_3_3
+    */
+    ConvertsToMultistepTableau( enum tableau_identifier id):m_t( dg::create::lmstableau<real_type>(id)){}
+    /*! @brief Create MultistepTableau from its name (very useful)
+    *
+    * Explicit methods
+    * @copydoc hide_explicit_multistep_tableaus
+    * Implicit methods
+    * @copydoc hide_implicit_multistep_tableaus
+    * Implicit-Explicit methods
+    * @copydoc hide_imex_multistep_tableaus
+    * @param name The name of the tableau as stated in the Name column above, as a string, for example "eBDF-3-3"
+    */
+    ConvertsToMultistepTableau( std::string name):m_t(
+            dg::create::lmstableau<real_type>(name)){}
+
+    ///@copydoc ConvertsToMultistepTableau(std::string)
+    ConvertsToMultistepTableau( const char* name):m_t(
+            dg::create::lmstableau<real_type>(std::string(name))){}
+    ///Convert to MultistepTableau
+    ///
+    ///which means an object can be directly assigned to a MultistepTableau
+    operator MultistepTableau<real_type>( )const{
+        return m_t;
+    }
+    private:
+    MultistepTableau<real_type> m_t;
+};
+
+}//namespace dg
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index 32643e7a0..6640c481c 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -27,7 +27,7 @@ namespace dg{
  * @copydoc hide_implicit_butcher_tableaus
  * @tparam real_type type of the coefficients
  * @sa RungeKutta, ERKStep, ARKStep
- * @ingroup time
+ * @ingroup time_utils_utils
  */
 template<class real_type>
 struct ButcherTableau{
@@ -174,7 +174,7 @@ struct ButcherTableau{
  *
  * @tparam real_type type of the coefficients
  * @sa ShuOsher
- * @ingroup time
+ * @ingroup time_utils_utils
  */
 template<class real_type>
 struct ShuOsherTableau
@@ -267,7 +267,6 @@ struct ShuOsherTableau
     unsigned m_stages, m_order;
     dg::Operator<real_type> m_alpha, m_beta;
 };
-
 ///@cond
 namespace tableau{
 ///%%%%%%%%%%%%%%%%%%%%%%%%%%%Classic Butcher tables%%%%%%%%%%%%%%%%%%
@@ -313,7 +312,7 @@ ButcherTableau<real_type> classic_4_4()
     return ButcherTableau<real_type>( 4,4, a,b,c);
 }
 //From Yoh and Zhong (AIAA 42, 2004)
-//!Attention! assumes another form of implementation
+// !Attention! assumes another form of implementation
 //than ARK tableaus
 template<class real_type>
 ButcherTableau<real_type> sirk3a_ex_3_3()
@@ -1072,7 +1071,7 @@ ShuOsherTableau<real_type> ssprk_5_4()
 *  - Q is the global order of the method
 *
 *  @note In some of the links below you might want to use the search function of your browser to find the indicated method
-*  @ingroup time
+*  @ingroup time_utils
 */
 enum tableau_identifier{
     EXPLICIT_EULER_1_1, //!< <a href="https://en.wikipedia.org/wiki/List_of_Runge%E2%80%93Kutta_methods">Euler</a>
@@ -1362,7 +1361,7 @@ ButcherTableau<real_type> tableau( std::string name)
  * Implicit methods
  * @copydoc hide_implicit_butcher_tableaus
  * @param real_type The type of the coefficients in the ButcherTableau
- * @ingroup time
+ * @ingroup time_utils
  */
 template<class real_type>
 struct ConvertsToButcherTableau
@@ -1410,7 +1409,7 @@ struct ConvertsToButcherTableau
  * Explicit methods (the ones that are marked with "Shu-Osher-Form"
  * @copydoc hide_explicit_butcher_tableaus
  * @param real_type The type of the coefficients in the ShuOsherTableau
- * @ingroup time
+ * @ingroup time_utils
  */
 template<class real_type>
 struct ConvertsToShuOsherTableau
-- 
GitLab


From 4f3fa1f008383376c34ba0e7043be32ab5c22333 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Feb 2021 14:31:14 +0100
Subject: [PATCH 448/540] Adapt Shu code

- to changed file namespace
- new timesteppers
---
 inc/dg/enums.h           | 50 ++++++++++++++++++++++++++++++++++++++--
 inc/dg/topology/filter.h |  3 ++-
 src/lamb_dipole/diag.h   | 14 +++++------
 src/lamb_dipole/init.h   | 48 +++++++++++++++++++-------------------
 src/lamb_dipole/shu.cuh  | 30 +++++++++++++-----------
 src/lamb_dipole/shu_b.cu | 44 +++++++++++++++++------------------
 6 files changed, 120 insertions(+), 69 deletions(-)

diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index e54c4bc29..44ffa871f 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -84,7 +84,7 @@ static inline bc str2bc( std::string s)
  * @param bound boundary condition to invert
  * @return NEU for DIR, DIR for NEU, NEU_DIR for DIR_NEU, DIR_NEU for NEU_DIR and PER for PER
  */
-bc inverse( bc bound)
+static inline bc inverse( bc bound)
 {
     if( bound == DIR) return NEU;
     if( bound == NEU) return DIR;
@@ -107,13 +107,59 @@ enum direction{
 };
 
 
+/**
+ * @brief convert a string to a direction
+ *
+ * converts
+ * - "forward" to forward
+ * - "backward" to backward
+ * - "centered" to centered
+ *
+ * @param s the input string
+ * @return a valid direction
+ * \throw std::runtime_error if string doesn't match any of the above
+ */
+static inline direction str2direction( std::string s)
+{
+    if( "forward" == s)
+        return forward;
+    if( "backward" == s)
+        return backward;
+    if( "centered" == s)
+        return centered;
+    throw std::runtime_error( "Direction '"+s+"' not recognized!");
+}
+/**
+ * @brief convert a direciton to string
+ *
+ * converts
+ * - forward to "forward"
+ * - backward to "backward"
+ * - centered to "centered"
+ *
+ * @param dir the input direction
+ * @return a string
+ */
+static inline std::string direction2str( enum direction dir)
+{
+    std::string s;
+    switch(dir)
+    {
+        case(dg::forward): s = "forward"; break;
+        case(dg::backward): s = "backward"; break;
+        case(dg::centered): s = "centered"; break;
+        default: s = "Not specified!!";
+    }
+    return s;
+}
+
 /**
  * @brief invert direction
  *
  * @param dir direction to invert
  * @return backward for forward, forward for backward, centered for centered
  */
-direction inverse( direction dir)
+static inline direction inverse( direction dir)
 {
     if( dir == forward) return backward;
     if( dir == backward) return forward;
diff --git a/inc/dg/topology/filter.h b/inc/dg/topology/filter.h
index 008d7dfb8..92c54a9eb 100644
--- a/inc/dg/topology/filter.h
+++ b/inc/dg/topology/filter.h
@@ -137,8 +137,9 @@ struct ModalFilter
      *
      * @tparam Topology Any grid
      * @tparam UnaryOp Model of Unary Function \c real_type \c sigma(unsigned) The input will be the modal number \c i where \f$ i=0,...,n-1\f$ and \c n is the number of polynomial coefficients in use. The output is the filter strength for the given mode number
-     * @param f The filter to evaluate on the normalized modal coefficients
+     * @param sigma The filter to evaluate on the normalized modal coefficients
      * @param t The topology to apply the modal filter on
+     * @param ps parameters that are forwarded to the creation of a ContainerType (e.g. when a std::vector is to be created it is the vector size)
      */
     template<class UnaryOp, class Topology, class ...Params>
     ModalFilter( UnaryOp sigma, const Topology& t, Params&& ...ps) : m_filter (
diff --git a/src/lamb_dipole/diag.h b/src/lamb_dipole/diag.h
index 007ec9637..8f6ba8995 100644
--- a/src/lamb_dipole/diag.h
+++ b/src/lamb_dipole/diag.h
@@ -13,7 +13,7 @@ struct Variables{
     const double& time;
     const dg::DVec& weights;
     double duration;
-    enum file::error mode;
+    enum dg::file::error mode;
     Json::Value& js;
 };
 
@@ -78,11 +78,11 @@ std::vector<Record1d> diagnostics1d_list = {
     },
     {"error", "Relative error to analytical solution (not available for every intitial condition)",
         []( Variables& v ) {
-            std::string initial = file::get( v.mode, v.js, "init", "type", "lamb").asString();
+            std::string initial = dg::file::get( v.mode, v.js, "init", "type", "lamb").asString();
             if( "mms" == initial)
             {
-                double R = file::get( v.mode, v.js, "init", "sigma", 0.1).asDouble();
-                double U = file::get( v.mode, v.js, "init", "velocity", 1).asDouble();
+                double R = dg::file::get( v.mode, v.js, "init", "sigma", 0.1).asDouble();
+                double U = dg::file::get( v.mode, v.js, "init", "velocity", 1).asDouble();
                 shu::MMSVorticity vortex( R, U, v.grid.ly(), v.time);
                 dg::DVec sol = dg::evaluate( vortex, v.grid);
                 dg::blas1::axpby( 1., v.y0, -1., sol);
@@ -92,11 +92,11 @@ std::vector<Record1d> diagnostics1d_list = {
             {
                 double nu = 0.;
                 unsigned order = 1;
-                std::string regularization = file::get( v.mode, v.js, "regularization", "type", "moddal").asString();
+                std::string regularization = dg::file::get( v.mode, v.js, "regularization", "type", "moddal").asString();
                 if( "viscosity" == regularization)
                 {
-                    nu = file::get( v.mode, v.js, "regularization", "nu_perp", 1e-3).asDouble();
-                    order = file::get( v.mode, v.js, "regularization", "order", 1).asUInt();
+                    nu = dg::file::get( v.mode, v.js, "regularization", "nu_perp", 1e-3).asDouble();
+                    order = dg::file::get( v.mode, v.js, "regularization", "order", 1).asUInt();
                 }
                 double time = v.time;
                 dg::DVec sol = dg::evaluate( [time,nu,order](double x, double y) {
diff --git a/src/lamb_dipole/init.h b/src/lamb_dipole/init.h
index d888ff1c2..fa3475559 100644
--- a/src/lamb_dipole/init.h
+++ b/src/lamb_dipole/init.h
@@ -4,21 +4,21 @@
 #include <functional>
 #include "json/json.h"
 #include "dg/algorithm.h"
-#include "file/json_utilities.h"
+#include "dg/file/json_utilities.h"
 
 namespace shu{
 
-dg::CartesianGrid2d createGrid( Json::Value& js, enum file::error mode)
+dg::CartesianGrid2d createGrid( Json::Value& js, enum dg::file::error mode)
 {
-    unsigned n = file::get( mode, js, "grid", "n",3).asUInt();
-    unsigned Nx = file::get( mode, js, "grid", "Nx", 48).asUInt();
-    unsigned Ny = file::get( mode, js, "grid", "Ny", 48).asUInt();
-    double x0 = file::get_idx( mode, js, "grid", "x", 0u, 0.).asDouble();
-    double x1 = file::get_idx( mode, js, "grid", "x", 1u, 1.).asDouble();
-    double y0 = file::get_idx( mode, js, "grid", "y", 0u, 0.).asDouble();
-    double y1 = file::get_idx( mode, js, "grid", "y", 1u, 1.).asDouble();
-    dg::bc bcx = dg::str2bc ( file::get_idx( mode, js, "grid", "bc", 0u, "DIR").asString());
-    dg::bc bcy = dg::str2bc ( file::get_idx( mode, js, "grid", "bc", 1u, "PER").asString());
+    unsigned n = dg::file::get( mode, js, "grid", "n",3).asUInt();
+    unsigned Nx = dg::file::get( mode, js, "grid", "Nx", 48).asUInt();
+    unsigned Ny = dg::file::get( mode, js, "grid", "Ny", 48).asUInt();
+    double x0 = dg::file::get_idx( mode, js, "grid", "x", 0u, 0.).asDouble();
+    double x1 = dg::file::get_idx( mode, js, "grid", "x", 1u, 1.).asDouble();
+    double y0 = dg::file::get_idx( mode, js, "grid", "y", 0u, 0.).asDouble();
+    double y1 = dg::file::get_idx( mode, js, "grid", "y", 1u, 1.).asDouble();
+    dg::bc bcx = dg::str2bc ( dg::file::get_idx( mode, js, "grid", "bc", 0u, "DIR").asString());
+    dg::bc bcy = dg::str2bc ( dg::file::get_idx( mode, js, "grid", "bc", 1u, "PER").asString());
     return dg::CartesianGrid2d( x0, x1, y0, y1, n, Nx, Ny, bcx, bcy);
 }
 
@@ -94,37 +94,37 @@ struct MMSSource{
 };
 
 std::map<std::string, std::function< dg::HVec(
-    Json::Value& js, enum file::error mode,
+    Json::Value& js, enum dg::file::error mode,
     const dg::CartesianGrid2d& grid) >
 > initial_conditions =
 {
     {"lamb", [](
-        Json::Value& js, enum file::error mode,
+        Json::Value& js, enum dg::file::error mode,
         const dg::CartesianGrid2d& grid)
         {
             dg::HVec omega;
-            double posX = file::get( mode, js, "init", "posX", 0.5).asDouble();
-            double posY = file::get( mode, js, "init", "posY", 0.8).asDouble();
-            double R = file::get( mode, js, "init", "sigma", 0.1).asDouble();
-            double U = file::get( mode, js, "init", "velocity", 1).asDouble();
+            double posX = dg::file::get( mode, js, "init", "posX", 0.5).asDouble();
+            double posY = dg::file::get( mode, js, "init", "posY", 0.8).asDouble();
+            double R = dg::file::get( mode, js, "init", "sigma", 0.1).asDouble();
+            double U = dg::file::get( mode, js, "init", "velocity", 1).asDouble();
             dg::Lamb lamb( posX*grid.lx(), posY*grid.ly(), R, U);
             omega = dg::evaluate ( lamb, grid);
             return omega;
         }
     },
     {"shear", [](
-        Json::Value& js, enum file::error mode,
+        Json::Value& js, enum dg::file::error mode,
         const dg::CartesianGrid2d& grid)
         {
             dg::HVec omega;
-            double rho = file::get( mode, js, "init", "rho", M_PI/15.).asDouble();
-            double delta = file::get( mode, js, "init", "delta", 0.05).asDouble();
+            double rho = dg::file::get( mode, js, "init", "rho", M_PI/15.).asDouble();
+            double delta = dg::file::get( mode, js, "init", "delta", 0.05).asDouble();
             omega = dg::evaluate ( ShearLayer( rho, delta), grid);
             return omega;
         }
     },
     {"sine", [](
-        Json::Value& js, enum file::error mode,
+        Json::Value& js, enum dg::file::error mode,
         const dg::CartesianGrid2d& grid)
         {
             dg::HVec omega;
@@ -133,12 +133,12 @@ std::map<std::string, std::function< dg::HVec(
         }
     },
     {"mms", [](
-        Json::Value& js, enum file::error mode,
+        Json::Value& js, enum dg::file::error mode,
         const dg::CartesianGrid2d& grid)
         {
             dg::HVec omega;
-            double sigma = file::get( mode, js, "init", "sigma", 0.2).asDouble();
-            double velocity = file::get( mode, js, "init", "velocity", 0.1).asDouble();
+            double sigma = dg::file::get( mode, js, "init", "sigma", 0.2).asDouble();
+            double velocity = dg::file::get( mode, js, "init", "velocity", 0.1).asDouble();
             std::cout << "Sigma "<<sigma<<" "<<velocity<<std::endl;
             omega = dg::evaluate ( MMSVorticity( sigma, velocity,grid.ly(), 0.), grid);
             return omega;
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index d5cedd239..f79049d53 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -5,7 +5,7 @@
 #include <cusp/ell_matrix.h>
 
 #include "json/json.h"
-#include "file/json_utilities.h"
+#include "dg/file/json_utilities.h"
 #include "dg/algorithm.h"
 #include "init.h"
 
@@ -25,13 +25,13 @@ struct Upwind{
 template< class Geometry, class Matrix, class Container>
 struct Diffusion
 {
-    Diffusion( const Geometry& g, Json::Value& js, enum file::error mode)
+    Diffusion( const Geometry& g, Json::Value& js, enum dg::file::error mode)
     {
-        std::string regularization = file::get( mode, js, "regularization", "type", "modal").asString();
+        std::string regularization = dg::file::get( mode, js, "regularization", "type", "modal").asString();
         if( "viscosity" == regularization)
         {
-            m_nu = file::get( mode, js, "regularization", "nu_perp", 1e-3).asDouble();
-            m_order = file::get( mode, js, "regularization", "order", 1).asUInt();
+            m_nu = dg::file::get( mode, js, "regularization", "nu_perp", 1e-3).asDouble();
+            m_order = dg::file::get( mode, js, "regularization", "order", 1).asUInt();
             m_temp = dg::evaluate( dg::zero, g);
             m_LaplacianM.construct( g, dg::normed);
         }
@@ -62,7 +62,7 @@ struct Shu
 {
     using value_type = dg::get_value_type<Container>;
 
-    Shu( const Geometry& grid, Json::Value& js, enum file::error mode);
+    Shu( const Geometry& grid, Json::Value& js, enum dg::file::error mode);
 
     const dg::Elliptic<Matrix, Container, Container>& lap() const { return m_multi_laplaceM[0];}
     dg::ArakawaX<Geometry, Matrix, Container>& arakawa() {return m_arakawa;}
@@ -103,14 +103,14 @@ struct Shu
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
 Shu< Geometry, IMatrix, Matrix, Container>::Shu(
-        const Geometry& g, Json::Value& js, enum file::error mode):
+        const Geometry& g, Json::Value& js, enum dg::file::error mode):
     m_old_psi( 2, dg::evaluate( dg::zero, g)),
     m_multigrid( g, 3),
     m_x( dg::evaluate( dg::cooX2d, g)),
     m_y( dg::evaluate( dg::cooY2d, g))
 {
-    m_advection = file::get( mode, js, "advection", "type", "arakawa").asString();
-    m_multiplication = file::get( mode, js, "advection", "multiplication", "pointwise").asString();
+    m_advection = dg::file::get( mode, js, "advection", "type", "arakawa").asString();
+    m_multiplication = dg::file::get( mode, js, "advection", "multiplication", "pointwise").asString();
     m_metric = g.metric();
 
     m_psi = dg::evaluate( dg::zero, g);
@@ -153,17 +153,21 @@ Shu< Geometry, IMatrix, Matrix, Container>::Shu(
         m_arakawa.construct( g);
     }
 
-    unsigned stages = file::get( mode, js, "elliptic", "stages", 3).asUInt();
+    unsigned stages = dg::file::get( mode, js, "elliptic", "stages", 3).asUInt();
     m_eps.resize(stages);
-    m_eps[0] = file::get_idx( mode, js, "elliptic", "eps_pol", 0, 1e-6).asDouble();
+    m_eps[0] = dg::file::get_idx( mode, js, "elliptic", "eps_pol", 0, 1e-6).asDouble();
     for( unsigned i=1;i<stages; i++)
     {
-        m_eps[i] = file::get_idx( mode, js, "elliptic", "eps_pol", i, 1).asDouble();
+        m_eps[i] = dg::file::get_idx( mode, js, "elliptic", "eps_pol", i, 1).asDouble();
         m_eps[i]*= m_eps[0];
     }
+    //this is a hidden parameter
+    //note that only centered works with double periodic boundary conditions
+    enum dg::direction dir = dg::str2direction( dg::file::get( dg::file::error::is_warning, js,
+                "elliptic", "direction", "centered").asString());
     m_multi_laplaceM.resize(stages);
     for( unsigned u=0; u<stages; u++)
-        m_multi_laplaceM[u].construct( m_multigrid.grid(u), dg::not_normed, dg::centered, 1);
+        m_multi_laplaceM[u].construct( m_multigrid.grid(u), dg::not_normed, dir, 1);
 }
 
 template< class Geometry, class IMatrix, class Matrix, class Container>
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index ae2864a32..9c7f82b43 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -20,11 +20,11 @@ int main( int argc, char* argv[])
 {
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
-    enum file::error mode = file::error::is_throw;
+    enum dg::file::error mode = dg::file::error::is_throw;
     if( argc == 1)
-        file::file2Json( "input/default.json", js, file::comments::are_discarded);
+        dg::file::file2Json( "input/default.json", js, dg::file::comments::are_discarded);
     else
-        file::file2Json( argv[1], js);
+        dg::file::file2Json( argv[1], js);
     std::cout << js <<std::endl;
 
     /////////////////////////////////////////////////////////////////
@@ -32,7 +32,7 @@ int main( int argc, char* argv[])
     dg::DVec w2d( dg::create::weights(grid));
     /////////////////////////////////////////////////////////////////
 
-    std::string initial = file::get( mode, js, "init", "type", "lamb").asString();
+    std::string initial = dg::file::get( mode, js, "init", "type", "lamb").asString();
     dg::HVec omega = shu::initial_conditions.at( initial)(js, mode, grid);
 
     dg::DVec y0( omega ), y1( y0);
@@ -48,8 +48,8 @@ int main( int argc, char* argv[])
     shu::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, js, mode);
     if( "mms" == initial)
     {
-        double sigma = file::get( mode, js, "init", "sigma", 0.2).asDouble();
-        double velocity = file::get( mode, js, "init", "velocity", 0.1).asDouble();
+        double sigma = dg::file::get( mode, js, "init", "sigma", 0.2).asDouble();
+        double velocity = dg::file::get( mode, js, "init", "velocity", 0.1).asDouble();
         shu.set_mms_source( sigma, velocity, grid.ly());
     }
 
@@ -67,8 +67,8 @@ int main( int argc, char* argv[])
     }
 
     /// ////////////// Initialize timestepper ///////////////////////
-    std::string stepper = file::get( mode, js, "timestepper", "stepper", "FilteredMultistep").asString();
-    std::string regularization = file::get( mode, js, "regularization", "type", "moddal").asString();
+    std::string stepper = dg::file::get( mode, js, "timestepper", "stepper", "FilteredMultistep").asString();
+    std::string regularization = dg::file::get( mode, js, "regularization", "type", "moddal").asString();
     dg::ModalFilter<dg::DMatrix, dg::DVec> filter;
     dg::IdentityFilter id;
     bool apply_filter = true;
@@ -77,9 +77,9 @@ int main( int argc, char* argv[])
     dg::FilteredExplicitMultistep<dg::DVec> multistep;
     if( regularization == "modal")
     {
-        double alpha = file::get( mode, js, "regularization", "alpha", 36).asDouble();
-        double order = file::get( mode, js, "regularization", "order", 8).asDouble();
-        double eta_c = file::get( mode, js, "regularization", "eta_c", 0.5).asDouble();
+        double alpha = dg::file::get( mode, js, "regularization", "alpha", 36).asDouble();
+        double order = dg::file::get( mode, js, "regularization", "order", 8).asDouble();
+        double eta_c = dg::file::get( mode, js, "regularization", "eta_c", 0.5).asDouble();
         filter.construct( dg::ExponentialFilter(alpha, eta_c, order, grid.n()), grid);
     }
     else
@@ -87,7 +87,7 @@ int main( int argc, char* argv[])
     if( regularization == "viscosity" && stepper != "Karniadakis")
         throw dg::Error(dg::Message(_ping_)<<"Error: Viscosity only works with Karniadakis! Exit now!");
 
-    double dt = file::get( mode, js, "timestepper", "dt", 2e-3).asDouble();
+    double dt = dg::file::get( mode, js, "timestepper", "dt", 2e-3).asDouble();
     if( "Karniadakis" == stepper)
     {
         if( regularization != "viscosity")
@@ -96,7 +96,7 @@ int main( int argc, char* argv[])
 
             return -1;
         }
-        double eps_time = file::get( mode, js, "timestepper", "eps_time", 1e-10).asDouble();
+        double eps_time = dg::file::get( mode, js, "timestepper", "eps_time", 1e-10).asDouble();
         karniadakis.construct( y0, y0.size(), eps_time);
         karniadakis.init( shu, diffusion, time, y0, dt);
     }
@@ -106,7 +106,7 @@ int main( int argc, char* argv[])
     }
     else if( "FilteredMultistep" == stepper)
     {
-        multistep.construct( "eBDF", 3, y0);
+        multistep.construct( "eBDF-3-3", y0);
         if( apply_filter)
             multistep.init( shu, filter, time, y0, dt);
         else
@@ -118,9 +118,9 @@ int main( int argc, char* argv[])
 
         return -1;
     }
-    unsigned maxout = file::get( mode, js, "output", "maxout", 100).asUInt();
-    unsigned itstp = file::get( mode, js, "output", "itstp", 5).asUInt();
-    std::string output = file::get( mode, js, "output", "type", "glfw").asString();
+    unsigned maxout = dg::file::get( mode, js, "output", "maxout", 100).asUInt();
+    unsigned itstp = dg::file::get( mode, js, "output", "itstp", 5).asUInt();
+    std::string output = dg::file::get( mode, js, "output", "type", "glfw").asString();
 #ifndef WITHOUT_GLFW
     if( "glfw" == output)
     {
@@ -188,7 +188,7 @@ int main( int argc, char* argv[])
         else
             outputfile = argv[2];
         /// //////////////////////set up netcdf/////////////////////////////////////
-        file::NC_Error_Handle err;
+        dg::file::NC_Error_Handle err;
         int ncid=-1;
         try{
             err = nc_create( outputfile.c_str(),NC_NETCDF4|NC_CLOBBER, &ncid);
@@ -221,7 +221,7 @@ int main( int argc, char* argv[])
 
         int dim_ids[3], tvarID;
         std::map<std::string, int> id1d, id3d;
-        err = file::define_dimensions( ncid, dim_ids, &tvarID, grid,
+        err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, grid,
                 {"time", "y", "x"});
 
         //Create field IDs
@@ -260,7 +260,7 @@ int main( int argc, char* argv[])
             err = nc_enddef(ncid);
             record.function( resultD, var);
             dg::assign( resultD, resultH);
-            file::put_var_double( ncid, staticID, grid, resultH);
+            dg::file::put_var_double( ncid, staticID, grid, resultH);
             err = nc_redef(ncid);
         }
         err = nc_enddef(ncid);
@@ -271,7 +271,7 @@ int main( int argc, char* argv[])
         {
             record.function( resultD, var);
             dg::assign( resultD, resultH);
-            file::put_vara_double( ncid, id3d.at(record.name), start[0], grid, resultH);
+            dg::file::put_vara_double( ncid, id3d.at(record.name), start[0], grid, resultH);
         }
         for( auto& record : shu::diagnostics1d_list)
         {
@@ -315,7 +315,7 @@ int main( int argc, char* argv[])
             {
                 record.function( resultD, var);
                 dg::assign( resultD, resultH);
-                file::put_vara_double( ncid, id3d.at(record.name), start[0], grid, resultH);
+                dg::file::put_vara_double( ncid, id3d.at(record.name), start[0], grid, resultH);
             }
             for( auto& record : shu::diagnostics1d_list)
             {
-- 
GitLab


From e4c2b220056200f559d9aa7505e6ff6518ec83f9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Feb 2021 16:41:31 +0100
Subject: [PATCH 449/540] Fix missing allocation in ImExMultistep

---
 inc/dg/multistep.h         | 67 +++++++++++++++++++-------------------
 inc/dg/multistep_tableau.h |  2 +-
 2 files changed, 35 insertions(+), 34 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 6512ee843..45c77c027 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -113,6 +113,7 @@ struct ImExMultistep
 
         m_u.assign( m_t.steps(), m_solver.copyable());
         m_ex.assign( m_t.steps(), m_solver.copyable());
+        m_tmp = m_solver.copyable();
         m_counter = 0;
     }
     /**
@@ -170,39 +171,6 @@ struct ImExMultistep
     unsigned m_counter; //counts how often step has been called after init
 };
 
-
-/** @brief Deprecated  (use ImExMultistep and select "Karniadakis" from the multistep tableaus)
-* @ingroup time
-* @sa dg::ImExMultistep
-*/
-template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
-struct Karniadakis
-{
-    using value_type = get_value_type<ContainerType>;
-    using container_type = ContainerType;
-    Karniadakis(){}
-    template<class ...SolverParams>
-    Karniadakis( SolverParams&& ...ps): m_imex( "Karniadakis", std::forward<SolverParams> (ps)...) { }
-    template<class ...Params>
-    void construct( Params&& ...ps)
-    {
-        *this = Karniadakis( std::forward<Params>( ps)...);
-    }
-    const ContainerType& copyable()const{ return m_imex.copyable();}
-    SolverType& solver() { return m_imex.solver();}
-    const SolverType& solver() const { return m_imex.solver();}
-    template< class Explicit, class Implicit>
-    void init( Explicit& ex, Implicit& im, value_type t0, const ContainerType& u0, value_type dt){
-        m_imex.init( ex, im, t0, u0, dt);
-    }
-    template< class Explicit, class Implicit>
-    void step( Explicit& ex, Implicit& im, value_type& t, ContainerType& u){
-        m_imex.step( ex, im, t, u);
-    }
-  private:
-    ImExMultistep<ContainerType, SolverType> m_imex;
-};
-
 ///@cond
 template< class ContainerType, class SolverType>
 template< class RHS, class Diffusion>
@@ -277,6 +245,39 @@ void ImExMultistep<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, va
 }
 ///@endcond
 
+/** @brief Deprecated  (use ImExMultistep and select "Karniadakis" from the multistep tableaus)
+* @ingroup time
+* @sa dg::ImExMultistep
+*/
+template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
+struct Karniadakis
+{
+    using value_type = get_value_type<ContainerType>;
+    using container_type = ContainerType;
+    Karniadakis(){}
+    template<class ...SolverParams>
+    Karniadakis( SolverParams&& ...ps): m_imex( "Karniadakis", std::forward<SolverParams> (ps)...) { }
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        *this = Karniadakis( std::forward<Params>( ps)...);
+    }
+    const ContainerType& copyable()const{ return m_imex.copyable();}
+    SolverType& solver() { return m_imex.solver();}
+    const SolverType& solver() const { return m_imex.solver();}
+    template< class Explicit, class Implicit>
+    void init( Explicit& ex, Implicit& im, value_type t0, const ContainerType& u0, value_type dt){
+        m_imex.init( ex, im, t0, u0, dt);
+    }
+    template< class Explicit, class Implicit>
+    void step( Explicit& ex, Implicit& im, value_type& t, ContainerType& u){
+        m_imex.step( ex, im, t, u);
+    }
+  private:
+    ImExMultistep<ContainerType, SolverType> m_imex;
+};
+
+
 /**
 * @brief Implicit multistep time-integration with Limiter/Filter
 * \f[
diff --git a/inc/dg/multistep_tableau.h b/inc/dg/multistep_tableau.h
index f07c36339..afe8bd375 100644
--- a/inc/dg/multistep_tableau.h
+++ b/inc/dg/multistep_tableau.h
@@ -571,7 +571,7 @@ MultistepTableau<real_type> lmstableau( std::string name)
  *   -------|------------| -----------
  *   ImEx-Euler-1-1         | dg::IMEX_EULER_1_1 | Explicit Euler combined with Implicit Euler
     ImEx-Adams-X-X | dg::IMEX_ADAMS_X_X | <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> @note **X can be 2 (C=0.44) or 3 (C=0.16)**
-    ImEx-BDF_X_X | dg::IMEX_BDF_X_X | The family of schems described in <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> <br>The implicit part is a normal BDF scheme https://en.wikipedia.org/wiki/Backward_differentiation_formula while the explicit part equals the Minimal Projecting method by <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a> or **extrapolated BDF** in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S. J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> <br> @note Possible values for **X: 1 ( C=1), 2 (C=0.63), 3 (C=0.39), 4 (C=0.22), 5( C=0.09), * 6, 7** <br> Note that X=3 is identical to the "Karniadakis" scheme
+    ImEx-BDF-X-X | dg::IMEX_BDF_X_X | The family of schems described in <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> <br>The implicit part is a normal BDF scheme https://en.wikipedia.org/wiki/Backward_differentiation_formula while the explicit part equals the Minimal Projecting method by <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a> or **extrapolated BDF** in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S. J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> <br> @note Possible values for **X: 1 ( C=1), 2 (C=0.63), 3 (C=0.39), 4 (C=0.22), 5( C=0.09), * 6, 7** <br> Note that X=3 is identical to the "Karniadakis" scheme
     * Karniadakis | dg::IMEX_BDF_3_3 | The ImEx-BDF-3-3 scheme is identical to the widely used "Karniadakis" scheme <a href = "https://dx.doi.org/10.1016/0021-9991(91)90007-8"> Karniadakis, et al. J. Comput. Phys. 97 (1991)</a>
     ImEx-TVB-X-X | dg::IMEX_TVB_X_X | The family of schems described in < <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> <br> The explicit part is a TVB scheme while the implicit part is optimized to maximize damping of high wavelength <br> @note Possible values for **X: 3 (C=0.54), 4 (C=0.458), 5 (C=0.376)**
     *
-- 
GitLab


From a92453c5552fb39fe71e786a4a061950a9f4fcf9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Feb 2021 23:08:34 +0100
Subject: [PATCH 450/540] Improvements to Multistep

- fix ImEx Adams schemes
- remove BDF-7 methods because they are not zero stable
- add new koto-2-2 method with a supposedly large stability region
---
 inc/dg/multistep.h         |  7 ++-
 inc/dg/multistep_t.cu      |  7 +--
 inc/dg/multistep_tableau.h | 88 ++++++++++++++++++++++----------------
 3 files changed, 58 insertions(+), 44 deletions(-)

diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 45c77c027..61cd3cf4e 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -295,7 +295,7 @@ struct Karniadakis
     You can use your own coefficients defined as a \c dg::MultistepTableau
     or use one of the predefined coefficients in
     @copydoc hide_implicit_multistep_tableaus
-    and
+    and (any imex tableau can be used in an implicit scheme, disregarding the explicit coefficients)
     @copydoc hide_imex_multistep_tableaus
 *
 * The necessary Inversion in the implicit part is provided by the \c SolverType class.
@@ -473,7 +473,7 @@ void FilteredImplicitMultistep<ContainerType, SolverType>::step(RHS& rhs, Limite
     You can use your own coefficients defined as a \c dg::MultistepTableau
     or use one of the predefined coefficients in
     @copydoc hide_implicit_multistep_tableaus
-    and
+    and (any imex tableau can be used in an implicit scheme, disregarding the explicit coefficients)
     @copydoc hide_imex_multistep_tableaus
 *
 * The necessary Inversion in the implicit part is provided by the \c SolverType class.
@@ -590,8 +590,7 @@ struct FilteredExplicitMultistep
      * @brief Reserve memory for the integration
      *
      * Set the coefficients \f$ a_i,\ b_i\f$
-     * @param method the name of the family of schemes to be used (a string can be converted to an enum with the same spelling) @sa multistep_identifier
-     * @param stages (global) stages (= number of steps in the multistep) of the method (Currently possible values depend on the method), does not necessarily coincide with the order of the method
+     * @param tableau Tableau, name or identifier that \c ConvertsToMultistepTableau
      * @param copyable ContainerType of the size that is used in \c step
      * @note it does not matter what values \c copyable contains, but its size is important
      */
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index bab30129a..4ba10b1c6 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -110,7 +110,7 @@ int main()
     std::cout << "### Test Explicit Multistep methods with "<<NT<<" steps\n";
     std::vector<std::string> ex_names{
     "AB-1-1", "AB-2-2", "AB-3-3", "AB-4-4", "AB-5-5",
-    "eBDF-1-1", "eBDF-2-2", "eBDF-3-3", "eBDF-4-4", "eBDF-5-5", "eBDF-6-6", "eBDF-7-7",
+    "eBDF-1-1", "eBDF-2-2", "eBDF-3-3", "eBDF-4-4", "eBDF-5-5", "eBDF-6-6",
     "TVB-1-1", "TVB-2-2", "TVB-3-3", "TVB-4-4", "TVB-5-5", "TVB-6-6",
     "SSP-1-1", "SSP-2-2", "SSP-3-2", "SSP-4-2", "SSP-5-3", "SSP-6-3",
     };
@@ -128,8 +128,8 @@ int main()
     }
     std::cout << "### Test implicit multistep methods with "<<NT<<" steps\n";
     std::vector<std::string> imex_names{
-    "Euler","ImEx-Adams-2-2", "ImEx-Adams-3-3", "ImEx-BDF-2-2",
-    "ImEx-BDF-3-3", "ImEx-BDF-4-4", "ImEx-BDF-5-5", "ImEx-BDF-6-6", "ImEx-BDF-7-7",
+    "Euler", "ImEx-Koto-2-2", "ImEx-Adams-2-2", "ImEx-Adams-3-3", "ImEx-BDF-2-2",
+    "ImEx-BDF-3-3", "ImEx-BDF-4-4", "ImEx-BDF-5-5", "ImEx-BDF-6-6",
     "ImEx-TVB-3-3", "ImEx-TVB-4-4", "ImEx-TVB-5-5",
     };
     for( auto name : imex_names)
@@ -144,6 +144,7 @@ int main()
         res.d = sqrt(dg::blas1::dot( y0, y0)/norm_sol);
         std::cout << "Relative error: "<<std::setw(20) <<name<<"\t"<< res.d<<"\t"<<res.i<<std::endl;
     }
+    std::cout << "### Test ImEx multistep methods with "<<NT<<" steps\n";
     Explicit ex( nu);
     Implicit im( nu);
     // Test Semi-Implicit methods
diff --git a/inc/dg/multistep_tableau.h b/inc/dg/multistep_tableau.h
index afe8bd375..d1fc67ba9 100644
--- a/inc/dg/multistep_tableau.h
+++ b/inc/dg/multistep_tableau.h
@@ -120,7 +120,7 @@ MultistepTableau<real_type> imex_adams_2_2()
     b[0] =  3./2.;
     b[1] = -1./2.;
     c[0] = 9./16.;
-    c[1] = -3./8.;
+    c[1] = 3./8.;
     c[2] = 1./16.;
     return MultistepTableau<real_type>( steps, order, a, b, c);
 }
@@ -136,11 +136,34 @@ MultistepTableau<real_type> imex_adams_3_3()
     b[1] = -4./3.;
     b[2] = 5./12.;
     c[0] =   4661./10000.;
-    c[1] = -15551./30000.;
+    c[1] =  15551./30000.;
     c[2] =     1949/30000;
     c[3] = -1483./30000.;
     return MultistepTableau<real_type>( steps, order, a, b, c);
 }
+template<class real_type>
+MultistepTableau<real_type> imex_koto_2_2()
+{
+    // stabilized 2nd order method
+    unsigned steps = 2, order = 2;
+    std::vector<real_type> am(steps,0), bm(steps, 0), cm(steps+1,0);
+    std::vector<real_type> ap(steps,0), bp(steps, 0), cp(steps+1,0);
+    //real_type a = 1.5, b = 1.5;
+    real_type a = 20., b = 20.;
+    ap[0] = a;
+    ap[1] = 1-2.*a;
+    ap[2] = a-1;
+    cp[0] =  b;
+    cp[1] = 0.5+a-2*b;
+    cp[2] = 0.5-a+b;
+    bp[1] = 0.5+a;
+    bp[2] = 0.5-a;
+    am[0] = -ap[1]/a, am[1] = -ap[2]/a;
+    bm[0] = bp[1]/a, bm[1] = bp[2]/a;
+    cm[0] = cp[0]/a, cm[1] = cp[1]/a, cm[2] = cp[2]/a;
+    return MultistepTableau<real_type>( steps, order, am, bm, cm);
+}
+
 template<class real_type>
 MultistepTableau<real_type> imex_bdf(unsigned steps)
 {
@@ -184,11 +207,6 @@ MultistepTableau<real_type> imex_bdf(unsigned steps)
         b = {360./147.,-900./147.,1200./147.,-900./147.,360./147.,-60./147.};
         c[0] = 60./147.;
         break;
-        case (7):
-        a = { 2940./1089.,-4410./1089.,4900./1089.,-3675./1089.,1764./1089.,-490./1089.,60./1089.};
-        b = { 2940./1089.,-8820./1089.,14700./1089.,-14700./1089.,8820./1089.,-2940./1089.,420./1089.};
-        c[0] = 420./1089.;
-        break;
     }
     return MultistepTableau<real_type>( steps, order, a, b, c);
 }
@@ -347,12 +365,12 @@ enum multistep_identifier{
     IMEX_EULER_1_1,
     IMEX_ADAMS_2_2,
     IMEX_ADAMS_3_3,
+    IMEX_KOTO_2_2,
     IMEX_BDF_2_2,
     IMEX_BDF_3_3,
     IMEX_BDF_4_4,
     IMEX_BDF_5_5,
     IMEX_BDF_6_6,
-    IMEX_BDF_7_7,
     IMEX_TVB_3_3,
     IMEX_TVB_4_4,
     IMEX_TVB_5_5,
@@ -368,7 +386,6 @@ enum multistep_identifier{
     eBDF_4_4,
     eBDF_5_5,
     eBDF_6_6,
-    eBDF_7_7,
     TVB_1_1,
     TVB_2_2,
     TVB_3_3,
@@ -382,14 +399,12 @@ enum multistep_identifier{
     SSP_5_3,
     SSP_6_3,
     // implicit methods
-    BDF_1_1, //!<
-    BDF_2_2, //!<
-    BDF_3_3, //!<
-    BDF_4_4, //!<
-    BDF_5_5, //!<
-    BDF_6_6, //!<
-    BDF_7_7, //!<
-
+    BDF_1_1,
+    BDF_2_2,
+    BDF_3_3,
+    BDF_4_4,
+    BDF_5_5,
+    BDF_6_6,
 };
 
 ///@cond
@@ -401,13 +416,13 @@ static std::unordered_map<std::string, enum multistep_identifier> str2lmsid{
     {"Euler-1-1", IMEX_EULER_1_1},
     {"ImEx-Adams-2-2", IMEX_ADAMS_2_2},
     {"ImEx-Adams-3-3", IMEX_ADAMS_3_3},
+    {"ImEx-Koto-2-2", IMEX_KOTO_2_2},
     {"ImEx-BDF-2-2", IMEX_BDF_2_2},
     {"ImEx-BDF-3-3", IMEX_BDF_3_3},
     {"Karniadakis",  IMEX_BDF_3_3},
     {"ImEx-BDF-4-4", IMEX_BDF_4_4},
     {"ImEx-BDF-5-5", IMEX_BDF_5_5},
     {"ImEx-BDF-6-6", IMEX_BDF_6_6},
-    {"ImEx-BDF-7-7", IMEX_BDF_7_7},
     {"ImEx-TVB-3-3", IMEX_TVB_3_3},
     {"ImEx-TVB-4-4", IMEX_TVB_4_4},
     {"ImEx-TVB-5-5", IMEX_TVB_5_5},
@@ -423,7 +438,6 @@ static std::unordered_map<std::string, enum multistep_identifier> str2lmsid{
     {"eBDF-4-4", eBDF_4_4},
     {"eBDF-5-5", eBDF_5_5},
     {"eBDF-6-6", eBDF_6_6},
-    {"eBDF-7-7", eBDF_7_7},
     {"TVB-1-1", TVB_1_1},
     {"TVB-2-2", TVB_2_2},
     {"TVB-3-3", TVB_3_3},
@@ -443,7 +457,6 @@ static std::unordered_map<std::string, enum multistep_identifier> str2lmsid{
     {"BDF-4-4", BDF_4_4},
     {"BDF-5-5", BDF_5_5},
     {"BDF-6-6", BDF_6_6},
-    {"BDF-7-7", BDF_7_7}
 };
 enum multistep_identifier str2lmstableau( std::string name)
 {
@@ -472,6 +485,8 @@ MultistepTableau<real_type> lmstableau( enum multistep_identifier id)
             return dg::tableau::imex_adams_2_2<real_type>();
         case IMEX_ADAMS_3_3:
             return dg::tableau::imex_adams_3_3<real_type>();
+        case IMEX_KOTO_2_2:
+            return dg::tableau::imex_koto_2_2<real_type>();
         case IMEX_BDF_2_2:
             return dg::tableau::imex_bdf<real_type>(2);
         case IMEX_BDF_3_3:
@@ -482,8 +497,6 @@ MultistepTableau<real_type> lmstableau( enum multistep_identifier id)
             return dg::tableau::imex_bdf<real_type>(5);
         case IMEX_BDF_6_6:
             return dg::tableau::imex_bdf<real_type>(6);
-        case IMEX_BDF_7_7:
-            return dg::tableau::imex_bdf<real_type>(7);
         case IMEX_TVB_3_3:
             return dg::tableau::imex_tvb<real_type>(3);
         case IMEX_TVB_4_4:
@@ -501,7 +514,7 @@ MultistepTableau<real_type> lmstableau( enum multistep_identifier id)
         case AB_5_5:
             return dg::tableau::ab<real_type>(5);
         case eBDF_1_1:
-            return dg::tableau::imex_bdf<real_type>(1);
+            return dg::tableau::imex_euler_1_1<real_type>();
         case eBDF_2_2:
             return dg::tableau::imex_bdf<real_type>(2);
         case eBDF_3_3:
@@ -512,10 +525,8 @@ MultistepTableau<real_type> lmstableau( enum multistep_identifier id)
             return dg::tableau::imex_bdf<real_type>(5);
         case eBDF_6_6:
             return dg::tableau::imex_bdf<real_type>(6);
-        case eBDF_7_7:
-            return dg::tableau::imex_bdf<real_type>(7);
         case TVB_1_1:
-            return dg::tableau::tvb<real_type>(1);
+            return dg::tableau::imex_euler_1_1<real_type>();
         case TVB_2_2:
             return dg::tableau::tvb<real_type>(2);
         case TVB_3_3:
@@ -550,8 +561,6 @@ MultistepTableau<real_type> lmstableau( enum multistep_identifier id)
             return dg::tableau::imex_bdf<real_type>(5);
         case BDF_6_6:
             return dg::tableau::imex_bdf<real_type>(6);
-        case BDF_7_7:
-            return dg::tableau::imex_bdf<real_type>(7);
     }
     return MultistepTableau<real_type>(); //avoid compiler warning
 }
@@ -570,12 +579,15 @@ MultistepTableau<real_type> lmstableau( std::string name)
  *    Name  | Identifier | Description
  *   -------|------------| -----------
  *   ImEx-Euler-1-1         | dg::IMEX_EULER_1_1 | Explicit Euler combined with Implicit Euler
-    ImEx-Adams-X-X | dg::IMEX_ADAMS_X_X | <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> @note **X can be 2 (C=0.44) or 3 (C=0.16)**
-    ImEx-BDF-X-X | dg::IMEX_BDF_X_X | The family of schems described in <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> <br>The implicit part is a normal BDF scheme https://en.wikipedia.org/wiki/Backward_differentiation_formula while the explicit part equals the Minimal Projecting method by <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a> or **extrapolated BDF** in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S. J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> <br> @note Possible values for **X: 1 ( C=1), 2 (C=0.63), 3 (C=0.39), 4 (C=0.22), 5( C=0.09), * 6, 7** <br> Note that X=3 is identical to the "Karniadakis" scheme
+ *   Euler                  | dg::IMEX_EULER_1_1 | For convenience
+    ImEx-Koto-2-2 | dg::IMEX_KOTO_2_2 | <a href="https://dx.doi.org/10.1007/s11464-009-0005-9">Koto T. Front. Math. China 2009, 4(1): 113-129</a> A stabilized 2nd order scheme with a large region of stability
+    ImEx-Adams-X-X | dg::IMEX_ADAMS_X_X | <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> @note **Possible values for X: 2 (C=0.44), 3 (C=0.16)**
+    ImEx-BDF-X-X | dg::IMEX_BDF_X_X | The family of schems described in <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> <br>The implicit part is a normal BDF scheme https://en.wikipedia.org/wiki/Backward_differentiation_formula while the explicit part equals the Minimal Projecting method by <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a> or **extrapolated BDF** in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S. J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> <br> @note Possible values for **X: 1 (C=1.00), 2 (C=0.63), 3 (C=0.39), 4 (C=0.22), 5 (C=0.09), 6** <br> Note that X=3 is identical to the "Karniadakis" scheme
     * Karniadakis | dg::IMEX_BDF_3_3 | The ImEx-BDF-3-3 scheme is identical to the widely used "Karniadakis" scheme <a href = "https://dx.doi.org/10.1016/0021-9991(91)90007-8"> Karniadakis, et al. J. Comput. Phys. 97 (1991)</a>
-    ImEx-TVB-X-X | dg::IMEX_TVB_X_X | The family of schems described in < <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> <br> The explicit part is a TVB scheme while the implicit part is optimized to maximize damping of high wavelength <br> @note Possible values for **X: 3 (C=0.54), 4 (C=0.458), 5 (C=0.376)**
+    ImEx-TVB-X-X | dg::IMEX_TVB_X_X | The family of schems described in < <a href="https://dx.doi.org/10.1016/j.jcp.2007.03.003">Hundsdorfer and Ruuth, Journal of Computational Physics 225 (2007)</a> <br> The explicit part is a TVB scheme while the implicit part is optimized to maximize damping of high wavelength <br> @note Possible values for **X: 3 (C=0.54), 4 (C=0.46), 5 (C=0.38)**
     *
- @note the CFL coefficient C is given relative to the forward Euler method: \f$ \Delta t < C \Delta_{FE}\f$.
+ @note the CFL coefficient C is given relative to the forward Euler method: \f$ \Delta t < C \Delta t_{FE}\f$.
+ @attention The coefficient C is the one that ensures the TVD property of the scheme and is **not** directly related to the stability region of the scheme
  */
 
 /*! @class hide_explicit_multistep_tableaus
@@ -583,19 +595,21 @@ MultistepTableau<real_type> lmstableau( std::string name)
  *    Name  | Identifier | Description
  *   -------|------------| -----------
  *   AB-X-X | dg::AB_X_X | The family of schemes described in <a href = "https://en.wikipedia.org/wiki/Linear_multistep_method"> Linear multistep methods </a> as **Adams-Bashforth** \f[ u^{n+1} = u^n + \Delta t\sum_{j=0}^{s-1} b_j f\left(t^n - j \Delta t, u^{n-j}\right) \f] @note **Possible stages are X: 1, 2,..., 5**, the order of the method is the same as its stages @note The Adams-Bashforth schemes implemented here need less storage but may have **a smaller region of absolute stability** than for example an extrapolated BDF method of the same order.
- * eBDF-X-X | dg::eBDF_X_X | The family of schemes described in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S.  J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> as **extrapolated BDF**  where it is found to be TVB (**total variation bound**). The schemes also appear as **Minimal Projecting** scheme described in <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a> * <br> @note **Possible stages are X: 1 (C=1), 2, 3 (C=0.39), 4, 5, 6, 7** with the order the same as the number of stages
- * TVB-X-X | dg::TVB_X_X | The family of schemes described in <a href="https://doi.org/10.1016/j.jcp.2005.02.029">S.J. Ruuth and W. Hundsdorfer, High-order linear multistep methods with general monotonicity and boundedness properties, Journal of Computational Physics, Volume 209, Issue 1, 2005 </a> as Total variation Bound. These schemes have **larger stable step sizes than the eBDF family**, <br> @note **Possible values for X are 1 (C=1),2 (C=0.5), 3 (C=0.54), 4 (C=0.46), 5 (C=0.38) 6 (C=0.33)**. We highlight that TVB-3-3 has 38% larger stepsize than eBDF-3-3 and TVB-4-4 has 109% larger stepsize than eBDF-4-4.
- * SSP-X-Y | dg::SSP_X_Y | The family of schemes described in <a href="https://doi.org/10.1007/BF02728985">Gottlieb, S. On high order strong stability preserving runge-kutta and multi step time discretizations. J Sci Comput 25, 105–128 (2005)</a> as Strong Stability preserving. We implement the lowest order schemes for each stage and disregard the remaining schemes in the paper since their CFL conditions are worse than the TVB scheme of the same order.  @note Possible values for **X-Y : 1-1 (C=1), 2-2 (C=0.5), 3-2 (C=0.5), 4-2 (C=0.66), 5-3: (C=0.5), 6-3 (C=0.567)**.@note These schemes are noteworthy because the coefficients b_i are all positive except for the 2-2 method and **the "4-2" and "6-3" methods allow slightly larger stepsize but increased storage requirements than TVB** of same order (2 and 3).
+ * eBDF-X-X | dg::eBDF_X_X | The family of schemes described in <a href = "https://doi.org/10.1137/S0036142902406326"> Hundsdorfer, W., Ruuth, S.  J., & Spiteri, R. J. (2003). Monotonicity-preserving linear multistep methods. SIAM Journal on Numerical Analysis, 41(2), 605-623 </a> as **extrapolated BDF**  where it is found to be TVB (**total variation bound**). The schemes also appear as **Minimal Projecting** scheme described in <a href = "https://www.ams.org/journals/mcom/1979-33-148/S0025-5718-1979-0537965-0/S0025-5718-1979-0537965-0.pdf"> Alfeld, P., Math. Comput. 33.148 1195-1212 (1979)</a> <br> @note **Possible stages are X: 1 (C=1), 2 (C=0.63), 3 (C=0.39), 4 (C=0.22), 5 (C=0.09), 6** with the order the same as the number of stages
+ * TVB-X-X | dg::TVB_X_X | The family of schemes described in <a href="https://doi.org/10.1016/j.jcp.2005.02.029">S.J. Ruuth and W. Hundsdorfer, High-order linear multistep methods with general monotonicity and boundedness properties, Journal of Computational Physics, Volume 209, Issue 1, 2005 </a> as Total variation Bound. These schemes have larger allowable step sizes than the eBDF family, <br> @note **Possible values for X are 1 (C=1), 2 (C=0.5), 3 (C=0.54), 4 (C=0.46), 5 (C=0.38) 6 (C=0.33)**. We highlight that TVB-3-3 has 38% larger allowable stepsize than eBDF-3-3 and TVB-4-4 has 109% larger stepsize than eBDF-4-4 (to ensure the TVB property, not stability).
+ * SSP-X-Y | dg::SSP_X_Y | The family of schemes described in <a href="https://doi.org/10.1007/BF02728985">Gottlieb, S. On high order strong stability preserving runge-kutta and multi step time discretizations. J Sci Comput 25, 105–128 (2005)</a> as Strong Stability preserving. We implement the lowest order schemes for each stage and disregard the remaining schemes in the paper since their CFL conditions are worse than the TVB scheme of the same order.  @note **Possible values for X-Y : 1-1 (C=1), 2-2 (C=0.5), 3-2 (C=0.5), 4-2 (C=0.66), 5-3 (C=0.5), 6-3 (C=0.567)**.@note These schemes are noteworthy because the coefficients b_i are all positive except for the 2-2 method and **the "4-2" and "6-3" methods allow slightly larger allowable stepsize but increased storage requirements than TVB** of same order (2 and 3).
 
  *@note Total variation bound (TVB) means \f$ || v^n|| \leq M ||v^0||\f$  where the norm signifies the total variation semi-norm. Total variation diminishing
-     (TVD) means M=1, and strong stability preserving (SSP) is the same as TVD, TVB schemes converge to the correct entropy solutions of hyperbolic conservation laws)
+     (TVD) means M=1, and strong stability preserving (SSP) is the same as TVD, TVB schemes converge to the correct entropy solutions of hyperbolic conservation laws
+ @note the CFL coefficient C is given relative to the forward Euler method: \f$ \Delta t < C \Delta t_{FE}\f$.
+ @attention The coefficient C is the one that ensures the TVD property of the scheme and is **not** directly related to the stability region of the scheme
  */
 
 /*! @class hide_implicit_multistep_tableaus
  *
  *    Name  | Identifier | Description
  *   -------|------------| -----------
- *   BDF-X-X | dg::BDF_X_X | The coefficients for backward differences can be found at https://en.wikipedia.org/wiki/Backward_differentiation_formula <br> @note A BDF scheme is simply constructed by discretizing the time derivative with a n-th order backward difference formula and evaluating the right hand side at the new timestep
+ *   BDF-X-X | dg::BDF_X_X | The coefficients for backward differences can be found at https://en.wikipedia.org/wiki/Backward_differentiation_formula <br> @note **Possible values for X: 1, 2, 3, 4, 5, 6** @note A BDF scheme is simply constructed by discretizing the time derivative with a n-th order backward difference formula and evaluating the right hand side at the new timestep. @note Methods with s>6 are not zero-stable so they cannot be used
  *
 */
 
-- 
GitLab


From 58297f56cdb6d4533b31d1f57e3ddfc3e577aa6a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 2 Feb 2021 23:10:44 +0100
Subject: [PATCH 451/540] Adapt Shu code and documentation

- new timesteppers
- ability to change to forward Laplace discretization
- viscosity possible also in explicit schemes
---
 src/lamb_dipole/shu.cuh  | 41 ++++++++++++++++++++++++++++++++++++----
 src/lamb_dipole/shu.tex  | 32 +++++++++++++++++--------------
 src/lamb_dipole/shu_b.cu | 29 ++++++++++++++--------------
 3 files changed, 69 insertions(+), 33 deletions(-)

diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index f79049d53..20b047a04 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -27,13 +27,17 @@ struct Diffusion
 {
     Diffusion( const Geometry& g, Json::Value& js, enum dg::file::error mode)
     {
+        //only allocate if needed
         std::string regularization = dg::file::get( mode, js, "regularization", "type", "modal").asString();
-        if( "viscosity" == regularization)
+        std::string timestepper = dg::file::get( mode, js, "timestepper", "type", "FilteredExplicitMultistep").asString();
+        if( "ImExMultistep" == timestepper &&  "viscosity" == regularization)
         {
-            m_nu = dg::file::get( mode, js, "regularization", "nu_perp", 1e-3).asDouble();
+            m_nu = dg::file::get( mode, js, "regularization", "nu", 1e-3).asDouble();
             m_order = dg::file::get( mode, js, "regularization", "order", 1).asUInt();
             m_temp = dg::evaluate( dg::zero, g);
-            m_LaplacianM.construct( g, dg::normed);
+            enum dg::direction dir = dg::str2direction( dg::file::get( mode, js,
+                        "regularization", "direction", "centered").asString());
+            m_LaplacianM.construct( g, dg::normed, dir, 1);
         }
     }
     void operator()(double t, const Container& x, Container& y)
@@ -86,6 +90,7 @@ struct Shu
   private:
     Container m_psi, m_v, m_temp[3], m_fine_psi, m_fine_v, m_fine_temp[3], m_fine_y, m_fine_yp;
     std::vector<dg::Elliptic<Geometry, Matrix, Container>> m_multi_laplaceM;
+    dg::Elliptic<Geometry, Matrix,Container> m_LaplacianM;
     dg::ArakawaX<Geometry, Matrix, Container> m_arakawa;
     dg::Extrapolation<Container> m_old_psi;
     dg::MultigridCG2d<Geometry, Matrix, Container> m_multigrid;
@@ -99,6 +104,9 @@ struct Shu
     shu::MMSSource m_mms;
     bool m_add_mms = false;
     Container m_x, m_y;
+    double m_nu; // for Diffusion
+    unsigned m_order; // for Diffusion
+    bool m_add_viscosity = false;
 };
 
 template<class Geometry, class IMatrix, class Matrix, class Container>
@@ -163,11 +171,23 @@ Shu< Geometry, IMatrix, Matrix, Container>::Shu(
     }
     //this is a hidden parameter
     //note that only centered works with double periodic boundary conditions
-    enum dg::direction dir = dg::str2direction( dg::file::get( dg::file::error::is_warning, js,
+    enum dg::direction dir = dg::str2direction( dg::file::get( mode, js,
                 "elliptic", "direction", "centered").asString());
     m_multi_laplaceM.resize(stages);
     for( unsigned u=0; u<stages; u++)
         m_multi_laplaceM[u].construct( m_multigrid.grid(u), dg::not_normed, dir, 1);
+    // explicit Diffusion term
+    std::string regularization = dg::file::get( mode, js, "regularization", "type", "modal").asString();
+    std::string timestepper = dg::file::get( mode, js, "timestepper", "type", "FilteredExplicitMultistep").asString();
+    if( !("ImExMultistep" == timestepper) &&  "viscosity" == regularization)
+    {
+        m_nu = dg::file::get( mode, js, "regularization", "nu", 1e-3).asDouble();
+        m_order = dg::file::get( mode, js, "regularization", "order", 1).asUInt();
+        enum dg::direction dir = dg::str2direction( dg::file::get( mode, js,
+                    "regularization", "direction", "centered").asString());
+        m_LaplacianM.construct( g, dg::normed, dir, 1);
+        m_add_viscosity = true;
+    }
 }
 
 template< class Geometry, class IMatrix, class Matrix, class Container>
@@ -249,6 +269,19 @@ void Shu<Geometry, IMatrix, Matrix, Container>::operator()(double t, const Conta
     }
     if( m_add_mms) //for the manufactured solution we need to add a source term
         dg::blas1::evaluate( yp, dg::plus_equals(), m_mms, m_x, m_y, t);
+    if( m_add_viscosity)
+    {
+        dg::blas1::copy( y, m_temp[1]);
+        for( unsigned p=0; p<m_order; p++)
+        {
+            using std::swap;
+            swap( m_temp[0], m_temp[1]);
+            dg::blas2::symv( m_nu, m_LaplacianM, m_temp[0], 0., m_temp[1]);
+        }
+        dg::blas1::axpby( -1., m_temp[1], 1., yp);
+    }
+
+
 
 }
 
diff --git a/src/lamb_dipole/shu.tex b/src/lamb_dipole/shu.tex
index 878829a0c..91a9cd911 100644
--- a/src/lamb_dipole/shu.tex
+++ b/src/lamb_dipole/shu.tex
@@ -183,25 +183,27 @@ grid & dict & & \\
 \bottomrule
 \end{longtable}
 \subsection{Time steppers}
-Possible time-steppers are
-\begin{longtable}{llll}
+Possible time-steppers are the explicit and semi-implicit multistep schemes
+as well as the Shu-Osher scheme that originally incorporated limiters in the dG scheme.
+\begin{longtable}{lllp{6cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
 timestepper & dict & & \\
-\qquad stepper  & string& Karniadakis & The \textit{dg::Karniadakis} class \\
-\qquad order    & float & 3 & Order of Karniadakis (1,2 or 3)\\
+\qquad type     & string& ImExMultistep & The semi-implicit multistep scheme (only in combination with viscosity regularization) \\
+\qquad tableau  & string & Any ImEx tableau & for example ImEx-BDF-3-3* \\
 \qquad dt       & float & 2e-3 & Fixed timestep \\
 \qquad eps\_time & float & 1e-9 & Accuracy requirement for implicit solver \\
 timestepper & dict & & \\
-\qquad stepper & string & Shu-Osher & The \textit{dg::ShuOsher} class \\
-\qquad order   & string & SSPRK-3-3 & Check dg documentation for possible values \\
+\qquad type & string & Shu-Osher & An explicit Runge Kutta method with filter (viscosity is treated explicitly) \\
+\qquad tableau   & string & Any Shu-Osher tableau & for example SSPRK-3-3* \\
 \qquad dt      & float & 1e-3 & Fixed Time-step \\
 timestepper & dict & & \\
-\qquad stepper & string & FilteredMultistep & The \textit{dg::FilteredExplicitMultistep} class \\
-\qquad order   & [string, integer] & [eBDF,3] & Check dg documentation for possible values \\
+\qquad type & string & FilteredExplicitMultistep & an explicit multistep class with the option to use a filter (viscosity is treated explicitly)\\
+\qquad tableau   & string & Any explicit multistep tableau & for example eBDF-3-3* \\
 \qquad dt      & float & 2e-3 & Fixed timestep \\
 \bottomrule
 \end{longtable}
+*See the dg documentation for what tableaus are available.
 \subsection{Regularization technique}
 Choose either no regularization or artificial viscosity or modal filtering by the following
 parameters in the input file.
@@ -218,19 +220,20 @@ regularization & dict & & \\
 For artificial viscosity Eqs.~\eqref{eq:euler_poisson} are modified to
 \begin{subequations}
 \begin{align}
-    \frac{\partial \omega}{\partial t} + \{ \phi, \omega\} = -(-\nu_\perp \Delta)^s \omega\\
+    \frac{\partial \omega}{\partial t} + \{ \phi, \omega\} = -(-\nu \Delta)^s \omega\\
  -\Delta \phi = \omega
 \end{align}
 \label{eq:euler_poisson_viscous}
 \end{subequations}
-where $\nu_\perp$ is the viscosity coefficient and $s=1,2,3,\cdots$ is the order
+where $\nu$ is the viscosity coefficient and $s=1,2,3,\cdots$ is the order
 \begin{longtable}{lllp{7.5cm}}
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
 regularization & dict & & \\
-\qquad type  & string& viscosity & Choosable only for \textbf{Karniadakis} timestepper (because it is solved implicitly) \\
-\qquad order    & integer & 2 & Order: 1 is normal diffusion, 2 is hyperdiffusion, can be arbitrarily high, but higher orders might take longer to solve \\
-\qquad nu\_perp    & float & 1e-3 & Viscosity coefficient \\
+\qquad type  & string& viscosity & the artificial viscosity \\
+\qquad order    & integer & 2 & Order: 1 is normal diffusion, 2 is hyperdiffusion, can be arbitrarily high, but higher orders might take longer to solve or restrict the CFL condition \\
+\qquad nu    & float & 1e-3 & Viscosity coefficient \\
+\qquad directoin & string & centered & Direction of the Laplacian: forward or centered
 \bottomrule
 \end{longtable}
 The other regularization method is the modal filter that applies an exponential filter
@@ -247,7 +250,7 @@ and is choosable with the following parameters
 \toprule
 \rowcolor{gray!50}\textbf{Name} &  \textbf{Type} & \textbf{Value}  & \textbf{Description}  \\ \midrule
 regularization & dict & & \\
-\qquad type  & string& modal & Not choosable for \textbf{Karniadakis} timestepper\\
+\qquad type  & string& modal & Not choosable for \textbf{ImExMultistep} timestepper\\
 \qquad order & integer & 8 & Order: normally 8 or 16 \\
 \qquad eta\_c & float & 0.5 & cutoff wavelength below which no damping is applied \\
 \qquad alpha & float & 36 & damping coefficient determining damping for highest wavenumber \\
@@ -264,6 +267,7 @@ elliptic & dict & & \\
 \qquad type  & string& multigrid & Actually a nested iterations class \\
 \qquad stages    & integer & 3 & Number of stages (3 is best in virtually all cases) \\
 \qquad eps\_pol    & float[stages] & [1e-6,10,10] & Accuracy requirement on each stage of the multigrid scheme. $\eps_0 = \eps_{pol,0}$, $\eps_i = \eps_{pol,i} \eps_{pol,0}$  for $i>1$. \\
+\qquad directoin & string & centered & Direction of the Laplacian: forward or centered
 \bottomrule
 \end{longtable}
 \subsection{Advection schemes}
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 9c7f82b43..fde965466 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -67,12 +67,12 @@ int main( int argc, char* argv[])
     }
 
     /// ////////////// Initialize timestepper ///////////////////////
-    std::string stepper = dg::file::get( mode, js, "timestepper", "stepper", "FilteredMultistep").asString();
+    std::string stepper = dg::file::get( mode, js, "timestepper", "type", "FilteredExplicitMultistep").asString();
     std::string regularization = dg::file::get( mode, js, "regularization", "type", "moddal").asString();
     dg::ModalFilter<dg::DMatrix, dg::DVec> filter;
     dg::IdentityFilter id;
     bool apply_filter = true;
-    dg::Karniadakis<dg::DVec> karniadakis;
+    dg::ImExMultistep<dg::DVec> imex;
     dg::ShuOsher<dg::DVec> shu_osher;
     dg::FilteredExplicitMultistep<dg::DVec> multistep;
     if( regularization == "modal")
@@ -84,27 +84,26 @@ int main( int argc, char* argv[])
     }
     else
         apply_filter = false;
-    if( regularization == "viscosity" && stepper != "Karniadakis")
-        throw dg::Error(dg::Message(_ping_)<<"Error: Viscosity only works with Karniadakis! Exit now!");
 
     double dt = dg::file::get( mode, js, "timestepper", "dt", 2e-3).asDouble();
-    if( "Karniadakis" == stepper)
+    if( "ImExMultistep" == stepper)
     {
         if( regularization != "viscosity")
         {
-            throw dg::Error(dg::Message(_ping_)<<"Error: Karniadakis only works with viscosity regularization! Exit now!");
+            throw dg::Error(dg::Message(_ping_)<<"Error: ImExMultistep only works with viscosity regularization! Exit now!");
 
             return -1;
         }
         double eps_time = dg::file::get( mode, js, "timestepper", "eps_time", 1e-10).asDouble();
-        karniadakis.construct( y0, y0.size(), eps_time);
-        karniadakis.init( shu, diffusion, time, y0, dt);
+        std::string tableau = dg::file::get( mode, js, "timestepper", "tableau", "ImEx-BDF-3-3").asString();
+        imex.construct( tableau, y0, y0.size(), eps_time);
+        imex.init( shu, diffusion, time, y0, dt);
     }
     else if( "Shu-Osher" == stepper)
     {
         shu_osher.construct( "SSPRK-3-3", y0);
     }
-    else if( "FilteredMultistep" == stepper)
+    else if( "FilteredExplicitMultistep" == stepper)
     {
         multistep.construct( "eBDF-3-3", y0);
         if( apply_filter)
@@ -151,9 +150,9 @@ int main( int argc, char* argv[])
             try{
                 for( unsigned j=0; j<itstp; j++)
                 {
-                    if( "Karniadakis" == stepper)
-                        karniadakis.step( shu, diffusion, time, y0);
-                    else if ( "FilteredMultistep" == stepper)
+                    if( "ImExMultistep" == stepper)
+                        imex.step( shu, diffusion, time, y0);
+                    else if ( "FilteredExplicitMultistep" == stepper)
                     {
                         if( apply_filter)
                             multistep.step( shu, filter, time, y0);
@@ -289,9 +288,9 @@ int main( int argc, char* argv[])
             ti.tic();
             for( unsigned j=0; j<itstp; j++)
             {
-                if( "Karniadakis" == stepper)
-                    karniadakis.step( shu, diffusion, time, y0);
-                else if ( "FilteredMultistep" == stepper)
+                if( "ImExMultistep" == stepper)
+                    imex.step( shu, diffusion, time, y0);
+                else if ( "FilteredExplicitMultistep" == stepper)
                 {
                     if( apply_filter)
                         multistep.step( shu, filter, time, y0);
-- 
GitLab


From 31d634e4750341c4971d93ef700a495aca7c576d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 3 Feb 2021 18:03:33 +0100
Subject: [PATCH 452/540] Add polynomial coefficients to fast_interpolation

- fast interpolation and projection can now operate between various n
grids (for example double n -> 2n)
- include unit tests
- they are now documented in the interpolation section
- adapt all codes that use them
- create template typedefs for host vectors and matrices to shorten docu
---
 .../dg_introduction/dg_introduction.tex       |  55 +++---
 inc/dg/backend/typedefs.h                     |  18 +-
 inc/dg/blas_b.cu                              |   4 +-
 inc/dg/blas_mpib.cu                           |   4 +-
 inc/dg/dg_doc.h                               |   2 +-
 inc/dg/multigrid.h                            |   6 +-
 inc/dg/topology/fast_interpolation.h          | 159 ++++++++++++------
 inc/dg/topology/interpolation.h               |  16 +-
 inc/dg/topology/interpolation_mpit.cu         | 118 +++++++++++++
 inc/dg/topology/mpi_projection.h              |  12 +-
 inc/dg/topology/projection.h                  |   1 +
 inc/dg/topology/projection_mpit.cu            | 131 +++++----------
 inc/dg/topology/projection_t.cu               |  63 ++++---
 src/lamb_dipole/diag.h                        |   2 +-
 src/lamb_dipole/shu.cuh                       |  19 ++-
 src/lamb_dipole/shu_b.cu                      |   2 +-
 16 files changed, 383 insertions(+), 229 deletions(-)
 create mode 100644 inc/dg/topology/interpolation_mpit.cu

diff --git a/doc/related_pages/dg_introduction/dg_introduction.tex b/doc/related_pages/dg_introduction/dg_introduction.tex
index bec264fc1..63ca07b5b 100644
--- a/doc/related_pages/dg_introduction/dg_introduction.tex
+++ b/doc/related_pages/dg_introduction/dg_introduction.tex
@@ -427,12 +427,12 @@ As a first step we rewrite Eq.~\eqref{eq:elliptic} into two first order differen
 \end{align}
 \end{subequations}
 %Our plan is to now simply use a forward discretization\footnote{
-%The choice of forward discretizing the first equation is arbitrary. A backward discretization 
-%works equally well} for the first equation 
-Our plan is to simply use one of the discretizations developed in the last 
-section for the first equation~\eqref{eq:polarisationa} 
-and its negative adjoint for the third equation~\eqref{eq:polarisationc}. 
-Recall that the adjoint of a square matrix $A$ is defined by the scalar product, i.e. 
+%The choice of forward discretizing the first equation is arbitrary. A backward discretization
+%works equally well} for the first equation
+Our plan is to simply use one of the discretizations developed in the last
+section for the first equation~\eqref{eq:polarisationa}
+and its negative adjoint for the third equation~\eqref{eq:polarisationc}.
+Recall that the adjoint of a square matrix $A$ is defined by the scalar product, i.e.
 \begin{align}
     \vec f^\mathrm{T} (\Eins\otimes W)\circ A\vec g = 
     \vec g^\mathrm{T} (A^\mathrm{T}\circ(1\otimes W)) \vec f =:     \vec g^\mathrm{T} (\Eins\otimes W) A^\dagger \vec f. \nonumber
@@ -444,9 +444,9 @@ From here we immediately get the relation
     \label{eq:adjoint}
 \end{align}
 There is a close connection between symmetric, $A=A^\mathrm{T}$, and self-adjoint, $A=A^\dagger$, matrices.
-If and only if the matrix $A$ is symmetric, then $(\Eins\otimes V)\circ A$ is self-adjoint. Of course, we have $(AB)^\dagger = B^\dagger A^\dagger$ and $(A^\dagger)^\dagger = A$. 
+If and only if the matrix $A$ is symmetric, then $(\Eins\otimes V)\circ A$ is self-adjoint. Of course, we have $(AB)^\dagger = B^\dagger A^\dagger$ and $(A^\dagger)^\dagger = A$.
 
-The function product in Eq.~\eqref{eq:polarisationb} is computed pointwisely on the Gaussian abscissas. 
+The function product in Eq.~\eqref{eq:polarisationb} is computed pointwisely on the Gaussian abscissas.
 \begin{align}
     \vec j' &= (\Eins\otimes V)\circ D_x \vec \phi, \nonumber \\
     \vec j &= \vec \chi \vec j', \nonumber\\
@@ -454,7 +454,7 @@ The function product in Eq.~\eqref{eq:polarisationb} is computed pointwisely on
     \label{eq:naive}
 \end{align}
 where $D_x$ is either $D_x^+$, $D_x^-$, or $D_x^0$ with the correct boundary terms.
-Note, that~\cite{Cockburn2002} originally only proposed to use the forward or backward discretization for $D_x$. 
+Note, that~\cite{Cockburn2002} originally only proposed to use the forward or backward discretization for $D_x$.
 Equation~\eqref{eq:naive} is indeed a self-adjoint
 discretization for the second derivative. However, it turns out that 
 Eq.~\eqref{eq:naive} is inconsistent for given $\rho(x)$. The solution does not converge. 
@@ -544,50 +544,45 @@ points to interpolate. $I$ has $P^d$ entries per line, where $d$ is the dimensio
 
 \subsection{Interpolation and Projection}
 A special case emerges when the list of points to interpolate is made up
-by the Gaussian abscissas of another grid. 
+by the Gaussian abscissas of another grid.
 Suppose we want to divide each cell $C_n$ of the original grid into $M$ equidistant subcells.
 We use the letter $c$ to denote the coarse grid and the letter $f$ to denote
-the fine grid. 
-For ease of discussion we assume that the number of polynomial coefficients
-in the fine and the coarse grid are the same. 
-We denote $q_{ml}(x)$ the polynomials on the fine grid and $y^P_{mj}$ the 
-corresponding Gaussian abscissas. 
+the fine grid.
+We denote $q_{ml}(x)$ the polynomials on the fine grid and $x^F_{mj}$ the
+corresponding Gaussian abscissas.
 If a vector $\vec f$ is given on the coarse grid, we can simply interpolate
 it onto the fine grid analogous to Eq.~\eqref{eq:basic_interpolation} via
-\begin{subequations}
 \begin{align}
-  f^F_{mj} &= f_h(y^P_{mj}) = p_{nk}(y^P_{mj})F^{ki} f^C_{ni} \\
-  f^F_{mj} &=: {Q_{mj}}^{ni} f^C_{ni}
+  f^F_{mj} = f_h(x^F_{mj}) = p_{nk}(x^F_{mj})F^{ki} f^C_{ni}  =: {Q_{mj}}^{ni} f^C_{ni}
   \label{eq:interpolation}
 \end{align}
-\end{subequations}
 where we denote the special interpolation matrix with $Q$ and implicitly assume the sum of repeated indices.
-No information is lost in this process if the cells $C_n$ are divided by an 
-integer number $M$ and the number of polynomials is the same, i.e. 
-$f^F$ and $f^C$ represent exactly the same expansion $f_h(x)$. 
+No information is lost in this process if the cells $C_n$ are divided by an
+integer number $M$ and the number of polynomials is the same, i.e.
+$f^F$ and $f^C$ represent exactly the same expansion $f_h(x)$.
 \begin{align}
   f^C(x) = \bar f_C^{nk}p_{nk}(x) = f^F(x) = \bar f_F^{ml}q_{ml}(x) 
   \label{eq:no_loss}
 \end{align}
 
-Vice versa, given a expansion $f^F(x)$ on the fine grid, we can the projection integrals from the fine grid to the coarse grid. 
+Vice versa, given an expansion $f^F(x)$ on the fine grid, we can compute the projection integrals from the fine grid to the coarse grid.
 It can be shown that
 \begin{align}
-  \bar f_C^{nk} &:= T_C^{ks}\int \dx f^F(x)p_{ns}(x) = T_C^{ks}  W_m^{ij}p_{ns}(y_{mj}) f^F_{mi} \\
+  \bar f_C^{nk} &:= T_C^{ks}\int \dx f^F(x)p_{ns}(x) = T_C^{ks}  W_m^{ij}p_{ns}(x_{mj}) f^F_{mi} \\
   f^C_{nt} &= B_{tk}\bar f_C^{nk} = V^c_{tk}  {{(Q^T)}^{nk}}_{mj}W_m^{ji} f^F_{mi} =: {P_{nt}}^{mi} f^F_{mi}
   %\bar f_C^{nk} = T_C^{ks}\int \dx f^F(x)p_{ns}(x) = T_C^{ks} \bar f_F^{ml} \int \dx q_{ml}(x)p_{ns}(x) \\
   %= T_C^{ks} F^{lo}f^F_{mo} \int \dx q_{ml}(x)p_{ns}(x)  \\
-  %= T_C^{ks} F^{lo}f^F_{mo} W_F^{ij}q_{ml}(y_{mi})p_{ns}(y_{mj})  \\
-  %= T_C^{ks} F^{lo}f^F_{mo} W_m^{ij}B_{il}p_{ns}(y_{mj})  \\
-  %= T_C^{ks} f^F_{mi} W_m^{ij}p_{ns}(y_{mj})  \\
-  %f^C_{nt} = B_{tk}\bar f_C^{nk} = B_{tk} T_C^{ks} W_m^{ij} p_{ns}(y_{mj}) f^F_{mi}\\
-  %= V^c_{tk} F^{sk} W_m^{ij} p_{ns}(y_{mj}) f^F_{mi}\\
+  %= T_C^{ks} F^{lo}f^F_{mo} W_F^{ij}q_{ml}(x_{mi})p_{ns}(x_{mj})  \\
+  %= T_C^{ks} F^{lo}f^F_{mo} W_m^{ij}B_{il}p_{ns}(x_{mj})  \\
+  %= T_C^{ks} f^F_{mi} W_m^{ij}p_{ns}(x_{mj})  \\
+  %f^C_{nt} = B_{tk}\bar f_C^{nk} = B_{tk} T_C^{ks} W_m^{ij} p_{ns}(x_{mj}) f^F_{mi}\\
+  %= V^c_{tk} F^{sk} W_m^{ij} p_{ns}(x_{mj}) f^F_{mi}\\
   %= V^c_{tk}  {Q_{mj}}^{nk}W_m^{ji} f^F_{mi}\\
   %= V^c_{tk}  {{(Q^T)}^{nk}}_{mj}W_m^{ji} f^F_{mi}\\
   %=: {P_{nt}}^{mi} f^F_{mi}
   \label{eq:basic_projection}
 \end{align}
-from where we directly conclude that 
+from where we directly conclude that
 \begin{align}
   P = Q^\dagger = V^C Q^T W_F
   \label{eq:projection_adjoint}
diff --git a/inc/dg/backend/typedefs.h b/inc/dg/backend/typedefs.h
index 1c08dd4eb..2e63de729 100644
--- a/inc/dg/backend/typedefs.h
+++ b/inc/dg/backend/typedefs.h
@@ -14,6 +14,8 @@ namespace dg{
 ///@addtogroup typedefs
 ///@{
 //vectors
+template<class T>
+using HVec_t  = thrust::host_vector<T>; //!< Host Vector
 using HVec  = thrust::host_vector<double>; //!< Host Vector
 using iHVec = thrust::host_vector<int>; //!< integer Host Vector
 using fHVec = thrust::host_vector<float>; //!< Host Vector
@@ -23,6 +25,8 @@ using iDVec = thrust::device_vector<int>; //!< integer Device Vector
 using fDVec = thrust::device_vector<float>; //!< Device Vector. The device can be an OpenMP parallelized cpu or a gpu. This depends on the value of the macro THRUST_DEVICE_SYSTEM, which can be either THRUST_DEVICE_SYSTEM_OMP for openMP or THRUST_DEVICE_SYSTEM_CUDA for a gpu.
 
 //derivative matrices
+template<class T>
+using HMatrix_t = EllSparseBlockMat<T>;
 using HMatrix = EllSparseBlockMat<double>; //!< Host Matrix for derivatives
 using fHMatrix = EllSparseBlockMat<float>; //!< Host Matrix for derivatives
 using DMatrix = EllSparseBlockMatDevice<double>; //!< Device Matrix for derivatives
@@ -38,25 +42,31 @@ namespace dg{
 ///@addtogroup typedefs
 ///@{
 //using MPI_Vector<thrust::device_vector<double> >  MDVec; //!< MPI Device Vector s.a. dg::DVec
+template<class T>
+using MHVec_t     = dg::MPI_Vector<dg::HVec_t<T> >; //!< MPI Host Vector s.a. dg::HVec_t
 using MHVec     = dg::MPI_Vector<dg::HVec >; //!< MPI Host Vector s.a. dg::HVec
 using fMHVec    = dg::MPI_Vector<dg::fHVec >; //!< MPI Host Vector s.a. dg::fHVec
 using MDVec     = dg::MPI_Vector<dg::DVec >; //!< MPI Device Vector s.a. dg::DVec
 using fMDVec    = dg::MPI_Vector<dg::fDVec >; //!< MPI Device Vector s.a. dg::fDVec
 
-template<class real_type>
-using NNCH = dg::NearestNeighborComm<dg::iHVec, thrust::host_vector<const real_type*>, thrust::host_vector<real_type> >; //!< host Communicator for the use in an mpi matrix for derivatives
-template<class real_type>
-using NNCD = dg::NearestNeighborComm<dg::iDVec, thrust::device_vector<const real_type*>, thrust::device_vector<real_type> >; //!< host Communicator for the use in an mpi matrix for derivatives
+template<class T>
+using NNCH = dg::NearestNeighborComm<dg::iHVec, thrust::host_vector<const T*>, thrust::host_vector<T> >; //!< host Communicator for the use in an mpi matrix for derivatives
+template<class T>
+using NNCD = dg::NearestNeighborComm<dg::iDVec, thrust::device_vector<const T*>, thrust::device_vector<T> >; //!< host Communicator for the use in an mpi matrix for derivatives
 using dNNCH = dg::NNCH<double>; //!< host Communicator for the use in an mpi matrix for derivatives
 using fNNCH = dg::NNCH<float>; //!< host Communicator for the use in an mpi matrix for derivatives
 using dNNCD = dg::NNCD<double>; //!< device Communicator for the use in an mpi matrix for derivatives
 using fNNCD = dg::NNCD<float>; //!< device Communicator for the use in an mpi matrix for derivatives
 
+template< class T>
+using CooMat_t    = dg::CooSparseBlockMat<T>;
 using CooMat    = dg::CooSparseBlockMat<double>;
 using fCooMat   = dg::CooSparseBlockMat<float>;
 using DCooMat   = dg::CooSparseBlockMatDevice<double>;
 using fDCooMat  = dg::CooSparseBlockMatDevice<float>;
 
+template<class T>
+using MHMatrix_t  = dg::RowColDistMat<dg::HMatrix_t<T>, dg::CooMat_t<T>, dg::NNCH<T>>; //!< MPI Host Matrix for derivatives
 using MHMatrix  = dg::RowColDistMat<dg::HMatrix, dg::CooMat, dg::dNNCH>; //!< MPI Host Matrix for derivatives
 using fMHMatrix = dg::RowColDistMat<dg::fHMatrix, dg::fCooMat, dg::fNNCH>; //!< MPI Host Matrix for derivatives
 using MDMatrix  = dg::RowColDistMat<dg::DMatrix, dg::DCooMat, dg::dNNCD>; //!< MPI Device Matrix for derivatives
diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 953f5ce2e..6f81ba11f 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -76,8 +76,8 @@ int main()
     value_type gbytes=(value_type)x.size()*x[0].size()*sizeof(value_type)/1e9;
     std::cout << "Size of vectors is "<<gbytes<<" GB\n";
     dg::MultiMatrix<Matrix, ArrayVec> inter, project;
-    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2), inter);
-    dg::blas2::transfer(dg::create::fast_projection( grid, 2,2), project);
+    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2,1), inter);
+    dg::blas2::transfer(dg::create::fast_projection( grid, 2,2,1), project);
     //dg::IDMatrix inter = dg::create::interpolation( grid, grid_half);
     //dg::IDMatrix project = dg::create::projection( grid_half, grid);
     int multi=100;
diff --git a/inc/dg/blas_mpib.cu b/inc/dg/blas_mpib.cu
index 3cd097431..2c6f610a4 100644
--- a/inc/dg/blas_mpib.cu
+++ b/inc/dg/blas_mpib.cu
@@ -84,8 +84,8 @@ int main( int argc, char* argv[])
     value_type gbytes=(value_type)x.size()*grid.size()*sizeof(value_type)/1e9;
     if(rank==0)std::cout << "Sizeof vectors is "<<gbytes<<" GB\n";
     dg::MultiMatrix<Matrix, ArrayVec> inter, project;
-    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2), inter);
-    dg::blas2::transfer(dg::create::fast_projection( grid, 2,2), project);
+    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2,1), inter);
+    dg::blas2::transfer(dg::create::fast_projection( grid, 2,2,1), project);
 
     int multi=100;
     if(rank==0)std::cout<<"\nNo communication\n";
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 050bdfb70..bdf26876a 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -1,7 +1,7 @@
 #error Documentation only
 /*! @namespace dg
  * @brief This is the namespace for all functions and
- * classes defined and used by the discontinuous Galerkin solvers.
+ * classes defined and used by the discontinuous Galerkin library.
  */
 /*!
  * @defgroup backend Level 1: Vectors, Matrices and basic operations
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 0a5ea48ec..155ca9f8d 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -91,9 +91,9 @@ struct MultigridCG2d
         {
             // Projecting from one grid to the next is the same as
             // projecting from the original grid to the coarse grids
-            m_project[u].construct( dg::create::fast_projection(*m_grids[u], 2, 2, dg::normed), std::forward<Params>(ps)...);
-            m_inter[u].construct( dg::create::fast_interpolation(*m_grids[u+1], 2, 2), std::forward<Params>(ps)...);
-            m_interT[u].construct( dg::create::fast_projection(*m_grids[u], 2, 2, dg::not_normed), std::forward<Params>(ps)...);
+            m_project[u].construct( dg::create::fast_projection(*m_grids[u], 2, 2, 1, dg::normed), std::forward<Params>(ps)...);
+            m_inter[u].construct( dg::create::fast_interpolation(*m_grids[u+1], 2, 2, 1), std::forward<Params>(ps)...);
+            m_interT[u].construct( dg::create::fast_projection(*m_grids[u], 2, 2, 1, dg::not_normed), std::forward<Params>(ps)...);
         }
 
         m_x.resize( m_stages);
diff --git a/inc/dg/topology/fast_interpolation.h b/inc/dg/topology/fast_interpolation.h
index 40f2ca1c9..033b24e76 100644
--- a/inc/dg/topology/fast_interpolation.h
+++ b/inc/dg/topology/fast_interpolation.h
@@ -2,6 +2,7 @@
 
 #include <thrust/host_vector.h>
 #include "dg/backend/memory.h"
+#include "dg/backend/typedefs.h"
 #include "dg/enums.h"
 #include "dg/blas.h"
 #include "grid.h"
@@ -25,6 +26,7 @@ namespace dg
  *
  * \f[ y = M_{N-1}(...M_1(M_0x))\f]
  * where \f$ M_i\f$ is the i-th matrix
+ * @sa mainly used by dg::create::fast_interpolation and dg::ModalFilter
  * @copydoc hide_matrix
  * @copydoc hide_ContainerType
  * @ingroup misc
@@ -89,166 +91,222 @@ struct TensorTraits<MultiMatrix<M, V> >
     using value_type  = get_value_type<V>;
     using tensor_category = SelfMadeMatrixTag;
 };
+///@endcond
 
 
 namespace create
 {
+///@addtogroup interpolation
+///@{
+
+/**
+ * @brief Create interpolation matrix for integer multipliers
+ *
+ * When creating an interpolation from a given dg grid to one that has
+ * an integer multiple of cells and/or polynomial coefficients, the
+ * resulting interpolation matrix fits into our \c dg::EllSparseBlockMat format,
+ * which is much faster to apply than the full sparse matrix format from
+ * the general purpose interpolation function, especially since it requires
+ * no communication from neighboring cells
+ * @sa dg::create::interpolation
+ * @sa For a derivation of the coefficients consult the dg manual <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @tparam real_type a floating point type
+ * @param t The existing (old/coarse) grid
+ * @param multiplyNx integer multiplier, the new grid has \c Nx*multiplyNx points
+ * @param multiplyn integer multiplier, the new grid has \c n*multiplyn polynomial coefficients
+ *
+ * @return a matrix that when applied to vectors on the old grid produces a vector on the new grid
+ */
 template<class real_type>
-MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > fast_interpolation( const RealGrid1d<real_type>& t, unsigned multiply)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolation( const RealGrid1d<real_type>& t, unsigned multiplyNx, unsigned multiplyn)
 {
     unsigned n=t.n();
     dg::RealGrid1d<real_type> g_old( -1., 1., n, 1);
-    dg::RealGrid1d<real_type> g_new( -1., 1., n, multiply);
+    dg::RealGrid1d<real_type> g_new( -1., 1., n*multiplyn, multiplyNx);
     cusp::coo_matrix<int, real_type, cusp::host_memory> interpolX = dg::create::interpolation( g_new, g_old);
-    EllSparseBlockMat<real_type> iX( multiply*t.N(), t.N(), 1, multiply, t.n());
-    for( unsigned  k=0; k<multiply; k++)
+    EllSparseBlockMat<real_type> iX( multiplyn*multiplyNx*t.N(), t.N(), 1, multiplyNx*multiplyn, t.n());
+    for( unsigned  k=0; k<multiplyNx*multiplyn; k++)
     for( unsigned  i=0; i<n; i++)
     for( unsigned  j=0; j<n; j++)
         iX.data[(k*n+i)*n+j] = interpolX.values[(k*n+i)*n+j];
-    for( unsigned i=0; i<multiply*t.N(); i++)
+    for( unsigned i=0; i<multiplyNx*multiplyn*t.N(); i++)
     {
-        iX.cols_idx[i] = i/multiply;
-        iX.data_idx[i] = i%multiply;
+        iX.cols_idx[i] = i/(multiplyNx*multiplyn);
+        iX.data_idx[i] = i%(multiplyNx*multiplyn);
     }
     MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > inter(1);
     inter.get_matrices()[0] = iX;
     return inter;
 }
 
+/**
+ * @brief Create projecton or interpolationT matrix for integer dividers
+ *
+ * When creating a projection from a given dg grid to one that has
+ * an integer division of cells and/or polynomial coefficients, the
+ * resulting projection matrix fits into our \c dg::EllSparseBlockMat format,
+ * which is much faster to apply than the full sparse matrix format from
+ * the general purpose projection function, especially since it requires
+ * no communication from neighboring cells
+ * @sa dg::create::projection dg::create::interpolationT
+ * @sa For a derivation of the coefficients consult the dg manual <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @tparam real_type a floating point type
+ * @param t The existing (old/fine) grid
+ * @param divideNx integer divisor, the new grid has \c Nx/multiplyNx points
+ * @param dividen integer divisor, the new grid has \c n/multiplyn polynomial coefficients
+ * @param no if dg::normed than a projection matrix is returned, if dg::not_normed the interpolationT is computed
+ *
+ * @return a matrix that when applied to vectors on the old grid produces a vector on the new grid
+ */
 template<class real_type>
-MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > fast_projection( const RealGrid1d<real_type>& t, unsigned divide, enum dg::norm no = normed)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection( const RealGrid1d<real_type>& t, unsigned divideNx, unsigned dividen, enum dg::norm no = normed)
 {
-    unsigned n=t.n();
-    if( t.N()%divide != 0) throw Error( Message(_ping_)<< "Nx and divide don't match: Nx: " << t.N()<< " divide "<< (unsigned)divide);
-    dg::RealGrid1d<real_type> g_old( -1., 1., n, divide);
+    if( t.N()%divideNx != 0) throw Error( Message(_ping_)<< "Nx and divideNx don't match: Nx: " << t.N()<< " divideNx "<< (unsigned)divideNx);
+    if( t.n()%dividen != 0) throw Error( Message(_ping_)<< "n and dividen don't match: Nx: " << t.n()<< " dividen "<< (unsigned)dividen);
+    unsigned n=t.n()/dividen;
+    dg::RealGrid1d<real_type> g_old( -1., 1., n*dividen, divideNx);
     dg::RealGrid1d<real_type> g_new( -1., 1., n, 1);
     dg::HVec w1d = dg::create::weights( g_old);
     dg::HVec v1d = dg::create::inv_weights( g_new);
     cusp::coo_matrix<int, real_type, cusp::host_memory> projectX;
     //Here, we cannot use create::projection because that would remove explicit zeros!!
     projectX = dg::create::interpolationT( g_new, g_old);
-    EllSparseBlockMat<real_type> pX( t.N()/divide, t.N(), divide, divide, t.n());
-    for( unsigned k=0; k<divide; k++)
+    EllSparseBlockMat<real_type> pX( t.N()/divideNx, t.N()*dividen, divideNx*dividen, divideNx*dividen, n);
+    for( unsigned k=0; k<divideNx; k++)
+    for( unsigned l=0; l<dividen; l++)
     for( unsigned i=0; i<n; i++)
     for( unsigned j=0; j<n; j++)
     {
-        pX.data[(k*n+i)*n+j] = projectX.values[(i*divide +k)*n+j];
+        pX.data[((k*dividen+l)*n+i)*n+j] = projectX.values[((i*divideNx+k)*dividen + l)*n+j];
         if( no == normed)
-            pX.data[(k*n+i)*n+j] *= v1d[i]*w1d[j];
+            pX.data[((k*dividen+l)*n+i)*n+j] *= v1d[i]*w1d[l*n+j];
     }
-    for( unsigned i=0; i<t.N()/divide; i++)
-        for( unsigned d=0; d<divide; d++)
+    for( unsigned i=0; i<t.N()/divideNx; i++)
+        for( unsigned d=0; d<divideNx*dividen; d++)
         {
-            pX.cols_idx[i*divide+d] = i*divide+d;
-            pX.data_idx[i*divide+d] = d;
+            pX.cols_idx[i*divideNx*dividen+d] = i*divideNx*dividen+d;
+            pX.data_idx[i*divideNx*dividen+d] = d;
         }
     MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > inter(1);
     inter.get_matrices()[0] = pX;
     return inter;
 }
 
+///@copydoc fast_interpolation(const RealGrid1d<real_type>&,unsigned,unsigned)
+///@param multiplyNy integer multiplier, the new grid has \c Ny*multiplyNy points
 template<class real_type>
-MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > fast_interpolation( const aRealTopology2d<real_type>& t, unsigned multiplyX, unsigned multiplyY)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolation( const aRealTopology2d<real_type>& t, unsigned multiplyNx, unsigned multiplyNy, unsigned multiplyn)
 {
     dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_interpolation( gx, multiplyX);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_interpolation( gx, multiplyNx, multiplyn);
     interX.get_matrices()[0].left_size = t.n()*t.Ny();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_interpolation( gy, multiplyY);
-    interY.get_matrices()[0].right_size = t.n()*t.Nx()*multiplyX;
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_interpolation( gy, multiplyNy, multiplyn);
+    interY.get_matrices()[0].right_size = t.n()*t.Nx()*multiplyNx*multiplyn;
     interY.get_matrices()[0].set_default_range();
 
     MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > inter(2);
     inter.get_matrices()[0] = interX.get_matrices()[0];
     inter.get_matrices()[1] = interY.get_matrices()[0];
-    thrust::host_vector<real_type> vec( t.size()*multiplyX);
+    thrust::host_vector<real_type> vec( t.size()*multiplyNx*multiplyn);
     inter.get_temp()[0] = Buffer<thrust::host_vector<real_type > >(vec);
     return inter;
 }
 
+///@copydoc fast_projection(const RealGrid1d<real_type>&,unsigned,unsigned,enum dg::norm)
+///@param divideNy integer multiplier, the new grid has \c Ny/divideNy points
 template<class real_type>
-MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > fast_projection( const aRealTopology2d<real_type>& t, unsigned divideX, unsigned divideY, enum dg::norm no = normed)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection( const aRealTopology2d<real_type>& t, unsigned divideNx, unsigned divideNy, unsigned dividen, enum dg::norm no = normed)
 {
     dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_projection( gx, divideX, no);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_projection( gx, divideNx, dividen, no);
     interX.get_matrices()[0].left_size = t.n()*t.Ny();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_projection( gy, divideY, no);
-    interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideX;
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_projection( gy, divideNy, dividen, no);
+    interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideNx/dividen;
     interY.get_matrices()[0].set_default_range();
 
     MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > inter(2);
     inter.get_matrices()[0] = interX.get_matrices()[0];
     inter.get_matrices()[1] = interY.get_matrices()[0];
-    thrust::host_vector<real_type> vec( t.size()/divideX);
+    thrust::host_vector<real_type> vec( t.size()/divideNx/dividen);
     inter.get_temp()[0] = Buffer<thrust::host_vector<real_type> >(vec);
     return inter;
 }
 
+///@copydoc fast_interpolation(const RealGrid1d<real_type>&,unsigned,unsigned)
+///@param multiplyNy integer multiplier, the new grid has \c Ny*multiplyNy points
 template<class real_type>
-MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > fast_interpolation( const aRealTopology3d<real_type>& t, unsigned multiplyX, unsigned multiplyY)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolation( const aRealTopology3d<real_type>& t, unsigned multiplyNx, unsigned multiplyNy, unsigned multiplyn)
 {
     dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_interpolation( gx, multiplyX);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_interpolation( gx, multiplyNx,multiplyn);
     interX.get_matrices()[0].left_size = t.n()*t.Ny()*t.Nz();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_interpolation( gy, multiplyY);
-    interY.get_matrices()[0].right_size = t.n()*t.Nx()*multiplyX;
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_interpolation( gy, multiplyNy,multiplyn);
+    interY.get_matrices()[0].right_size = t.n()*t.Nx()*multiplyNx*multiplyn;
     interY.get_matrices()[0].left_size = t.Nz();
     interY.get_matrices()[0].set_default_range();
 
     MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > inter(2);
     inter.get_matrices()[0] = interX.get_matrices()[0];
     inter.get_matrices()[1] = interY.get_matrices()[0];
-    thrust::host_vector<real_type> vec( t.size()*multiplyX);
+    thrust::host_vector<real_type> vec( t.size()*multiplyNx*multiplyn);
     inter.get_temp()[0] = Buffer<thrust::host_vector<real_type > >(vec);
     return inter;
 }
 
+///@copydoc fast_projection(const RealGrid1d<real_type>&,unsigned,unsigned,enum dg::norm)
+///@param divideNy integer multiplier, the new grid has \c Ny/divideNy points
 template<class real_type>
-MultiMatrix< EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > fast_projection( const aRealTopology3d<real_type>& t, unsigned divideX, unsigned divideY, enum dg::norm no = normed)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection( const aRealTopology3d<real_type>& t, unsigned divideNx, unsigned divideNy, unsigned dividen, enum dg::norm no = normed)
 {
     dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_projection( gx, divideX, no);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_projection( gx, divideNx, dividen, no);
     interX.get_matrices()[0].left_size = t.n()*t.Ny()*t.Nz();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_projection( gy, divideY, no);
-    interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideX;
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_projection( gy, divideNy, dividen, no);
+    interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideNx/dividen;
     interY.get_matrices()[0].left_size = t.Nz();
     interY.get_matrices()[0].set_default_range();
 
     MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > inter(2);
     inter.get_matrices()[0] = interX.get_matrices()[0];
     inter.get_matrices()[1] = interY.get_matrices()[0];
-    thrust::host_vector<real_type> vec( t.size()/divideX);
+    thrust::host_vector<real_type> vec( t.size()/divideNx/dividen);
     inter.get_temp()[0] = Buffer<thrust::host_vector<real_type> >(vec);
     return inter;
 }
 
 #ifdef MPI_VERSION
 //very elaborate way of telling the compiler to just apply the local matrix to the local vector
+///@copydoc fast_interpolation(const RealGrid1d<real_type>&,unsigned,unsigned)
+///@param multiplyNy integer multiplier, the new grid has \c Ny*multiplyNy points
 template<class real_type>
-MultiMatrix< RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type> >, MPI_Vector<thrust::host_vector<real_type> > > fast_interpolation( const aRealMPITopology2d<real_type>& t, unsigned divideX, unsigned divideY)
+MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_interpolation( const aRealMPITopology2d<real_type>& t, unsigned multiplyNx, unsigned multiplyNy, unsigned multiplyn)
 {
     typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
     typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
-    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_interpolation( t.local(), divideX, divideY);
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_interpolation( t.local(), multiplyNx, multiplyNy, multiplyn);
     MultiMatrix< Matrix, Vector > inter(2);
     inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
     inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
     inter.get_temp()[0] = Buffer<Vector> ( Vector( temp.get_temp()[0].data(), t.communicator())  );
     return inter;
 }
+
+///@copydoc fast_projection(const RealGrid1d<real_type>&,unsigned,unsigned,enum dg::norm)
+///@param divideNy integer multiplier, the new grid has \c Ny/divideNy points
 template<class real_type>
-MultiMatrix< RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type> >, MPI_Vector<thrust::host_vector<real_type> > > fast_projection( const aRealMPITopology2d<real_type>& t, unsigned divideX, unsigned divideY, enum dg::norm no = normed)
+MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_projection( const aRealMPITopology2d<real_type>& t, unsigned divideNx, unsigned divideNy, unsigned dividen, enum dg::norm no = normed)
 {
     typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
     typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
-    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_projection( t.local(), divideX, divideY, no);
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_projection( t.local(), divideNx, divideNy, dividen, no);
     MultiMatrix< Matrix, Vector > inter(2);
     inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
     inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
@@ -256,12 +314,14 @@ MultiMatrix< RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_
     return inter;
 }
 
+///@copydoc fast_interpolation(const RealGrid1d<real_type>&,unsigned,unsigned)
+///@param multiplyNy integer multiplier, the new grid has \c Ny*multiplyNy points
 template<class real_type>
-MultiMatrix< RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type> >, MPI_Vector<thrust::host_vector<real_type> > > fast_interpolation( const aRealMPITopology3d<real_type>& t, unsigned divideX, unsigned divideY)
+MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_interpolation( const aRealMPITopology3d<real_type>& t, unsigned multiplyNx, unsigned multiplyNy, unsigned multiplyn)
 {
     typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
     typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
-    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_interpolation( t.local(), divideX, divideY);
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_interpolation( t.local(), multiplyNx, multiplyNy, multiplyn);
     MultiMatrix< Matrix, Vector > inter(2);
     inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
     inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
@@ -269,12 +329,14 @@ MultiMatrix< RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_
     return inter;
 }
 
+///@copydoc fast_projection(const RealGrid1d<real_type>&,unsigned,unsigned,enum dg::norm)
+///@param divideNy integer multiplier, the new grid has \c Ny/divideNy points
 template<class real_type>
-MultiMatrix< RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type> >, MPI_Vector<thrust::host_vector<real_type> > > fast_projection( const aRealMPITopology3d<real_type>& t, unsigned divideX, unsigned divideY, enum dg::norm no = normed)
+MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_projection( const aRealMPITopology3d<real_type>& t, unsigned divideNx, unsigned divideNy, unsigned dividen, enum dg::norm no = normed)
 {
     typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
     typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
-    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_projection( t.local(), divideX, divideY, no);
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_projection( t.local(), divideNx, divideNy, dividen, no);
     MultiMatrix< Matrix, Vector > inter(2);
     inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
     inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
@@ -283,7 +345,8 @@ MultiMatrix< RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_
 }
 
 #endif //MPI_VERSION
+///@}
+
 }//namespace create
 
-///@endcond
 }//namespace dg
diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index 859c76685..a81514e38 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -435,12 +435,14 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const thrust:
  *
  * This matrix interpolates vectors on the old grid \c g_old to the %Gaussian nodes of the new grid \c g_new. The interpolation is of the order \c g_old.n()
  * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @sa for integer multiples between old and new %grid you may want to consider the dg::create::fast_interpolation %functions
  *
  * @param g_new The new grid
  * @param g_old The old grid
  *
  * @return Interpolation matrix with \c g_old.size() columns and \c g_new.size() rows
  * @note The boundaries of the old grid must lie within the boundaries of the new grid
+ * @note When interpolating a 2d grid to a 3d grid the third coordinate is simply ignored, i.e. the 2d vector will be trivially copied Nz times into the 3d vector
  * @note also check the transformation matrix, which is the more general solution
  */
 template<class real_type>
@@ -486,19 +488,7 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const aRealTo
     return interpolation( pointsX, pointsY, pointsZ, g_old);
 
 }
-/**
- * @brief Create interpolation between two grids
- *
- * This matrix interpolates vectors on the old grid \c g_old to the %Gaussian nodes of the new grid \c g_new. The interpolation is of the order \c g_old.n()
- * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
- *
- * @param g_new The new grid
- * @param g_old The old grid
- *
- * @return Interpolation matrix with \c g_old.size() columns and \c g_new.size() rows
- * @note When interpolating a 2d grid to a 3d grid the third coordinate is simply ignored, i.e. the 2d vector will be trivially copied Nz times into the 3d vector
- * @note also check the transformation matrix, which is the more general solution
- */
+///@copydoc interpolation(const RealGrid1d<real_type>&,const RealGrid1d<real_type>&)
 template<class real_type>
 cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const aRealTopology3d<real_type>& g_new, const aRealTopology2d<real_type>& g_old)
 {
diff --git a/inc/dg/topology/interpolation_mpit.cu b/inc/dg/topology/interpolation_mpit.cu
new file mode 100644
index 000000000..5d2703893
--- /dev/null
+++ b/inc/dg/topology/interpolation_mpit.cu
@@ -0,0 +1,118 @@
+#include <iostream>
+#include <sstream>
+#include <mpi.h>
+#include "dg/backend/mpi_init.h"
+#include "dg/backend/transpose.h"
+#include "dg/blas.h"
+#include "mpi_projection.h"
+#include "mpi_evaluation.h"
+
+
+double shift = 0.2;
+double function( double x, double y){ return sin(2*M_PI*x)*sin(2*M_PI*y);}
+
+int main(int argc, char* argv[])
+{
+
+    MPI_Init( &argc, &argv);
+    int rank, size;
+    MPI_Comm_size( MPI_COMM_WORLD, &size);
+    if(size!=4)
+    {
+        std::cerr << "Please run with 4 processes!\n";
+        MPI_Finalize();
+        return 0;
+    }
+    unsigned n, Nx, Ny;
+    MPI_Comm comm;
+    std::stringstream ss;
+    ss<< "2 2 3 8 8";
+    mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm, ss);
+    MPI_Comm_rank( comm, &rank);
+    ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%//
+    if(rank==0) std::cout << "Test NON-COMMUNICATING MPI matrix-creation!\n";
+    dg::MPIGrid2d g2d( 0,1,0,1, n,Nx,Ny, comm);
+    dg::MPIGrid2d g2d_half = g2d;
+    g2d_half.multiplyCellNumbers(0.5, 0.5);
+    dg::MIHMatrix direct_p= dg::create::interpolation( g2d, g2d_half);
+    dg::HVec x = dg::evaluate( dg::cooX2d, g2d.local());
+    dg::HVec y = dg::evaluate( dg::cooY2d, g2d.local());
+    dg::IHMatrix global_projection = dg::create::interpolation( x,y, g2d_half.global());
+    dg::MIHMatrix converted_p = dg::convert(global_projection, g2d_half);
+
+    //now compare
+    bool equal_cols=true, equal_rows=true, equal_values=true;
+    for( unsigned i=0; i<direct_p.matrix().values.size(); i++)
+    {
+        if( direct_p.matrix().column_indices[i] - converted_p.matrix().column_indices[i] > 1e-15) equal_cols = false;
+        if( direct_p.matrix().values[i] - converted_p.matrix().values[i] > 1e-15) equal_values = false;
+    }
+    for( unsigned i=0; i<direct_p.matrix().num_rows+1; i++)
+        if( direct_p.matrix().row_offsets[i] - converted_p.matrix().row_offsets[i] > 1e-15) equal_rows = false;
+
+    if( !equal_cols || !equal_rows || !equal_values || direct_p.collective().buffer_size() != 0 || converted_p.collective().buffer_size() != 0 )
+        std::cout << "FAILED from rank "<<rank<<"!\n";
+    else
+        std::cout << "SUCCESS from rank "<<rank<<"!\n";
+
+    MPI_Barrier(comm);
+    ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%//
+    if(rank==0) std::cout << "Now test COMMUNICATING MPI matrix-creation!\n";
+    x = dg::evaluate( dg::cooX2d, g2d.local());
+    y = dg::evaluate( dg::cooY2d, g2d.local());
+    for( unsigned i=0; i<x.size(); i++)
+    {
+        x[i] +=shift;
+        y[i] +=shift;
+        bool negative = false;
+        g2d.global().shift( negative, x[i], y[i]);
+    }
+    dg::MIHMatrix converted_i = dg::create::interpolation( x,y,g2d);
+    dg::IHMatrix  direct_i = dg::create::interpolation( x,y,g2d.global());
+    dg::MHVec sine = dg::evaluate( function, g2d);
+    dg::MHVec temp(sine);
+    converted_i.symv( sine, temp);
+    dg::HVec global_sine = dg::evaluate( function, g2d.global());
+    dg::HVec g_temp( g2d.local().size());
+    dg::blas2::symv( direct_i, global_sine, g_temp);
+    //now compare
+    bool success = true;
+    for( unsigned i=0; i<temp.size(); i++)
+        if( fabs(temp.data()[i] - g_temp[i]) > 1e-14)
+            success = false;
+    if( !success)
+        std::cout << "FAILED from rank "<<rank<<"!\n";
+    else
+        std::cout << "SUCCESS from rank "<<rank<<"!\n";
+    MPI_Barrier(comm);
+    ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%//
+    if(rank==0) std::cout << "Now test TRANSPOSITION!\n";
+    converted_i = dg::transpose( converted_i);
+    converted_i.symv( sine, temp);
+    //Compute global transposition and distribute among processes
+    x = dg::evaluate( dg::cooX2d, g2d.global());
+    y = dg::evaluate( dg::cooY2d, g2d.global());
+    for( unsigned i=0; i<x.size(); i++)
+    {
+        x[i] +=shift;
+        y[i] +=shift;
+        bool negative = false;
+        g2d.global().shift( negative, x[i], y[i]);
+    }
+    direct_i = dg::transpose(dg::create::interpolation( x,y,g2d.global()));
+    g_temp.resize( g2d.global().size());
+    dg::blas2::symv( direct_i, global_sine, g_temp);
+    dg::MHVec mpi_g_temp = dg::global2local( g_temp, g2d);
+    //now compare
+    success = true;
+    for( unsigned i=0; i<temp.size(); i++)
+        if( fabs(temp.data()[i] - mpi_g_temp.data()[i]) > 1e-14)
+            success = false;
+    if( !success)
+        std::cout << "FAILED from rank "<<rank<<"!\n";
+    else
+        std::cout << "SUCCESS from rank "<<rank<<"!\n";
+
+    MPI_Finalize();
+    return 0;
+}
diff --git a/inc/dg/topology/mpi_projection.h b/inc/dg/topology/mpi_projection.h
index 6b1b52b39..70fcaa8ad 100644
--- a/inc/dg/topology/mpi_projection.h
+++ b/inc/dg/topology/mpi_projection.h
@@ -122,39 +122,39 @@ dg::tMIHMatrix<real_type> interpolation( const aRealMPITopology2d<real_type>& g_
 {
     return tMIHMatrix<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
-///@copydoc interpolation(const RealGrid1d&,const RealGrid1d&)
+///@copydoc dg::create::interpolation(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
 dg::tMIHMatrix<real_type> interpolation( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology3d<real_type>& g_old)
 {
     return tMIHMatrix<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
-///@copydoc dg::create::interpolation(const aRealTopology3d<real_type>&,const aRealTopology2d<real_type>&)
+///@copydoc dg::create::interpolation(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
 dg::tMIHMatrix<real_type> interpolation( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
 {
     return tMIHMatrix<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
 
-///@copydoc interpolationT(const RealGrid1d&,const RealGrid1d&)
+///@copydoc dg::create::interpolationT(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
 dg::tMIHMatrix<real_type> interpolationT( const aRealMPITopology2d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
 {
     return tMIHMatrix<real_type>( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
-///@copydoc interpolationT(const RealGrid1d&,const RealGrid1d&)
+///@copydoc dg::create::interpolationT(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
 dg::tMIHMatrix<real_type> interpolationT( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology3d<real_type>& g_old)
 {
     return tMIHMatrix<real_type>( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
 
-///@copydoc projection(const RealGrid1d&,const RealGrid1d&)
+///@copydoc dg::create::projection(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
 dg::tMIHMatrix<real_type> projection( const aRealMPITopology2d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
 {
     return tMIHMatrix<real_type>( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
-///@copydoc projection(const RealGrid1d&,const RealGrid1d&)
+///@copydoc dg::create::projection(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
 dg::tMIHMatrix<real_type> projection( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology3d<real_type>& g_old)
 {
diff --git a/inc/dg/topology/projection.h b/inc/dg/topology/projection.h
index f0cce22da..e5c83e4c1 100644
--- a/inc/dg/topology/projection.h
+++ b/inc/dg/topology/projection.h
@@ -104,6 +104,7 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolationT( const aRealT
  between old and new vector small.
  * The projection matrix is the adjoint of the interpolation matrix
  * @sa <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @sa for integer multiples between old and new grid you may want to consider the dg::create::fast_projection functions
  *
  * @param g_new The new (coarse) grid
  * @param g_old The old (fine) grid
diff --git a/inc/dg/topology/projection_mpit.cu b/inc/dg/topology/projection_mpit.cu
index 5d2703893..edf9fd054 100644
--- a/inc/dg/topology/projection_mpit.cu
+++ b/inc/dg/topology/projection_mpit.cu
@@ -4,16 +4,19 @@
 #include "dg/backend/mpi_init.h"
 #include "dg/backend/transpose.h"
 #include "dg/blas.h"
+#include "dg/blas1.h"
+#include "mpi_weights.h"
 #include "mpi_projection.h"
 #include "mpi_evaluation.h"
+#include "fast_interpolation.h"
 
-
-double shift = 0.2;
-double function( double x, double y){ return sin(2*M_PI*x)*sin(2*M_PI*y);}
+double sine( double x){ return sin(x);}
+double sine( double x, double y){return sin(x)*sin(y);}
+double sine( double x, double y, double z){return sin(x)*sin(y);}
+//Actually this file is a test for fast_interpolation
 
 int main(int argc, char* argv[])
 {
-
     MPI_Init( &argc, &argv);
     int rank, size;
     MPI_Comm_size( MPI_COMM_WORLD, &size);
@@ -23,95 +26,47 @@ int main(int argc, char* argv[])
         MPI_Finalize();
         return 0;
     }
-    unsigned n, Nx, Ny;
+    unsigned n, Nx, Ny, Nz;
     MPI_Comm comm;
     std::stringstream ss;
-    ss<< "2 2 3 8 8";
-    mpi_init2d( dg::PER, dg::PER, n, Nx, Ny, comm, ss);
+    ss<< "2 2 1 3 8 8 8";
+    mpi_init3d( dg::PER, dg::PER, dg::PER, n, Nx, Ny, Nz, comm, ss);
     MPI_Comm_rank( comm, &rank);
-    ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%//
-    if(rank==0) std::cout << "Test NON-COMMUNICATING MPI matrix-creation!\n";
-    dg::MPIGrid2d g2d( 0,1,0,1, n,Nx,Ny, comm);
-    dg::MPIGrid2d g2d_half = g2d;
-    g2d_half.multiplyCellNumbers(0.5, 0.5);
-    dg::MIHMatrix direct_p= dg::create::interpolation( g2d, g2d_half);
-    dg::HVec x = dg::evaluate( dg::cooX2d, g2d.local());
-    dg::HVec y = dg::evaluate( dg::cooY2d, g2d.local());
-    dg::IHMatrix global_projection = dg::create::interpolation( x,y, g2d_half.global());
-    dg::MIHMatrix converted_p = dg::convert(global_projection, g2d_half);
-
-    //now compare
-    bool equal_cols=true, equal_rows=true, equal_values=true;
-    for( unsigned i=0; i<direct_p.matrix().values.size(); i++)
-    {
-        if( direct_p.matrix().column_indices[i] - converted_p.matrix().column_indices[i] > 1e-15) equal_cols = false;
-        if( direct_p.matrix().values[i] - converted_p.matrix().values[i] > 1e-15) equal_values = false;
-    }
-    for( unsigned i=0; i<direct_p.matrix().num_rows+1; i++)
-        if( direct_p.matrix().row_offsets[i] - converted_p.matrix().row_offsets[i] > 1e-15) equal_rows = false;
 
-    if( !equal_cols || !equal_rows || !equal_values || direct_p.collective().buffer_size() != 0 || converted_p.collective().buffer_size() != 0 )
-        std::cout << "FAILED from rank "<<rank<<"!\n";
-    else
-        std::cout << "SUCCESS from rank "<<rank<<"!\n";
-
-    MPI_Barrier(comm);
-    ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%//
-    if(rank==0) std::cout << "Now test COMMUNICATING MPI matrix-creation!\n";
-    x = dg::evaluate( dg::cooX2d, g2d.local());
-    y = dg::evaluate( dg::cooY2d, g2d.local());
-    for( unsigned i=0; i<x.size(); i++)
-    {
-        x[i] +=shift;
-        y[i] +=shift;
-        bool negative = false;
-        g2d.global().shift( negative, x[i], y[i]);
-    }
-    dg::MIHMatrix converted_i = dg::create::interpolation( x,y,g2d);
-    dg::IHMatrix  direct_i = dg::create::interpolation( x,y,g2d.global());
-    dg::MHVec sine = dg::evaluate( function, g2d);
-    dg::MHVec temp(sine);
-    converted_i.symv( sine, temp);
-    dg::HVec global_sine = dg::evaluate( function, g2d.global());
-    dg::HVec g_temp( g2d.local().size());
-    dg::blas2::symv( direct_i, global_sine, g_temp);
-    //now compare
-    bool success = true;
-    for( unsigned i=0; i<temp.size(); i++)
-        if( fabs(temp.data()[i] - g_temp[i]) > 1e-14)
-            success = false;
-    if( !success)
-        std::cout << "FAILED from rank "<<rank<<"!\n";
-    else
-        std::cout << "SUCCESS from rank "<<rank<<"!\n";
-    MPI_Barrier(comm);
     ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%//
-    if(rank==0) std::cout << "Now test TRANSPOSITION!\n";
-    converted_i = dg::transpose( converted_i);
-    converted_i.symv( sine, temp);
-    //Compute global transposition and distribute among processes
-    x = dg::evaluate( dg::cooX2d, g2d.global());
-    y = dg::evaluate( dg::cooY2d, g2d.global());
-    for( unsigned i=0; i<x.size(); i++)
-    {
-        x[i] +=shift;
-        y[i] +=shift;
-        bool negative = false;
-        g2d.global().shift( negative, x[i], y[i]);
-    }
-    direct_i = dg::transpose(dg::create::interpolation( x,y,g2d.global()));
-    g_temp.resize( g2d.global().size());
-    dg::blas2::symv( direct_i, global_sine, g_temp);
-    dg::MHVec mpi_g_temp = dg::global2local( g_temp, g2d);
-    //now compare
-    success = true;
-    for( unsigned i=0; i<temp.size(); i++)
-        if( fabs(temp.data()[i] - mpi_g_temp.data()[i]) > 1e-14)
-            success = false;
-    if( !success)
-        std::cout << "FAILED from rank "<<rank<<"!\n";
-    else
-        std::cout << "SUCCESS from rank "<<rank<<"!\n";
+    if(rank==0)std::cout << "TEST 2D and 3D\n";
+    unsigned n_old = 6, n_new = 3, N_old = 40, N_new = 20;
+    if(rank==0)std::cout << "Fine   Grid "<< n_old << " x "<<N_old <<"\n";
+    if(rank==0)std::cout << "Coarse Grid "<< n_new << " x "<<N_new <<"\n";
+    dg::MPIGrid3d g2o (0, M_PI, 0, M_PI, 0,1, n_old, N_old, N_old, 4, comm);
+    dg::MPIGrid3d g2n (0, M_PI, 0, M_PI, 0,1, n_new, N_new, N_new, 4, comm);
+    dg::MIHMatrix inte2d = dg::create::interpolation( g2n, g2o);
+    dg::MultiMatrix< dg::MHMatrix, dg::MHVec > proj2d = dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new, n_old/n_new);
+    dg::MultiMatrix< dg::MHMatrix, dg::MHVec > fast_inte2d = dg::create::fast_interpolation( g2n, N_old/N_new, N_old/N_new, n_old/n_new);
+    const dg::MHVec sinO( dg::evaluate( sine, g2o)),
+                                sinN( dg::evaluate( sine, g2n));
+    dg::MHVec w2do = dg::create::weights( g2o);
+    dg::MHVec w2dn = dg::create::weights( g2n);
+    dg::MHVec sinP( sinN), sinI(sinO);
+    dg::blas2::gemv( proj2d, sinO, sinP); //FAST PROJECTION
+    double value0 = sqrt(dg::blas2::dot( sinO, w2do, sinO));
+    if(rank==0)std::cout << "Original vector     "<<value0 << "\n";
+    double value1 = sqrt(dg::blas2::dot( sinP, w2dn, sinP));
+    if(rank==0)std::cout << "Projected vector    "<<value1 << "\n";
+    if(rank==0)std::cout << "Difference in Norms "<<value0-value1 << std::endl;
+    dg::blas1::axpby( 1., sinN, -1., sinP);
+    double value2 = sqrt(dg::blas2::dot( sinP, w2dn, sinP)/dg::blas2::dot(sinN, w2dn, sinN));
+    if(rank==0)std::cout << "Difference between projection and evaluation      "<<value2<<"\n";
+    dg::blas2::gemv( inte2d, sinO, sinP);
+    value1 = sqrt(dg::blas2::dot( sinP, w2dn, sinP));
+    if(rank==0)std::cout << "Interpolated vec    "<<value1 << "\n";
+    value0 = sqrt(dg::blas2::dot( sinO, w2do, sinO));
+    if(rank==0)std::cout << "Difference in Norms "<<value0 - value1 << "\n" << std::endl;
+    dg::blas2::gemv( fast_inte2d, sinN, sinI);
+    value2 = sqrt(dg::blas2::dot( sinI, w2do, sinI));
+    if(rank==0)std::cout << "Fast Interpolated vec "<< value2 << "\n";
+    double value3 = sqrt(dg::blas2::dot( sinN, w2dn, sinN));
+    if(rank==0)std::cout << "Difference in Norms   "<<value2 - value3  << "\n" << std::endl;
 
     MPI_Finalize();
     return 0;
diff --git a/inc/dg/topology/projection_t.cu b/inc/dg/topology/projection_t.cu
index 11bd4d985..aa1da28a7 100644
--- a/inc/dg/topology/projection_t.cu
+++ b/inc/dg/topology/projection_t.cu
@@ -8,21 +8,24 @@
 double sine( double x){ return sin(x);}
 double sine( double x, double y){return sin(x)*sin(y);}
 double sine( double x, double y, double z){return sin(x)*sin(y);}
+//Actually this file is a test for fast_interpolation
 
 int main()
 {
     std::cout << "TEST 1D\n";
-    unsigned n_old = 4, n_new = 3, N_old = 10, N_new = 1;
-    std::cout << "Type n and N of old (fine) grid!\n";
-    std::cin >> n_old >> N_old;
-    std::cout << "Type n and N of new (coarser) grid!\n";
-    std::cin >> n_new >> N_new;
+    unsigned n_old = 6, n_new = 3, N_old = 40, N_new = 20;
+    //std::cout << "Type n and N of old (fine) grid!\n";
+    //std::cin >> n_old >> N_old;
+    //std::cout << "Type n and N of new (coarser) grid!\n";
+    //std::cin >> n_new >> N_new;
+    std::cout << "Fine   Grid "<< n_old << " x "<<N_old <<"\n";
+    std::cout << "Coarse Grid "<< n_new << " x "<<N_new <<"\n";
     dg::Grid1d go ( 0, M_PI/2., n_old, N_old);
     dg::Grid1d gn ( 0, M_PI/2., n_new, N_new);
     //cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::projection( gn, go);
     //cusp::coo_matrix<int, double, cusp::host_memory> inte = dg::create::interpolation( go, gn);
-    dg::MultiMatrix< dg::DMatrix, dg::DVec > proj = dg::create::fast_projection( go,  N_old/N_new);
-    dg::MultiMatrix< dg::DMatrix, dg::DVec > inte = dg::create::fast_interpolation( gn, N_old/N_new);
+    dg::MultiMatrix< dg::DMatrix, dg::DVec > proj = dg::create::fast_projection( go,  N_old/N_new, n_old/n_new);
+    dg::MultiMatrix< dg::DMatrix, dg::DVec > inte = dg::create::fast_interpolation( gn, N_old/N_new, n_old/n_new);
     dg::DVec v = dg::evaluate( sine, go);
     dg::DVec w1do = dg::create::weights( go);
     dg::DVec w1dn = dg::create::weights( gn);
@@ -32,12 +35,12 @@ int main()
     dg::blas2::symv( proj, v, w);
     std::cout << "Original vector  "<<dg::blas2::dot( oneo, w1do, v) << "\n";
     std::cout << "Projected vector "<<dg::blas2::dot( onen, w1dn, w) << "\n";
-    std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << "\n"<<std::endl;
+    std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << " (Must be 0)\n"<<std::endl;
     w = dg::evaluate( sine, gn);
     dg::blas2::symv( inte, w, v);
     std::cout << "Original vector  "<<dg::blas2::dot( onen, w1dn, w) << "\n";
     std::cout << "Interpolated vec "<<dg::blas2::dot( oneo, w1do, v) << "\n";
-    std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << "\n"<<std::endl;
+    std::cout << "Difference       "<<dg::blas2::dot( oneo, w1do, v) - dg::blas2::dot( onen, w1dn, w) << " (Must be 0)\n"<<std::endl;
 
     std::cout << "TEST 2D and 3D\n";
 
@@ -46,28 +49,46 @@ int main()
     //cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::transformation( g2n, g2o);
     cusp::coo_matrix<int, double, cusp::host_memory> inte2d = dg::create::interpolation( g2n, g2o);
     //dg::MultiMatrix< dg::HMatrix, std::vector<thrust::host_vector<double>> > proj2d;
-    //proj2d.construct( dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new), 2);
+    //proj2d.construct( dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new, n_old/n_new), 2);
     //dg::IHMatrix proj2d = dg::create::projection( g2n, g2o);
-    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > proj2d = dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new);
-    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > fast_inte2d = dg::create::fast_interpolation( g2n, N_old/N_new, N_old/N_new);
+    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > proj2d = dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new, n_old/n_new);
+    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > fast_inte2d = dg::create::fast_interpolation( g2n, N_old/N_new, N_old/N_new, n_old/n_new);
     const dg::HVec sinO( dg::evaluate( sine, g2o)),
                                 sinN( dg::evaluate( sine, g2n));
     dg::HVec w2do = dg::create::weights( g2o);
     dg::HVec w2dn = dg::create::weights( g2n);
     dg::HVec sinP( sinN), sinI(sinO);
     dg::blas2::gemv( proj2d, sinO, sinP); //FAST PROJECTION
-    std::cout << "Original vector     "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) << "\n";
-    std::cout << "Projected vector    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
-    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << std::endl;
-    std::cout << "Difference between projection and evaluation      ";
+    //std::cout << "Original vector     "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) << "\n";
+    //std::cout << "Projected vector    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
+    //std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << std::endl;
+    //std::cout << "Difference between projection and evaluation      ";
+    //dg::blas1::axpby( 1., sinN, -1., sinP);
+    //std::cout << sqrt(dg::blas2::dot( sinP, w2dn, sinP)/dg::blas2::dot(sinN, w2dn, sinN))<<"\n";
+    //dg::blas2::gemv( inte2d, sinO, sinP);
+    //std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
+    //std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
+    //dg::blas2::gemv( fast_inte2d, sinN, sinI);
+    //std::cout << "Fast Interpolated vec "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) << "\n";
+    //std::cout << "Difference in Norms   "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) - sqrt(dg::blas2::dot( sinN, w2dn, sinN)) << "\n" << std::endl;
+    double value0 = sqrt(dg::blas2::dot( sinO, w2do, sinO));
+    std::cout << "Original vector     "<<value0 << "\n";
+    double value1 = sqrt(dg::blas2::dot( sinP, w2dn, sinP));
+    std::cout << "Projected vector    "<<value1 << "\n";
+    std::cout << "Difference in Norms "<<value0-value1 << std::endl;
     dg::blas1::axpby( 1., sinN, -1., sinP);
-    std::cout << sqrt(dg::blas2::dot( sinP, w2dn, sinP)/dg::blas2::dot(sinN, w2dn, sinN))<<"\n";
+    double value2 = sqrt(dg::blas2::dot( sinP, w2dn, sinP)/dg::blas2::dot(sinN, w2dn, sinN));
+    std::cout << "Difference between projection and evaluation      "<<value2<<"\n";
     dg::blas2::gemv( inte2d, sinO, sinP);
-    std::cout << "Interpolated vec    "<<sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n";
-    std::cout << "Difference in Norms "<<sqrt(dg::blas2::dot( sinO, w2do, sinO)) - sqrt(dg::blas2::dot( sinP, w2dn, sinP)) << "\n" << std::endl;
+    value1 = sqrt(dg::blas2::dot( sinP, w2dn, sinP));
+    std::cout << "Interpolated vec    "<<value1 << "\n";
+    value0 = sqrt(dg::blas2::dot( sinO, w2do, sinO));
+    std::cout << "Difference in Norms "<<value0 - value1 << "\n" << std::endl;
     dg::blas2::gemv( fast_inte2d, sinN, sinI);
-    std::cout << "Fast Interpolated vec "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) << "\n";
-    std::cout << "Difference in Norms   "<<sqrt(dg::blas2::dot( sinI, w2do, sinI)) - sqrt(dg::blas2::dot( sinN, w2dn, sinN)) << "\n" << std::endl;
+    value2 = sqrt(dg::blas2::dot( sinI, w2do, sinI));
+    std::cout << "Fast Interpolated vec "<< value2 << "\n";
+    double value3 = sqrt(dg::blas2::dot( sinN, w2dn, sinN));
+    std::cout << "Difference in Norms   "<<value2 - value3  << "\n" << std::endl;
 
     return 0;
 }
diff --git a/src/lamb_dipole/diag.h b/src/lamb_dipole/diag.h
index 8f6ba8995..f69165116 100644
--- a/src/lamb_dipole/diag.h
+++ b/src/lamb_dipole/diag.h
@@ -7,7 +7,7 @@
 namespace shu{
 
 struct Variables{
-    shu::Shu<dg::CartesianGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec>& shu;
+    shu::Shu<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>& shu;
     const dg::CartesianGrid2d& grid;
     const dg::DVec& y0;
     const double& time;
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index 20b047a04..29b989264 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -61,7 +61,7 @@ struct Diffusion
     dg::Elliptic<Geometry, Matrix,Container> m_LaplacianM;
 };
 
-template< class Geometry, class IMatrix, class Matrix, class Container >
+template< class Geometry, class Matrix, class Container >
 struct Shu
 {
     using value_type = dg::get_value_type<Container>;
@@ -94,7 +94,7 @@ struct Shu
     dg::ArakawaX<Geometry, Matrix, Container> m_arakawa;
     dg::Extrapolation<Container> m_old_psi;
     dg::MultigridCG2d<Geometry, Matrix, Container> m_multigrid;
-    IMatrix m_inter, m_project;
+    dg::MultiMatrix<Matrix,Container> m_inter, m_project;
     Matrix m_forward[2], m_backward[2], m_centered[2];
     Matrix m_centered_phi[2]; // for variation
     std::vector<double> m_eps;
@@ -109,8 +109,8 @@ struct Shu
     bool m_add_viscosity = false;
 };
 
-template<class Geometry, class IMatrix, class Matrix, class Container>
-Shu< Geometry, IMatrix, Matrix, Container>::Shu(
+template<class Geometry, class Matrix, class Container>
+Shu< Geometry, Matrix, Container>::Shu(
         const Geometry& g, Json::Value& js, enum dg::file::error mode):
     m_old_psi( 2, dg::evaluate( dg::zero, g)),
     m_multigrid( g, 3),
@@ -131,9 +131,10 @@ Shu< Geometry, IMatrix, Matrix, Container>::Shu(
     if( "projection" == m_multiplication)
     {
         Geometry fine_grid = g;
-        fine_grid.set( 2*g.n()-1, g.Nx(), g.Ny());
-        m_inter = dg::create::interpolation( fine_grid, g);
-        m_project = dg::create::projection( g, fine_grid);
+        fine_grid.set( 2*g.n(), g.Nx(), g.Ny());
+        //theoretically we only need 2n-1 but it isn't wrong to take more
+        m_inter = dg::create::fast_interpolation( g, 1, 1, 2);
+        m_project = dg::create::fast_projection( fine_grid, 1, 1, 2);
 
         m_centered[0] = dg::create::dx( fine_grid, g.bcx(), dg::centered);
         m_centered[1] = dg::create::dy( fine_grid, g.bcy(), dg::centered);
@@ -190,8 +191,8 @@ Shu< Geometry, IMatrix, Matrix, Container>::Shu(
     }
 }
 
-template< class Geometry, class IMatrix, class Matrix, class Container>
-void Shu<Geometry, IMatrix, Matrix, Container>::operator()(double t, const Container& y, Container& yp)
+template< class Geometry, class Matrix, class Container>
+void Shu<Geometry, Matrix, Container>::operator()(double t, const Container& y, Container& yp)
 {
     //solve elliptic equation
     m_old_psi.extrapolate( t, m_psi);
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index fde965466..de7fc2f2b 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -43,7 +43,7 @@ int main( int argc, char* argv[])
         dg::blas1::axpby( -meanMass, 1., 1., y0);
     }
     //make solver and stepper
-    shu::Shu<dg::CartesianGrid2d, dg::IDMatrix, dg::DMatrix, dg::DVec>
+    shu::Shu<dg::CartesianGrid2d, dg::DMatrix, dg::DVec>
         shu( grid, js, mode);
     shu::Diffusion<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> diffusion( grid, js, mode);
     if( "mms" == initial)
-- 
GitLab


From 753edd07381d8f56859dffafb90d92d689f4577a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 3 Feb 2021 22:11:48 +0100
Subject: [PATCH 453/540] Adapt default input file for shu

---
 src/lamb_dipole/input/default.json | 33 +++++++++++++++---------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/src/lamb_dipole/input/default.json b/src/lamb_dipole/input/default.json
index 9e594aad8..37bd1d9ed 100644
--- a/src/lamb_dipole/input/default.json
+++ b/src/lamb_dipole/input/default.json
@@ -13,25 +13,22 @@
     },
     //"timestepper":
     //{
-    //    "stepper": "Karniadakis",
-    //    "order": 3,
-    //    "dt" : 2e-3,
-    //    "eps_time" : 1e-9, // (stopping for implicit part)
-    //    "solver":{
-    //        "type": "default"
-    //    }
+    //    "type": "ImExMultistep",
+    //    "tableau": "ImEx-BDF-3-3",
+    //    "dt" : 5e-3,
+    //    "eps_time" : 1e-9 // (stopping for implicit part)
     //},
     //"timestepper":
     //{
-    //    "stepper": "Shu-Osher",
-    //    "order": "SSPRK-3-3",
-    //    "dt" : 1e-3
+    //    "type": "Shu-Osher",
+    //    "tableau": "SSPRK-3-3",
+    //    "dt" : 1.5e-2
     //},
     "timestepper":
     {
-        "stepper": "FilteredMultistep",
-        "order": ["eBDF", 3],
-        "dt" : 5e-4
+        "type": "FilteredExplicitMultistep",
+        "tableau": "ImEx-BDF-3-3",
+        "dt" : 1e-3
     },
     "regularization":
     {
@@ -44,7 +41,9 @@
     //{
     //    "type" : "viscosity",
     //    "order": 2,
-    //    "nu_perp": 1.0e-3  //(viscosity)  
+    //    "direction": "forward",
+    //    //"direction": "centered",
+    //    "nu": 1.0e-3  //(viscosity)
     //},
     "output":
     {
@@ -58,7 +57,9 @@
     {
         "type" : "multigrid",
         "stages": 3,
-        "eps_pol" : [1e-6,10,10] // (stopping on each steage )
+        "eps_pol" : [1e-6,10.0,10.0], // (stopping on each steage )
+        "direction" : "forward" //does not work with double PER
+        //"direction" : "centered"
     },
     "advection":
     {
@@ -82,7 +83,7 @@
     //    "rho":  0.20943951023931953, //pi/15
     //    "delta": 0.05
     //}
-    //"init": //use [-1,1]x[-1,1]
+    //"init":
     //{
     //    "type" : "mms",
     //    "sigma":  0.2,
-- 
GitLab


From 8de38fd21f5d21db2f23cd9d7e21f5db244620f0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 3 Feb 2021 22:14:05 +0100
Subject: [PATCH 454/540] adapt bc in shu.tex

---
 src/lamb_dipole/shu.tex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lamb_dipole/shu.tex b/src/lamb_dipole/shu.tex
index 91a9cd911..03128f952 100644
--- a/src/lamb_dipole/shu.tex
+++ b/src/lamb_dipole/shu.tex
@@ -67,7 +67,7 @@ The Lamb dipole is chosen with the following parameters in the input file
 grid & dict &  & Grid parameters \\
 \qquad x & float[2]& [0,1] & Choose x box boundaries appropriately \\
 \qquad y & float[2]& [0,1] & Choose y box boundaries appropriately \\
-\qquad bc & string[2] & [PER, PER] & Choose boundary conditions [x,y] appropriately \\
+\qquad bc & string[2] & [DIR, PER] & Choose boundary conditions [x,y] appropriately \\
 init &  dict &   & Parameters for initialization \\
 \qquad type      & string & lamb & This choice necessitates the following parameters \\
 \qquad velocity  & float &  1    &  blob speed \\
@@ -156,7 +156,7 @@ The double shear layer initialization is chosen with the following parameters in
 grid & dict &  & Grid parameters \\
 \qquad x & float[2]& [0, 6.283185307179586] & Choose x box boundaries appropriately \\
 \qquad y & float[2]& [0, 6.283185307179586] & Choose y box boundaries appropriately \\
-\qquad bc & string[2] & [PER, PER] & Choose boundary conditions appropriately \\
+\qquad bc & string[2] & [PER, DIR] & Choose boundary conditions appropriately \\
 init &  dict &   & Parameters for initialization \\
 \qquad type      & string & shear & This choice necessitates the following parameters \\
 \qquad rho    & float & 0.20943951023931953 & The width $\rho = \pi/15$ \\
-- 
GitLab


From b6c0f41729098391916143054bcec2c1aa9dcaef Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 4 Feb 2021 21:37:49 +0100
Subject: [PATCH 455/540] Add upwind advection scheme

in contrast to upwind in the conservative formulation
---
 inc/dg/arakawa.h        |  18 ++++--
 src/lamb_dipole/shu.cuh | 138 +++++++++++++++++++++++++++++++---------
 src/lamb_dipole/shu.tex |  18 ++++--
 3 files changed, 132 insertions(+), 42 deletions(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 6111a24c8..542bd00c5 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -77,7 +77,11 @@ struct ArakawaX
      * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
      */
     template<class ContainerType0, class ContainerType1, class ContainerType2>
-    void operator()( const ContainerType0& lhs, const ContainerType1& rhs, ContainerType2& result);
+    void operator()( const ContainerType0& lhs, const ContainerType1& rhs, ContainerType2& result){
+        return this->operator()( 1., lhs, rhs, 0., result);
+    }
+    template<class ContainerType0, class ContainerType1, class ContainerType2>
+    void operator()( value_type alpha, const ContainerType0& lhs, const ContainerType1& rhs, value_type beta, ContainerType2& result);
     /**
      * @brief Change Chi
      *
@@ -151,18 +155,18 @@ struct ArakawaFunctor
 
 template< class Geometry, class Matrix, class Container>
 template<class ContainerType0, class ContainerType1, class ContainerType2>
-void ArakawaX< Geometry, Matrix, Container>::operator()( const ContainerType0& lhs, const ContainerType1& rhs, ContainerType2& result)
+void ArakawaX< Geometry, Matrix, Container>::operator()( value_type alpha, const ContainerType0& lhs, const ContainerType1& rhs, value_type beta, ContainerType2& result)
 {
     //compute derivatives in x-space
     blas2::symv( m_bdxf, lhs, m_dxlhs);
     blas2::symv( m_bdyf, lhs, m_dylhs);
     blas2::symv( m_bdxf, rhs, m_dxrhs);
-    blas2::symv( m_bdyf, rhs, result);
-    blas1::subroutine( ArakawaFunctor<get_value_type<Container>>(), lhs, rhs, m_dxlhs, m_dylhs, m_dxrhs, result);
+    blas2::symv( m_bdyf, rhs, m_dyrhs);
+    blas1::subroutine( ArakawaFunctor<get_value_type<Container>>(), lhs, rhs, m_dxlhs, m_dylhs, m_dxrhs, m_dyrhs);
 
-    blas2::symv( 1., m_bdxf, m_dylhs, 1., result);
-    blas2::symv( 1., m_bdyf, m_dxrhs, 1., result);
-    blas1::pointwiseDot( m_chi, result, result);
+    blas2::symv( 1., m_bdxf, m_dylhs, 1., m_dyrhs);
+    blas2::symv( 1., m_bdyf, m_dxrhs, 1., m_dyrhs);
+    blas1::pointwiseDot( alpha, m_chi, m_dyrhs, beta, result);
 }
 ///@endcond
 
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index 29b989264..e7d756f0c 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -16,10 +16,20 @@ struct Upwind{
     DG_DEVICE
     void operator()( double& result, double fw, double bw, double v)
     {
-        if( v > 0)
-            result -= bw; // yp = - Div( F)
+        if( v < 0)
+            result -= fw; // yp = - Div( F)
         else
-            result -= fw;
+            result -= bw;
+    }
+};
+struct UpwindAdvection{
+    DG_DEVICE
+    void operator()( double& result, double fw, double bw, double v)
+    {
+        if( v < 0)
+            result -= v*fw; // yp = - v Grad f
+        else
+            result -= v*bw;
     }
 };
 template< class Geometry, class Matrix, class Container>
@@ -96,6 +106,7 @@ struct Shu
     dg::MultigridCG2d<Geometry, Matrix, Container> m_multigrid;
     dg::MultiMatrix<Matrix,Container> m_inter, m_project;
     Matrix m_forward[2], m_backward[2], m_centered[2];
+    Matrix m_fine_forward[2], m_fine_backward[2], m_fine_centered[2];
     Matrix m_centered_phi[2]; // for variation
     std::vector<double> m_eps;
     std::string m_advection, m_multiplication;
@@ -136,12 +147,12 @@ Shu< Geometry, Matrix, Container>::Shu(
         m_inter = dg::create::fast_interpolation( g, 1, 1, 2);
         m_project = dg::create::fast_projection( fine_grid, 1, 1, 2);
 
-        m_centered[0] = dg::create::dx( fine_grid, g.bcx(), dg::centered);
-        m_centered[1] = dg::create::dy( fine_grid, g.bcy(), dg::centered);
-        m_forward[0] = dg::create::dx( fine_grid, dg::inverse( g.bcx()), dg::forward);
-        m_forward[1] = dg::create::dy( fine_grid, dg::inverse( g.bcy()), dg::forward);
-        m_backward[0] = dg::create::dx( fine_grid, dg::inverse( g.bcx()), dg::backward);
-        m_backward[1] = dg::create::dy( fine_grid, dg::inverse( g.bcy()), dg::backward);
+        m_fine_centered[0] = dg::create::dx( fine_grid, g.bcx(), dg::centered);
+        m_fine_centered[1] = dg::create::dy( fine_grid, g.bcy(), dg::centered);
+        m_fine_forward[0] = dg::create::dx( fine_grid, dg::inverse( g.bcx()), dg::forward);
+        m_fine_forward[1] = dg::create::dy( fine_grid, dg::inverse( g.bcy()), dg::forward);
+        m_fine_backward[0] = dg::create::dx( fine_grid, dg::inverse( g.bcx()), dg::backward);
+        m_fine_backward[1] = dg::create::dy( fine_grid, dg::inverse( g.bcy()), dg::backward);
         m_fine_psi = dg::evaluate( dg::zero, fine_grid);
         m_fine_y = dg::evaluate( dg::zero, fine_grid);
         m_fine_yp = dg::evaluate( dg::zero, fine_grid);
@@ -151,16 +162,13 @@ Shu< Geometry, Matrix, Container>::Shu(
         m_fine_temp[2] = dg::evaluate( dg::zero, fine_grid);
         m_arakawa.construct( fine_grid);
     }
-    else
-    {
-        m_centered[0] = dg::create::dx( g, g.bcx(), dg::centered);
-        m_centered[1] = dg::create::dy( g, g.bcy(), dg::centered);
-        m_forward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::forward);
-        m_forward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::forward);
-        m_backward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::backward);
-        m_backward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::backward);
-        m_arakawa.construct( g);
-    }
+    m_centered[0] = dg::create::dx( g, g.bcx(), dg::centered);
+    m_centered[1] = dg::create::dy( g, g.bcy(), dg::centered);
+    m_forward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::forward);
+    m_forward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::forward);
+    m_backward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::backward);
+    m_backward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::backward);
+    m_arakawa.construct( g);
 
     unsigned stages = dg::file::get( mode, js, "elliptic", "stages", 3).asUInt();
     m_eps.resize(stages);
@@ -232,39 +240,107 @@ void Shu<Geometry, Matrix, Container>::operator()(double t, const Container& y,
             dg::blas1::pointwiseDot( y, m_v, m_temp[0]); //f_y
             dg::blas2::symv( -1., m_centered[1], m_temp[0], 1., yp);
         }
+        else if( "upwind-advection" == m_advection)
+        {
+            dg::blas1::copy( 0., yp);
+            // v_x dx n
+            dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
+            dg::blas2::symv( m_forward[0], y, m_temp[1]);
+            dg::blas2::symv( m_backward[0], y, m_temp[2]);
+            dg::blas1::subroutine( shu::UpwindAdvection(), yp, m_temp[1], m_temp[2], m_v);
+            // v_y dy n
+            dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_v); //v_y
+            dg::blas2::symv( m_forward[1], y, m_temp[1]);
+            dg::blas2::symv( m_backward[1], y, m_temp[2]);
+            dg::blas1::subroutine( shu::UpwindAdvection(), yp, m_temp[1], m_temp[2], m_v);
+        }
+        else if( "centered-advection" == m_advection)
+        {
+            //dx ( nv_x)
+            dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
+            dg::blas2::symv( -1., m_centered[0], y, 0., m_temp[0]); // -v_x dx omega
+            dg::blas1::pointwiseDot( m_v, m_temp[0], yp);
+            //dy ( nv_y)
+            dg::blas2::symv(  1., m_centered[0], m_psi, 0., m_v); //v_y
+            dg::blas2::symv( -1., m_centered[1], y, 0., m_temp[0]);
+            dg::blas1::pointwiseDot( 1., m_v, m_temp[0], 1., yp); //f_y
+        }
     }
     else // "projection " == multiplication
     {
-        dg::blas2::symv( m_inter, y, m_fine_y);
-        dg::blas2::symv( m_inter, m_psi, m_fine_psi);
         if( "arakawa" == m_advection)
+        {
+            dg::blas2::symv( m_inter, y, m_fine_y);
+            dg::blas2::symv( m_inter, m_psi, m_fine_psi);
             m_arakawa( m_fine_y, m_fine_psi, m_fine_yp); //A(y,psi)-> yp
+        }
         else if( "upwind" == m_advection)
         {
+            dg::blas2::symv( m_inter, y, m_fine_y);
+            dg::blas2::symv( m_inter, m_psi, m_fine_psi);
             dg::blas1::copy( 0., m_fine_yp);
             //dx ( nv_x)
-            dg::blas2::symv( -1., m_centered[1], m_fine_psi, 0., m_fine_v); //v_x
+            dg::blas2::symv( -1., m_fine_centered[1], m_fine_psi, 0., m_fine_v); //v_x
             dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_x
-            dg::blas2::symv( m_forward[0], m_fine_temp[0], m_fine_temp[1]);
-            dg::blas2::symv( m_backward[0], m_fine_temp[0], m_fine_temp[2]);
+            dg::blas2::symv( m_fine_forward[0], m_fine_temp[0], m_fine_temp[1]);
+            dg::blas2::symv( m_fine_backward[0], m_fine_temp[0], m_fine_temp[2]);
             dg::blas1::subroutine( shu::Upwind(), m_fine_yp, m_fine_temp[1], m_fine_temp[2], m_fine_v);
             //dy ( nv_y)
-            dg::blas2::symv( 1., m_centered[0], m_fine_psi, 0., m_fine_v); //v_y
+            dg::blas2::symv( 1., m_fine_centered[0], m_fine_psi, 0., m_fine_v); //v_y
             dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_y
-            dg::blas2::symv( m_forward[1], m_fine_temp[0], m_fine_temp[1]);
-            dg::blas2::symv( m_backward[1], m_fine_temp[0], m_fine_temp[2]);
+            dg::blas2::symv( m_fine_forward[1], m_fine_temp[0], m_fine_temp[1]);
+            dg::blas2::symv( m_fine_backward[1], m_fine_temp[0], m_fine_temp[2]);
             dg::blas1::subroutine( shu::Upwind(), m_fine_yp, m_fine_temp[1], m_fine_temp[2], m_fine_v);
         }
         else if( "centered" == m_advection)
         {
+            dg::blas2::symv( m_inter, y, m_fine_y);
+            dg::blas2::symv( m_inter, m_psi, m_fine_psi);
             //dx ( nv_x)
-            dg::blas2::symv( -1., m_centered[1], m_fine_psi, 0., m_fine_v); //v_x
+            dg::blas2::symv( -1., m_fine_centered[1], m_fine_psi, 0., m_fine_v); //v_x
             dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_x
-            dg::blas2::symv( -1., m_centered[0], m_fine_temp[0], 0., m_fine_yp);
+            dg::blas2::symv( -1., m_fine_centered[0], m_fine_temp[0], 0., m_fine_yp);
             //dy ( nv_y)
-            dg::blas2::symv( 1., m_centered[0], m_fine_psi, 0., m_fine_v); //v_y
+            dg::blas2::symv( 1., m_fine_centered[0], m_fine_psi, 0., m_fine_v); //v_y
             dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_y
-            dg::blas2::symv( -1., m_centered[1], m_fine_temp[0], 1., m_fine_yp);
+            dg::blas2::symv( -1., m_fine_centered[1], m_fine_temp[0], 1., m_fine_yp);
+        }
+        else if( "upwind-advection" == m_advection)
+        {
+            // v_x dx n
+            dg::blas1::copy( 0., m_fine_yp);
+            dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
+            dg::blas2::symv( m_forward[0], y, m_temp[1]);
+            dg::blas2::symv( m_backward[0], y, m_temp[2]);
+            dg::blas2::symv( m_inter, m_v, m_fine_v);
+            dg::blas2::symv( m_inter, m_temp[1], m_fine_temp[1]);
+            dg::blas2::symv( m_inter, m_temp[2], m_fine_temp[2]);
+            dg::blas1::subroutine( shu::UpwindAdvection(), m_fine_yp,
+                    m_fine_temp[1], m_fine_temp[2], m_fine_v);
+            // v_y dy n
+            dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_v); //v_y
+            dg::blas2::symv( m_forward[1], y, m_temp[1]);
+            dg::blas2::symv( m_backward[1], y, m_temp[2]);
+            dg::blas2::symv( m_inter, m_temp[1], m_fine_temp[1]);
+            dg::blas2::symv( m_inter, m_temp[2], m_fine_temp[2]);
+            dg::blas2::symv( m_inter, m_v, m_fine_v);
+            dg::blas1::subroutine( shu::UpwindAdvection(), m_fine_yp,
+                    m_fine_temp[1], m_fine_temp[2], m_fine_v);
+        }
+        else if( "centered-advection" == m_advection)
+        {
+            // v_x dx n
+            dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
+            dg::blas2::symv( -1., m_centered[0], y, 0., m_temp[0]);
+            dg::blas2::symv( m_inter, m_temp[0], m_fine_temp[0]);
+            dg::blas2::symv( m_inter, m_v, m_fine_v);
+            dg::blas1::pointwiseDot( m_fine_v, m_fine_temp[0], m_fine_yp);
+            // v_y dy n
+            dg::blas2::symv(  1., m_centered[0], m_psi, 0., m_v); //v_y
+            dg::blas2::symv( -1., m_centered[1], y, 0., m_temp[0]);
+            dg::blas2::symv( m_inter, m_temp[0], m_fine_temp[0]);
+            dg::blas2::symv( m_inter, m_v, m_fine_v);
+            dg::blas1::pointwiseDot( 1., m_fine_v, m_fine_temp[0], 1., m_fine_yp);
         }
         dg::blas2::symv( m_project, m_fine_yp, yp);
     }
diff --git a/src/lamb_dipole/shu.tex b/src/lamb_dipole/shu.tex
index 03128f952..1bcb9e564 100644
--- a/src/lamb_dipole/shu.tex
+++ b/src/lamb_dipole/shu.tex
@@ -34,7 +34,15 @@ Eq.~\eqref{eq:euler_poisson} is a reformulation of the standard conservative for
 \end{align}
 \label{eq:euler_conservative}
 \end{subequations}
-with $v_x = - \phi_y$ and $v_y = \phi_x$.
+with $v_x = - \phi_y$ and $v_y = \phi_x$
+or since the divergence of $\vec v$ is $0$, $\nc \vec v = 0$ we have the advection form
+\begin{subequations}
+\begin{align}
+    \frac{\partial \omega}{\partial t} + \vec v\cn \omega = 0 \\
+    -\Delta\phi = \omega \quad v_x = -\phi_y \quad v_y = \phi_x
+\end{align}
+\label{eq:euler_advection}
+\end{subequations}
 
 Eqs.~\eqref{eq:euler_poisson} have an infinite amount of conserved quantities
 among them the total vorticity $V$, the kinetic energy $E$ and the enstrophy $\Omega$
@@ -233,7 +241,7 @@ regularization & dict & & \\
 \qquad type  & string& viscosity & the artificial viscosity \\
 \qquad order    & integer & 2 & Order: 1 is normal diffusion, 2 is hyperdiffusion, can be arbitrarily high, but higher orders might take longer to solve or restrict the CFL condition \\
 \qquad nu    & float & 1e-3 & Viscosity coefficient \\
-\qquad directoin & string & centered & Direction of the Laplacian: forward or centered
+\qquad direction & string & centered & Direction of the Laplacian: forward or centered \\
 \bottomrule
 \end{longtable}
 The other regularization method is the modal filter that applies an exponential filter
@@ -267,7 +275,7 @@ elliptic & dict & & \\
 \qquad type  & string& multigrid & Actually a nested iterations class \\
 \qquad stages    & integer & 3 & Number of stages (3 is best in virtually all cases) \\
 \qquad eps\_pol    & float[stages] & [1e-6,10,10] & Accuracy requirement on each stage of the multigrid scheme. $\eps_0 = \eps_{pol,0}$, $\eps_i = \eps_{pol,i} \eps_{pol,0}$  for $i>1$. \\
-\qquad directoin & string & centered & Direction of the Laplacian: forward or centered
+\qquad direction & string & centered & Direction of the Laplacian: forward or centered \\
 \bottomrule
 \end{longtable}
 \subsection{Advection schemes}
@@ -279,10 +287,12 @@ advection & dict & & \\
 \qquad type  & string& arakawa & Discretize Eqs.~\eqref{eq:euler_poisson} using Arakawa's scheme \cite{Einkemmer2014} \\
 \qquad type  & string& centered & Discretize Eqs.~\eqref{eq:euler_conservative} using the centered flux \\
 \qquad type  & string& upwind & Discretize Eqs.~\eqref{eq:euler_conservative} using the upwind flux \\
+\qquad type  & string& centered-advection & Discretize Eqs.~\eqref{eq:euler_advection} using the centered flux \\
+\qquad type  & string& upwind-advection & Discretize Eqs.~\eqref{eq:euler_advection} using the upwind flux \\
 \qquad multiplication    & string & pointwise & The multiplications in the scheme
 are done pointwise in nodal space\\
 \qquad multiplication    & string & projection & The multiplications in the scheme
-are done by first interpolating to a higher polynomial grid with $n_{fine} = 2n -1$, and projecting the result back to the coarse grid\\
+are done by first interpolating to a higher polynomial grid with $n_{fine} = 2n$, and projecting the result back to the coarse grid\\
 \bottomrule
 \end{longtable}
 %\subsection{Forward time and centered space}
-- 
GitLab


From 4ba0e708483ff85610eabd6f01f07e175cc85196 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 5 Feb 2021 11:26:28 +0100
Subject: [PATCH 456/540] Update geometries on new dg::file namespace

---
 inc/geometries/average_t.cu                   |  4 +-
 inc/geometries/conformalX_elliptic_b.cu       |  2 +-
 inc/geometries/flux_t.cu                      |  4 +-
 inc/geometries/geometryX_elliptic_b.cu        |  4 +-
 .../geometryX_refined_elliptic_b.cu           |  4 +-
 inc/geometries/geometry_advection_b.cu        | 10 ----
 inc/geometries/geometry_advection_mpib.cu     | 10 ----
 inc/geometries/geometry_diag.cu               | 24 +++++-----
 inc/geometries/geometry_elliptic_b.cu         |  4 +-
 inc/geometries/geometry_elliptic_mpib.cu      | 14 +++---
 inc/geometries/hector_t.cu                    |  4 +-
 inc/geometries/make_field.h                   | 48 +++++++++----------
 inc/geometries/normalize_params_t.cu          |  8 ++--
 inc/geometries/polynomial_parameters.h        | 22 ++++-----
 inc/geometries/ribeiroX_t.cu                  |  8 ++--
 inc/geometries/ribeiro_mpit.cu                | 14 +++---
 inc/geometries/ribeiro_t.cu                   |  4 +-
 inc/geometries/separatrix_orthogonal_t.cu     |  4 +-
 inc/geometries/simple_orthogonal_t.cu         |  4 +-
 inc/geometries/solovev_parameters.h           | 20 ++++----
 20 files changed, 98 insertions(+), 118 deletions(-)

diff --git a/inc/geometries/average_t.cu b/inc/geometries/average_t.cu
index d258683ca..6d4d77e1b 100644
--- a/inc/geometries/average_t.cu
+++ b/inc/geometries/average_t.cu
@@ -146,10 +146,10 @@ int main( int argc, char* argv[])
     }
     ///////////Write file
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     err = nc_create( "average.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim1d;
-    err = file::define_dimension(  ncid, &dim1d, grid1d, "psi");
+    err = dg::file::define_dimension(  ncid, &dim1d, grid1d, "psi");
     for(auto tp : map1d)
     {
         int vid;
diff --git a/inc/geometries/conformalX_elliptic_b.cu b/inc/geometries/conformalX_elliptic_b.cu
index 233ebba59..1041d8be9 100644
--- a/inc/geometries/conformalX_elliptic_b.cu
+++ b/inc/geometries/conformalX_elliptic_b.cu
@@ -127,7 +127,7 @@ int main(int argc, char**argv)
     for( unsigned i=1; i<nIter; i++)
     {
         Nx*=2; Ny*=2;
-        dg::MultiMatrix<dg::DMatrix, dg::DVec >  inter = dg::create::fast_interpolation(g2d.grid(), 2, 2);
+        dg::MultiMatrix<dg::DMatrix, dg::DVec >  inter = dg::create::fast_interpolation(g2d.grid(), 2, 2, 1);
         dg::geo::CurvilinearGridX2d g2d_new( generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::DIR);
         std::cout << "Computing on "<<n<<" x "<<Nx<<" x "<<Ny<<"\n";
         dg::DVec x_new = dg::evaluate( dg::zero, g2d_new);
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 5b0e0eaef..36e6d9550 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -90,10 +90,10 @@ int main( int argc, char* argv[])
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     //////////////////////////////setup and write netcdf//////////////////
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     err = nc_create( "flux.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
-    err = file::define_dimensions(  ncid, dim2d, g2d_periodic);
+    err = dg::file::define_dimensions(  ncid, dim2d, g2d_periodic);
     int coordsID[2];
     err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index fb7a27f5b..aa53883a7 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -72,10 +72,10 @@ int main(int argc, char**argv)
     std::cout << "Computing on "<<n<<" x "<<Nx<<" x "<<Ny<<"\n";
     ///////////////////////////////////////////////////////////////////////////
     int ncid;
-    file::NC_Error_Handle ncerr;
+    dg::file::NC_Error_Handle ncerr;
     ncerr = nc_create( "testX.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
-    ncerr = file::define_dimensions(  ncid, dim2d, g2d.grid());
+    ncerr = dg::file::define_dimensions(  ncid, dim2d, g2d.grid());
     int coordsID[2], psiID, functionID, function2ID, divBID;
     ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
diff --git a/inc/geometries/geometryX_refined_elliptic_b.cu b/inc/geometries/geometryX_refined_elliptic_b.cu
index 242d1b34b..ef231e0a1 100644
--- a/inc/geometries/geometryX_refined_elliptic_b.cu
+++ b/inc/geometries/geometryX_refined_elliptic_b.cu
@@ -80,10 +80,10 @@ int main(int argc, char**argv)
     std::cout << "Computing on "<<n<<" x "<<Nx<<" x "<<Ny<<" + "<<add_x<<" x "<<add_y<<" x "<<howmanyX<<" x "<<howmanyY<<"\n";
     ///////////////////////////////////////////////////////////////////////////
     int ncid;
-    file::NC_Error_Handle ncerr;
+    dg::file::NC_Error_Handle ncerr;
     ncerr = nc_create( "testXrefined.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
-    ncerr = file::define_dimensions(  ncid, dim2d, g2d_fine.grid());
+    ncerr = dg::file::define_dimensions(  ncid, dim2d, g2d_fine.grid());
     int coordsID[2], psiID, functionID, function2ID;
     ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index d799543f5..dffe33b49 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -173,12 +173,6 @@ int main(int argc, char** argv)
     dg::blas1::axpby( 1., sol, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     std::cout << "          Rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
-    arakawa.variation( lhs, jac);
-    const double normVar = dg::blas2::dot( vol, variation);
-    //std::cout << "norm of variation "<<normVar<<"\n";
-    dg::blas1::axpby( 1., variation, -1., jac);
-    result = dg::blas2::dot( jac, vol, jac);
-    std::cout << "Variation rel. distance to solution "<<sqrt( result/normVar)<<std::endl; //don't forget sqrt when comuting errors
     ///////////////////////////////////////////////////////////////////////
     std::cout << "TESTING POISSON\n";
     dg::Poisson<dg::aGeometry2d, dg::DMatrix, dg::DVec> poisson( grid);
@@ -195,10 +189,6 @@ int main(int argc, char** argv)
     dg::blas1::axpby( 1., sol, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     std::cout << "          Rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
-    poisson.variationRHS( lhs, jac);
-    dg::blas1::axpby( 1., variation, -1., jac);
-    result = dg::blas2::dot( jac, vol, jac);
-    std::cout << "Variation rel. distance to solution "<<sqrt( result/normVar)<<std::endl; //don't forget sqrt when comuting errors
 
     ////////////////////////////transform curvature components////////
     std::cout << "TESTING CURVATURE 3D\n";
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index a36a8dead..21f246d08 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -172,11 +172,6 @@ int main(int argc, char** argv)
     dg::blas1::axpby( 1., sol, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     if(rank==0)std::cout << "          Rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
-    arakawa.variation( lhs, jac);
-    norm = dg::blas2::dot( vol, jac);
-    dg::blas1::axpby( 1., variation, -1., jac);
-    result = dg::blas2::dot( jac, vol, jac);
-    if(rank==0)std::cout << "Variation rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
     ///////////////////////////////////////////////////////////////////////
     if(rank==0)std::cout << "TESTING POISSON 3D\n";
     dg::Poisson<Geometry, dg::MDMatrix, dg::MDVec> poisson( grid);
@@ -191,11 +186,6 @@ int main(int argc, char** argv)
     dg::blas1::axpby( 1., sol, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     if(rank==0)std::cout << "          Rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
-    poisson.variationRHS( lhs, jac);
-    norm = dg::blas2::dot( vol, jac);
-    dg::blas1::axpby( 1., variation, -1., jac);
-    result = dg::blas2::dot( jac, vol, jac);
-    if(rank==0)std::cout << "Variation rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
 
     ////////////////////////////transform curvature components////////
     if(rank==0)std::cout << "TESTING CURVATURE 3D\n";
diff --git a/inc/geometries/geometry_diag.cu b/inc/geometries/geometry_diag.cu
index e58a6c38d..23bb40517 100644
--- a/inc/geometries/geometry_diag.cu
+++ b/inc/geometries/geometry_diag.cu
@@ -82,14 +82,14 @@ int main( int argc, char* argv[])
     {
         newfilename = argv[3];
         std::cout << argv[0]<< " "<<argv[1]<<" & "<<argv[2]<<" -> " <<argv[3]<<std::endl;
-        file::file2Json( argv[1], input_js, file::comments::are_discarded);
-        file::file2Json( argv[2], geom_js, file::comments::are_discarded);
+        dg::file::file2Json( argv[1], input_js, dg::file::comments::are_discarded);
+        dg::file::file2Json( argv[2], geom_js, dg::file::comments::are_discarded);
     }
     else if( argc == 3)
     {
         newfilename = argv[2];
         std::cout << argv[0]<< " "<<argv[1]<<" -> " <<argv[2]<<std::endl;
-        file::NC_Error_Handle err;
+        dg::file::NC_Error_Handle err;
         int ncid_in;
         err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
         size_t length;
@@ -101,8 +101,8 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geomfile[0]);
         err = nc_close( ncid_in);
         Json::Value js,gs;
-        file::string2Json(inputfile, input_js, file::comments::are_discarded);
-        file::string2Json(geomfile, geom_js, file::comments::are_discarded);
+        dg::file::string2Json(inputfile, input_js, dg::file::comments::are_discarded);
+        dg::file::string2Json(geomfile, geom_js, dg::file::comments::are_discarded);
     }
     else
     {
@@ -119,9 +119,9 @@ int main( int argc, char* argv[])
     //Test coefficients
     dg::geo::CylindricalFunctor wall, transition, sheath, direction;
     dg::geo::TokamakMagneticField mag = dg::geo::createMagneticField(geom_js,
-            file::error::is_throw);
+            dg::file::error::is_throw);
     dg::geo::TokamakMagneticField mod_mag =
-        dg::geo::createModifiedField(geom_js, input_js, file::error::is_throw,
+        dg::geo::createModifiedField(geom_js, input_js, dg::file::error::is_throw,
                 wall, transition);
     std::string input = input_js.toStyledString();
     std::string geom = geom_js.toStyledString();
@@ -131,7 +131,7 @@ int main( int argc, char* argv[])
     double Zmin=-p.boxscaleZm*mag.params().a()*mag.params().elongation();
     double Rmax=mag.R0()+p.boxscaleRp*mag.params().a();
     double Zmax=p.boxscaleZp*mag.params().a()*mag.params().elongation();
-    dg::geo::createSheathRegion( input_js, file::error::is_warning,
+    dg::geo::createSheathRegion( input_js, dg::file::error::is_warning,
             mag, wall, Rmin, Rmax, Zmin, Zmax, sheath, direction);
 
     dg::geo::description mag_description = mag.params().getDescription();
@@ -335,7 +335,7 @@ int main( int argc, char* argv[])
 
     /////////////////////////////set up netcdf/////////////////////////////////////
     std::cout << "CREATING/OPENING FILE AND WRITING ... \n";
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_create( newfilename.data(), NC_NETCDF4|NC_CLOBBER, &ncid);
     /// Set global attributes
@@ -364,7 +364,7 @@ int main( int argc, char* argv[])
     if( mag_description == dg::geo::description::standardX)
     {
         int dim_idsX[2] = {0,0};
-        err = file::define_dimensions( ncid, dim_idsX, gX2d->grid(), {"eta", "zeta"} );
+        err = dg::file::define_dimensions( ncid, dim_idsX, gX2d->grid(), {"eta", "zeta"} );
         std::string long_name = "Flux surface label";
         err = nc_put_att_text( ncid, dim_idsX[0], "long_name",
             long_name.size(), long_name.data());
@@ -388,7 +388,7 @@ int main( int argc, char* argv[])
     }
     else
     {
-        err = file::define_dimension( ncid, &dim1d_ids[0], grid1d, "zeta");
+        err = dg::file::define_dimension( ncid, &dim1d_ids[0], grid1d, "zeta");
         std::string psi_long_name = "Flux surface label";
         err = nc_put_att_text( ncid, dim1d_ids[0], "long_name",
             psi_long_name.size(), psi_long_name.data());
@@ -396,7 +396,7 @@ int main( int argc, char* argv[])
     dg::CylindricalGrid3d grid3d(Rmin,Rmax,Zmin,Zmax, 0, 2.*M_PI, n,Nx,Ny,Nz);
     dg::RealCylindricalGrid3d<float> fgrid3d(Rmin,Rmax,Zmin,Zmax, 0, 2.*M_PI, n,Nx,Ny,Nz);
 
-    err = file::define_dimensions( ncid, &dim3d_ids[0], fgrid3d);
+    err = dg::file::define_dimensions( ncid, &dim3d_ids[0], fgrid3d);
     dim2d_ids[0] = dim3d_ids[1], dim2d_ids[1] = dim3d_ids[2];
 
     //write 1d vectors
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index 17a0c6b4d..885a0f527 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -47,10 +47,10 @@ int main(int argc, char**argv)
     std::cout << "Construction took "<<t.diff()<<"s\n";
     ///////////////////////////////////////////////////////////////////////////
     int ncid;
-    file::NC_Error_Handle ncerr;
+    dg::file::NC_Error_Handle ncerr;
     ncerr = nc_create( "testE.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
-    ncerr = file::define_dimensions(  ncid, dim2d, *g2d);
+    ncerr = dg::file::define_dimensions(  ncid, dim2d, *g2d);
     int coordsID[2], psiID, functionID, function2ID;
     ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 5632568ef..ab2bfeb77 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -54,10 +54,10 @@ int main(int argc, char**argv)
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
     ///////////////////////////////////////////////////////////////////////////
     int ncid;
-    file::NC_Error_Handle ncerr;
+    dg::file::NC_Error_Handle ncerr;
     if(rank==0)ncerr = nc_create( "testE_mpi.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim2d[2];
-    if(rank==0)ncerr = file::define_dimensions(  ncid, dim2d, *g2d);
+    if(rank==0)ncerr = dg::file::define_dimensions(  ncid, dim2d, *g2d);
     int coordsID[2], psiID, functionID, function2ID;
     if(rank==0)ncerr = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim2d, &coordsID[0]);
     if(rank==0)ncerr = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim2d, &coordsID[1]);
@@ -66,8 +66,8 @@ int main(int argc, char**argv)
     if(rank==0)ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
 
     dg::MHVec X( g2d->map()[0]), Y( g2d->map()[1]);
-    file::put_var_double( ncid, coordsID[0], *g2d, X);
-    file::put_var_double( ncid, coordsID[1], *g2d, Y);
+    dg::file::put_var_double( ncid, coordsID[0], *g2d, X);
+    dg::file::put_var_double( ncid, coordsID[1], *g2d, Y);
     ///////////////////////////////////////////////////////////////////////////
     dg::MDVec x =    dg::evaluate( dg::zero, *g2d);
     const dg::MDVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), *g2d);
@@ -105,11 +105,11 @@ int main(int argc, char**argv)
 
     dg::MHVec transfer;
     dg::assign( error, transfer);
-    file::put_var_double( ncid, psiID, *g2d, transfer);
+    dg::file::put_var_double( ncid, psiID, *g2d, transfer);
     dg::assign( x, transfer);
-    file::put_var_double( ncid, functionID, *g2d, transfer);
+    dg::file::put_var_double( ncid, functionID, *g2d, transfer);
     dg::assign( solution, transfer);
-    file::put_var_double( ncid, function2ID, *g2d, transfer);
+    dg::file::put_var_double( ncid, function2ID, *g2d, transfer);
     if(rank==0)ncerr = nc_close( ncid);
     MPI_Finalize();
 
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index a2c671842..89e0f0d90 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -105,10 +105,10 @@ int main( int argc, char* argv[])
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     std::cout << "Length in u is    "<<hector->width()<<std::endl;
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     err = nc_create( "conformal.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim3d[2];
-    err = file::define_dimensions(  ncid, dim3d, g2d_periodic);
+    err = dg::file::define_dimensions(  ncid, dim3d, g2d_periodic);
     int coordsID[2], onesID, defID, confID,volID,divBID;
     err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
     err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
index 35b17206c..8f82d773c 100644
--- a/inc/geometries/make_field.h
+++ b/inc/geometries/make_field.h
@@ -30,9 +30,9 @@ namespace geo{
  * @return A magnetic field object
  * @attention This function is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
  */
-static inline TokamakMagneticField createMagneticField( Json::Value gs, file::error mode)
+static inline TokamakMagneticField createMagneticField( Json::Value gs, dg::file::error mode)
 {
-    std::string e = file::get( mode, gs, "equilibrium", "solovev" ).asString();
+    std::string e = dg::file::get( mode, gs, "equilibrium", "solovev" ).asString();
     equilibrium equi = str2equilibrium.at( e);
     switch( equi){
         case equilibrium::polynomial:
@@ -42,19 +42,19 @@ static inline TokamakMagneticField createMagneticField( Json::Value gs, file::er
         }
         case equilibrium::toroidal:
         {
-            double R0 = file::get( mode, gs, "R_0", 10).asDouble();
+            double R0 = dg::file::get( mode, gs, "R_0", 10).asDouble();
             return createToroidalField( R0);
         }
         case equilibrium::guenther:
         {
-            double I0 = file::get( mode, gs, "I_0", 20).asDouble();
-            double R0 = file::get( mode, gs, "R_0", 10).asDouble();
+            double I0 = dg::file::get( mode, gs, "I_0", 20).asDouble();
+            double R0 = dg::file::get( mode, gs, "R_0", 10).asDouble();
             return createGuentherField( R0, I0);
         }
         case equilibrium::circular:
         {
-            double I0 = file::get( mode, gs, "I_0", 20).asDouble();
-            double R0 = file::get( mode, gs, "R_0", 10).asDouble();
+            double I0 = dg::file::get( mode, gs, "I_0", 20).asDouble();
+            double R0 = dg::file::get( mode, gs, "R_0", 10).asDouble();
             return createCircularField( R0, I0);
         }
         default:
@@ -104,13 +104,13 @@ void transform_psi( TokamakMagneticField mag, double& psi0, double& alpha0, doub
  * @return A magnetic field object
  * @attention This function is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
  */
-static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Value jsmod, file::error mode, CylindricalFunctor& wall, CylindricalFunctor& transition)
+static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Value jsmod, dg::file::error mode, CylindricalFunctor& wall, CylindricalFunctor& transition)
 {
-    std::string e = file::get( mode, gs, "equilibrium", "solovev" ).asString();
+    std::string e = dg::file::get( mode, gs, "equilibrium", "solovev" ).asString();
     equilibrium equi = str2equilibrium.at( e);
-    std::string m = file::get( mode, jsmod, "wall", "type", "heaviside" ).asString();
+    std::string m = dg::file::get( mode, jsmod, "wall", "type", "heaviside" ).asString();
     modifier mod = str2modifier.at( m);
-    std::string d = file::get( mode, gs, "description", "standardX" ).asString();
+    std::string d = dg::file::get( mode, gs, "description", "standardX" ).asString();
     description desc = str2description.at( d);
     TokamakMagneticField mag = createMagneticField( gs, mode);
     const MagneticFieldParameters& inp = mag.params();
@@ -126,14 +126,14 @@ static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Va
         }
         case modifier::heaviside:
         {
-            double psi0 = file::get( mode, jsmod, "wall", "boundary", 1.1 ).asDouble();
-            double alpha = file::get( mode, jsmod, "wall", "alpha", 0.2 ).asDouble();
+            double psi0 = dg::file::get( mode, jsmod, "wall", "boundary", 1.1 ).asDouble();
+            double alpha = dg::file::get( mode, jsmod, "wall", "alpha", 0.2 ).asDouble();
             double sign = +1;
             if( desc == description::standardX || desc == description::standardO ||
                     desc == description::doubleX)
                 detail::transform_psi( mag, psi0, alpha, sign);
             else
-                sign = file::get( mode, jsmod, "wall", "sign", -1. ).asDouble();
+                sign = dg::file::get( mode, jsmod, "wall", "sign", -1. ).asDouble();
 
             mod_psip = mod::createPsip( mod::everywhere, mag.get_psip(), psi0, alpha, sign);
             wall = mod::DampingRegion( mod::everywhere, mag.psip(), psi0, alpha, -sign);
@@ -142,10 +142,10 @@ static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Va
         }
         case modifier::sol_pfr:
         {
-            double psi0 = file::get_idx( mode, jsmod, "wall", "boundary",0, 1.1 ).asDouble();
-            double alpha0 = file::get_idx( mode, jsmod, "wall", "alpha",0, 0.2 ).asDouble();
-            double psi1 = file::get_idx( mode, jsmod, "wall", "boundary",1, 0.97 ).asDouble();
-            double alpha1 = file::get_idx( mode, jsmod, "wall", "alpha",1, 0.2 ).asDouble();
+            double psi0 = dg::file::get_idx( mode, jsmod, "wall", "boundary",0, 1.1 ).asDouble();
+            double alpha0 = dg::file::get_idx( mode, jsmod, "wall", "alpha",0, 0.2 ).asDouble();
+            double psi1 = dg::file::get_idx( mode, jsmod, "wall", "boundary",1, 0.97 ).asDouble();
+            double alpha1 = dg::file::get_idx( mode, jsmod, "wall", "alpha",1, 0.2 ).asDouble();
             switch( desc){
                 case description::standardX:
                 {
@@ -200,8 +200,8 @@ static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Va
                 }
                 default:
                 {
-                    double sign0 = file::get_idx( mode, jsmod, "wall", "sign",0, -1. ).asDouble();
-                    double sign1 = file::get_idx( mode, jsmod, "wall", "sign", 1, +1. ).asDouble();
+                    double sign0 = dg::file::get_idx( mode, jsmod, "wall", "sign",0, -1. ).asDouble();
+                    double sign1 = dg::file::get_idx( mode, jsmod, "wall", "sign", 1, +1. ).asDouble();
                     CylindricalFunctorsLvl2 mod0_psip;
                     mod0_psip = mod::createPsip(
                             mod::everywhere, mag.get_psip(), psi0, alpha0, sign0);
@@ -237,7 +237,7 @@ static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Va
 ///@addtogroup profiles
 ///@{
 
-static inline CylindricalFunctor createWallRegion( Json::Value gs, Json::Value jsmod, file::error mode)
+static inline CylindricalFunctor createWallRegion( Json::Value gs, Json::Value jsmod, dg::file::error mode)
 {
     CylindricalFunctor wall, transition;
     TokamakMagneticField mag = createModifiedField( gs, jsmod, mode, wall, transition);
@@ -268,7 +268,7 @@ static inline CylindricalFunctor createWallRegion( Json::Value gs, Json::Value j
  * @return sheath region
  */
 static inline void createSheathRegion(
-        Json::Value jsmod, file::error mode, TokamakMagneticField mag,
+        Json::Value jsmod, dg::file::error mode, TokamakMagneticField mag,
         CylindricalFunctor wall, double R0, double R1, double Z0, double Z1,
         CylindricalFunctor& sheath, CylindricalFunctor& direction )
 {
@@ -299,8 +299,8 @@ static inline void createSheathRegion(
     direction = dg::geo::WallDirection( mag, vertical_sheath, horizontal_sheath);
     //sheath
     CylindricalFunctor dist = dg::WallDistance( vertical_sheath, horizontal_sheath);
-    double boundary = file::get( mode, jsmod, "sheath", "boundary", 0.1 ).asDouble();
-    double alpha = file::get( mode, jsmod, "sheath", "alpha", 0.01 ).asDouble();
+    double boundary = dg::file::get( mode, jsmod, "sheath", "boundary", 0.1 ).asDouble();
+    double alpha = dg::file::get( mode, jsmod, "sheath", "alpha", 0.01 ).asDouble();
     double a = mag.params().a();
     dg::PolynomialHeaviside poly( boundary*a - alpha*a/2., alpha*a/2., -1);
     sheath = dg::compose( poly, dist);
diff --git a/inc/geometries/normalize_params_t.cu b/inc/geometries/normalize_params_t.cu
index 23634ad0a..72bd4c417 100644
--- a/inc/geometries/normalize_params_t.cu
+++ b/inc/geometries/normalize_params_t.cu
@@ -10,7 +10,7 @@ int main( int argc, char* argv[])
     if( argc == 3)
     {
         std::cout << argv[0]<< " "<<argv[1]<<" -> " <<argv[2]<<std::endl;
-        file::file2Json( argv[1], geom_js, file::comments::are_discarded);
+        dg::file::file2Json( argv[1], geom_js, dg::file::comments::are_discarded);
     }
     else
     {
@@ -21,20 +21,20 @@ int main( int argc, char* argv[])
 
     std::cout << "Input file: \n"<< geom_js.toStyledString();
     dg::geo::TokamakMagneticField mag;
-    std::string e = file::get( file::error::is_throw, geom_js, "equilibrium", "solovev" ).asString();
+    std::string e = dg::file::get( dg::file::error::is_throw, geom_js, "equilibrium", "solovev" ).asString();
     dg::geo::equilibrium equi = dg::geo::str2equilibrium.at( e);
     switch( equi){
         case dg::geo::equilibrium::polynomial:
         {
             std::cout << "Creating polynomial Field!\n";
-            dg::geo::polynomial::Parameters gp( geom_js, file::error::is_warning);
+            dg::geo::polynomial::Parameters gp( geom_js, dg::file::error::is_warning);
             mag = dg::geo::createPolynomialField( gp);
             break;
         }
         case dg::geo::equilibrium::solovev:
         {
             std::cout << "Creating Solovev Field!\n";
-            dg::geo::solovev::Parameters gp( geom_js, file::error::is_warning);
+            dg::geo::solovev::Parameters gp( geom_js, dg::file::error::is_warning);
             mag = dg::geo::createSolovevField( gp);
             break;
         }
diff --git a/inc/geometries/polynomial_parameters.h b/inc/geometries/polynomial_parameters.h
index 2892fe64d..b5eee42b3 100644
--- a/inc/geometries/polynomial_parameters.h
+++ b/inc/geometries/polynomial_parameters.h
@@ -41,20 +41,20 @@ struct Parameters
      * @note the default values in brackets are taken if the variables are not found in the input file
      * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
      */
-    Parameters( const Json::Value& js, file::error mode = file::error::is_silent) {
-        pp  = file::get( mode, js, "PP", 1).asDouble();
-        pi  = file::get( mode, js, "PI", 1).asDouble();
-        M = file::get( mode, js, "M", 1).asUInt();
-        N = file::get( mode, js, "N", 1).asUInt();
+    Parameters( const Json::Value& js, dg::file::error mode = dg::file::error::is_silent) {
+        pp  = dg::file::get( mode, js, "PP", 1).asDouble();
+        pi  = dg::file::get( mode, js, "PI", 1).asDouble();
+        M = dg::file::get( mode, js, "M", 1).asUInt();
+        N = dg::file::get( mode, js, "N", 1).asUInt();
         c.resize(M*N);
         for (unsigned i=0;i<M*N;i++)
-            c[i] = file::get_idx( mode, js, "c", i, 0.).asDouble();
+            c[i] = dg::file::get_idx( mode, js, "c", i, 0.).asDouble();
 
-        R_0  = file::get( mode, js, "R_0", 0.).asDouble();
-        a  = R_0*file::get( mode, js, "inverseaspectratio", 0.).asDouble();
-        elongation=file::get( mode, js, "elongation", 1.).asDouble();
-        triangularity=file::get( mode, js, "triangularity", 0.).asDouble();
-        description = file::get( mode, js, "description", "standardX").asString();
+        R_0  = dg::file::get( mode, js, "R_0", 0.).asDouble();
+        a  = R_0*dg::file::get( mode, js, "inverseaspectratio", 0.).asDouble();
+        elongation=dg::file::get( mode, js, "elongation", 1.).asDouble();
+        triangularity=dg::file::get( mode, js, "triangularity", 0.).asDouble();
+        description = dg::file::get( mode, js, "description", "standardX").asString();
     }
     /**
      * @brief Put values into a json string
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index e1618ef13..c4cf681a1 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -110,12 +110,12 @@ int main( int argc, char* argv[])
     dg::HVec x_left = dg::evaluate( sine, g1d), x_right(x_left);
     dg::HVec y_left = dg::evaluate( cosine, g1d);
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     err = nc_create( "ribeiroX.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim3d[3], dim1d[1];
-    err = file::define_dimensions(  ncid, dim3d, g3d_periodic.grid());
-    //err = file::define_dimensions(  ncid, dim3d, g2d.grid());
-    err = file::define_dimension(  ncid, dim1d, g1d, "i");
+    err = dg::file::define_dimensions(  ncid, dim3d, g3d_periodic.grid());
+    //err = dg::file::define_dimensions(  ncid, dim3d, g2d.grid());
+    err = dg::file::define_dimension(  ncid, dim1d, g1d, "i");
     int coordsID[2], onesID, defID, volID, divBID;
     int coord1D[5];
     err = nc_def_var( ncid, "xc", NC_DOUBLE, 3, dim3d, &coordsID[0]);
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 4d9352690..a1b50732e 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -62,10 +62,10 @@ int main( int argc, char* argv[])
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     if(rank==0)err = nc_create( "test_mpi.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim3d[2];
-    if(rank==0)err = file::define_dimensions(  ncid, dim3d, *g2d);
+    if(rank==0)err = dg::file::define_dimensions(  ncid, dim3d, *g2d);
     int coordsID[2], onesID, defID,confID, volID, divBID;
     if(rank==0)err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
     if(rank==0)err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
@@ -77,10 +77,10 @@ int main( int argc, char* argv[])
 
     dg::MHVec psi_p = dg::pullback( psip.f(), *g2d);
     //g.display();
-    file::put_var_double( ncid, onesID, *g2d, psi_p);
+    dg::file::put_var_double( ncid, onesID, *g2d, psi_p);
     dg::MHVec X = g2d->map()[0], Y = g2d->map()[1];
-    file::put_var_double( ncid, coordsID[0], *g2d, X);
-    file::put_var_double( ncid, coordsID[1], *g2d, Y);
+    dg::file::put_var_double( ncid, coordsID[0], *g2d, X);
+    dg::file::put_var_double( ncid, coordsID[1], *g2d, Y);
 
     dg::MHVec temp0( dg::evaluate(dg::zero, *g2d)), temp1(temp0);
     dg::MHVec w2d = dg::create::weights( *g2d);
@@ -91,7 +91,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     const dg::MHVec ones = dg::evaluate( dg::one, *g2d);
     dg::blas1::axpby( 1., ones, -1., temp0, temp0);
-    file::put_var_double( ncid, defID, *g2d, temp0);
+    dg::file::put_var_double( ncid, defID, *g2d, temp0);
 
     if(rank==0)std::cout << "Construction successful!\n";
 
@@ -111,7 +111,7 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
-    file::put_var_double( ncid, volID, *g2d, temp0);
+    dg::file::put_var_double( ncid, volID, *g2d, temp0);
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     if(rank==0)std::cout << "Rel Consistency  of volume is "<<error<<"\n";
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index d6ac73499..d76c4d916 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -74,10 +74,10 @@ int main( int argc, char* argv[])
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     err = nc_create( "ribeiro.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim3d[2];
-    err = file::define_dimensions(  ncid, dim3d, g2d_periodic);
+    err = dg::file::define_dimensions(  ncid, dim3d, g2d_periodic);
     int coordsID[2], onesID, defID, confID, volID,divBID;
     err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
     err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index baf29002a..483e4bcef 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -178,10 +178,10 @@ int main( int argc, char* argv[])
     map2d.emplace_back( "FuncDirNeu2", X, "FuncDirNeu");
     std::cout << "OPEN FILE orthogonalX.nc ...\n";
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     err = nc_create( "orthogonalX.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim3d[3];
-    err = file::define_dimensions(  ncid, dim3d, g3d_periodic.grid());
+    err = dg::file::define_dimensions(  ncid, dim3d, g3d_periodic.grid());
     for(auto tp : map2d)
     {
         int vid;
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 004109dba..e84621399 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -74,10 +74,10 @@ int main( int argc, char* argv[])
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     int ncid;
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     err = nc_create( "orthogonal.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
     int dim3d[2];
-    err = file::define_dimensions(  ncid, dim3d, g2d_periodic);
+    err = dg::file::define_dimensions(  ncid, dim3d, g2d_periodic);
     int coordsID[2], onesID, defID, confID,volID,divBID;
     err = nc_def_var( ncid, "xc", NC_DOUBLE, 2, dim3d, &coordsID[0]);
     err = nc_def_var( ncid, "yc", NC_DOUBLE, 2, dim3d, &coordsID[1]);
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index be01241a2..f050b1ece 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -40,20 +40,20 @@ struct Parameters
      * @note the default values in brackets are taken if the variables are not found in the input file
      * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
      */
-    Parameters( const Json::Value& js, file::error mode = file::error::is_silent) {
-        A  = file::get( mode, js, "A", 0).asDouble();
-        pp  = file::get( mode, js, "PP", 1).asDouble();
-        pi  = file::get( mode, js, "PI", 1).asDouble();
+    Parameters( const Json::Value& js, dg::file::error mode = dg::file::error::is_silent) {
+        A  = dg::file::get( mode, js, "A", 0).asDouble();
+        pp  = dg::file::get( mode, js, "PP", 1).asDouble();
+        pi  = dg::file::get( mode, js, "PI", 1).asDouble();
         c.resize(12);
         for (unsigned i=0;i<12;i++)
-            c[i] = file::get_idx( mode, js, "c", i, 0.).asDouble();
+            c[i] = dg::file::get_idx( mode, js, "c", i, 0.).asDouble();
 
-        R_0  = file::get( mode, js, "R_0", 0.).asDouble();
-        a  = R_0*file::get( mode, js, "inverseaspectratio", 0.).asDouble();
-        elongation=file::get( mode, js, "elongation", 1.).asDouble();
-        triangularity=file::get( mode, js, "triangularity", 0.).asDouble();
+        R_0  = dg::file::get( mode, js, "R_0", 0.).asDouble();
+        a  = R_0*dg::file::get( mode, js, "inverseaspectratio", 0.).asDouble();
+        elongation=dg::file::get( mode, js, "elongation", 1.).asDouble();
+        triangularity=dg::file::get( mode, js, "triangularity", 0.).asDouble();
         try{
-            description = file::get( file::error::is_throw, js, "description", "standardX").asString();
+            description = dg::file::get( dg::file::error::is_throw, js, "description", "standardX").asString();
         } catch ( std::exception& err)
         {
             if( isToroidal())
-- 
GitLab


From e8709412f6cc006b6ce4c6248bd9c458dd86d518 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 5 Feb 2021 11:35:01 +0100
Subject: [PATCH 457/540] Change diag programs to dg::file

---
 diag/compare.cpp           | 10 +++++-----
 diag/crosscoherencdiag.cpp | 12 ++++++------
 diag/feltorSHdiag.cpp      | 10 +++++-----
 diag/feltorSHdiag.cu       | 10 +++++-----
 diag/feltorSHvmaxdiag.cu   |  4 ++--
 diag/feltorSesoldiag.cpp   | 10 +++++-----
 diag/feltorShwdiag.cpp     | 10 +++++-----
 diag/feltorShwmerger.cpp   |  8 ++++----
 diag/feltorShwradstat.cpp  |  4 ++--
 diag/feltorShwstat.cpp     |  4 ++--
 diag/fftwdiag.cpp          | 12 ++++++------
 diag/growthrate.cpp        |  4 ++--
 diag/histdiag.cpp          |  8 ++++----
 diag/impRdiag.cu           | 10 +++++-----
 diag/normdiag.cu           |  4 ++--
 diag/reco2Ddiag.cu         |  4 ++--
 diag/toeflEPdiag.cu        | 12 ++++++------
 diag/toeflRdiag.cu         | 12 ++++++------
 diag/vmaxnc.cu             |  2 +-
 19 files changed, 75 insertions(+), 75 deletions(-)

diff --git a/diag/compare.cpp b/diag/compare.cpp
index 4c36de377..f1b2334ee 100644
--- a/diag/compare.cpp
+++ b/diag/compare.cpp
@@ -22,7 +22,7 @@ int main( int argc, char** argv)
     }
     std::cout << "Compare "<<argv[1]<<" with "<<argv[2]<<"\n";
     //////////////////////////////open nc files//////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid1, ncid2;
     err = nc_open( argv[1], NC_NOWRITE, &ncid1);
     err = nc_open( argv[2], NC_NOWRITE, &ncid2);
@@ -37,9 +37,9 @@ int main( int argc, char** argv)
     err = nc_inq_dimid( ncid1, "y", &dimIDs1[1]);
     err = nc_inq_dimid( ncid2, "y", &dimIDs2[1]);
     try{ err = nc_inq_dimid( ncid1, "z", &dimIDs1[2]);}
-    catch( file::NC_Error) { numDims1=2; }
+    catch( dg::file::NC_Error) { numDims1=2; }
     try{ err = nc_inq_dimid( ncid2, "z", &dimIDs2[2]);}
-    catch( file::NC_Error) { numDims2=2; }
+    catch( dg::file::NC_Error) { numDims2=2; }
     if( numDims1 != numDims2)
     {
         std::cerr << "Files not of same dimensionality!!\n";
@@ -67,13 +67,13 @@ int main( int argc, char** argv)
         err = nc_inq_varid(ncid1, "electrons", &dataID1);
         err = nc_inq_varid(ncid2, "electrons", &dataID2);
     }
-    catch( file::NC_Error)
+    catch( dg::file::NC_Error)
     {
         try{
             err = nc_inq_varid(ncid1, "T", &dataID1);
             err = nc_inq_varid(ncid2, "T", &dataID2);
         }
-        catch( file::NC_Error)
+        catch( dg::file::NC_Error)
         {
             std::cerr <<"Neither electrons nor T found!\n";
             return -1;
diff --git a/diag/crosscoherencdiag.cpp b/diag/crosscoherencdiag.cpp
index b136bc84f..f3d67529e 100644
--- a/diag/crosscoherencdiag.cpp
+++ b/diag/crosscoherencdiag.cpp
@@ -43,7 +43,7 @@ int main( int argc, char* argv[])
     }
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;   
     ///////////////////read in and show inputfile//////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     size_t length;
@@ -52,7 +52,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
 
@@ -126,11 +126,11 @@ int main( int argc, char* argv[])
     int dataIDs1[2],dataIDs2[2],dataIDs12[1];
     int dim_ids1[1],dim_ids2[1],dim_ids12[2];
     int ncidout;
-    file::NC_Error_Handle errout; 
+    dg::file::NC_Error_Handle errout; 
     errout = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncidout); 
     //plot 1
     std::cout << "1d plot of Ne"<<std::endl;
-    errout = file::define_dimension( ncidout, &dim_ids1[0],  g1d1, "Ne_");
+    errout = dg::file::define_dimension( ncidout, &dim_ids1[0],  g1d1, "Ne_");
     errout = nc_def_var( ncidout, "P(Ne)",   NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[0]);
     errout = nc_def_var( ncidout, "Ne",    NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[1]);
     errout = nc_enddef( ncidout);
@@ -139,7 +139,7 @@ int main( int argc, char* argv[])
     //plot 2
     std::cout << "1d plot of Phi"<<std::endl;
     errout = nc_redef(ncidout);
-    errout = file::define_dimension( ncidout, &dim_ids2[0],  g1d2,"Phi_");
+    errout = dg::file::define_dimension( ncidout, &dim_ids2[0],  g1d2,"Phi_");
     errout = nc_def_var( ncidout, "P(Phi)",   NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[0]);
     errout = nc_def_var( ncidout, "Phi",    NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[1]);
     errout = nc_enddef( ncidout);
@@ -150,7 +150,7 @@ int main( int argc, char* argv[])
     errout = nc_redef(ncidout);
     dim_ids12[0]=dataIDs1[0];
     dim_ids12[1]=dataIDs2[0];
-    errout = file::define_dimensions( ncidout, &dim_ids12[0],  g2d);
+    errout = dg::file::define_dimensions( ncidout, &dim_ids12[0],  g2d);
     errout = nc_def_var( ncidout, "P(Ne,Phi)",   NC_DOUBLE, 2, &dim_ids12[0], &dataIDs12[0]);
     errout = nc_enddef( ncidout);
     errout = nc_put_var_double( ncidout, dataIDs12[0], PA1A2.data() );
diff --git a/diag/feltorSHdiag.cpp b/diag/feltorSHdiag.cpp
index e562eed88..737c3071f 100644
--- a/diag/feltorSHdiag.cpp
+++ b/diag/feltorSHdiag.cpp
@@ -22,7 +22,7 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     size_t length;
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     
@@ -64,16 +64,16 @@ int main( int argc, char* argv[])
     size_t start1d[2]  = {0, 0};    
     //1d netcdf output file    
 
-    file::NC_Error_Handle err1d;
+    dg::file::NC_Error_Handle err1d;
     int ncid1d,dim_ids1d[2], tvarID1d,namescomID[6],timeID, timevarID;
     err1d = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid1d);
     err1d = nc_put_att_text( ncid1d, NC_GLOBAL, "inputfile", input.size(), input.data());
-    err1d = file::define_dimensions( ncid1d, dim_ids1d, &tvarID1d, g1d);
+    err1d = dg::file::define_dimensions( ncid1d, dim_ids1d, &tvarID1d, g1d);
     err1d = nc_close(ncid1d); 
     err1d = nc_open( argv[2], NC_WRITE, &ncid1d);
     err1d = nc_redef(ncid1d);
 
-    err1d = file::define_time( ncid1d, "ptime", &timeID, &timevarID);
+    err1d = dg::file::define_time( ncid1d, "ptime", &timeID, &timevarID);
     std::string namescom[6] = {"posX" , "posY" , "velX" , "velY" , "accX" , "accY" };
 
     for( unsigned i=0; i<6; i++){
diff --git a/diag/feltorSHdiag.cu b/diag/feltorSHdiag.cu
index c51365099..eaf66d719 100644
--- a/diag/feltorSHdiag.cu
+++ b/diag/feltorSHdiag.cu
@@ -45,7 +45,7 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     size_t length;
@@ -54,7 +54,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     
@@ -105,7 +105,7 @@ int main( int argc, char* argv[])
     size_t start1d[2]  = {0, 0};    
     //1d netcdf output file    
 
-    file::NC_Error_Handle err_out;
+    dg::file::NC_Error_Handle err_out;
     int ncid_out;
     int namescomID[12],names1dID[4],names2dID[4],tvarID1d,timeID,timevarID;
     int dim_ids2d[3];
@@ -115,13 +115,13 @@ int main( int argc, char* argv[])
     
     err_out = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
     err_out = nc_put_att_text( ncid_out, NC_GLOBAL, "inputfile", input.size(), input.data());
-    err_out = file::define_dimensions( ncid_out, dim_ids2d, &tvarID1d, g2d);
+    err_out = dg::file::define_dimensions( ncid_out, dim_ids2d, &tvarID1d, g2d);
     err_out = nc_close(ncid_out); 
     
     
     err_out = nc_open( argv[2], NC_WRITE, &ncid_out);
     err_out = nc_redef(ncid_out);
-    err_out = file::define_time( ncid_out, "ptime", &timeID, &timevarID);
+    err_out = dg::file::define_time( ncid_out, "ptime", &timeID, &timevarID);
     for( unsigned i=0; i<12; i++){
         err_out = nc_def_var( ncid_out, namescom[i].data(),  NC_DOUBLE, 1, dim_ids2d, &namescomID[i]);
     }   
diff --git a/diag/feltorSHvmaxdiag.cu b/diag/feltorSHvmaxdiag.cu
index 8679dc936..fd780e355 100644
--- a/diag/feltorSHvmaxdiag.cu
+++ b/diag/feltorSHvmaxdiag.cu
@@ -18,7 +18,7 @@ int main( int argc, char* argv[])
         std::cerr << "Usage: "<<argv[0]<<" [input1.nc] [input2.nc] ...\n";
         return -1;
     }
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int timeID, varID,varID2,varID3;
     size_t start = {0}, numOut;
     for( int i=1; i< argc; i++)
@@ -34,7 +34,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
 //         std::cout << "input "<<input<<std::endl;
         Json::Value js;
-        file::string2Json( input, js, file::comments::are_forbidden);
+        dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
         const eule::Parameters p(js);
         err = nc_inq_dimid( ncid, "time", &timeID);
         err = nc_inq_dimlen( ncid, timeID, &numOut);
diff --git a/diag/feltorSesoldiag.cpp b/diag/feltorSesoldiag.cpp
index 34b3310b8..ca409e9b6 100644
--- a/diag/feltorSesoldiag.cpp
+++ b/diag/feltorSesoldiag.cpp
@@ -22,7 +22,7 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     size_t length;
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
 
@@ -70,7 +70,7 @@ int main( int argc, char* argv[])
     std::string names[4] = {"electrons", "ions",  "potential","vor"}; 
     int dataIDs[4]; 
     //1d profiles
-    file::NC_Error_Handle err_out;
+    dg::file::NC_Error_Handle err_out;
     int ncid_out,dataIDs1d[8], tvarIDout;
     std::string names1d[8] =  {"neavg", "Niavg",  "ln(ne)avg","ln(Ni)avg","potentialavg","voravg","x_","vyavg"}; 
     int dim_ids2d[3],dataIDs2d[4];
@@ -81,7 +81,7 @@ int main( int argc, char* argv[])
     size_t start2d_out[3]  = {0, 0, 0};
     err_out = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
     err_out= nc_put_att_text( ncid_out, NC_GLOBAL, "inputfile", input.size(), input.data());
-    err_out= file::define_dimensions( ncid_out, dim_ids2d, &tvarIDout, g2d);
+    err_out= dg::file::define_dimensions( ncid_out, dim_ids2d, &tvarIDout, g2d);
  int dim_ids1d[2] = {dim_ids2d[0],dim_ids2d[2]};
     for( unsigned i=0; i<8; i++){
         err_out = nc_def_var( ncid_out, names1d[i].data(), NC_DOUBLE, 2, dim_ids1d, &dataIDs1d[i]);
@@ -121,7 +121,7 @@ int main( int argc, char* argv[])
     std::string phi_probes_names[num_probes] ;
     std::string gamma_probes_names[num_probes];
     int timeID, timevarID;
-    err_out = file::define_time( ncid_out, "ptime", &timeID, &timevarID);
+    err_out = dg::file::define_time( ncid_out, "ptime", &timeID, &timevarID);
     for( unsigned i=0; i<num_probes; i++){
         std::stringstream ss1,ss2,ss3;
         ss1<<"Ne_p"<<i;
diff --git a/diag/feltorShwdiag.cpp b/diag/feltorShwdiag.cpp
index ffa41ff31..a328e9ed4 100644
--- a/diag/feltorShwdiag.cpp
+++ b/diag/feltorShwdiag.cpp
@@ -22,7 +22,7 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     size_t length;
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     ///////////////////////////////////////////////////////////////////////////
@@ -80,7 +80,7 @@ int main( int argc, char* argv[])
     std::string names[4] = {"electrons", "ions",  "potential","vor"}; 
     int dataIDs[4]; 
     //1d profiles
-    file::NC_Error_Handle err_out;
+    dg::file::NC_Error_Handle err_out;
     int ncid_out,dataIDs1d[33], tvarIDout;
     //Rfx = -\partial_x\overbar{\overbar{\delta} u_x \overbar{\delta} u_y } = -\partial_x R_favre
     //A   = -\overbar{u_x} \partial_x \overbar{u_y} 
@@ -98,7 +98,7 @@ int main( int argc, char* argv[])
     size_t start1d[2]  = {0, 0};    
     err_out = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
     err_out= nc_put_att_text( ncid_out, NC_GLOBAL, "inputfile", input.size(), input.data());
-    err_out= file::define_dimensions( ncid_out, dim_ids2d, &tvarIDout, g2d);
+    err_out= dg::file::define_dimensions( ncid_out, dim_ids2d, &tvarIDout, g2d);
      int dim_ids1d[2] = {dim_ids2d[0],dim_ids2d[2]};
     for( unsigned i=0; i<33; i++){
         err_out = nc_def_var( ncid_out, names1d[i].data(), NC_DOUBLE, 2, dim_ids1d, &dataIDs1d[i]);
@@ -148,7 +148,7 @@ int main( int argc, char* argv[])
     std::string phi_probes_names[num_probes] ;
     std::string gamma_probes_names[num_probes];
     int timeID, timevarID;
-    err_out = file::define_time( ncid_out, "ptime", &timeID, &timevarID);
+    err_out = dg::file::define_time( ncid_out, "ptime", &timeID, &timevarID);
     for( unsigned i=0; i<num_probes; i++){
         std::stringstream ss1,ss2,ss3;
         ss1<<"Ne_p"<<i;
diff --git a/diag/feltorShwmerger.cpp b/diag/feltorShwmerger.cpp
index 8c2a11aa2..102a05630 100644
--- a/diag/feltorShwmerger.cpp
+++ b/diag/feltorShwmerger.cpp
@@ -25,7 +25,7 @@ int main( int argc, char* argv[])
     
     
     //nc defs
-    file::NC_Error_Handle err, err_out;
+    dg::file::NC_Error_Handle err, err_out;
     int ncid, ncid_out, tvarIDout,EtimeID, EtimevarID;
     int dim_ids2d[3];
     size_t start2d_out[3]  = {0, 0, 0};
@@ -44,7 +44,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        file::string2Json( input, js, file::comments::are_forbidden);
+        dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
         const eule::Parameters p(js);
         
         dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
@@ -54,8 +54,8 @@ int main( int argc, char* argv[])
         if (i==1) {
             err_out = nc_create(argv[argc-1],NC_NETCDF4|NC_CLOBBER, &ncid_out);
             err_out = nc_put_att_text( ncid_out, NC_GLOBAL, "inputfile", input.size(), input.data());
-            err_out = file::define_dimensions( ncid_out, dim_ids2d, &tvarIDout, g2d);
-            err_out = file::define_time( ncid_out, "energy_time", &EtimeID, &EtimevarID);
+            err_out = dg::file::define_dimensions( ncid_out, dim_ids2d, &tvarIDout, g2d);
+            err_out = dg::file::define_time( ncid_out, "energy_time", &EtimeID, &EtimevarID);
             for( unsigned j=0; j<4; j++) {
                 err_out  = nc_def_var(ncid_out, names[j].data(),  NC_DOUBLE, 3, dim_ids2d, &dataIDs_out[j]);
             }
diff --git a/diag/feltorShwradstat.cpp b/diag/feltorShwradstat.cpp
index 72c08e6cc..62da58d50 100644
--- a/diag/feltorShwradstat.cpp
+++ b/diag/feltorShwradstat.cpp
@@ -19,7 +19,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     //nc defs
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     int dataIDs[14];
     std::string names[14] = {"Rx","Guynx","Tnx","A","Rfn","difffauy1","difffauy2","Sfauy",
@@ -35,7 +35,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        file::string2Json( input, js, file::comments::are_forbidden);
+        dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
         const eule::Parameters p(js);
         
         dg::Grid1d g1d( 0., p.lx,p.n_out, p.Nx_out, p.bc_x);
diff --git a/diag/feltorShwstat.cpp b/diag/feltorShwstat.cpp
index 907aef4e3..2be088129 100644
--- a/diag/feltorShwstat.cpp
+++ b/diag/feltorShwstat.cpp
@@ -34,7 +34,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     //nc defs
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     int dataIDs[31];
     std::string names[31] = {"Rfxnorm","Anorm","Rfnnorm","Annorm","dtfauynorm","Rxnorm","invkappaavg","Rnxnorm","Guyxnorm","Txnorm","Guynxnorm","Tnxnorm","neatnorm","Gamma","Rxnormscal","Guynxnormscal","Tnxnormscal","Anormscal","Annormscal","Rfnnormscal","neatsupnorm","nuturbnorm","Rnnormscal","dfnormscal","Rnffnormscal","difflnnnorm","difffauy1norm","difffauy2norm","Slnnnorm","Sfauynorm","vyfavgnorm"}; 
@@ -49,7 +49,7 @@ int main( int argc, char* argv[])
         err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
         
         Json::Value js;
-        file::string2Json( input, js, file::comments::are_forbidden);
+        dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
         const eule::Parameters p(js);
         
 	size_t start0d  = 0;    
diff --git a/diag/fftwdiag.cpp b/diag/fftwdiag.cpp
index f8bbb76c1..e1bb2c3ff 100644
--- a/diag/fftwdiag.cpp
+++ b/diag/fftwdiag.cpp
@@ -23,7 +23,7 @@ int main( int argc, char* argv[])
 //     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     ///////////////////read in and show inputfile//////////////////
@@ -33,7 +33,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     
@@ -70,7 +70,7 @@ int main( int argc, char* argv[])
     
     
     //2d field netcdf vars of outputkxky.nc
-    file::NC_Error_Handle err2d_f;
+    dg::file::NC_Error_Handle err2d_f;
     int ncid2d_f,dim_ids2d_f[3],dataIDs2d_f[3], tvarID2d_f;
     std::string names2d_f[3] = {"S(ne)","S(phi)","gamma(phi)"};    
 
@@ -78,21 +78,21 @@ int main( int argc, char* argv[])
     size_t start2d_f[3]  = {0, 0, 0};    
     err2d_f = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid2d_f);
     err2d_f = nc_put_att_text( ncid2d_f, NC_GLOBAL, "inputfile", input.size(), input.data());
-    err2d_f = file::define_dimensions( ncid2d_f, dim_ids2d_f, &tvarID2d_f, g2d_f);
+    err2d_f = dg::file::define_dimensions( ncid2d_f, dim_ids2d_f, &tvarID2d_f, g2d_f);
     for( unsigned i=0; i<3; i++){
         err2d_f = nc_def_var( ncid2d_f, names2d_f[i].data(), NC_DOUBLE, 3, dim_ids2d_f, &dataIDs2d_f[i]);
     }   
     err2d_f = nc_close(ncid2d_f); 
     
     //1d file netcdf vars of outputk.nc
-    file::NC_Error_Handle err1d_f;
+    dg::file::NC_Error_Handle err1d_f;
     int ncid1d_f,dim_ids1d_f[2],dataIDs1d_f[4], tvarID1d_f;
     std::string names1d_f[4] = {"Sk(ne)","Sk(phi)","gamma(phi)","k",}; //may  goto ln(n/<n>)
     size_t count1d_f[2]  = {1, g1d_f.N()};
     size_t start1d_f[2]  = {0, 0};    
     err1d_f = nc_create(argv[3],NC_NETCDF4|NC_CLOBBER, &ncid1d_f);
     err1d_f = nc_put_att_text( ncid1d_f, NC_GLOBAL, "inputfile", input.size(), input.data());
-    err1d_f = file::define_dimensions( ncid1d_f, dim_ids1d_f, &tvarID1d_f, g1d_f);
+    err1d_f = dg::file::define_dimensions( ncid1d_f, dim_ids1d_f, &tvarID1d_f, g1d_f);
     for( unsigned i=0; i<4; i++){
         err1d_f = nc_def_var( ncid1d_f, names1d_f[i].data(), NC_DOUBLE, 2, dim_ids1d_f, &dataIDs1d_f[i]);
     }   
diff --git a/diag/growthrate.cpp b/diag/growthrate.cpp
index 35bd1259e..6131a653c 100644
--- a/diag/growthrate.cpp
+++ b/diag/growthrate.cpp
@@ -21,7 +21,7 @@ int main( int argc, char* argv[])
     }
 
     ///////////////////read in and show inputfile//////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     size_t length;
@@ -30,7 +30,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const eule::Parameters p(js);
     p.display(std::cout);
     
diff --git a/diag/histdiag.cpp b/diag/histdiag.cpp
index 784a5293d..99686dcfe 100644
--- a/diag/histdiag.cpp
+++ b/diag/histdiag.cpp
@@ -95,10 +95,10 @@ int main( int argc, char* argv[])
     int dataIDs1[2],dataIDs2[2],dataIDs12[1];
     int dim_ids1[1],dim_ids2[1],dim_ids12[2];
     int ncid;
-    file::NC_Error_Handle err; 
+    dg::file::NC_Error_Handle err; 
     err = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid); 
     //plot 1
-    err = file::define_dimension( ncid, &dim_ids1[0],  g1d1,"A1_");
+    err = dg::file::define_dimension( ncid, &dim_ids1[0],  g1d1,"A1_");
     err = nc_def_var( ncid, "P(A1)",   NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[0]);
     err = nc_def_var( ncid, "A1",    NC_DOUBLE, 1, &dim_ids1[0], &dataIDs1[1]);
     err = nc_enddef( ncid);
@@ -106,7 +106,7 @@ int main( int argc, char* argv[])
     err = nc_put_var_double( ncid, dataIDs1[1], A1.data() );
     err = nc_redef(ncid);
     //plot 2
-    err = file::define_dimension( ncid, &dim_ids2[0],  g1d2,"A2_");
+    err = dg::file::define_dimension( ncid, &dim_ids2[0],  g1d2,"A2_");
     err = nc_def_var( ncid, "P(A2)",   NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[0]);
     err = nc_def_var( ncid, "A2",    NC_DOUBLE, 1, &dim_ids2[0], &dataIDs2[1]);
     err = nc_enddef( ncid);
@@ -118,7 +118,7 @@ int main( int argc, char* argv[])
 //     dim_ids12[1]=dim_ids2[0];
     dim_ids12[0]=dataIDs1[0];
     dim_ids12[1]=dataIDs2[0];
-    err = file::define_dimensions( ncid, &dim_ids12[0],  g2d);
+    err = dg::file::define_dimensions( ncid, &dim_ids12[0],  g2d);
     err = nc_def_var( ncid, "P(A1,A2)",   NC_DOUBLE, 2, &dim_ids12[0], &dataIDs12[0]);
     err = nc_enddef( ncid);
     err = nc_put_var_double( ncid, dataIDs12[0], PA1A2.data() );
diff --git a/diag/impRdiag.cu b/diag/impRdiag.cu
index b0e7bba24..9688db9db 100644
--- a/diag/impRdiag.cu
+++ b/diag/impRdiag.cu
@@ -34,7 +34,7 @@ int main( int argc, char* argv[])
     }
     std::cout << argv[1] << " -> " << argv[2]<<std::endl;
     ////////process parameter from .nc datafile////////
-    file::NC_Error_Handle err_in;
+    dg::file::NC_Error_Handle err_in;
     int ncid_in;
     err_in = nc_open(argv[1], NC_NOWRITE, &ncid_in);
     //read & print parameter string
@@ -45,7 +45,7 @@ int main( int argc, char* argv[])
     std::cout << "input "<< input << std::endl;
     //parse: parameter string--json-->p.xxx
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
 
   const imp::Parameters p(js);
   p.display(std::cout);
@@ -163,14 +163,14 @@ int main( int argc, char* argv[])
   int cache_size = (transfer2d.capacity()*sizeof(transfer2d[0])+sizeof(transfer2d))*nelems;
   int cache_nelems = nelems;
   double cache_preemption = 0.9;
-  file::NC_Error_Handle err_out;
+  dg::file::NC_Error_Handle err_out;
   int ncid_out, dim_t_x_y_id[3], tvar_id, etvar_w_id, etdim_w_id;
   err_out = nc_create(argv[2], NC_NETCDF4|NC_CLOBBER, &ncid_out);
   err_out = nc_put_att_text(ncid_out, NC_GLOBAL, "inputfile",
                             input.size(), input.data());
-  err_out = file::define_limtime_xy(ncid_out, dim_t_x_y_id, p.maxout+1,
+  err_out = dg::file::define_limtime_xy(ncid_out, dim_t_x_y_id, p.maxout+1,
                                     &tvar_id, g2d);
-  err_out = file::define_limited_time(ncid_out, "etime", num_etime,
+  err_out = dg::file::define_limited_time(ncid_out, "etime", num_etime,
                                       &etdim_w_id, &etvar_w_id);
   int k = 0;
   for (unsigned i = 0; i < num_species; i++)
diff --git a/diag/normdiag.cu b/diag/normdiag.cu
index 557d4bb0b..9c439a4bc 100644
--- a/diag/normdiag.cu
+++ b/diag/normdiag.cu
@@ -22,7 +22,7 @@ int main( int argc, char* argv[])
     }
 
     //////////////////////////////open nc file//////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     ///////////////////read in and show inputfile //////////////////
@@ -33,7 +33,7 @@ int main( int argc, char* argv[])
     err = nc_close(ncid); 
 
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const eule::Parameters p(js);
     
     //////////////////////////////Grids//////////////////////////////////////
diff --git a/diag/reco2Ddiag.cu b/diag/reco2Ddiag.cu
index f6f8b00ae..69bfc8b5c 100644
--- a/diag/reco2Ddiag.cu
+++ b/diag/reco2Ddiag.cu
@@ -22,7 +22,7 @@ int main( int argc, char* argv[])
     }
 
     ///////////////////read in and show inputfile//////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     size_t length;
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const asela::Parameters p(js);
 
     //////////////////////////////Grids//////////////////////////////////////
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index 6e0b54685..7ad81ff30 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -63,7 +63,7 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     size_t length;
@@ -72,7 +72,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const Parameters p(js);
     p.display(std::cout);
     
@@ -125,7 +125,7 @@ int main( int argc, char* argv[])
     //size_t start1d[2]  = {0, 0};    
     //1d netcdf output file    
 
-    file::NC_Error_Handle err_out;
+    dg::file::NC_Error_Handle err_out;
     const size_t number_of_names = 17;
     int ncid_out;
     int namescomID[number_of_names],tvarID1d;
@@ -139,9 +139,9 @@ int main( int argc, char* argv[])
     
     err_out = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
     err_out = nc_put_att_text( ncid_out, NC_GLOBAL, "inputfile", input.size(), input.data());
-    //err_out = file::define_dimensions( ncid_out, dim_ids2d, &tvarID1d, g2d);
-    err_out = file::define_limited_time( ncid_out, "time", p.maxout+1, &dim_ids2d[0], &tvarID1d);
-    //err_out = file::define_time( ncid_out, "ptime", &timeID, &timevarID);
+    //err_out = dg::file::define_dimensions( ncid_out, dim_ids2d, &tvarID1d, g2d);
+    err_out = dg::file::define_limited_time( ncid_out, "time", p.maxout+1, &dim_ids2d[0], &tvarID1d);
+    //err_out = dg::file::define_time( ncid_out, "ptime", &timeID, &timevarID);
     for( unsigned i=0; i<number_of_names; i++){
         err_out = nc_def_var( ncid_out, namescom[i].data(),  NC_DOUBLE, 1, &dim_ids2d[0], &namescomID[i]);
     }   
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index 3853fddeb..a752c4515 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -58,7 +58,7 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     ///////////////////read in and show inputfile//////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     size_t length;
@@ -67,7 +67,7 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid, NC_GLOBAL, "inputfile", &input[0]);
     std::cout << "input "<<input<<std::endl;
     Json::Value js;
-    file::string2Json( input, js, file::comments::are_forbidden);
+    dg::file::string2Json( input, js, dg::file::comments::are_forbidden);
     const Parameters p(js);
     p.display(std::cout);
     
@@ -120,7 +120,7 @@ int main( int argc, char* argv[])
     //size_t start1d[2]  = {0, 0};    
     //1d netcdf output file    
 
-    file::NC_Error_Handle err_out;
+    dg::file::NC_Error_Handle err_out;
     const size_t number_of_names = 17;
     int ncid_out;
     int namescomID[number_of_names],tvarID1d;
@@ -134,9 +134,9 @@ int main( int argc, char* argv[])
     
     err_out = nc_create(argv[2],NC_NETCDF4|NC_CLOBBER, &ncid_out);
     err_out = nc_put_att_text( ncid_out, NC_GLOBAL, "inputfile", input.size(), input.data());
-    //err_out = file::define_dimensions( ncid_out, dim_ids2d, &tvarID1d, g2d);
-    err_out = file::define_limited_time( ncid_out, "time", p.maxout+1, &dim_ids2d[0], &tvarID1d);
-    //err_out = file::define_time( ncid_out, "ptime", &timeID, &timevarID);
+    //err_out = dg::file::define_dimensions( ncid_out, dim_ids2d, &tvarID1d, g2d);
+    err_out = dg::file::define_limited_time( ncid_out, "time", p.maxout+1, &dim_ids2d[0], &tvarID1d);
+    //err_out = dg::file::define_time( ncid_out, "ptime", &timeID, &timevarID);
     for( unsigned i=0; i<number_of_names; i++){
         err_out = nc_def_var( ncid_out, namescom[i].data(),  NC_DOUBLE, 1, &dim_ids2d[0], &namescomID[i]);
     }   
diff --git a/diag/vmaxnc.cu b/diag/vmaxnc.cu
index 3895d27f1..7ad996e1d 100644
--- a/diag/vmaxnc.cu
+++ b/diag/vmaxnc.cu
@@ -15,7 +15,7 @@ int main( int argc, char* argv[])
         std::cerr << "Usage: "<<argv[0]<<" [input1.nc] [input2.nc] ...\n";
         return -1;
     }
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int timeID, varID;
     size_t start = {0}, numOut;
     for( int i=1; i< argc; i++)
-- 
GitLab


From 5016c496f4803d504c5a9dfb7a988cd0bf0d4cf7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 5 Feb 2021 11:42:07 +0100
Subject: [PATCH 458/540] Change all src files to dg::file

- I think the deleted variation function from Poisson still makes
problems in some folders
---
 src/ep/toeflR.cu                |   6 +-
 src/ep/toefl_mpi.cu             |  20 ++---
 src/feltor/feltor.cu            |  14 ++--
 src/feltor/feltor_hpc.cu        |  42 +++++------
 src/feltor/feltordiag.cu        |  20 ++---
 src/feltor/implicit_t.cu        |   6 +-
 src/feltor/init_from_file.h     |   4 +-
 src/feltor/interpolate_in_3d.cu |  14 ++--
 src/feltor/manufactured.cu      |   6 +-
 src/feltor/parameters.h         | 130 ++++++++++++++++----------------
 src/feltorSH/feltor.cu          |   6 +-
 src/feltorSH/feltor_hpc.cu      |   8 +-
 src/feltorSH/feltor_mpi.cu      |   8 +-
 src/feltorSHp/feltor.cu         |   6 +-
 src/feltorSHp/feltor_hpc.cu     |   8 +-
 src/feltorSHp/feltor_mpi.cu     |   8 +-
 src/feltorSesol/feltor.cu       |   6 +-
 src/feltorSesol/feltor_hpc.cu   |  12 +--
 src/feltorSesol/feltor_mpi.cu   |  12 +--
 src/feltorShw/feltor.cu         |   6 +-
 src/feltorShw/feltor_hpc.cu     |  12 +--
 src/feltorShw/feltor_mpi.cu     |  12 +--
 src/hasegawa/hw.cu              |   6 +-
 src/hasegawa/mima.cu            |   6 +-
 src/heat/heat.cu                |  10 +--
 src/heat/heat_hpc.cu            |  16 ++--
 src/impurities/toeflI.cu        |   6 +-
 src/impurities/toefl_hpc.cu     |   8 +-
 src/impurities/toefl_mpi.cu     |   8 +-
 src/polar/polar.cu              |   4 +-
 src/polar/polar_mpi.cu          |   4 +-
 src/reco2D/reconnection.cu      |   6 +-
 src/reco2D/reconnection_hpc.cu  |   8 +-
 src/reco2D/reconnection_mpi.cu  |   8 +-
 src/toefl/toeflR.cu             |   6 +-
 src/toefl/toefl_hpc.cu          |  12 +--
 36 files changed, 237 insertions(+), 237 deletions(-)

diff --git a/src/ep/toeflR.cu b/src/ep/toeflR.cu
index e696978d7..179712301 100644
--- a/src/ep/toeflR.cu
+++ b/src/ep/toeflR.cu
@@ -18,9 +18,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json("input.json", js, file::comments::are_discarded);
+        dg::file::file2Json("input.json", js, dg::file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json(argv[1], js, file::comments::are_discarded);
+        dg::file::file2Json(argv[1], js, dg::file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -29,7 +29,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json("window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json("window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/ep/toefl_mpi.cu b/src/ep/toefl_mpi.cu
index 1a1295ee7..23b948936 100644
--- a/src/ep/toefl_mpi.cu
+++ b/src/ep/toefl_mpi.cu
@@ -56,7 +56,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const Parameters p( js);
     if(rank==0)p.display( std::cout);
@@ -86,12 +86,12 @@ int main( int argc, char* argv[])
     karniadakis.init( test, diffusion, 0., y0, p.dt);
     y0.swap( y1); //y1 now contains value at zero time
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     if(rank==0)err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     if(rank==0)err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
-    if(rank==0)err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
+    if(rank==0)err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
     //field IDs
     std::string names[4] = {"electrons", "positrons", "potential", "vorticity"}; 
     int dataIDs[4]; 
@@ -100,7 +100,7 @@ int main( int argc, char* argv[])
 
     //energy IDs
     int EtimeID, EtimevarID;
-    if(rank==0)err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    if(rank==0)err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, dissID, dEdtID;
     if(rank==0)err = nc_def_var( ncid, "energy",      NC_DOUBLE, 1, &EtimeID, &energyID);
     if(rank==0)err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
@@ -123,19 +123,19 @@ int main( int argc, char* argv[])
     {
         dg::blas2::gemv( interpolate, y0[i], transferD);
         dg::blas1::transfer( transferD, transferH);
-        file::put_vara_double( ncid, dataIDs[i], 0, grid_out, transferH);
+        dg::file::put_vara_double( ncid, dataIDs[i], 0, grid_out, transferH);
     }
     //pot
     transfer = test.potential();
     dg::blas2::gemv( interpolate, transfer, transferD);
     dg::blas1::transfer( transferD, transferH);
-    file::put_vara_double( ncid, dataIDs[2], 0, grid_out, transferH );
+    dg::file::put_vara_double( ncid, dataIDs[2], 0, grid_out, transferH );
     //Vor
     transfer = test.potential();
     dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);
     dg::blas2::gemv( interpolate,y1[1], transferD);
     dg::blas1::transfer( transferD, transferH);
-    file::put_vara_double( ncid, dataIDs[3], 0, grid_out, transferH );
+    dg::file::put_vara_double( ncid, dataIDs[3], 0, grid_out, transferH );
     if(rank==0)err = nc_put_vara_double( ncid, tvarID, start, count, &time);
     if(rank==0)err = nc_close(ncid);
     ///////////////////////////////////////Timeloop/////////////////////////////////
@@ -192,17 +192,17 @@ int main( int argc, char* argv[])
         {
             dg::blas2::gemv( interpolate, y0[j], transferD);
             dg::blas1::transfer( transferD, transferH);
-            file::put_vara_double( ncid, dataIDs[j], i, grid, transferH);
+            dg::file::put_vara_double( ncid, dataIDs[j], i, grid, transferH);
         }
         transfer = test.potential();
         dg::blas2::gemv( interpolate, transfer, transferD);
         dg::blas1::transfer( transferD, transferH);
-        file::put_vara_double( ncid, dataIDs[2], i, grid, transferH );
+        dg::file::put_vara_double( ncid, dataIDs[2], i, grid, transferH );
         transfer = test.potential();
         dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);        //correct?    
         dg::blas2::gemv( interpolate,y1[1], transferD);
         dg::blas1::transfer( transferD, transferH);
-        file::put_vara_double( ncid, dataIDs[3], i, grid, transferH );
+        dg::file::put_vara_double( ncid, dataIDs[3], i, grid, transferH );
         if(rank==0)err = nc_put_vara_double( ncid, tvarID, start, count, &time);
         if(rank==0)err = nc_close(ncid);
 
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index 15fe0e998..ea2fc2c16 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -37,8 +37,8 @@ int main( int argc, char* argv[])
         return -1;
     }
     try{
-        file::file2Json( inputfile, js, file::comments::are_forbidden, file::error::is_throw);
-        feltor::Parameters(js, file::error::is_throw);
+        dg::file::file2Json( inputfile, js, dg::file::comments::are_forbidden, dg::file::error::is_throw);
+        feltor::Parameters(js, dg::file::error::is_throw);
     }catch(std::runtime_error& e)
     {
 
@@ -47,7 +47,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     try{
-        file::file2Json( geomfile, gs, file::comments::are_discarded, file::error::is_throw);
+        dg::file::file2Json( geomfile, gs, dg::file::comments::are_discarded, dg::file::error::is_throw);
     }catch(std::runtime_error& e)
     {
 
@@ -62,8 +62,8 @@ int main( int argc, char* argv[])
     dg::geo::TokamakMagneticField mag, mod_mag;
     dg::geo::CylindricalFunctor wall, transition, sheath, direction;
     try{
-        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
-        mod_mag = dg::geo::createModifiedField(gs, js, file::error::is_throw, wall, transition);
+        mag = dg::geo::createMagneticField(gs, dg::file::error::is_throw);
+        mod_mag = dg::geo::createModifiedField(gs, js, dg::file::error::is_throw, wall, transition);
     }catch(std::runtime_error& e)
     {
         std::cerr << "ERROR in geometry file "<<geomfile<<std::endl;
@@ -79,7 +79,7 @@ int main( int argc, char* argv[])
     dg::CylindricalGrid3d grid( Rmin,Rmax, Zmin,Zmax, 0, 2.*M_PI,
         p.n, p.Nx, p.Ny, p.symmetric ? 1 : p.Nz, p.bcxN, p.bcyN, dg::PER);
     try{
-        dg::geo::createSheathRegion( js, file::error::is_throw, mag, wall,
+        dg::geo::createSheathRegion( js, dg::file::error::is_throw, mag, wall,
                 Rmin, Rmax, Zmin, Zmax, sheath, direction);
     }catch(std::runtime_error& e)
     {
@@ -170,7 +170,7 @@ int main( int argc, char* argv[])
     /////////glfw initialisation ////////////////////////////////////////////
     //
     std::stringstream title;
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     unsigned red = js.get("reduction", 1).asUInt();
     double rows = js["rows"].asDouble(), cols = p.Nz/red+1,
            width = js["width"].asDouble(), height = js["height"].asDouble();
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 3c81ff908..9201ada7d 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -114,8 +114,8 @@ int main( int argc, char* argv[])
     else
     {
         try{
-            file::file2Json( argv[1], js, file::comments::are_discarded, file::error::is_throw);
-            feltor::Parameters( js, file::error::is_throw);
+            dg::file::file2Json( argv[1], js, dg::file::comments::are_discarded, dg::file::error::is_throw);
+            feltor::Parameters( js, dg::file::error::is_throw);
         } catch( std::exception& e) {
             MPI_OUT std::cerr << "ERROR in input parameter file "<<argv[1]<<std::endl;
             MPI_OUT std::cerr << e.what()<<std::endl;
@@ -125,7 +125,7 @@ int main( int argc, char* argv[])
             return -1;
         }
         try{
-            file::file2Json( argv[2], gs, file::comments::are_discarded, file::error::is_throw);
+            dg::file::file2Json( argv[2], gs, dg::file::comments::are_discarded, dg::file::error::is_throw);
         } catch( std::exception& e) {
             MPI_OUT std::cerr << "ERROR in geometry file "<<argv[2]<<std::endl;
             MPI_OUT std::cerr << e.what()<<std::endl;
@@ -142,8 +142,8 @@ int main( int argc, char* argv[])
     dg::geo::TokamakMagneticField mag, mod_mag;
     dg::geo::CylindricalFunctor wall, transition, sheath, direction;
     try{
-        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
-        mod_mag = dg::geo::createModifiedField(gs, js, file::error::is_throw, wall, transition);
+        mag = dg::geo::createMagneticField(gs, dg::file::error::is_throw);
+        mod_mag = dg::geo::createModifiedField(gs, js, dg::file::error::is_throw, wall, transition);
     }catch(std::runtime_error& e)
     {
         std::cerr << "ERROR in geometry file "<<argv[2]<<std::endl;
@@ -187,7 +187,7 @@ int main( int argc, char* argv[])
 #endif
 
     try{
-        dg::geo::createSheathRegion( js, file::error::is_throw, mag, wall,
+        dg::geo::createSheathRegion( js, dg::file::error::is_throw, mag, wall,
                 Rmin, Rmax, Zmin, Zmax, sheath, direction);
     }catch(std::runtime_error& e)
     {
@@ -277,7 +277,7 @@ int main( int argc, char* argv[])
     }
 
     /// //////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     std::string file_name = argv[3];
     int ncid=-1;
     try{
@@ -315,8 +315,8 @@ int main( int argc, char* argv[])
 
     // Define dimensions (t,z,y,x)
     int dim_ids[4], restart_dim_ids[3], tvarID;
-    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out, {"time", "z", "y", "x"});
-    MPI_OUT err = file::define_dimensions( ncid, restart_dim_ids, grid, {"zr", "yr", "xr"});
+    MPI_OUT err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, g3d_out, {"time", "z", "y", "x"});
+    MPI_OUT err = dg::file::define_dimensions( ncid, restart_dim_ids, grid, {"zr", "yr", "xr"});
     int dim_ids3d[3] = {dim_ids[0], dim_ids[2], dim_ids[3]};
     bool write2d = true;
 #ifdef FELTOR_MPI
@@ -337,7 +337,7 @@ int main( int argc, char* argv[])
         record.function( transferH, var, g3d_out);
         //record.function( resultH, var, grid);
         //dg::blas2::symv( projectH, resultH, transferH);
-        file::put_var_double( ncid, vecID, g3d_out, transferH);
+        dg::file::put_var_double( ncid, vecID, g3d_out, transferH);
         MPI_OUT err = nc_redef(ncid);
     }
     //create & output static 2d variables into file
@@ -353,7 +353,7 @@ int main( int argc, char* argv[])
         //record.function( transferH, var, g3d_out); //ATTENTION: This does not work because feltor internal varialbes return full grid functions
         record.function( resultH, var, grid);
         dg::blas2::symv( projectH, resultH, transferH);
-        if(write2d)file::put_var_double( ncid, vecID, *g2d_out_ptr, transferH);
+        if(write2d)dg::file::put_var_double( ncid, vecID, *g2d_out_ptr, transferH);
         MPI_OUT err = nc_redef(ncid);
     }
 
@@ -422,13 +422,13 @@ int main( int argc, char* argv[])
         record.function( resultD, var);
         dg::blas2::symv( projectD, resultD, transferD);
         dg::assign( transferD, transferH);
-        file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
+        dg::file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
     }
     for( auto& record : feltor::restart3d_list)
     {
         record.function( resultD, var);
         dg::assign( resultD, resultH);
-        file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
+        dg::file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
     }
     for( auto& record : feltor::diagnostics2d_list)
     {
@@ -446,7 +446,7 @@ int main( int argc, char* argv[])
         tti.toc();
         MPI_OUT std::cout<< name << " Computing average took "<<tti.diff()<<"\n";
         tti.tic();
-        if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+        if(write2d) dg::file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
         tti.tic();
@@ -456,7 +456,7 @@ int main( int argc, char* argv[])
         feltor::slice_vector3d( transferD, transferD2d, local_size2d);
         dg::assign( transferD2d, transferH2d);
         if( record.integral) time_integrals[name].init( time, transferH2d);
-        if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+        if(write2d) dg::file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
         tti.toc();
         MPI_OUT std::cout<< name << " 2d output took "<<tti.diff()<<"\n";
     }
@@ -586,13 +586,13 @@ int main( int argc, char* argv[])
             record.function( resultD, var);
             dg::blas2::symv( projectD, resultD, transferD);
             dg::assign( transferD, transferH);
-            file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
+            dg::file::put_vara_double( ncid, id4d.at(record.name), start, g3d_out, transferH);
         }
         for( auto& record : feltor::restart3d_list)
         {
             record.function( resultD, var);
             dg::assign( resultD, resultH);
-            file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
+            dg::file::put_var_double( ncid, restart_ids.at(record.name), grid, resultH);
         }
         for( auto& record : feltor::diagnostics2d_list)
         {
@@ -601,12 +601,12 @@ int main( int argc, char* argv[])
                 std::string name = record.name+"_ta2d";
                 transferH2d = time_integrals.at(name).get_integral();
                 time_integrals.at(name).flush();
-                if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+                if(write2d) dg::file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
 
                 name = record.name+"_2d";
                 transferH2d = time_integrals.at(name).get_integral( );
                 time_integrals.at(name).flush( );
-                if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+                if(write2d) dg::file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
             }
             else // compute from scratch
             {
@@ -616,13 +616,13 @@ int main( int argc, char* argv[])
                 std::string name = record.name+"_ta2d";
                 dg::assign( transferD, transferH);
                 toroidal_average( transferH, transferH2d, false);
-                if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+                if(write2d) dg::file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
 
                 // 2d data of plane varphi = 0
                 name = record.name+"_2d";
                 feltor::slice_vector3d( transferD, transferD2d, local_size2d);
                 dg::assign( transferD2d, transferH2d);
-                if(write2d) file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
+                if(write2d) dg::file::put_vara_double( ncid, id3d.at(name), start, *g2d_out_ptr, transferH2d);
             }
         }
         MPI_OUT err = nc_close(ncid);
diff --git a/src/feltor/feltordiag.cu b/src/feltor/feltordiag.cu
index a2fb52f4f..db4aafaf2 100644
--- a/src/feltor/feltordiag.cu
+++ b/src/feltor/feltordiag.cu
@@ -30,7 +30,7 @@ int main( int argc, char* argv[])
     std::cout << " -> "<<argv[argc-1]<<std::endl;
 
     //------------------------open input nc file--------------------------------//
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid_in;
     err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
     size_t length;
@@ -42,10 +42,10 @@ int main( int argc, char* argv[])
     err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geomfile[0]);
     err = nc_close( ncid_in);
     Json::Value js,gs;
-    file::string2Json(inputfile, js, file::comments::are_forbidden);
-    file::string2Json(geomfile, gs, file::comments::are_forbidden);
+    dg::file::string2Json(inputfile, js, dg::file::comments::are_forbidden);
+    dg::file::string2Json(geomfile, gs, dg::file::comments::are_forbidden);
     //we only need some parameters from p, not all
-    const feltor::Parameters p(js, file::error::is_warning);
+    const feltor::Parameters p(js, dg::file::error::is_warning);
     const dg::geo::solovev::Parameters gp(gs);
     p.display();
     gp.display();
@@ -96,7 +96,7 @@ int main( int argc, char* argv[])
 
     dg::geo::CylindricalFunctor wall, transition;
     dg::geo::TokamakMagneticField mag =
-        dg::geo::createModifiedField(gs, js, file::error::is_warning, wall, transition);
+        dg::geo::createModifiedField(gs, js, dg::file::error::is_warning, wall, transition);
     dg::HVec psipog2d = dg::evaluate( mag.psip(), g2d_out);
     // Construct weights and temporaries
 
@@ -213,13 +213,13 @@ int main( int argc, char* argv[])
 
     // define 2d and 1d and 0d dimensions and variables
     int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
+    err = dg::file::define_dimensions( ncid_out, dim_ids, &tvarID, g2d_out);
     //Write long description
     std::string long_name = "Time at which 2d fields are written";
     err = nc_put_att_text( ncid_out, tvarID, "long_name", long_name.size(),
             long_name.data());
     int dim_ids1d[2] = {dim_ids[0], 0}; //time,  psi
-    err = file::define_dimension( ncid_out, &dim_ids1d[1], g1d_out, {"psi"} );
+    err = dg::file::define_dimension( ncid_out, &dim_ids1d[1], g1d_out, {"psi"} );
     std::map<std::string, int> id0d, id1d, id2d;
 
     size_t count1d[2] = {1, g1d_out.n()*g1d_out.N()};
@@ -316,7 +316,7 @@ int main( int argc, char* argv[])
         std::cout << "Opening file "<<argv[j]<<"\n";
         try{
             err = nc_open( argv[j], NC_NOWRITE, &ncid); //open 3d file
-        } catch ( file::NC_Error& error)
+        } catch ( dg::file::NC_Error& error)
         {
             std::cerr << "An error occurded opening file "<<argv[j]<<"\n";
             std::cerr << error.what()<<std::endl;
@@ -349,7 +349,7 @@ int main( int argc, char* argv[])
                 bool available = true;
                 try{
                     err = nc_inq_varid(ncid, (record.name+"_ta2d").data(), &dataID);
-                } catch ( file::NC_Error& error)
+                } catch ( dg::file::NC_Error& error)
                 {
                     if(  i == 0)
                     {
@@ -398,7 +398,7 @@ int main( int argc, char* argv[])
                 available = true;
                 try{
                     err = nc_inq_varid(ncid, (record.name+"_2d").data(), &dataID);
-                } catch ( file::NC_Error& error)
+                } catch ( dg::file::NC_Error& error)
                 {
                     if(  i == 0)
                     {
diff --git a/src/feltor/implicit_t.cu b/src/feltor/implicit_t.cu
index 348628e28..f583a2b01 100644
--- a/src/feltor/implicit_t.cu
+++ b/src/feltor/implicit_t.cu
@@ -16,15 +16,15 @@ int main( int argc, char* argv[])
 {
     Json::Value js, gs;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_forbidden);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile]\n";
         return -1;
     }
-    const feltor::Parameters p( js, file::error::is_throw);
+    const feltor::Parameters p( js, dg::file::error::is_throw);
     p.display( std::cout);
     const double R_0 = 10;
     const double I_0 = 20; //q factor at r=1 is I_0/R_0
diff --git a/src/feltor/init_from_file.h b/src/feltor/init_from_file.h
index 180c0631c..b697df9a1 100644
--- a/src/feltor/init_from_file.h
+++ b/src/feltor/init_from_file.h
@@ -16,7 +16,7 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     std::array<std::array<DVec,2>,2> y0;
     ///////////////////read in and show inputfile
 
-    file::NC_Error_Handle errIN;
+    dg::file::NC_Error_Handle errIN;
     int ncidIN;
     errIN = nc_open( file_name.data(), NC_NOWRITE, &ncidIN);
     Json::Value jsIN;
@@ -24,7 +24,7 @@ std::array<std::array<DVec,2>,2> init_from_file( std::string file_name, const Ge
     errIN = nc_inq_attlen( ncidIN, NC_GLOBAL, "inputfile", &length);
     std::string input(length, 'x');
     errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &input[0]);
-    file::string2Json( input, jsIN, file::comments::are_forbidden);
+    dg::file::string2Json( input, jsIN, dg::file::comments::are_forbidden);
     unsigned  pINn  = jsIN["n"].asUInt();
     unsigned  pINNx = jsIN["Nx"].asUInt();
     unsigned  pINNy = jsIN["Ny"].asUInt();
diff --git a/src/feltor/interpolate_in_3d.cu b/src/feltor/interpolate_in_3d.cu
index e84ce0fd1..816a370bb 100644
--- a/src/feltor/interpolate_in_3d.cu
+++ b/src/feltor/interpolate_in_3d.cu
@@ -42,7 +42,7 @@ int main( int argc, char* argv[])
     std::cout << argv[1]<< " -> "<<argv[2]<<std::endl;
 
     //------------------------open input nc file--------------------------------//
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid_in;
     err = nc_open( argv[1], NC_NOWRITE, &ncid_in); //open 3d file
     size_t length;
@@ -53,14 +53,14 @@ int main( int argc, char* argv[])
     std::string geomfile(length, 'x');
     err = nc_get_att_text( ncid_in, NC_GLOBAL, "geomfile", &geomfile[0]);
     Json::Value js,gs;
-    file::string2Json(inputfile, js, file::comments::are_forbidden);
-    file::string2Json(geomfile, gs, file::comments::are_forbidden);
-    const feltor::Parameters p(js, file::error::is_warning);
+    dg::file::string2Json(inputfile, js, dg::file::comments::are_forbidden);
+    dg::file::string2Json(geomfile, gs, dg::file::comments::are_forbidden);
+    const feltor::Parameters p(js, dg::file::error::is_warning);
     p.display();
     std::cout << gs.toStyledString() << std::endl;
     dg::geo::TokamakMagneticField mag;
     try{
-        mag = dg::geo::createMagneticField(gs, file::error::is_throw);
+        mag = dg::geo::createMagneticField(gs, dg::file::error::is_throw);
     }catch(std::runtime_error& e)
     {
         std::cerr << "ERROR in geometry file "<<geomfile<<std::endl;
@@ -120,7 +120,7 @@ int main( int argc, char* argv[])
 
     // define 4d dimension
     int dim_ids[4], tvarID;
-    err = file::define_dimensions( ncid_out, dim_ids, &tvarID, g3d_out_periodic_equidistant, {"time", "z", "y", "x"});
+    err = dg::file::define_dimensions( ncid_out, dim_ids, &tvarID, g3d_out_periodic_equidistant, {"time", "z", "y", "x"});
     std::map<std::string, int> id4d;
 
     /////////////////////////////////////////////////////////////////////////
@@ -206,7 +206,7 @@ int main( int argc, char* argv[])
             bool available = true;
             try{
                 err = nc_inq_varid(ncid_in, record.name.data(), &dataID);
-            } catch ( file::NC_Error& error)
+            } catch ( dg::file::NC_Error& error)
             {
                 if(  i == 0)
                 {
diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 0486a409e..3b5a580cf 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -20,15 +20,15 @@ int main( int argc, char* argv[])
 {
     Json::Value js, gs;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_forbidden);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Wrong number of arguments!\nUsage: "<< argv[0]<<" [inputfile]\n";
         return -1;
     }
-    const feltor::Parameters p( js, file::error::is_throw);
+    const feltor::Parameters p( js, dg::file::error::is_throw);
     p.display( std::cout);
     const double R_0 = 10;
     const double I_0 = 20; //q factor at r=1 is I_0/R_0
diff --git a/src/feltor/parameters.h b/src/feltor/parameters.h
index d2365f87b..7967df819 100644
--- a/src/feltor/parameters.h
+++ b/src/feltor/parameters.h
@@ -51,46 +51,46 @@ struct Parameters
     std::string source_type, sheath_bc;
     bool symmetric, periodify, explicit_diffusion ;
     Parameters() = default;
-    Parameters( const Json::Value& js, enum file::error mode = file::error::is_warning ) {
+    Parameters( const Json::Value& js, enum dg::file::error mode = dg::file::error::is_warning ) {
         //We need to check if a member is present
-        n       = file::get(mode, js,"n", 3).asUInt();
-        Nx      = file::get(mode, js,"Nx", 0).asUInt();
-        Ny      = file::get(mode, js,"Ny", 0).asUInt();
-        Nz      = file::get(mode, js,"Nz", 0).asUInt();
-        dt      = file::get(mode, js,"dt", 0.).asDouble();
-        cx      = file::get_idx(mode, js,"compression",0u,1).asUInt();
-        cy      = file::get_idx(mode, js,"compression",1u,1).asUInt();
+        n       = dg::file::get(mode, js,"n", 3).asUInt();
+        Nx      = dg::file::get(mode, js,"Nx", 0).asUInt();
+        Ny      = dg::file::get(mode, js,"Ny", 0).asUInt();
+        Nz      = dg::file::get(mode, js,"Nz", 0).asUInt();
+        dt      = dg::file::get(mode, js,"dt", 0.).asDouble();
+        cx      = dg::file::get_idx(mode, js,"compression",0u,1).asUInt();
+        cy      = dg::file::get_idx(mode, js,"compression",1u,1).asUInt();
         n_out = n, Nx_out = Nx/cx, Ny_out = Ny/cy, Nz_out = Nz;
-        inner_loop = file::get(mode, js, "inner_loop",1).asUInt();
-        itstp   = file::get( mode, js, "itstp", 0).asUInt();
-        maxout  = file::get( mode, js, "maxout", 0).asUInt();
-        eps_time    = file::get( mode, js, "eps_time", 1e-10).asDouble();
+        inner_loop = dg::file::get(mode, js, "inner_loop",1).asUInt();
+        itstp   = dg::file::get( mode, js, "itstp", 0).asUInt();
+        maxout  = dg::file::get( mode, js, "maxout", 0).asUInt();
+        eps_time    = dg::file::get( mode, js, "eps_time", 1e-10).asDouble();
 
-        stages      = file::get( mode, js, "stages", 3).asUInt();
+        stages      = dg::file::get( mode, js, "stages", 3).asUInt();
         eps_pol.resize(stages);
-        eps_pol[0] = file::get_idx( mode, js, "eps_pol", 0, 1e-6).asDouble();
+        eps_pol[0] = dg::file::get_idx( mode, js, "eps_pol", 0, 1e-6).asDouble();
         for( unsigned i=1;i<stages; i++)
         {
-            eps_pol[i] = file::get_idx( mode, js, "eps_pol", i, 1).asDouble();
+            eps_pol[i] = dg::file::get_idx( mode, js, "eps_pol", i, 1).asDouble();
             eps_pol[i]*=eps_pol[0];
         }
-        jfactor     = file::get( mode, js, "jumpfactor", 1).asDouble();
+        jfactor     = dg::file::get( mode, js, "jumpfactor", 1).asDouble();
 
-        eps_gamma   = file::get( mode, js, "eps_gamma", 1e-6).asDouble();
-        mx          = file::get_idx( mode, js,"FCI","refine", 0u, 1).asUInt();
-        my          = file::get_idx( mode, js,"FCI","refine", 1u, 1).asUInt();
-        rk4eps      = file::get( mode, js,"FCI", "rk4eps", 1e-6).asDouble();
-        periodify   = file::get( mode, js,"FCI", "periodify", true).asBool();
+        eps_gamma   = dg::file::get( mode, js, "eps_gamma", 1e-6).asDouble();
+        mx          = dg::file::get_idx( mode, js,"FCI","refine", 0u, 1).asUInt();
+        my          = dg::file::get_idx( mode, js,"FCI","refine", 1u, 1).asUInt();
+        rk4eps      = dg::file::get( mode, js,"FCI", "rk4eps", 1e-6).asDouble();
+        periodify   = dg::file::get( mode, js,"FCI", "periodify", true).asBool();
 
-        mu[0]       = file::get( mode, js, "mu", -0.000272121).asDouble();
+        mu[0]       = dg::file::get( mode, js, "mu", -0.000272121).asDouble();
         mu[1]       = +1.;
         tau[0]      = -1.;
-        tau[1]      = file::get( mode, js, "tau", 0.).asDouble();
-        beta        = file::get( mode, js, "beta", 0.).asDouble();
-        eta         = file::get( mode, js, "resistivity", 0.).asDouble();
-        nu_perp     = file::get( mode, js, "nu_perp", 0.).asDouble();
-        perp_diff   = file::get_idx( mode, js, "perp_diff", 0, "viscous").asString();
-        std::string temp = file::get_idx( mode, js, "perp_diff", 1, "").asString();
+        tau[1]      = dg::file::get( mode, js, "tau", 0.).asDouble();
+        beta        = dg::file::get( mode, js, "beta", 0.).asDouble();
+        eta         = dg::file::get( mode, js, "resistivity", 0.).asDouble();
+        nu_perp     = dg::file::get( mode, js, "nu_perp", 0.).asDouble();
+        perp_diff   = dg::file::get_idx( mode, js, "perp_diff", 0, "viscous").asString();
+        std::string temp = dg::file::get_idx( mode, js, "perp_diff", 1, "").asString();
         explicit_diffusion = true;
         if(temp == "implicit")
             explicit_diffusion = false;
@@ -98,53 +98,53 @@ struct Parameters
             explicit_diffusion = true;
         else
         {
-            if( file::error::is_throw == mode)
+            if( dg::file::error::is_throw == mode)
                 throw std::runtime_error( "Value "+temp+" for perp_diff[1] is invalid! Must be either explicit or implicit\n");
-            else if ( file::error::is_warning == mode)
+            else if ( dg::file::error::is_warning == mode)
                 std::cerr << "Value "+temp+" for perp_diff[1] is invalid!\n";
             else
                 ;
         }
 
-        //nu_parallel = file::get( mode, js, "nu_parallel", 0.).asDouble();
+        //nu_parallel = dg::file::get( mode, js, "nu_parallel", 0.).asDouble();
         //Init after reading in eta and mu[0]
         nu_parallel[0] = 0.73/eta;
         nu_parallel[1] = sqrt(fabs(mu[0]))*1.36/eta;
 
-        initne      = file::get( mode, js, "initne", "blob").asString();
-        initphi     = file::get( mode, js, "initphi", "zero").asString();
-        amp         = file::get( mode, js, "amplitude", 0.).asDouble();
-        sigma       = file::get( mode, js, "sigma", 0.).asDouble();
-        posX        = file::get( mode, js, "posX", 0.).asDouble();
-        posY        = file::get( mode, js, "posY", 0.).asDouble();
-        sigma_z     = file::get( mode, js, "sigma_z", 0.).asDouble();
-        k_psi       = file::get( mode, js, "k_psi", 0.).asDouble();
-
-        nprofamp   = file::get( mode, js, "profile", "amp", 0.).asDouble();
-        profile_alpha = file::get( mode, js, "profile", "alpha", 0.2).asDouble();
-
-        source_rate     = file::get( mode, js, "source", "rate", 0.).asDouble();
-        source_type     = file::get( mode, js, "source", "type", "profile").asString();
-        sheath_bc       = file::get( mode, js, "sheath", "bc", "bohm").asString();
-        source_boundary = file::get( mode, js, "source", "boundary", 0.5).asDouble();
-        source_alpha    = file::get( mode, js, "source", "alpha", 0.2).asDouble();
-        wall_rate = file::get( mode, js, "wall", "penalization", 0.).asDouble();
-        sheath_rate  = file::get( mode, js, "sheath", "penalization", 0.).asDouble();
-
-        bcxN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 0, "").asString());
-        bcyN = dg::str2bc(file::get_idx( mode, js, "bc", "density", 1, "").asString());
-        bcxU = dg::str2bc(file::get_idx( mode, js, "bc", "velocity", 0, "").asString());
-        bcyU = dg::str2bc(file::get_idx( mode, js, "bc", "velocity", 1, "").asString());
-        bcxP = dg::str2bc(file::get_idx( mode, js, "bc", "potential", 0, "").asString());
-        bcyP = dg::str2bc(file::get_idx( mode, js, "bc", "potential", 1, "").asString());
-
-        boxscaleRm  = file::get_idx( mode, js, "box", "scaleR", 0u, 1.05).asDouble();
-        boxscaleRp  = file::get_idx( mode, js, "box", "scaleR", 1u, 1.05).asDouble();
-        boxscaleZm  = file::get_idx( mode, js, "box", "scaleZ", 0u, 1.05).asDouble();
-        boxscaleZp  = file::get_idx( mode, js, "box", "scaleZ", 1u, 1.05).asDouble();
-
-        curvmode    = file::get( mode, js, "curvmode", "toroidal").asString();
-        symmetric   = file::get( mode, js, "symmetric", false).asBool();
+        initne      = dg::file::get( mode, js, "initne", "blob").asString();
+        initphi     = dg::file::get( mode, js, "initphi", "zero").asString();
+        amp         = dg::file::get( mode, js, "amplitude", 0.).asDouble();
+        sigma       = dg::file::get( mode, js, "sigma", 0.).asDouble();
+        posX        = dg::file::get( mode, js, "posX", 0.).asDouble();
+        posY        = dg::file::get( mode, js, "posY", 0.).asDouble();
+        sigma_z     = dg::file::get( mode, js, "sigma_z", 0.).asDouble();
+        k_psi       = dg::file::get( mode, js, "k_psi", 0.).asDouble();
+
+        nprofamp   = dg::file::get( mode, js, "profile", "amp", 0.).asDouble();
+        profile_alpha = dg::file::get( mode, js, "profile", "alpha", 0.2).asDouble();
+
+        source_rate     = dg::file::get( mode, js, "source", "rate", 0.).asDouble();
+        source_type     = dg::file::get( mode, js, "source", "type", "profile").asString();
+        sheath_bc       = dg::file::get( mode, js, "sheath", "bc", "bohm").asString();
+        source_boundary = dg::file::get( mode, js, "source", "boundary", 0.5).asDouble();
+        source_alpha    = dg::file::get( mode, js, "source", "alpha", 0.2).asDouble();
+        wall_rate = dg::file::get( mode, js, "wall", "penalization", 0.).asDouble();
+        sheath_rate  = dg::file::get( mode, js, "sheath", "penalization", 0.).asDouble();
+
+        bcxN = dg::str2bc(dg::file::get_idx( mode, js, "bc", "density", 0, "").asString());
+        bcyN = dg::str2bc(dg::file::get_idx( mode, js, "bc", "density", 1, "").asString());
+        bcxU = dg::str2bc(dg::file::get_idx( mode, js, "bc", "velocity", 0, "").asString());
+        bcyU = dg::str2bc(dg::file::get_idx( mode, js, "bc", "velocity", 1, "").asString());
+        bcxP = dg::str2bc(dg::file::get_idx( mode, js, "bc", "potential", 0, "").asString());
+        bcyP = dg::str2bc(dg::file::get_idx( mode, js, "bc", "potential", 1, "").asString());
+
+        boxscaleRm  = dg::file::get_idx( mode, js, "box", "scaleR", 0u, 1.05).asDouble();
+        boxscaleRp  = dg::file::get_idx( mode, js, "box", "scaleR", 1u, 1.05).asDouble();
+        boxscaleZm  = dg::file::get_idx( mode, js, "box", "scaleZ", 0u, 1.05).asDouble();
+        boxscaleZp  = dg::file::get_idx( mode, js, "box", "scaleZ", 1u, 1.05).asDouble();
+
+        curvmode    = dg::file::get( mode, js, "curvmode", "toroidal").asString();
+        symmetric   = dg::file::get( mode, js, "symmetric", false).asBool();
     }
     void display( std::ostream& os = std::cout ) const
     {
diff --git a/src/feltorSH/feltor.cu b/src/feltorSH/feltor.cu
index 62ffb7e14..943bde82d 100644
--- a/src/feltorSH/feltor.cu
+++ b/src/feltorSH/feltor.cu
@@ -17,9 +17,9 @@ int main( int argc, char* argv[])
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_forbidden);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -28,7 +28,7 @@ int main( int argc, char* argv[])
     const eule::Parameters p(  js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     std::stringstream title;
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
diff --git a/src/feltorSH/feltor_hpc.cu b/src/feltorSH/feltor_hpc.cu
index 5c0b5ea2e..16b6177fb 100644
--- a/src/feltorSH/feltor_hpc.cu
+++ b/src/feltorSH/feltor_hpc.cu
@@ -21,7 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     p.display( std::cout);
@@ -84,7 +84,7 @@ int main( int argc, char* argv[])
     dg::Karniadakis< std::vector<dg::DVec> > karniadakis( y0, y0[0].size(), p.eps_time);
     karniadakis.init( feltor, rolkar, 0., y0, p.dt);
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
@@ -93,7 +93,7 @@ int main( int argc, char* argv[])
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     err = nc_enddef( ncid);
     err = nc_redef(ncid);
 
@@ -105,7 +105,7 @@ int main( int argc, char* argv[])
 
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, energyIDs[3], dissID, dEdtID, accuracyID;
     err = nc_def_var( ncid, "energy",   NC_DOUBLE, 1, &EtimeID, &energyID);
     err = nc_def_var( ncid, "mass",   NC_DOUBLE, 1, &EtimeID, &massID);
diff --git a/src/feltorSH/feltor_mpi.cu b/src/feltorSH/feltor_mpi.cu
index 5f49667a2..299de47e2 100644
--- a/src/feltorSH/feltor_mpi.cu
+++ b/src/feltorSH/feltor_mpi.cu
@@ -40,7 +40,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     if(rank==0)p.display( std::cout);
@@ -120,7 +120,7 @@ int main( int argc, char* argv[])
     karniadakis.init( feltor, rolkar, 0., y0, p.dt);
     if(rank==0) std::cout << "Done!\n";    
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     MPI_Info info = MPI_INFO_NULL;
 //     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);//MPI OFF
@@ -132,7 +132,7 @@ int main( int argc, char* argv[])
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     dg::Grid2d global_grid_out ( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);  
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
     err = nc_enddef( ncid);
     err = nc_redef(ncid);
 
@@ -146,7 +146,7 @@ int main( int argc, char* argv[])
     err = nc_var_par_access( ncid, tvarID, NC_COLLECTIVE);
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     err = nc_var_par_access( ncid, EtimevarID, NC_COLLECTIVE);
 
     int energyID, massID, energyIDs[3], dissID, dEdtID, accuracyID;
diff --git a/src/feltorSHp/feltor.cu b/src/feltorSHp/feltor.cu
index f24eb18a6..91ce80bb6 100644
--- a/src/feltorSHp/feltor.cu
+++ b/src/feltorSHp/feltor.cu
@@ -17,9 +17,9 @@ int main( int argc, char* argv[])
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_forbidden);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -29,7 +29,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorSHp/feltor_hpc.cu b/src/feltorSHp/feltor_hpc.cu
index 66e510e68..f69ab0e81 100644
--- a/src/feltorSHp/feltor_hpc.cu
+++ b/src/feltorSHp/feltor_hpc.cu
@@ -21,7 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     p.display( std::cout);
@@ -79,7 +79,7 @@ int main( int argc, char* argv[])
     std::cout << "Done!\n";
     
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
@@ -88,7 +88,7 @@ int main( int argc, char* argv[])
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version", NC_INT, 1, &version[1]);
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     err = nc_enddef( ncid);
     err = nc_redef(ncid);
 
@@ -100,7 +100,7 @@ int main( int argc, char* argv[])
 
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, energyIDs[3], dissID, dEdtID, accuracyID;
     err = nc_def_var( ncid, "energy",   NC_DOUBLE, 1, &EtimeID, &energyID);
     err = nc_def_var( ncid, "mass",   NC_DOUBLE, 1, &EtimeID, &massID);
diff --git a/src/feltorSHp/feltor_mpi.cu b/src/feltorSHp/feltor_mpi.cu
index a66edd018..d4cf2aa0c 100644
--- a/src/feltorSHp/feltor_mpi.cu
+++ b/src/feltorSHp/feltor_mpi.cu
@@ -40,7 +40,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const eule::Parameters p( js);
     if(rank==0)p.display( std::cout);
@@ -113,7 +113,7 @@ int main( int argc, char* argv[])
     karniadakis.init( feltor, rolkar, 0., y0, p.dt);
     if(rank==0) std::cout << "Done!\n";    
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     MPI_Info info = MPI_INFO_NULL;
 //     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);//MPI OFF
@@ -125,7 +125,7 @@ int main( int argc, char* argv[])
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
     dg::Grid2d global_grid_out ( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);  
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
     err = nc_enddef( ncid);
     err = nc_redef(ncid);
 
@@ -139,7 +139,7 @@ int main( int argc, char* argv[])
     err = nc_var_par_access( ncid, tvarID, NC_COLLECTIVE);
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     err = nc_var_par_access( ncid, EtimevarID, NC_COLLECTIVE);
 
     int energyID, massID, energyIDs[3], dissID, dEdtID, accuracyID;
diff --git a/src/feltorSesol/feltor.cu b/src/feltorSesol/feltor.cu
index 611afbda0..0ffc72674 100644
--- a/src/feltorSesol/feltor.cu
+++ b/src/feltorSesol/feltor.cu
@@ -18,9 +18,9 @@ int main( int argc, char* argv[])
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json("input.json", js, file::comments::are_discarded);
+        dg::file::file2Json("input.json", js, dg::file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json(argv[1], js, file::comments::are_discarded);
+        dg::file::file2Json(argv[1], js, dg::file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -30,7 +30,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorSesol/feltor_hpc.cu b/src/feltorSesol/feltor_hpc.cu
index 80c1f788b..8a45c47b7 100644
--- a/src/feltorSesol/feltor_hpc.cu
+++ b/src/feltorSesol/feltor_hpc.cu
@@ -21,7 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     p.display( std::cout);
@@ -74,7 +74,7 @@ int main( int argc, char* argv[])
       std::cout << "Done!\n";
     }
     if (argc==4) {
-        file::NC_Error_Handle errIN;
+        dg::file::NC_Error_Handle errIN;
         int ncidIN;
         errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
@@ -83,7 +83,7 @@ int main( int argc, char* argv[])
         std::string inputIN(length, 'x');
         errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json(inputIN, jsIN, file::comments::are_forbidden);
+        dg::file::string2Json(inputIN, jsIN, dg::file::comments::are_forbidden);
 
         const eule::Parameters pIN(  jsIN);
         std::cout << "[input.nc] file parameters" << std::endl;
@@ -127,12 +127,12 @@ int main( int argc, char* argv[])
     karniadakis.init( feltor, rolkar, 0., y0, p.dt);
 //     feltor.energies( y0);//now energies and potential are at time 0
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_create( argv[2], NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids_field[3], tvarID_field;
-    err = file::define_dimensions(ncid, dim_ids_field, &tvarID_field, grid_out);
+    err = dg::file::define_dimensions(ncid, dim_ids_field, &tvarID_field, grid_out);
     err = nc_enddef(ncid);
     err = nc_redef(ncid);
 
@@ -144,7 +144,7 @@ int main( int argc, char* argv[])
         err = nc_def_var(ncid, varname_fields[i].data(), NC_DOUBLE, 3, dim_ids_field, &dataIDs[i]);
     //energy IDs, used for small time-step diagnostic
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, energyIDs[3], dissID, dEdtID, accuracyID, couplingID; 
 
     err = nc_def_var(ncid, "energy", NC_DOUBLE, 1, &EtimeID, &energyID);
diff --git a/src/feltorSesol/feltor_mpi.cu b/src/feltorSesol/feltor_mpi.cu
index beb8cf455..1808ecccd 100644
--- a/src/feltorSesol/feltor_mpi.cu
+++ b/src/feltorSesol/feltor_mpi.cu
@@ -38,7 +38,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     if(rank==0)p.display( std::cout);
@@ -103,7 +103,7 @@ int main( int argc, char* argv[])
         if(rank==0) std::cout << "Done!\n";
     }
     if (argc==4) {
-        file::NC_Error_Handle errIN;
+        dg::file::NC_Error_Handle errIN;
         int ncidIN;
         errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
@@ -112,7 +112,7 @@ int main( int argc, char* argv[])
         std::string inputIN(length, 'x');
         errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json(inputIN, jsIN, file::comments::are_forbidden);
+        dg::file::string2Json(inputIN, jsIN, dg::file::comments::are_forbidden);
         const eule::Parameters pIN(  jsIN);    
         std::cout << "[input.nc] file parameters" << std::endl;
         pIN.display( std::cout);   
@@ -154,7 +154,7 @@ int main( int argc, char* argv[])
     karniadakis.init( feltor, rolkar, 0., y0, p.dt);
     if(rank==0) std::cout << "Done!\n";
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     MPI_Info info = MPI_INFO_NULL;
     //  err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);//MPI OFF
@@ -162,7 +162,7 @@ int main( int argc, char* argv[])
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
     dg::Grid2d global_grid_out ( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);  
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
     err = nc_enddef( ncid);
     err = nc_redef(ncid);
 
@@ -177,7 +177,7 @@ int main( int argc, char* argv[])
 
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     err = nc_var_par_access( ncid, EtimevarID, NC_COLLECTIVE);
 
     int energyID, massID, energyIDs[3], dissID, dEdtID, accuracyID;
diff --git a/src/feltorShw/feltor.cu b/src/feltorShw/feltor.cu
index 61f94e94d..dbcbcbdb5 100644
--- a/src/feltorShw/feltor.cu
+++ b/src/feltorShw/feltor.cu
@@ -19,9 +19,9 @@ int main( int argc, char* argv[])
     ////////////////////////Parameter initialisation//////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_forbidden);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -31,7 +31,7 @@ int main( int argc, char* argv[])
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
     std::stringstream title;
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["cols"].asUInt()*js["width"].asUInt()*p.lx/p.ly, js["rows"].asUInt()*js["height"].asUInt(), "");
     draw::RenderHostData render(js["rows"].asUInt(), js["cols"].asUInt());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/feltorShw/feltor_hpc.cu b/src/feltorShw/feltor_hpc.cu
index b8989b28d..1a19388bb 100644
--- a/src/feltorShw/feltor_hpc.cu
+++ b/src/feltorShw/feltor_hpc.cu
@@ -21,7 +21,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     p.display( std::cout);
@@ -98,7 +98,7 @@ int main( int argc, char* argv[])
       }
     }
     if (argc==4) {
-        file::NC_Error_Handle errIN;
+        dg::file::NC_Error_Handle errIN;
         int ncidIN;
         errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
@@ -107,7 +107,7 @@ int main( int argc, char* argv[])
         std::string inputIN(length, 'x');
         errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json(inputIN, jsIN, file::comments::are_forbidden);
+        dg::file::string2Json(inputIN, jsIN, dg::file::comments::are_forbidden);
 
         const eule::Parameters pIN(  jsIN);    
         std::cout << "[input.nc] file parameters" << std::endl;
@@ -150,12 +150,12 @@ int main( int argc, char* argv[])
     karniadakis.init( feltor, rolkar, 0., y0, p.dt);
 //     feltor.energies( y0);//now energies and potential are at time 0
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_create( argv[2], NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids_field[3], tvarID_field;
-    err = file::define_dimensions(ncid, dim_ids_field, &tvarID_field, grid_out);
+    err = dg::file::define_dimensions(ncid, dim_ids_field, &tvarID_field, grid_out);
     err = nc_enddef(ncid);
     err = nc_redef(ncid);
 
@@ -167,7 +167,7 @@ int main( int argc, char* argv[])
         err = nc_def_var(ncid, varname_fields[i].data(), NC_DOUBLE, 3, dim_ids_field, &dataIDs[i]);
     //energy IDs, used for small time-step diagnostic
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, energyIDs[3], dissID, dEdtID, accuracyID, couplingID,radtransID; 
 
     err = nc_def_var(ncid, "energy", NC_DOUBLE, 1, &EtimeID, &energyID);
diff --git a/src/feltorShw/feltor_mpi.cu b/src/feltorShw/feltor_mpi.cu
index 6d0badd89..744327fc6 100644
--- a/src/feltorShw/feltor_mpi.cu
+++ b/src/feltorShw/feltor_mpi.cu
@@ -61,7 +61,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); 
     const eule::Parameters p( js);
     if(rank==0) p.display( std::cout);
@@ -151,7 +151,7 @@ int main( int argc, char* argv[])
       }
     }
     if (argc==4) {
-        file::NC_Error_Handle errIN;
+        dg::file::NC_Error_Handle errIN;
         int ncidIN;
         errIN = nc_open( argv[3], NC_NOWRITE, &ncidIN);
         ///////////////////read in and show inputfile und geomfile//////////////////
@@ -160,7 +160,7 @@ int main( int argc, char* argv[])
         std::string inputIN(length, 'x');
         errIN = nc_get_att_text( ncidIN, NC_GLOBAL, "inputfile", &inputIN[0]);
         Json::Value jsIN;
-        file::string2Json(inputIN, jsIN, file::comments::are_forbidden);
+        dg::file::string2Json(inputIN, jsIN, dg::file::comments::are_forbidden);
         const eule::Parameters pIN(  jsIN);    
         if(rank==0) std::cout << "[input.nc] file parameters" << std::endl;
         if(rank==0) pIN.display( std::cout);   
@@ -203,7 +203,7 @@ int main( int argc, char* argv[])
     karniadakis.init( feltor, rolkar, 0., y0, p.dt);
     if(rank==0) std::cout << "Done!\n";
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     MPI_Info info = MPI_INFO_NULL;
 //         err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);//MPI OFF
@@ -212,7 +212,7 @@ int main( int argc, char* argv[])
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
     dg::Grid2d global_grid_out ( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);  
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
     err = nc_enddef( ncid);
     err = nc_redef(ncid);
 
@@ -227,7 +227,7 @@ int main( int argc, char* argv[])
 
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     err = nc_var_par_access( ncid, EtimevarID, NC_COLLECTIVE);
 
     int energyID, massID, energyIDs[3], dissID, dEdtID, accuracyID;
diff --git a/src/hasegawa/hw.cu b/src/hasegawa/hw.cu
index 1a0a11cf8..b7ab354b1 100644
--- a/src/hasegawa/hw.cu
+++ b/src/hasegawa/hw.cu
@@ -17,9 +17,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_discarded);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_discarded);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -28,7 +28,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/hasegawa/mima.cu b/src/hasegawa/mima.cu
index a29cdc594..691eafb3c 100644
--- a/src/hasegawa/mima.cu
+++ b/src/hasegawa/mima.cu
@@ -16,9 +16,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_discarded);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_discarded);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -27,7 +27,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/heat/heat.cu b/src/heat/heat.cu
index eef8bbf20..4dbddae75 100644
--- a/src/heat/heat.cu
+++ b/src/heat/heat.cu
@@ -22,13 +22,13 @@ int main( int argc, char* argv[])
     Json::Value js, gs;
     if( argc == 1)
     {
-        file::file2Json("input/default.json", js, file::comments::are_discarded);
-        file::file2Json("geometry/geometry_params.json", gs, file::comments::are_discarded);
+        dg::file::file2Json("input/default.json", js, dg::file::comments::are_discarded);
+        dg::file::file2Json("geometry/geometry_params.json", gs, dg::file::comments::are_discarded);
     }
     else if( argc == 3)
     {
-        file::file2Json(argv[1], js, file::comments::are_forbidden);
-        file::file2Json(argv[2], gs, file::comments::are_forbidden);
+        dg::file::file2Json(argv[1], js, dg::file::comments::are_forbidden);
+        dg::file::file2Json(argv[2], gs, dg::file::comments::are_forbidden);
     }
     else
     {
@@ -53,7 +53,7 @@ int main( int argc, char* argv[])
     std::cout << "Initialize implicit" << std::endl;
     heat::Implicit<dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec > im( grid, p, mag);
     /////////glfw initialisation ////////////////////////////////////////
-    file::file2Json("window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json("window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     //////////////////////////////////////////////////////////////////////
diff --git a/src/heat/heat_hpc.cu b/src/heat/heat_hpc.cu
index 14bff4b44..55d1ba268 100644
--- a/src/heat/heat_hpc.cu
+++ b/src/heat/heat_hpc.cu
@@ -32,8 +32,8 @@ int main( int argc, char* argv[])
     }
     else
     {
-        file::file2Json(argv[1], js, file::comments::are_forbidden);
-        file::file2Json(argv[2], gs, file::comments::are_forbidden);
+        dg::file::file2Json(argv[1], js, dg::file::comments::are_forbidden);
+        dg::file::file2Json(argv[2], gs, dg::file::comments::are_forbidden);
     }
     const heat::Parameters p( js); p.display( std::cout);
     const dg::geo::solovev::Parameters gp(gs); gp.display( std::cout);
@@ -58,7 +58,7 @@ int main( int argc, char* argv[])
     //////////////////////////////open nc file//////////////////////////////////
     if (argc == 5)
     {
-        file::NC_Error_Handle errin;
+        dg::file::NC_Error_Handle errin;
         int ncidin;
         errin = nc_open( argv[4], NC_NOWRITE, &ncidin);
         //////////////read in and show inputfile und geomfile////////////
@@ -70,8 +70,8 @@ int main( int argc, char* argv[])
         std::string geomin(length, 'x');
         errin = nc_get_att_text( ncidin, NC_GLOBAL, "geomfile", &geomin[0]);
         Json::Value js,gs;
-        file::string2Json(inputin, js, file::comments::are_forbidden);
-        file::string2Json(geomin, gs, file::comments::are_forbidden);
+        dg::file::string2Json(inputin, js, dg::file::comments::are_forbidden);
+        dg::file::string2Json(geomin, gs, dg::file::comments::are_forbidden);
         std::cout << "input in"<<inputin<<std::endl;
         std::cout << "geome in"<<geomin <<std::endl;
         const heat::Parameters pin(js);
@@ -139,7 +139,7 @@ int main( int argc, char* argv[])
     };
     //////////////////set up netcdf for output/////////////////////////////////////
 
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_create( argv[3],NC_NETCDF4|NC_CLOBBER, &ncid);
     std::string input = js.toStyledString();
@@ -147,11 +147,11 @@ int main( int argc, char* argv[])
     std::string geom = gs.toStyledString();
     err = nc_put_att_text( ncid, NC_GLOBAL, "geomfile", geom.size(), geom.data());
     int dim_ids[4], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
 
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     std::map<std::string, int> id0d;
     for( auto name_value : v0d)
         err = nc_def_var( ncid, name_value.first.data(), NC_DOUBLE, 1, &EtimeID, &id0d[name_value.first]);
diff --git a/src/impurities/toeflI.cu b/src/impurities/toeflI.cu
index 3deddbf12..9bc6bab57 100644
--- a/src/impurities/toeflI.cu
+++ b/src/impurities/toeflI.cu
@@ -17,9 +17,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_discarded);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_discarded);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -28,7 +28,7 @@ int main( int argc, char* argv[])
     const imp::Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/impurities/toefl_hpc.cu b/src/impurities/toefl_hpc.cu
index 72ccf6ee1..1e1d2fc7a 100644
--- a/src/impurities/toefl_hpc.cu
+++ b/src/impurities/toefl_hpc.cu
@@ -20,7 +20,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     std::cout << js<<std::endl;
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const imp::Parameters p( js);
@@ -97,7 +97,7 @@ int main( int argc, char* argv[])
     dg::Karniadakis< std::vector<dg::DVec> > karniadakis( y0, y0[0].size(), p.eps_time);
     karniadakis.init( toeflI, diffusion, time, y0, p.dt);
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
@@ -106,7 +106,7 @@ int main( int argc, char* argv[])
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version",    NC_INT, 1, &version[1]);
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     //field IDs
     std::string names[5] = {"electrons", "ions", "impurities", "potential", "vorticity"};
     int dataIDs[5];
@@ -114,7 +114,7 @@ int main( int argc, char* argv[])
         err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, dissID, dEdtID, accuracyID;
     err = nc_def_var( ncid, "energy",      NC_DOUBLE, 1, &EtimeID, &energyID);
     err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
diff --git a/src/impurities/toefl_mpi.cu b/src/impurities/toefl_mpi.cu
index 32ef33c1a..0d6cffdb7 100644
--- a/src/impurities/toefl_mpi.cu
+++ b/src/impurities/toefl_mpi.cu
@@ -48,7 +48,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else
-        file::file2Json(argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json(argv[1], js, dg::file::comments::are_forbidden);
     std::string input = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
     const imp::Parameters p( js);
     if(rank==0)p.display( std::cout);
@@ -124,7 +124,7 @@ int main( int argc, char* argv[])
     dg::Karniadakis< std::vector<dg::MDVec> > karniadakis( y0, y0[0].size(), p.eps_time);
     karniadakis.init( toeflI, diffusion, time, y0, p.dt);
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     MPI_Info info = MPI_INFO_NULL;
     err = nc_create_par( argv[2], NC_NETCDF4|NC_MPIIO|NC_CLOBBER, comm, info, &ncid); //MPI ON
@@ -134,7 +134,7 @@ int main( int argc, char* argv[])
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_minor_version",    NC_INT, 1, &version[1]);
     //err = nc_put_att_int( ncid, NC_GLOBAL, "feltor_subminor_version", NC_INT, 1, &version[2]);
     int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, grid_out.global());
     //field IDs
     std::string names[5] = {"electrons", "ions", "impurities", "potential", "vorticity"};
     int dataIDs[5];
@@ -142,7 +142,7 @@ int main( int argc, char* argv[])
         err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, dissID, dEdtID, accuracyID;
     err = nc_def_var( ncid, "energy",      NC_DOUBLE, 1, &EtimeID, &energyID);
     err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
diff --git a/src/polar/polar.cu b/src/polar/polar.cu
index fbc088fd1..4fbe5c7a9 100644
--- a/src/polar/polar.cu
+++ b/src/polar/polar.cu
@@ -98,9 +98,9 @@ int main(int argc, char* argv[])
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_discarded);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_discarded);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
diff --git a/src/polar/polar_mpi.cu b/src/polar/polar_mpi.cu
index a556eb90d..b1fa7c16f 100644
--- a/src/polar/polar_mpi.cu
+++ b/src/polar/polar_mpi.cu
@@ -38,9 +38,9 @@ int main(int argc, char* argv[])
     ////Parameter initialisation ////////////////////////////////////////////
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_discarded);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_discarded);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
diff --git a/src/reco2D/reconnection.cu b/src/reco2D/reconnection.cu
index 6a897ca62..6857d2371 100644
--- a/src/reco2D/reconnection.cu
+++ b/src/reco2D/reconnection.cu
@@ -19,9 +19,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_discarded);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_discarded);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_discarded);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_discarded);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -30,7 +30,7 @@ int main( int argc, char* argv[])
     const asela::Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     //////////////////////////////////////////////////////////////////////////
diff --git a/src/reco2D/reconnection_hpc.cu b/src/reco2D/reconnection_hpc.cu
index 98189f386..e7e7306a4 100644
--- a/src/reco2D/reconnection_hpc.cu
+++ b/src/reco2D/reconnection_hpc.cu
@@ -19,7 +19,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     const asela::Parameters p( js);
     p.display( std::cout);
     std::string input = js.toStyledString();
@@ -87,12 +87,12 @@ int main( int argc, char* argv[])
     dg::Karniadakis< std::vector<dg::DVec> > karniadakis( y0, y0[0].size(), p.eps_time);
     karniadakis.init( asela, rolkar, 0., y0, p.dt);
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     
     //field IDs
     std::string names[8] = {"electrons", "ions", "Ue", "Ui", "potential","Aparallel","Vor","Jparallel"}; 
@@ -101,7 +101,7 @@ int main( int argc, char* argv[])
         err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);}
     //energy IDs
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, energyIDs[6], dissID, alignedID, dEdtID, accuracyID;
     err = nc_def_var( ncid, "energy",   NC_DOUBLE, 1, &EtimeID, &energyID);
     err = nc_def_var( ncid, "mass",   NC_DOUBLE, 1, &EtimeID, &massID);
diff --git a/src/reco2D/reconnection_mpi.cu b/src/reco2D/reconnection_mpi.cu
index aee79ec94..b057ad9e1 100644
--- a/src/reco2D/reconnection_mpi.cu
+++ b/src/reco2D/reconnection_mpi.cu
@@ -52,7 +52,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else 
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     const asela::Parameters p( js);
     if(rank==0)p.display( std::cout);
     std::string input = js.toStyledString();
@@ -111,7 +111,7 @@ int main( int argc, char* argv[])
     dg::Karniadakis< std::vector<dg::MDVec> > karniadakis( y0, y0[0].size(), p.eps_time);
     karniadakis.init( asela, rolkar,0., y0, p.dt);
     /////////////////////////////set up netcdf/////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     MPI_Info info = MPI_INFO_NULL;
     err = nc_create_par( argv[2], NC_NETCDF4|NC_MPIIO|NC_CLOBBER, comm, info, &ncid); //MPI ON
@@ -120,7 +120,7 @@ int main( int argc, char* argv[])
     err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
     dg::Grid2d global_grid_out (  -p.lxhalf, p.lxhalf, -p.lyhalf, p.lyhalf , p.n, p.Nx, p.Ny, dg::DIR, dg::PER);  
-    err = file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
+    err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, global_grid_out);
     
     
     //field IDs 
@@ -130,7 +130,7 @@ int main( int argc, char* argv[])
         err = nc_def_var( ncid, names[i].data(), NC_DOUBLE, 3, dim_ids, &dataIDs[i]);
     //energy IDs 
     int EtimeID, EtimevarID;
-    err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, energyIDs[8], dissID, alignedID, dEdtID, accuracyID;
     err = nc_def_var( ncid, "energy",   NC_DOUBLE, 1, &EtimeID, &energyID);
     err = nc_def_var( ncid, "mass",   NC_DOUBLE, 1, &EtimeID, &massID);
diff --git a/src/toefl/toeflR.cu b/src/toefl/toeflR.cu
index 537f17a06..10714cf8e 100644
--- a/src/toefl/toeflR.cu
+++ b/src/toefl/toeflR.cu
@@ -24,9 +24,9 @@ int main( int argc, char* argv[])
     std::stringstream title;
     Json::Value js;
     if( argc == 1)
-        file::file2Json( "input.json", js, file::comments::are_forbidden);
+        dg::file::file2Json( "input.json", js, dg::file::comments::are_forbidden);
     else if( argc == 2)
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     else
     {
         std::cerr << "ERROR: Too many arguments!\nUsage: "<< argv[0]<<" [filename]\n";
@@ -35,7 +35,7 @@ int main( int argc, char* argv[])
     const Parameters p( js);
     p.display( std::cout);
     /////////glfw initialisation ////////////////////////////////////////////
-    file::file2Json( "window_params.json", js, file::comments::are_discarded);
+    dg::file::file2Json( "window_params.json", js, dg::file::comments::are_discarded);
     GLFWwindow* w = draw::glfwInitAndCreateWindow( js["width"].asDouble(), js["height"].asDouble(), "");
     draw::RenderHostData render(js["rows"].asDouble(), js["cols"].asDouble());
     /////////////////////////////////////////////////////////////////////////
diff --git a/src/toefl/toefl_hpc.cu b/src/toefl/toefl_hpc.cu
index 88ec3ddf5..e2b963360 100644
--- a/src/toefl/toefl_hpc.cu
+++ b/src/toefl/toefl_hpc.cu
@@ -73,7 +73,7 @@ int main( int argc, char* argv[])
         return -1;
     }
     else
-        file::file2Json( argv[1], js, file::comments::are_forbidden);
+        dg::file::file2Json( argv[1], js, dg::file::comments::are_forbidden);
     MPI_OUT std::cout << js<<std::endl;
     const Parameters p( js);
     MPI_OUT p.display( std::cout);
@@ -109,13 +109,13 @@ int main( int argc, char* argv[])
     karniadakis.init( exp, imp, time, y0, p.dt);
     y1 = y0;
     /////////////////////////////set up netcdf/////////////////////////////////////
-    file::NC_Error_Handle err;
+    dg::file::NC_Error_Handle err;
     int ncid;
     MPI_OUT err = nc_create( argv[2],NC_NETCDF4|NC_CLOBBER, &ncid);
     std::string input = js.toStyledString();
     MPI_OUT err = nc_put_att_text( ncid, NC_GLOBAL, "inputfile", input.size(), input.data());
     int dim_ids[3], tvarID;
-    MPI_OUT err = file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
+    MPI_OUT err = dg::file::define_dimensions( ncid, dim_ids, &tvarID, grid_out);
     //field IDs
     std::string names[4] = {"electrons", "ions", "potential", "vorticity"};
     int dataIDs[4];
@@ -124,7 +124,7 @@ int main( int argc, char* argv[])
 
     //energy IDs
     int EtimeID, EtimevarID;
-    MPI_OUT err = file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
+    MPI_OUT err = dg::file::define_time( ncid, "energy_time", &EtimeID, &EtimevarID);
     int energyID, massID, dissID, dEdtID;
     MPI_OUT err = nc_def_var( ncid, "energy",      NC_DOUBLE, 1, &EtimeID, &energyID);
     MPI_OUT err = nc_def_var( ncid, "mass",        NC_DOUBLE, 1, &EtimeID, &massID);
@@ -147,7 +147,7 @@ int main( int argc, char* argv[])
     for( int k=0;k<4; k++)
     {
         dg::blas1::transfer( transferD[k], transferH);
-        file::put_vara_double( ncid, dataIDs[k], start, grid_out, transferH);
+        dg::file::put_vara_double( ncid, dataIDs[k], start, grid_out, transferH);
     }
     MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
     MPI_OUT err = nc_close(ncid);
@@ -204,7 +204,7 @@ int main( int argc, char* argv[])
         for( int k=0;k<4; k++)
         {
             dg::blas1::transfer( transferD[k], transferH);
-            file::put_vara_double( ncid, dataIDs[k], start, grid_out, transferH);
+            dg::file::put_vara_double( ncid, dataIDs[k], start, grid_out, transferH);
         }
         MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
         MPI_OUT err = nc_close(ncid);
-- 
GitLab


From 361bb8c64bb466eeb4c8c000222a7bfee5a8fae5 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 5 Feb 2021 11:45:15 +0100
Subject: [PATCH 459/540] Fix warning about unused nowhere function

---
 inc/geometries/modified.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inc/geometries/modified.h b/inc/geometries/modified.h
index 5fe811069..55513d5ee 100644
--- a/inc/geometries/modified.h
+++ b/inc/geometries/modified.h
@@ -204,7 +204,7 @@ struct MagneticTransition : public aCylindricalFunctor<MagneticTransition>
 };
 //some possible predicates
 static bool nowhere( double R, double Z){return false;}
-static bool everywhere( double R, double Z){return true;}
+static bool everywhere( double R, double Z){return !nowhere(R,Z);}
 struct HeavisideZ{
     HeavisideZ( double Z_X, int side): m_ZX( Z_X), m_side(side) {}
     bool operator()(double R, double Z){
-- 
GitLab


From cc18fd74b2373cf7c0f60923aee95df34ce6f507 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 5 Feb 2021 12:35:01 +0100
Subject: [PATCH 460/540] Reorganize documentation of functors

---
 inc/dg/blas1.h               |   22 +-
 inc/dg/dg_doc.h              |   22 +-
 inc/dg/functors.h            | 2210 +++++++++++++++++-----------------
 inc/dg/subroutines.h         |  144 ++-
 inc/dg/topology/evaluation.h |    6 +-
 inc/dg/topology/functions.h  |   44 +-
 6 files changed, 1246 insertions(+), 1202 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 4f78c5376..2aabff4e0 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -27,7 +27,7 @@ namespace dg{
 /*! @brief BLAS Level 1 routines
  *
  * @ingroup blas1
- * Only those routines that are actually called need to be implemented.
+ *
  * @note successive calls to blas routines are executed sequentially
  * @note A manual synchronization of threads or devices is never needed in an application
  * using these functions. All functions returning a value block until the value is ready.
@@ -40,7 +40,7 @@ namespace blas1
 
 /**
  * @class hide_iterations
- * where \c i iterates over @b all elements inside the given vectors. The order of iterations is indetermined. Scalar arguments to container types are interpreted as vectors with all elements constant. If \c ContainerType has the \c RecursiveVectorTag, \c i recursively loops over all entries.
+ * where \c i iterates over @b all elements inside the given vectors. The order of iterations is undefined. Scalar arguments to container types are interpreted as vectors with all elements constant. If \c ContainerType has the \c RecursiveVectorTag, \c i recursively loops over all entries.
  * If the vector sizes do not match, the result is undefined.
  * The compiler chooses the implementation and parallelization of this function based on given template parameters. For a full set of rules please refer to \ref dispatch.
  */
@@ -489,15 +489,15 @@ dg::HVec pi2(20, M_PI/2.), pi3( 20, 3*M_PI/2.), result(20, 0);
 dg::blas1::evaluate( result, dg::equals(), function, pi2, pi3);
 // result[i] = sin(M_PI/2.)*sin(3*M_PI/2.) = -1
 @endcode
+ * @tparam BinarySubroutine Functor with signature: <tt> void operator()( value_type_y&, value_type_g) </tt> i.e. it writes into the first and reads from its (first and) second argument
+ * @tparam Functor signature: <tt> value_type_g operator()( value_type_x0, value_type_x1, ...) </tt>
+ * @attention Both \c BinarySubroutine and \c Functor must be callable on the device in use. In particular, with CUDA they must be functor tpyes (@b not functions) and their signatures must contain the \__device__ specifier. (s.a. \ref DG_DEVICE)
  * @param y contains result
- * @param f The subroutine, for example \c dg::equals or \c dg::plus_equals
- * @param g The functor to evaluate
+ * @param f The subroutine, for example \c dg::equals or \c dg::plus_equals, see @ref binary_operators for a collection of predefined functors to use here
+ * @param g The functor to evaluate, see @ref functions and @ref variadic_evaluates for a collection of predefined functors to use here
  * @param x0 first input
  * @param xs more input
  * @note all aliases allowed
- * @tparam BinarySubroutine Functor with signature: <tt> void operator()( value_type_y&, value_type_g) </tt> i.e. it writes into the first and reads from its (first and) second argument
- * @tparam Functor signature: <tt> value_type_g operator()( value_type_x0, value_type_x1, ...) </tt>
- * @note Both \c BinarySubroutine and \c Functor must be callable on the device in use. In particular, with CUDA they must be functor tpyes (@b not functions) and their signatures must contain the \__device__ specifier. (s.a. \ref DG_DEVICE)
  * @copydoc hide_ContainerType
  *
  */
@@ -550,14 +550,14 @@ dg::blas1::subroutine( Routine(), two, 3., four);
 // four[i] now has the value 21 (7*2+3+4)
 @endcode
 
- * @param f the subroutine
+ * @param f the subroutine, see @ref variadic_subroutines for a collection of predefind subroutines to use here
  * @param x the first argument
  * @param xs other arguments
 @note This function can compute @b any trivial parallel expression for @b any
-number of input and output arguments. In this sense it replaces all other \c blas1 functions
-except the scalar product, which is not trivial parallel.
+number of input and output arguments, which is quite remarkable really. In this sense it replaces all other \c blas1 functions
+except the scalar product, which is not trivially parallel.
 @attention The user has to decide whether or not it is safe to alias input or output vectors. If in doubt, do not alias output vectors.
- * @tparam Subroutine a function or functor taking a \c value_type argument for each input argument in the call
+ * @tparam Subroutine a function or functor with an arbitrary number of arguments and no return type; taking a \c value_type argument for each input argument in the call
  * and a <tt> value_type&  </tt> argument for each output argument.
  * \c Subroutine must be callable on the device in use. In particular, with CUDA it must be a functor (@b not a function) and its signature must contain the \__device__ specifier. (s.a. \ref DG_DEVICE)
  * @copydoc hide_ContainerType
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index bdf26876a..58f3c4191 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -110,11 +110,25 @@
  * @defgroup misc Level 0: Miscellaneous additions
  * @{
  *     @defgroup timer Timer class
- *     @defgroup functions Functions and Functors
+ *     @defgroup blas1_helpers Functions and functors for subroutine and evaluate
+ *     @{
+ *          @defgroup basics Simple
+ *              For the dg::evaluate and dg::blas1::evaluate functions
+ *
+ *          @defgroup functions A large collection
+ *              For the dg::evaluate and dg::blas1::evaluate functions
+ *
+ *          @defgroup composition Composition of two or more functors
  *
- *         The functions are useful mainly in the constructor of Operator objects.
- *         The functors are useful for either vector transformations or
- *         as init functions in the evaluate routines.
+ *          @defgroup binary_operators blas1::evaluate binary operators
+ *              Binary subroutines for the dg::blas1::evaluate function
+ *
+ *          @defgroup variadic_evaluates blas1::evaluate variadic subroutines
+ *              Functors with an arbitrary number of arguments to use in the dg::blas1::evaluate function
+ *
+ *          @defgroup variadic_subroutines blas1::subroutine subroutines
+ *              Functors with a variable number of arguments for use in the dg::blas1::subroutine functions
+ *     @}
  *     @defgroup lowlevel Lowlevel helper functions and classes
  *
  *         Low level helper routines.
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index ddb8c43ff..e9d8a2cfe 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -18,13 +18,166 @@
 namespace dg
 {
 
-///@addtogroup functions
+///@addtogroup basics
 ///@{
+//Everything that is quite basic and simple
+/**
+ * @brief \f$ f(x) = f(x,y) = f(x,y,z) = 0\f$
+ */
+struct ZERO
+{
+    DG_DEVICE
+    double operator()(double x)const{return 0.;}
+    DG_DEVICE
+    double operator()(double x, double y)const{return 0.;}
+    DG_DEVICE
+    double operator()(double x, double y, double z)const{return 0.;}
+};
+
+/**
+ * @brief \f$ f(x) = f(x,y) = f(x,y,z) = 1\f$
+ */
+struct ONE
+{
+    DG_DEVICE
+    double operator()(double x)const{return 1.;}
+    DG_DEVICE
+    double operator()(double x, double y)const{return 1.;}
+    DG_DEVICE
+    double operator()(double x, double y, double z)const{return 1.;}
+};
+/**
+ * @brief \f$ f(x) = f(x,y) = f(x,y,z) = c\f$
+ */
+struct CONSTANT
+{
+    /**
+     * @brief Construct with a value
+     *
+     * @param cte the constant value
+     */
+    CONSTANT( double cte): m_value(cte){}
+
+    DG_DEVICE
+    double operator()(double x)const{return m_value;}
+    DG_DEVICE
+    double operator()(double x, double y)const{return m_value;}
+    DG_DEVICE
+    double operator()(double x, double y, double z)const{return m_value;}
+    private:
+    double m_value;
+};
+/**
+ * @brief \f$ f(x) = x + c\f$
+ *
+ * Add a constant value
+ * @tparam T value type
+ */
+template <class T = double>
+struct PLUS
+{
+    /**
+     * @brief Construct
+     *
+     * @param value to be added
+     */
+    PLUS( T value): x_(value){}
+    /**
+     * @brief Add a constant value
+     *
+     * @param x  the input
+     *
+     * @return  x + value
+     */
+    DG_DEVICE
+    T operator()( T x)const{ return x + x_;}
+    private:
+    T x_;
+};
+
+///@brief \f$ f(x) = \exp( x)\f$
+template< class T = double >
+struct EXP
+{
+    EXP( ) {}
+    /**
+     * @brief return exponential
+     *
+     * @param x x
+     *
+     * @return \f$ \exp( x)\f$
+     */
+    DG_DEVICE T operator() ( T x) const
+    {
+        return exp(x);
+    }
+};
+
+///@brief \f$ f(x) = \ln(x)\f$
+template < class T = double>
+struct LN
+{
+    /**
+     * @brief The natural logarithm
+     *
+     * @param x of x
+     * @return  \f$ \ln(x) \f$
+     */
+    DG_DEVICE T operator() (const T& x) const
+    {
+        return log(x);
+    }
 
+};
+
+///@brief \f$ f(x) = \sqrt{x}\f$
+template < class T = double>
+struct SQRT
+{
+    /**
+     * @brief Square root
+     *
+     * @param x of x
+     *
+     * @return sqrt(x)
+     */
+    DG_DEVICE T operator() (T x) const
+    {
+        return sqrt(x);
+    }
+};
+
+
+///@brief \f$ f(x) = 1/x \f$
+template <class T = double>
+struct INVERT
+{
+    /**
+     * @brief Invert the given value
+     *
+     * @param x  the input
+     * @return  1/x
+     */
+    DG_DEVICE T operator()( T x)const{ return 1./x;}
+};
+
+///@brief \f$ f(x) = |x|\f$
+template <class T = double>
+struct ABS
+{
+    /**
+     * @brief The absolute value
+     *
+     * @param x of x
+     *
+     * @return  abs(x)
+     */
+    DG_DEVICE T operator()(T x)const{ return fabs(x);}
+};
 /**
- * @brief Absolute maximum
- * \f$ f(x,y) = \max(|x|,|y|)\f$
+ * @brief \f$ f(x,y) = \max(|x|,|y|)\f$
  *
+ * Absolute maximum
  */
 template <class T = double>
 struct AbsMax
@@ -37,8 +190,7 @@ struct AbsMax
      *
      * @return absolute maximum
      */
-DG_DEVICE
-    T operator() ( T x, T y) const
+    DG_DEVICE T operator() ( T x, T y) const
     {
         T absx = x>0 ? x : -x;
         T absy = y>0 ? y : -y;
@@ -46,8 +198,9 @@ DG_DEVICE
     }
 };
 /**
- * @brief Absolute minimum
- * \f$ f(x,y) = \min(|x|,|y|)\f$
+ * @brief \f$ f(x,y) = \min(|x|,|y|)\f$
+ *
+ * Absolute minimum
  */
 template <class T = double>
 struct AbsMin
@@ -60,8 +213,7 @@ struct AbsMin
      *
      * @return absolute minimum
      */
-DG_DEVICE
-    T operator() (T x, T y) const
+    DG_DEVICE T operator() (T x, T y) const
     {
         T absx = x<0 ? -x : x;
         T absy = y<0 ? -y : y;
@@ -69,591 +221,282 @@ DG_DEVICE
     }
 };
 
+
+
 /**
- * @brief Functor returning a 2d %Gaussian
- * \f$
-   f(x,y) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}\right)}
-   \f$
+ * @brief
+ \f$ f(x) = \begin{cases}
+         x \text{ for } x>0 \\
+         0 \text{ else}
+ \end{cases}
+ \f$
+ *
  */
-struct Gaussian
+template <class T = double>
+struct POSVALUE
 {
     /**
-     * @brief Functor returning a %Gaussian
-     *
-     * @param x0 x-center-coordinate
-     * @param y0 y-center-coordinate
-     * @param sigma_x x - variance (must be !=0)
-     * @param sigma_y y - variance (must be !=0)
-     * @param amp Amplitude
-     */
-    Gaussian( double x0, double y0, double sigma_x, double sigma_y, double amp)
-        : m_x0(x0), m_y0(y0), m_sigma_x(sigma_x), m_sigma_y(sigma_y), m_amp(amp){
-            assert( m_sigma_x != 0  &&  "sigma_x must not be 0 in Gaussian");
-            assert( m_sigma_y != 0  &&  "sigma_y must not be 0 in Gaussian");
-    }
-    /**
-     * @brief Return the value of the %Gaussian
+     * @brief Returns positive values of x
      *
-     * \f[
-       f(x,y) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}\right)}
-       \f]
-     * @param x x - coordinate
-     * @param y y - coordinate
+     * @param x of x
      *
-     * @return gaussian
+     * @return  x*0.5*(1+sign(x))
      */
-    DG_DEVICE
-    double operator()(double x, double y) const
-    {
-        return  m_amp*
-                   exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x +
-                          (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
+    DG_DEVICE T operator()( T x)const{
+        if (x >= 0.0) return x;
+        return 0.0;
     }
+};
+/**
+ * @brief \f$ f(x) = \f$ \c x mod m > 0 ? x mod m : x mod m + m
+ *
+ * returns (positive) modulo
+ * @tparam T value type
+ */
+template <class T= double>
+struct MOD
+{
     /**
-     * @brief Return the value of the %Gaussian
-     * \f[
-       f(x,y,z) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
-       \f]
-     * @param x x - coordinate
-     * @param y y - coordinate
-     * @param z z - coordinate
+     * @brief Construct from modulo
      *
-     * @return gaussian
+     * @param m modulo basis
      */
+    MOD( T m): m_m(m){}
+
     DG_DEVICE
-    double operator()(double x, double y, double z) const
-    {
-        return  this->operator()(x,y);
+    T operator()( T x)const{
+        return (fmod(x,m_m) < 0 ) ? (m_m + fmod(x,m_m)) : fmod(x,m_m);
     }
-  private:
-    double  m_x0, m_y0, m_sigma_x, m_sigma_y, m_amp;
+    private:
+    T m_m;
 
 };
+/**
+ * @brief \f$ f(x) =\f$ \c std::isnan(x)
+ *
+ * Check for NaN
+ */
+template <class T>
+struct ISNAN
+{
+#ifdef __CUDACC__
+    DG_DEVICE bool operator()(T x){return isnan(x);}
+#else
+    bool operator()( T x){ return std::isnan(x);}
+#endif
+};
+
 
 /**
- * @brief A bump that drops to zero and is infinitely continuously differentiable
- * \f$
-   f(x,y) = \begin{cases}
-   Ae^{1 + \left(\frac{(x-x_0)^2}{\sigma_x^2} + \frac{(y-y_0)^2}{\sigma_y^2} - 1\right)^{-1}} \text{ if } \frac{(x-x_0)^2}{\sigma_x^2} + \frac{(y-y_0)^2}{\sigma_y^2} < 1\\
-   0 \text{ else}
-   \end{cases}
-   \f$
+ * @brief
+ \f$ f(x_1, x_2, x_3) = \begin{cases}
+         \min(x_1, x_2, x_3) \text{ for } x_1, x_2, x_3 >0 \\
+         \max(x_1, x_2, x_3) \text{ for } x_1, x_2, x_3 <0 \\
+         0 \text{ else}
+ \end{cases}
+ \f$
+ *
+ * might be useful for flux limiter schemes
  */
-struct Cauchy
+template < class T>
+struct MinMod
 {
     /**
-     * @brief A blob that drops to zero
+     * @brief Minmod of three numbers
      *
-     * @param x0 x-center-coordinate
-     * @param y0 y-center-coordinate
-     * @param sigma_x radius in x (must be !=0)
-     * @param sigma_y radius in y (must be !=0)
-     * @param amp Amplitude
+     * @param a1 a1
+     * @param a2 a2
+     * @param a3 a3
+     *
+     * @return minmod(a1, a2, a3)
      */
-    Cauchy( double x0, double y0, double sigma_x, double sigma_y, double amp): x0_(x0), y0_(y0), sigmaX_(sigma_x), sigmaY_(sigma_y), amp_(amp){
-        assert( sigma_x != 0  &&  "sigma_x must be !=0 in Cauchy");
-        assert( sigma_y != 0  &&  "sigma_y must be !=0 in Cauchy");
-    }
-    DG_DEVICE
-    double operator()(double x, double y )const{
-        double xbar = (x-x0_)/sigmaX_;
-        double ybar = (y-y0_)/sigmaY_;
-        if( xbar*xbar + ybar*ybar < 1.)
-            return amp_*exp( 1. +  1./( xbar*xbar + ybar*ybar -1.) );
-        return 0.;
-    }
-    bool inside( double x, double y)const
+    DG_DEVICE T operator() ( T a1, T a2, T a3)const
     {
-        double xbar = (x-x0_)/sigmaX_;
-        double ybar = (y-y0_)/sigmaY_;
-        if( xbar*xbar + ybar*ybar < 1.)
-            return true;
-        return false;
-    }
+        if( a1*a2 > 0)
+            if( a1*a3 > 0)
+            {
+                if( a1 > 0)
+                    return min( a1, a2, a3, +1.);
+                else
+                    return min( a1, a2, a3, -1.);
+            }
+        return 0.;
 
-    double dx( double x, double y )const{
-        double xbar = (x-x0_)/sigmaX_;
-        double ybar = (y-y0_)/sigmaY_;
-        double temp = sigmaX_*(xbar*xbar + ybar*ybar  - 1.);
-        return -2.*(x-x0_)*this->operator()(x,y)/temp/temp;
-    }
-    double dxx( double x, double y)const{
-        double temp = sigmaY_*sigmaY_*(x-x0_)*(x-x0_) + sigmaX_*sigmaX_*((y-y0_)*(y-y0_) - sigmaY_*sigmaY_);
-        double bracket = sigmaX_*sigmaX_*((y-y0_)*(y-y0_)-sigmaY_*sigmaY_)*sigmaX_*sigmaX_*((y-y0_)*(y-y0_)-sigmaY_*sigmaY_)
-            -3.*sigmaY_*sigmaY_*sigmaY_*sigmaY_*(x-x0_)*(x-x0_)*(x-x0_)*(x-x0_)
-            -2.*sigmaY_*sigmaY_*sigmaX_*sigmaX_*(x-x0_)*(x-x0_)*(y-y0_)*(y-y0_);
-        return -2.*sigmaX_*sigmaX_*sigmaY_*sigmaY_*sigmaY_*sigmaY_*this->operator()(x,y)*bracket/temp/temp/temp/temp;
-    }
-    double dy( double x, double y)const{
-        double xbar = (x-x0_)/sigmaX_;
-        double ybar = (y-y0_)/sigmaY_;
-        double temp = sigmaY_*(xbar*xbar + ybar*ybar  - 1.);
-        return -2.*(y-y0_)*this->operator()(x,y)/temp/temp;
-    }
-    double dyy( double x, double y)const{
-        double temp = sigmaX_*sigmaX_*(y-y0_)*(y-y0_) + sigmaY_*sigmaY_*((x-x0_)*(x-x0_) - sigmaX_*sigmaX_);
-        double bracket = sigmaY_*sigmaY_*((x-x0_)*(x-x0_)-sigmaX_*sigmaX_)*sigmaY_*sigmaY_*((x-x0_)*(x-x0_)-sigmaX_*sigmaX_)
-            -3.*sigmaX_*sigmaX_*sigmaX_*sigmaX_*(y-y0_)*(y-y0_)*(y-y0_)*(y-y0_)
-            -2.*sigmaX_*sigmaX_*sigmaY_*sigmaY_*(y-y0_)*(y-y0_)*(x-x0_)*(x-x0_);
-        return -2.*sigmaY_*sigmaY_*sigmaX_*sigmaX_*sigmaX_*sigmaX_*this->operator()(x,y)*bracket/temp/temp/temp/temp;
-    }
-    double dxy( double x, double y )const{
-        double xbar = (x-x0_)/sigmaX_;
-        double ybar = (y-y0_)/sigmaY_;
-        double temp = (xbar*xbar + ybar*ybar  - 1.);
-        return 8.*xbar*ybar*this->operator()(x,y)/temp/temp/temp/sigmaX_/sigmaY_
-            + 4.*xbar*ybar*this->operator()(x,y)/temp/temp/temp/temp/sigmaX_/sigmaY_
-;
-    }
-    private:
-    double x0_, y0_, sigmaX_, sigmaY_, amp_;
-};
 
-/**
-* @brief The 3d %Gaussian
-* \f$
-f(x,y,z) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2} + \frac{(z-z_0)^2}{2\sigma_z^2}\right)}
-\f$
-*/
-struct Gaussian3d
-{
-    /**
-     * @brief Functor returning a %Gaussian
-     *
-     * @param x0 x-center-coordinate
-     * @param y0 y-center-coordinate
-     * @param z0 z-center-coordinate
-     * @param sigma_x x - variance (must be !=0)
-     * @param sigma_y y - variance (must be !=0)
-     * @param sigma_z z - variance (must be !=0)
-     * @param amp Amplitude
-     */
-    Gaussian3d( double x0, double y0, double z0, double sigma_x, double sigma_y, double sigma_z, double amp)
-        : m_x0(x0), m_y0(y0), m_z0(z0), m_sigma_x(sigma_x), m_sigma_y(sigma_y), m_sigma_z(sigma_z), m_amp(amp){
-            assert( m_sigma_x != 0  &&  "sigma_x must be !=0 in Gaussian3d");
-            assert( m_sigma_y != 0  &&  "sigma_y must be !=0 in Gaussian3d");
-            assert( m_sigma_z != 0  &&  "sigma_z must be !=0 in Gaussian3d");
     }
-    /**
-     * @brief Return a 2d %Gaussian
-     *
-     * \f[
-       f(x,y) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
-       \f]
-     * @param x x - coordinate
-     * @param y y - coordinate
-     *
-     * @return gaussian
-     */
-    DG_DEVICE
-    double operator()(double x, double y) const
-    {
-        return  m_amp*
-                   exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x +
-                          (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
-    }
-    /**
-     * @brief Return the value of the %Gaussian
-     *
-     * \f[
-       f(x,y) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}+\frac{(z-z_0)^2}{2\sigma_z^2})}
-       \f]
-     * @param x x - coordinate
-     * @param y y - coordinate
-     * @param z z - coordinate
-     *
-     * @return gaussian
-     */
-    DG_DEVICE
-    double operator()(double x, double y, double z) const
+    private:
+    T min( T a1, T a2, T a3, T sign)const
     {
-        return  m_amp*
-                exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x +
-                       (z-m_z0)*(z-m_z0)/2./m_sigma_z/m_sigma_z +
-                       (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
-    }
-  private:
-    double  m_x0, m_y0, m_z0, m_sigma_x, m_sigma_y, m_sigma_z, m_amp;
+        T temp = sign*a1;
+        if( sign*a2 < temp)
+            temp = sign*a2;
+        if( sign*a3 < temp)
+            temp = sign*a3;
+        return sign*temp;
 
-};
-/**
- * @brief A %Gaussian in x-direction
- * \f$
-   f(x,y) = Ae^{-\frac{(x-x_0)^2}{2\sigma_x^2} }
-   \f$
- */
-struct GaussianX
-{
-    /**
-     * @brief A %Gaussian in x
-     *
-     * @param x0 x-center-coordinate
-     * @param sigma_x x - variance (must be !=0)
-     * @param amp Amplitude
-     */
-    GaussianX( double x0, double sigma_x, double amp)
-        :m_x0(x0), m_sigma_x(sigma_x), m_amp(amp){
-            assert( m_sigma_x != 0  &&  "sigma_x must be !=0 in GaussianX");
-    }
-    DG_DEVICE
-    double operator()(double x) const
-    {
-        return  m_amp* exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x ));
     }
-    DG_DEVICE
-    double operator()(double x, double y) const
-    {
-        return this->operator()(x);
-    }
-    DG_DEVICE
-    double operator()(double x, double y, double z) const
-    {
-        return this->operator()(x);
-    }
-  private:
-    double  m_x0, m_sigma_x, m_amp;
-
 };
 /**
- * @brief A %Gaussian in y-direction
- * \f$
-   f(x,y) = Ae^{-\frac{(y-y_0)^2}{2\sigma_y^2}}
-   \f$
+ * @brief \f$ f(v, a, b ) = \begin{cases}  a \text{ if } v \geq 0 \\
+ *  b \text{ else}
+ *  \end{cases}
+ *  \f$
  */
-struct GaussianY
+struct Upwind
 {
-    /**
-     * @brief Functor returning a gaussian
-     *
-     * @param y0 y-center-coordinate
-     * @param sigma_y y - variance (must be !=0)
-     * @param amp Amplitude
-     */
-    GaussianY( double y0, double sigma_y, double amp)
-        : m_y0(y0), m_sigma_y(sigma_y), m_amp(amp){
-            assert( m_sigma_y != 0  &&  "sigma_x must be !=0 in GaussianY");
-    }
-    /**
-     * @brief Return the value of the gaussian
-     *
-     * \f[
-       f(x,y) = Ae^{-\frac{(y-y_0)^2}{2\sigma_y^2}}
-       \f]
-     * @param x x - coordinate
-     * @param y y - coordinate
-     *
-     * @return gaussian
-     */
-    DG_DEVICE
-    double operator()(double x, double y) const
-    {
-        return  m_amp*exp( -((y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
+    template<class T>
+    DG_DEVICE T operator()( T v, T a, T b){
+        if( v >= 0)
+            return a;
+        else
+            return b;
     }
-  private:
-    double  m_y0, m_sigma_y, m_amp;
-
 };
-/**
- * @brief A %Gaussian in z-direction
- * \f$
-   f(x,y,z) = Ae^{-\frac{(z-z_0)^2}{2\sigma_z^2}}
-   \f$
- */
-struct GaussianZ
-{
-    /**
-     * @brief Functor returning a gaussian
-     *
-     * @param z0 z-center-coordinate
-     * @param sigma_z z - variance (must be !=0)
-     * @param amp Amplitude
-     */
-    GaussianZ( double z0, double sigma_z, double amp)
-        : m_z0(z0), m_sigma_z(sigma_z), m_amp(amp){
-            assert( m_sigma_z != 0  &&  "sigma_z must be !=0 in GaussianZ");
-    }
-    /**
-     * @brief Return the value of the gaussian
-     *
-     * \f[
-       f(z) = Ae^{-\frac{(z-z_0)^2}{2\sigma_z^2}}
-       \f]
-     * @param z z - coordinate
-     *
-     * @return gaussian
-     */
-    DG_DEVICE
-    double operator()( double z) const
-    {
-        return  m_amp*exp( -((z-m_z0)*(z-m_z0)/2./m_sigma_z/m_sigma_z) );
-    }
-    /**
-     * @brief Return the value of the gaussian
-     *
-     * \f[
-       f(x,y,z) = Ae^{-\frac{(z-z_0)^2}{2\sigma_z^2}}
-       \f]
-     * @param x x - coordinate
-     * @param y y - coordinate
-     * @param z z - coordinate
-     *
-     * @return gaussian
-     */
-    DG_DEVICE
-    double operator()(double x, double y, double z) const
-    {
-        return  m_amp*exp( -((z-m_z0)*(z-m_z0)/2./m_sigma_z/m_sigma_z) );
-    }
-  private:
-    double  m_z0, m_sigma_z, m_amp;
 
-};
 /**
- * @brief Island function
- * \f$ f(x,y) = \lambda \ln{(\cosh{(x/\lambda) } +\epsilon \cos(y/\lambda)) } \f$
+ * @brief \f$ f(v, a, b ) = \begin{cases}  av \text{ if } v \geq 0 \\
+ *  bv \text{ else}
+ *  \end{cases}
+ *  \f$
+ *
+ * @tparam T a real value
  */
-struct IslandXY
+struct UpwindProduct
 {
-    /**
-     * @brief Construct Island
-     *
-     * @param lambda amplitude (must be != 0)
-     * @param eps y-amplitude
-     */
-     IslandXY( double lambda, double eps):lambda_(lambda), eps_(eps){
-         assert( lambda != 0 && "Lambda parameter in IslandXY must not be zero!");
-     }
-    /**
-     * @brief Return profile
-     *
-     * @param x x - coordinate
-     * @param y y - coordinate
-     * @return \f$ f(x,y)\f$
-     */
-    DG_DEVICE
-    double operator()( double x, double y)const{ return lambda_*log(cosh(x/lambda_)+eps_*cos(y/lambda_));}
-  private:
-    double lambda_,eps_;
+    template<class T>
+    DG_DEVICE T operator()( T v, T a, T b){
+        if( v >= 0)
+            return a*v;
+        else
+            return b*v;
+    }
 };
-/**
- * @brief A sin prof in x and y-direction
- * \f$ f(x,y) =B+ A \sin(k_x x) \sin(k_y y) \f$
- */
-struct SinXSinY
-{
-    /**
-     * @brief Construct with two coefficients
-     *
-     * @param amp amplitude A
-     * @param bamp backgroundamp B
-     * @param kx  kx
-     * @param ky  ky
-     */
-    SinXSinY( double amp, double bamp, double kx, double ky):amp_(amp), bamp_(bamp),kx_(kx),ky_(ky){}
-    /**
-     * @brief Return profile
-     *
-     * @param x x - coordinate
-     * @param y y - coordinate
+///@}
 
-     * @return \f$ f(x,y)\f$
-     */
-    DG_DEVICE
-    double operator()( double x, double y)const{ return bamp_+amp_*sin(x*kx_)*sin(y*ky_);}
-  private:
-    double amp_,bamp_,kx_,ky_;
-};
-/**
- * @brief A cos prof in x and y-direction
- * \f$ f(x,y) =B+ A \cos(k_x x) \cos(k_y y) \f$
- */
-struct CosXCosY
-{
-    /**
-     * @brief Construct with two coefficients
-     *
-     * @param amp amplitude A
-     * @param bamp backgroundamp B
-     * @param kx  kx
-     * @param ky  ky
-     */
-    CosXCosY( double amp, double bamp, double kx, double ky):amp_(amp), bamp_(bamp),kx_(kx),ky_(ky){}
-    /**
-     * @brief Return profile
-     *
-     * @param x x - coordinate
-     * @param y y - coordinate
 
-     * @return \f$ f(x,y)\f$
-     */
-    DG_DEVICE
-    double operator()( double x, double y)const{ return bamp_+amp_*cos(x*kx_)*cos(y*ky_);}
-  private:
-    double amp_,bamp_,kx_,ky_;
-};
-/**
- * @brief A sin prof in x- and cos prof in  y-direction
- * \f$ f(x,y) =B+ A \sin(k_x x) \cos(k_y y) \f$
- */
-struct SinXCosY
-{
-    /**
-     * @brief Construct with two coefficients
-     *
-     * @param amp amplitude
-     * @param bamp backgroundamp
-     * @param kx  kx
-     * @param ky  ky
-     */
-    SinXCosY( double amp, double bamp, double kx, double ky):amp_(amp), bamp_(bamp),kx_(kx),ky_(ky){}
-    /**
-     * @brief Return profile
-     *
-     * @param x x - coordinate
-     * @param y y - coordinate
+/////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////
+
+///@addtogroup functions
+///@{
+//
 
-     * @return \f$ f(x,y)\f$
-     */
-    DG_DEVICE
-    double operator()( double x, double y)const{ return bamp_+amp_*sin(x*kx_)*cos(y*ky_);}
-  private:
-    double amp_,bamp_,kx_,ky_;
-};
-/**
- * @brief A sin prof in x-direction
- * \f$ f(x) = f(x,y) = f(x,y,z) =B+ A \sin(k_x x) \f$
- */
-struct SinX
-{
-    /**
-     * @brief Construct with two coefficients
-     *
-     * @param amp amplitude A
-     * @param bamp backgroundamp B
-     * @param kx  kx
-     */
-    SinX( double amp, double bamp, double kx):amp_(amp), bamp_(bamp),kx_(kx){}
-    DG_DEVICE
-    double operator()( double x)const{ return bamp_+amp_*sin(x*kx_);}
-    DG_DEVICE
-    double operator()( double x, double y)const{ return this->operator()(x);}
-    DG_DEVICE
-    double operator()( double x, double y, double z)const{ return this->operator()(x);}
-  private:
-    double amp_,bamp_,kx_;
-};
 /**
- * @brief A sin prof in y-direction
- * \f$ f(x,y) =B+ A \sin(k_y y) \f$
+ * @brief
+     \f$ f(\psi) = \begin{cases}
+        1  \text{ if } \psi_{\min} < \psi < \psi_{\max}\\
+        0  \text{ else}
+     \end{cases}\f$
  */
-struct SinY
+struct Iris
 {
-    /**
-     * @brief Construct with two coefficients
-     *
-     * @param amp amplitude
-     * @param bamp backgroundamp
-     * @param ky  ky
-     */
-    SinY( double amp, double bamp, double ky):amp_(amp), bamp_(bamp),ky_(ky){}
+    Iris( double psi_min, double psi_max ):
+        m_psimin(psi_min), m_psimax(psi_max) { }
     DG_DEVICE
-    double operator()( double x, double y)const{ return bamp_+amp_*sin(y*ky_);}
-  private:
-    double amp_,bamp_,ky_;
+    double operator()(double psi)const
+    {
+        if( psi > m_psimax) return 0.;
+        if( psi < m_psimin) return 0.;
+        return 1.;
+    }
+    private:
+    double m_psimin, m_psimax;
 };
 /**
- * @brief A sin prof in x-direction
- * \f$ f(x,y) =B+ A \cos(k_y y) \f$
+ * @brief
+     \f$ f(\psi) = \begin{cases}
+        0  \text{ if } \psi > \psi_{\max} \\
+        1  \text{ else}
+     \end{cases}\f$
  */
-struct CosY
+struct Pupil
 {
-    /**
-     * @brief Construct with two coefficients
-     *
-     * @param amp amplitude A
-     * @param bamp backgroundamp B
-     * @param ky  ky
-     */
-    CosY( double amp, double bamp, double ky):amp_(amp), bamp_(bamp),ky_(ky){}
+    Pupil( double psimax):
+        psimax_(psimax) { }
     DG_DEVICE
-    double operator()( double x, double y)const{ return bamp_+amp_*cos(y*ky_);}
-  private:
-    double amp_,bamp_,ky_;
+    double operator()(double psi)const
+    {
+        if( psi > psimax_) return 0.;
+        return 1.;
+    }
+    private:
+    double psimax_;
 };
 /**
- * @brief Inverse cosh profile
- * \f$ f(x,y) =A/\cosh^2(k_x x) \f$
+ * @brief
+     \f$ f(\psi) = \begin{cases}
+        \psi_{\max}  \text{ if } \psi > \psi_{\max} \\
+        \psi \text{ else}
+     \end{cases}\f$
  */
-struct InvCoshXsq
+struct PsiPupil
 {
-    /**
-     * @brief Construct with two coefficients
-     *
-     * @param amp amplitude
-     * @param kx  kx
-     */
-    InvCoshXsq( double amp, double kx):m_amp(amp), m_kx(kx){}
-    DG_DEVICE
-    double operator()( double x)const{ return m_amp/cosh(x*m_kx)/cosh(x*m_kx);}
-    DG_DEVICE
-    double operator()( double x, double y)const{ return this->operator()(x);}
+    PsiPupil(double psimax):
+        psimax_(psimax){ }
     DG_DEVICE
-    double operator()( double x, double y, double z)const{ return this->operator()(x);}
-  private:
-    double m_amp, m_kx;
+    double operator()(double psi)const
+    {
+        if( psi > psimax_) return psimax_;
+        return  psi;
+    }
+    private:
+    double psimax_;
 };
 /**
- * @brief Sin prof in x-direction
- * \f$ f(x) = f(x,y) = f(x,y,z) = B + A(1 - \sin(k_xx )) \f$
+ * @brief
+     \f$ f(x) = \begin{cases}
+        0  \text{ if } x < x_b \\
+        1  \text{ else}
+     \end{cases}\f$
+  @note the 1 is inclusive i.e if x==x_b the functor always returns 1
  */
-struct SinProfX
+struct Heaviside
 {
+
     /**
-     * @brief Construct with two coefficients
+     * @brief Construct with xb and sign
      *
-     * @param amp amplitude A
-     * @param bamp backgroundamp B
-     * @param kx  kx
+     * @param xb boundary value
+     * @param sign either +1 or -1, If -1, we mirror the Heaviside at
+     *  the \c x=x_b axis, i.e. we swap the < sign in the definition to >
+     * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
      */
-    SinProfX( double amp, double bamp, double kx):m_amp(amp), m_bamp(bamp),m_kx(kx){}
-    DG_DEVICE
-    double operator()( double x)const{ return m_bamp+m_amp*(1.-sin(x*m_kx));}
-    DG_DEVICE
-    double operator()( double x, double y)const{ return this->operator()(x);}
+    Heaviside( double xb, int sign = +1):
+        m_xb(xb), m_s(sign){ }
+
     DG_DEVICE
-    double operator()( double x, double y, double z)const{ return this->operator()(x);}
-  private:
-    double m_amp, m_bamp, m_kx;
+    double operator()(double x)const
+    {
+        if( (x < m_xb && m_s == 1) || (x > m_xb && m_s == -1)) return 0.;
+        return 1.;
+    }
+    private:
+    double m_xb;
+    int m_s;
 };
+
+
 /**
- * @brief Exp prof in x-direction
- * \f$ f(x) = f(x,y) = f(x,y,z) = B + A\exp(-x/L_n) \f$
+ * @brief \f$ f(x,y) = \sqrt{ (x-x_0)^2 + (y-y_0)^2} \f$
  */
-struct ExpProfX
+struct Distance
 {
-    /**
-     * @brief Construct with two coefficients
-     *
-     * @param amp amplitude B
-     * @param bamp backgroundamp A (choose zero for constant gradient length
-     * @param ln  ln (must be !=0)
-     */
-    ExpProfX( double amp, double bamp, double ln):m_amp(amp),m_bamp(bamp),m_ln(ln){
-        assert( ln!=0 && "ln parameter must be != 0 in ExpProfX!");
-    }
-    DG_DEVICE
-    double operator()( double x)const{ return m_bamp+m_amp*exp(-x/m_ln);}
-    DG_DEVICE
-    double operator()( double x, double y)const{ return this->operator()(x);}
+    Distance( double x0, double y0): m_x0(x0), m_y0(y0){}
     DG_DEVICE
-    double operator()( double x, double y, double z)const{ return this->operator()(x);}
-  private:
-    double m_amp, m_bamp, m_ln;
+    double operator()(double x, double y){
+        return sqrt( (x-m_x0)*(x-m_x0) + (y-m_y0)*(y-m_y0));
+    }
+    private:
+    double m_x0, m_y0;
 };
-
 /**
- * @brief The linear interpolation polynomial
+ * @brief
  * \f$ f(x) = y_1\frac{x-x_0}{x_1-x_0} + y_0\frac{x-x_1}{x_0-x_1}\f$
+ *
+ * The linear interpolation polynomial
  */
 struct Line{
     Line(double x0, double y0, double x1, double y1) :
@@ -666,7 +509,7 @@ struct Line{
 };
 
 /**
- * @brief A linear function in x-direction
+ * @brief
  * \f$ f(x) = f(x,y) = f(x,y,z) = ax+b \f$
  */
 struct LinearX
@@ -688,7 +531,7 @@ struct LinearX
     double a_,b_;
 };
 /**
- * @brief A linear polynomial in y-direction
+ * @brief
  * \f$ f(x,y) = f(x,y,z) = ay+b \f$
  */
 struct LinearY
@@ -708,7 +551,7 @@ struct LinearY
     double a_,b_;
 };
 /**
- * @brief A linear function in z-direction
+ * @brief
  * \f$ f(x,y,z) = az+b \f$
  */
 struct LinearZ
@@ -725,730 +568,867 @@ struct LinearZ
   private:
     double a_,b_;
 };
-
 /**
  * @brief
-     \f$ \begin{cases}
-        1  \text{ if } \psi_{\min} < \psi < \psi_{\max}\\
-        0  \text{ else}
-     \end{cases}\f$ Zero outside psimax and inside psimin, otherwise 1
+ * \f$
+   f(x,y) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}\right)}
+   \f$
  */
-struct Iris
+struct Gaussian
 {
-    Iris( double psi_min, double psi_max ):
-        m_psimin(psi_min), m_psimax(psi_max) { }
+    /**
+     * @brief Functor returning a %Gaussian
+     *
+     * @param x0 x-center-coordinate
+     * @param y0 y-center-coordinate
+     * @param sigma_x x - variance (must be !=0)
+     * @param sigma_y y - variance (must be !=0)
+     * @param amp Amplitude
+     */
+    Gaussian( double x0, double y0, double sigma_x, double sigma_y, double amp)
+        : m_x0(x0), m_y0(y0), m_sigma_x(sigma_x), m_sigma_y(sigma_y), m_amp(amp){
+            assert( m_sigma_x != 0  &&  "sigma_x must not be 0 in Gaussian");
+            assert( m_sigma_y != 0  &&  "sigma_y must not be 0 in Gaussian");
+    }
+    /**
+     * @brief Return the value of the %Gaussian
+     *
+     * \f[
+       f(x,y) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}\right)}
+       \f]
+     * @param x x - coordinate
+     * @param y y - coordinate
+     *
+     * @return gaussian
+     */
     DG_DEVICE
-    double operator()(double psi)const
+    double operator()(double x, double y) const
     {
-        if( psi > m_psimax) return 0.;
-        if( psi < m_psimin) return 0.;
-        return 1.;
+        return  m_amp*
+                   exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x +
+                          (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
-    private:
-    double m_psimin, m_psimax;
+    /**
+     * @brief Return the value of the %Gaussian
+     * \f[
+       f(x,y,z) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
+       \f]
+     * @param x x - coordinate
+     * @param y y - coordinate
+     * @param z z - coordinate
+     *
+     * @return gaussian
+     */
+    DG_DEVICE
+    double operator()(double x, double y, double z) const
+    {
+        return  this->operator()(x,y);
+    }
+  private:
+    double  m_x0, m_y0, m_sigma_x, m_sigma_y, m_amp;
+
 };
+
 /**
  * @brief
-     \f$ \begin{cases}
-        0  \text{ if } \psi > \psi_{\max} \\
-        1  \text{ else}
-     \end{cases}\f$ Zero outside psimax, otherwise 1
+ * \f$
+   f(x,y) = \begin{cases}
+   Ae^{1 + \left(\frac{(x-x_0)^2}{\sigma_x^2} + \frac{(y-y_0)^2}{\sigma_y^2} - 1\right)^{-1}} \text{ if } \frac{(x-x_0)^2}{\sigma_x^2} + \frac{(y-y_0)^2}{\sigma_y^2} < 1\\
+   0 \text{ else}
+   \end{cases}
+   \f$
+
+   A bump that drops to zero and is infinitely continuously differentiable
  */
-struct Pupil
+struct Cauchy
 {
-    Pupil( double psimax):
-        psimax_(psimax) { }
+    /**
+     * @brief A blob that drops to zero
+     *
+     * @param x0 x-center-coordinate
+     * @param y0 y-center-coordinate
+     * @param sigma_x radius in x (must be !=0)
+     * @param sigma_y radius in y (must be !=0)
+     * @param amp Amplitude
+     */
+    Cauchy( double x0, double y0, double sigma_x, double sigma_y, double amp): x0_(x0), y0_(y0), sigmaX_(sigma_x), sigmaY_(sigma_y), amp_(amp){
+        assert( sigma_x != 0  &&  "sigma_x must be !=0 in Cauchy");
+        assert( sigma_y != 0  &&  "sigma_y must be !=0 in Cauchy");
+    }
     DG_DEVICE
-    double operator()(double psi)const
+    double operator()(double x, double y )const{
+        double xbar = (x-x0_)/sigmaX_;
+        double ybar = (y-y0_)/sigmaY_;
+        if( xbar*xbar + ybar*ybar < 1.)
+            return amp_*exp( 1. +  1./( xbar*xbar + ybar*ybar -1.) );
+        return 0.;
+    }
+    bool inside( double x, double y)const
     {
-        if( psi > psimax_) return 0.;
-        return 1.;
+        double xbar = (x-x0_)/sigmaX_;
+        double ybar = (y-y0_)/sigmaY_;
+        if( xbar*xbar + ybar*ybar < 1.)
+            return true;
+        return false;
+    }
+
+    double dx( double x, double y )const{
+        double xbar = (x-x0_)/sigmaX_;
+        double ybar = (y-y0_)/sigmaY_;
+        double temp = sigmaX_*(xbar*xbar + ybar*ybar  - 1.);
+        return -2.*(x-x0_)*this->operator()(x,y)/temp/temp;
+    }
+    double dxx( double x, double y)const{
+        double temp = sigmaY_*sigmaY_*(x-x0_)*(x-x0_) + sigmaX_*sigmaX_*((y-y0_)*(y-y0_) - sigmaY_*sigmaY_);
+        double bracket = sigmaX_*sigmaX_*((y-y0_)*(y-y0_)-sigmaY_*sigmaY_)*sigmaX_*sigmaX_*((y-y0_)*(y-y0_)-sigmaY_*sigmaY_)
+            -3.*sigmaY_*sigmaY_*sigmaY_*sigmaY_*(x-x0_)*(x-x0_)*(x-x0_)*(x-x0_)
+            -2.*sigmaY_*sigmaY_*sigmaX_*sigmaX_*(x-x0_)*(x-x0_)*(y-y0_)*(y-y0_);
+        return -2.*sigmaX_*sigmaX_*sigmaY_*sigmaY_*sigmaY_*sigmaY_*this->operator()(x,y)*bracket/temp/temp/temp/temp;
+    }
+    double dy( double x, double y)const{
+        double xbar = (x-x0_)/sigmaX_;
+        double ybar = (y-y0_)/sigmaY_;
+        double temp = sigmaY_*(xbar*xbar + ybar*ybar  - 1.);
+        return -2.*(y-y0_)*this->operator()(x,y)/temp/temp;
+    }
+    double dyy( double x, double y)const{
+        double temp = sigmaX_*sigmaX_*(y-y0_)*(y-y0_) + sigmaY_*sigmaY_*((x-x0_)*(x-x0_) - sigmaX_*sigmaX_);
+        double bracket = sigmaY_*sigmaY_*((x-x0_)*(x-x0_)-sigmaX_*sigmaX_)*sigmaY_*sigmaY_*((x-x0_)*(x-x0_)-sigmaX_*sigmaX_)
+            -3.*sigmaX_*sigmaX_*sigmaX_*sigmaX_*(y-y0_)*(y-y0_)*(y-y0_)*(y-y0_)
+            -2.*sigmaX_*sigmaX_*sigmaY_*sigmaY_*(y-y0_)*(y-y0_)*(x-x0_)*(x-x0_);
+        return -2.*sigmaY_*sigmaY_*sigmaX_*sigmaX_*sigmaX_*sigmaX_*this->operator()(x,y)*bracket/temp/temp/temp/temp;
+    }
+    double dxy( double x, double y )const{
+        double xbar = (x-x0_)/sigmaX_;
+        double ybar = (y-y0_)/sigmaY_;
+        double temp = (xbar*xbar + ybar*ybar  - 1.);
+        return 8.*xbar*ybar*this->operator()(x,y)/temp/temp/temp/sigmaX_/sigmaY_
+            + 4.*xbar*ybar*this->operator()(x,y)/temp/temp/temp/temp/sigmaX_/sigmaY_
+;
+    }
+    private:
+    double x0_, y0_, sigmaX_, sigmaY_, amp_;
+};
+
+/**
+* @brief
+* \f$
+f(x,y,z) = Ae^{-\left(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2} + \frac{(z-z_0)^2}{2\sigma_z^2}\right)}
+\f$
+*/
+struct Gaussian3d
+{
+    /**
+     * @brief Functor returning a %Gaussian
+     *
+     * @param x0 x-center-coordinate
+     * @param y0 y-center-coordinate
+     * @param z0 z-center-coordinate
+     * @param sigma_x x - variance (must be !=0)
+     * @param sigma_y y - variance (must be !=0)
+     * @param sigma_z z - variance (must be !=0)
+     * @param amp Amplitude
+     */
+    Gaussian3d( double x0, double y0, double z0, double sigma_x, double sigma_y, double sigma_z, double amp)
+        : m_x0(x0), m_y0(y0), m_z0(z0), m_sigma_x(sigma_x), m_sigma_y(sigma_y), m_sigma_z(sigma_z), m_amp(amp){
+            assert( m_sigma_x != 0  &&  "sigma_x must be !=0 in Gaussian3d");
+            assert( m_sigma_y != 0  &&  "sigma_y must be !=0 in Gaussian3d");
+            assert( m_sigma_z != 0  &&  "sigma_z must be !=0 in Gaussian3d");
+    }
+    /**
+     * @brief Return a 2d %Gaussian
+     *
+     * \f[
+       f(x,y) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2})}
+       \f]
+     * @param x x - coordinate
+     * @param y y - coordinate
+     *
+     * @return gaussian
+     */
+    DG_DEVICE
+    double operator()(double x, double y) const
+    {
+        return  m_amp*
+                   exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x +
+                          (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
-    private:
-    double psimax_;
-};
-/**
- * @brief
-     \f$ \begin{cases}
-        \psi_{\max}  \text{ if } \psi > \psi_{\max} \\
-        \psi \text{ else}
-     \end{cases}\f$ Psi inside psimax and psimax outside psimax
- */
-struct PsiPupil
-{
-    PsiPupil(double psimax):
-        psimax_(psimax){ }
+    /**
+     * @brief Return the value of the %Gaussian
+     *
+     * \f[
+       f(x,y) = Ae^{-(\frac{(x-x_0)^2}{2\sigma_x^2} + \frac{(y-y_0)^2}{2\sigma_y^2}+\frac{(z-z_0)^2}{2\sigma_z^2})}
+       \f]
+     * @param x x - coordinate
+     * @param y y - coordinate
+     * @param z z - coordinate
+     *
+     * @return gaussian
+     */
     DG_DEVICE
-    double operator()(double psi)const
+    double operator()(double x, double y, double z) const
     {
-        if( psi > psimax_) return psimax_;
-        return  psi;
+        return  m_amp*
+                exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x +
+                       (z-m_z0)*(z-m_z0)/2./m_sigma_z/m_sigma_z +
+                       (y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
-    private:
-    double psimax_;
+  private:
+    double  m_x0, m_y0, m_z0, m_sigma_x, m_sigma_y, m_sigma_z, m_amp;
+
 };
 /**
  * @brief
-     \f$ \begin{cases}
-        0  \text{ if } x < x_b \\
-        1  \text{ else}
-     \end{cases}\f$ Zero up to xb, then one
-  @note the 1 is inclusive i.e if x==x_b the functor always returns 1
+ * \f$
+   f(x,y) = Ae^{-\frac{(x-x_0)^2}{2\sigma_x^2} }
+   \f$
  */
-struct Heaviside
+struct GaussianX
 {
-
     /**
-     * @brief Construct with xb and sign
+     * @brief A %Gaussian in x
      *
-     * @param xb boundary value
-     * @param sign either +1 or -1, If -1, we mirror the Heaviside at
-     *  the \c x=x_b axis, i.e. we swap the < sign in the definition to >
-     * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
+     * @param x0 x-center-coordinate
+     * @param sigma_x x - variance (must be !=0)
+     * @param amp Amplitude
      */
-    Heaviside( double xb, int sign = +1):
-        m_xb(xb), m_s(sign){ }
-
+    GaussianX( double x0, double sigma_x, double amp)
+        :m_x0(x0), m_sigma_x(sigma_x), m_amp(amp){
+            assert( m_sigma_x != 0  &&  "sigma_x must be !=0 in GaussianX");
+    }
     DG_DEVICE
-    double operator()(double x)const
+    double operator()(double x) const
     {
-        if( (x < m_xb && m_s == 1) || (x > m_xb && m_s == -1)) return 0.;
-        return 1.;
+        return  m_amp* exp( -((x-m_x0)*(x-m_x0)/2./m_sigma_x/m_sigma_x ));
     }
-    private:
-    double m_xb;
-    int m_s;
-};
-
-
-/**
- * @brief \f$ \sqrt{ (x-x_0)^2 + (y-y_0)^2} \f$
- */
-struct Distance
-{
-    Distance( double x0, double y0): m_x0(x0), m_y0(y0){}
     DG_DEVICE
-    double operator()(double x, double y){
-        return sqrt( (x-m_x0)*(x-m_x0) + (y-m_y0)*(y-m_y0));
+    double operator()(double x, double y) const
+    {
+        return this->operator()(x);
     }
-    private:
-    double m_x0, m_y0;
-};
-
-/**
- * @brief
-     \f$ \begin{cases}
- 1 \text{ if } \psi < \psi_{\max}\\
- 0 \text{ if } \psi > (\psi_{\max} + 4\alpha) \\
- \exp\left( - \frac{(\psi - \psi_{\max})^2}{2\alpha^2}\right), \text{ else}
- \end{cases}
-   \f$
-
-   One up to \c psimax, then a %Gaussian down to zero
- */
-struct GaussianDamping
-{
-    GaussianDamping( double psimax, double alpha):
-        m_psimax(psimax), m_alpha(alpha) {
-            assert( alpha!= 0 && "Damping width in GaussianDamping must not be zero");
-        }
     DG_DEVICE
-    double operator()(double psi)const
+    double operator()(double x, double y, double z) const
     {
-        if( psi > m_psimax + 4.*m_alpha) return 0.;
-        if( psi < m_psimax) return 1.;
-        return exp( -( psi-m_psimax)*( psi-m_psimax)/2./m_alpha/m_alpha);
+        return this->operator()(x);
     }
-    private:
-    double m_psimax, m_alpha;
+  private:
+    double  m_x0, m_sigma_x, m_amp;
+
 };
 /**
- * @brief An approximation to Heaviside using tanh
- * \f$ f(x) = B + 0.5 A(1+ \text{sign} \tanh((x-x_b)/\alpha ) ) \f$
+ * @brief
+ * \f$
+   f(x,y) = Ae^{-\frac{(y-y_0)^2}{2\sigma_y^2}}
+   \f$
  */
-struct TanhProfX {
+struct GaussianY
+{
     /**
-     * @brief Construct with xb, width and sign
+     * @brief Functor returning a gaussian
      *
-     * @param xb boundary value
-     * @param width damping width \c alpha (must be !=0)
-     * @param sign sign of the Tanh, defines the damping direction
-     * @param bgamp background amplitude \c B
-     * @param profamp profile amplitude \c A
-     * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
+     * @param y0 y-center-coordinate
+     * @param sigma_y y - variance (must be !=0)
+     * @param amp Amplitude
      */
-    TanhProfX(double xb, double width, int sign =1,double bgamp = 0.,
-        double profamp = 1.) :
-        xb_(xb),w_(width), s_(sign),bga_(bgamp),profa_(profamp)  {
-            assert( width != 0&& "Width in TanhProfX must not be zero!");
+    GaussianY( double y0, double sigma_y, double amp)
+        : m_y0(y0), m_sigma_y(sigma_y), m_amp(amp){
+            assert( m_sigma_y != 0  &&  "sigma_x must be !=0 in GaussianY");
     }
+    /**
+     * @brief Return the value of the gaussian
+     *
+     * \f[
+       f(x,y) = Ae^{-\frac{(y-y_0)^2}{2\sigma_y^2}}
+       \f]
+     * @param x x - coordinate
+     * @param y y - coordinate
+     *
+     * @return gaussian
+     */
     DG_DEVICE
-    double operator() (double x)const
+    double operator()(double x, double y) const
     {
-        return profa_*0.5*(1.+s_*tanh((x-xb_)/w_))+bga_;
+        return  m_amp*exp( -((y-m_y0)*(y-m_y0)/2./m_sigma_y/m_sigma_y) );
     }
-    DG_DEVICE
-    double operator()( double x, double y)const{ return this->operator()(x);}
-    DG_DEVICE
-    double operator()( double x, double y, double z)const{ return this->operator()(x);}
-    private:
-    double xb_;
-    double w_;
-    int s_;
-    double bga_;
-    double profa_;
-};
+  private:
+    double  m_y0, m_sigma_y, m_amp;
 
+};
 /**
- * @brief \f$ \sum_{i=0}^{M-1} \sum_{j=0}^{N-1} c_{iN+j} x^i y^j  \f$
- *
- * Evaluated using [Horner's method](https://en.wikipedia.org/wiki/Horner%27s_method)
+ * @brief
+ * \f$
+   f(x,y,z) = Ae^{-\frac{(z-z_0)^2}{2\sigma_z^2}}
+   \f$
  */
-struct Horner2d
+struct GaussianZ
 {
-    ///Initialize 1 coefficient to 1
-    Horner2d(): m_c( 1, 1), m_M(1), m_N(1){}
-
     /**
-     * @brief Initialize coefficients and dimensions
+     * @brief Functor returning a gaussian
      *
-     * @param c vector of size MN containing coefficientc c (accessed as c[i*N+j] i.e. y-direction is contiguous)
-     * @param M number of polynomials in x
-     * @param N number of polynomials in y
+     * @param z0 z-center-coordinate
+     * @param sigma_z z - variance (must be !=0)
+     * @param amp Amplitude
      */
-    Horner2d( std::vector<double> c, unsigned M, unsigned N): m_c(c), m_M(M), m_N(N){}
-    double operator()( double x, double y) const
-    {
-        std::vector<double> cx( m_M);
-        for( unsigned i=0; i<m_M; i++)
-            cx[i] = horner( &m_c[i*m_N], m_N, y);
-        return horner( &cx[0], m_M, x);
+    GaussianZ( double z0, double sigma_z, double amp)
+        : m_z0(z0), m_sigma_z(sigma_z), m_amp(amp){
+            assert( m_sigma_z != 0  &&  "sigma_z must be !=0 in GaussianZ");
     }
-    private:
-    double horner( const double * c, unsigned M, double x) const
+    /**
+     * @brief Return the value of the gaussian
+     *
+     * \f[
+       f(z) = Ae^{-\frac{(z-z_0)^2}{2\sigma_z^2}}
+       \f]
+     * @param z z - coordinate
+     *
+     * @return gaussian
+     */
+    DG_DEVICE
+    double operator()( double z) const
     {
-        double b = c[M-1];
-        for( unsigned i=0; i<M-1; i++)
-            b = c[M-2-i] + b*x;
-        return b;
+        return  m_amp*exp( -((z-m_z0)*(z-m_z0)/2./m_sigma_z/m_sigma_z) );
     }
-    std::vector<double> m_c;
-    unsigned m_M, m_N;
-};
-
-
-/**
- * @brief \f$ \begin{cases}
-     0 \text{ if } x < x_b-a \\
-        ((16 a^3 - 29 a^2 (x - x_b) + 20 a (x - x_b)^2 - 5 (x - x_b)^3) (a + x -
-   x_b)^4)/(32 a^7) \text{ if } |x-x_b| < a \\
-        1  \text{ if } x > x_b + a
-     \end{cases}\f$
-
- An approximation to Heaviside using polynomials.
-     This function is 3 times continuously differentiable, takes the value 0.5 at xb and
-     has a transition width a on both sides of xb.
- */
-struct PolynomialHeaviside {
     /**
-     * @brief Construct with xb, width and sign
+     * @brief Return the value of the gaussian
      *
-     * @param xb boundary value
-     * @param a transition width (must be != 0)
-     * @param sign either +1 (original Heaviside) or -1 (the function is mirrored at the \c x=xb axis: f(2xb-x))
-     * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
+     * \f[
+       f(x,y,z) = Ae^{-\frac{(z-z_0)^2}{2\sigma_z^2}}
+       \f]
+     * @param x x - coordinate
+     * @param y y - coordinate
+     * @param z z - coordinate
+     *
+     * @return gaussian
      */
-    PolynomialHeaviside(double xb, double a, int sign = +1) :
-        x0(xb), a(a), m_s(sign){
-            assert( a!=0 && "PolynomialHeaviside width must not be zero");
-    }
     DG_DEVICE
-    double operator() (double x)const
+    double operator()(double x, double y, double z) const
     {
-        if( m_s == -1) x = 2*x0-x; //mirror
-        if ( x < x0-a) return 0;
-        if ( x > x0+a) return 1;
-        return ((16.*a*a*a - 29.*a*a*(x - x0)
-               + 20.*a*(x - x0)*(x - x0)
-               - 5.*(x - x0)*(x-x0)*(x-x0))
-               *(a + x - x0)*(a + x - x0)
-               *(a + x - x0)*(a + x - x0))/(32.*a*a*a * a*a*a*a);
+        return  m_amp*exp( -((z-m_z0)*(z-m_z0)/2./m_sigma_z/m_sigma_z) );
     }
-    private:
-    double x0, a;
-    int m_s;
+  private:
+    double  m_z0, m_sigma_z, m_amp;
+
 };
 /**
  * @brief
-     \f$ \begin{cases}
-     0 \text{ if } x < x_l-a_l \\
-        ((16 a_l^3 - 29 a_l^2 (x - x_l) + 20 a_l (x - x_l)^2 - 5 (x - x_l)^3) (a_l + x -
-   x_l)^4)/(32 a_l^7) \text{ if } |x-x_l| < a_l \\
-        1  \text{ if } x_l + a_l < x < x_r-a_r \\
-        ((16 a_r^3 - 29 a_r^2 (x - x_r) + 20 a_r (x - x_r)^2 - 5 (x - x_r)^3) (a_r + x -
-   x_l)^4)/(32 a_r^7) \text{ if } |x-x_r| < a_r \\
-   0 \text{ if } x > x_r + a_r
-     \end{cases}\f$
-
- An approximation to the Rectangle function using polynomials
-     Basically just the product of two PolynomialHeaviside functions
-
-     This function is 3 times continuously differentiable, takes the value 0.5 at xl and xr and
-     has a transition width a_l on both sides of xl and a width a_r on both sides of xr.
+ * \f$ f(x,y) = \lambda \ln{(\cosh{(x/\lambda) } +\epsilon \cos(y/\lambda)) } \f$
  */
-struct PolynomialRectangle {
+struct IslandXY
+{
     /**
-     * @brief Construct with xb, width and sign
+     * @brief Construct Island
      *
-     * @param xl left boundary value
-     * @param al left transition width (must be != 0)
-     * @param xr right boundary value
-     * @param ar right transition width (must be != 0)
+     * @param lambda amplitude (must be != 0)
+     * @param eps y-amplitude
+     */
+     IslandXY( double lambda, double eps):lambda_(lambda), eps_(eps){
+         assert( lambda != 0 && "Lambda parameter in IslandXY must not be zero!");
+     }
+    /**
+     * @brief Return profile
+     *
+     * @param x x - coordinate
+     * @param y y - coordinate
+     * @return \f$ f(x,y)\f$
      */
-    PolynomialRectangle(double xl, double al, double xr, double ar) :
-        m_hl( xl, al, +1), m_hr( xr, ar, -1) {
-        assert( xl < xr && "left boundary must be left of right boundary");
-    }
     DG_DEVICE
-    double operator() (double x)const
-    {
-        return m_hl(x)*m_hr(x);
-    }
-    private:
-    PolynomialHeaviside m_hl, m_hr;
+    double operator()( double x, double y)const{ return lambda_*log(cosh(x/lambda_)+eps_*cos(y/lambda_));}
+  private:
+    double lambda_,eps_;
 };
-
 /**
- * @brief \f$ \begin{cases}
-     x_b \text{ if } x < x_b-a \\
-     x_b + ((35 a^3 - 47 a^2 (x - x_b) + 25 a (x - x_b)^2 - 5 (x - x_b)^3) (a + x - x_b)^5)/(256 a^7)
-        \text{ if } |x-x_b| < a \\
-        x  \text{ if } x > x_b + a
-     \end{cases}\f$
- The integral of PolynomialHeaviside approximates xH(x)
-
-     This function is 4 times continuously differentiable,
-     has a transition width \c a on both sides of \c xb, where it transitions from the
-     constant \c xb to the linear function \c x.
+ * @brief \f$ f(x,y) =B+ A \sin(k_x x) \sin(k_y y) \f$
  */
-struct IPolynomialHeaviside {
+struct SinXSinY
+{
     /**
-     * @brief Construct with xb, width and sign
+     * @brief Construct with two coefficients
      *
-     * @param xb boundary value
-     * @param a transition width (must be != 0)
-     * @param sign either +1 (original) or -1 (the function is point mirrored at \c x=xb: 2*xb-f(2xb-x))
+     * @param amp amplitude A
+     * @param bamp backgroundamp B
+     * @param kx  kx
+     * @param ky  ky
      */
-    IPolynomialHeaviside(double xb, double a, int sign = +1) :
-        x0(xb), a(a), m_s(sign){
-            assert( a!=0 && "IPolynomialHeaviside width must not be zero");
-        }
-    DG_DEVICE
-    double operator() (double x)const
-    {
-        if( m_s == -1) x = 2*x0-x; //mirror
-        double result;
-        if ( x < x0-a) result =  x0;
-        else if ( x > x0+a) result =  x;
-        else
-            result =  x0 + ((35.* a*a*a - 47.* a*a*(x - x0) + 25.*a*(x - x0)*(x-x0)
-                - 5.*(x - x0)*(x-x0)*(x-x0))
-                *(a+x-x0)*(a+x-x0)*(a+x-x0)*(a+x-x0)*(a+x-x0))
-            /(256.*a*a*a * a*a*a*a);
-        if ( m_s == +1) return result;
-        return 2*x0 - result;
+    SinXSinY( double amp, double bamp, double kx, double ky):amp_(amp), bamp_(bamp),kx_(kx),ky_(ky){}
+    /**
+     * @brief Return profile
+     *
+     * @param x x - coordinate
+     * @param y y - coordinate
 
-    }
-    private:
-    double x0, a;
-    int m_s;
+     * @return \f$ f(x,y)\f$
+     */
+    DG_DEVICE
+    double operator()( double x, double y)const{ return bamp_+amp_*sin(x*kx_)*sin(y*ky_);}
+  private:
+    double amp_,bamp_,kx_,ky_;
 };
-
 /**
- * @brief \f$ \begin{cases}
-     0 \text{ if } x < x_b-a || x > x_b+a \\
-     (35 (a + x - x_b)^3 (a - x + x_b)^3)/(32 a^7)
-        \text{ if } |x-x_b| < a
-     \end{cases}\f$
-     The derivative of PolynomialHeaviside approximates delta(x)
-
-     This function is 2 times continuously differentiable, is symmetric around \c xb
-     and has a width \c a on both sides of \c x0.
-     The integral over this function yields 1.
+ * @brief
+ * \f$ f(x,y) =B+ A \cos(k_x x) \cos(k_y y) \f$
  */
-struct DPolynomialHeaviside {
+struct CosXCosY
+{
     /**
-     * @brief Construct with xb, width and sign
+     * @brief Construct with two coefficients
      *
-     * @param xb boundary value
-     * @param a transition width ( must be !=0)
-     * @param sign either +1 (original) or -1 (the function is mirrored at \c x=x0)
-     * (since this function is symmetric this parameter is ignored, it's there to be
-     * consistent with PolynomialHeaviside)
+     * @param amp amplitude A
+     * @param bamp backgroundamp B
+     * @param kx  kx
+     * @param ky  ky
+     */
+    CosXCosY( double amp, double bamp, double kx, double ky):amp_(amp), bamp_(bamp),kx_(kx),ky_(ky){}
+    /**
+     * @brief Return profile
+     *
+     * @param x x - coordinate
+     * @param y y - coordinate
+
+     * @return \f$ f(x,y)\f$
      */
-    DPolynomialHeaviside(double xb, double a, int sign = +1) :
-        x0(xb), a(a){
-            assert( a!=0 && "DPolynomialHeaviside width must not be zero");
-    }
     DG_DEVICE
-    double operator() (double x)const
-    {
-        if ( (x < x0-a) || (x > x0+a)) return 0;
-        return (35.*(a+x-x0)*(a+x-x0)*(a+x-x0)*(a-x+x0)*(a-x+x0)*(a-x+x0))
-            /(32.*a*a*a * a*a*a*a);
-    }
-    private:
-    double x0, a;
+    double operator()( double x, double y)const{ return bamp_+amp_*cos(x*kx_)*cos(y*ky_);}
+  private:
+    double amp_,bamp_,kx_,ky_;
 };
-
 /**
- * @brief \f$ \begin{cases}
-    1 \text{ if } \eta < \eta_c \\
-    \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
-    0 \text{ else} \\
-    \eta=\frac{i}{1-n}
-    \end{cases}\f$
-
-    where n is the number of polynomial coefficients
+ * @brief
+ * \f$ f(x,y) =B+ A \sin(k_x x) \cos(k_y y) \f$
+ */
+struct SinXCosY
+{
+    /**
+     * @brief Construct with two coefficients
+     *
+     * @param amp amplitude
+     * @param bamp backgroundamp
+     * @param kx  kx
+     * @param ky  ky
+     */
+    SinXCosY( double amp, double bamp, double kx, double ky):amp_(amp), bamp_(bamp),kx_(kx),ky_(ky){}
+    /**
+     * @brief Return profile
+     *
+     * @param x x - coordinate
+     * @param y y - coordinate
 
-    This function is s times continuously differentiable everywhere
-    @sa Its main use comes from the application in dg::ModalFilter
+     * @return \f$ f(x,y)\f$
+     */
+    DG_DEVICE
+    double operator()( double x, double y)const{ return bamp_+amp_*sin(x*kx_)*cos(y*ky_);}
+  private:
+    double amp_,bamp_,kx_,ky_;
+};
+/**
+ * @brief
+ * \f$ f(x) = f(x,y) = f(x,y,z) =B+ A \sin(k_x x) \f$
  */
-struct ExponentialFilter
+struct SinX
 {
     /**
-     * @brief Create exponential filter \f$ \begin{cases}
-    1 \text{ if } \eta < \eta_c \\
-    \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
-    0 \text{ else} \\
-    \eta := \frac{i}{n-1}
-    \end{cases}\f$
+     * @brief Construct with two coefficients
      *
-     * @param alpha damping for the highest mode is \c exp( -alpha)
-     * @param eta_c cutoff frequency (0<eta_c<1), 0.5 or 0 are good starting values
-     * @param order 8 or 16 are good values
-     * @param n The number of polynomial coefficients
+     * @param amp amplitude A
+     * @param bamp backgroundamp B
+     * @param kx  kx
      */
-    ExponentialFilter( double alpha, double eta_c, unsigned order, unsigned n):
-        m_alpha(alpha), m_etac(eta_c), m_s(order), m_n(n) {}
-    double operator()( unsigned i) const
-    {
-        double eta = (double)i/(double)(m_n-1);
-        if( m_n == 1) eta = 0.;
-        if( eta < m_etac)
-            return 1.;
-        if( eta <= 1.)
-            return exp( -m_alpha*pow( (eta-m_etac)/(1.-m_etac), 2*m_s));
-        return 0;
-    }
-    private:
-    double m_alpha, m_etac;
-    unsigned m_s, m_n;
+    SinX( double amp, double bamp, double kx):amp_(amp), bamp_(bamp),kx_(kx){}
+    DG_DEVICE
+    double operator()( double x)const{ return bamp_+amp_*sin(x*kx_);}
+    DG_DEVICE
+    double operator()( double x, double y)const{ return this->operator()(x);}
+    DG_DEVICE
+    double operator()( double x, double y, double z)const{ return this->operator()(x);}
+  private:
+    double amp_,bamp_,kx_;
 };
-
 /**
- * @brief Exponential \f$ f(x) = A \exp(\lambda x)\f$
- *
- * @tparam T value-type
+ * @brief
+ * \f$ f(x,y) =B+ A \sin(k_y y) \f$
  */
-template< class T = double >
-struct EXP
+struct SinY
 {
     /**
-     * @brief Coefficients of \f$ A\exp(\lambda x) \f$
-     *
-     * @param amp Amplitude
-     * @param lambda coefficient
-     */
-    EXP( T amp = 1., T lambda = 1.): m_amp(amp), m_lambda(lambda){}
-    /**
-     * @brief return exponential
-     *
-     * @param x x
+     * @brief Construct with two coefficients
      *
-     * @return \f$ A\exp(\lambda x)\f$
+     * @param amp amplitude
+     * @param bamp backgroundamp
+     * @param ky  ky
      */
+    SinY( double amp, double bamp, double ky):amp_(amp), bamp_(bamp),ky_(ky){}
     DG_DEVICE
-    T operator() ( T x) const
-    {
-        return m_amp*exp(m_lambda*x);
-    }
+    double operator()( double x, double y)const{ return bamp_+amp_*sin(y*ky_);}
   private:
-    T m_amp, m_lambda;
+    double amp_,bamp_,ky_;
 };
 /**
  * @brief
- * \f$ f(x) = \ln(x)\f$ natural logarithm
- *
- * @tparam T value-type
+ * \f$ f(x,y) =B+ A \cos(k_y y) \f$
  */
-template < class T = double>
-struct LN
+struct CosY
 {
     /**
-     * @brief The natural logarithm
+     * @brief Construct with two coefficients
      *
-     * @param x of x
-     * @return  \f$ \ln(x) \f$
+     * @param amp amplitude A
+     * @param bamp backgroundamp B
+     * @param ky  ky
      */
+    CosY( double amp, double bamp, double ky):amp_(amp), bamp_(bamp),ky_(ky){}
     DG_DEVICE
-    T operator() (const T& x) const
-    {
-        return log(x);
-    }
-
+    double operator()( double x, double y)const{ return bamp_+amp_*cos(y*ky_);}
+  private:
+    double amp_,bamp_,ky_;
 };
 /**
- * @brief Square root
- * \f$ f(x) = \sqrt{x}\f$
- *
- * @tparam T value-type
+ * @brief
+ * \f$ f(x,y) =A/\cosh^2(k_x x) \f$
  */
-template < class T = double>
-struct SQRT
+struct InvCoshXsq
 {
     /**
-     * @brief Square root
-     *
-     * @param x of x
+     * @brief Construct with two coefficients
      *
-     * @return sqrt(x)
+     * @param amp amplitude
+     * @param kx  kx
      */
+    InvCoshXsq( double amp, double kx):m_amp(amp), m_kx(kx){}
     DG_DEVICE
-    T operator() (T x) const
-    {
-        return sqrt(x);
-    }
-
+    double operator()( double x)const{ return m_amp/cosh(x*m_kx)/cosh(x*m_kx);}
+    DG_DEVICE
+    double operator()( double x, double y)const{ return this->operator()(x);}
+    DG_DEVICE
+    double operator()( double x, double y, double z)const{ return this->operator()(x);}
+  private:
+    double m_amp, m_kx;
 };
-
 /**
- * @brief Minmod function
- \f$ f(x_1, x_2, x_3) = \begin{cases}
-         \min(x_1, x_2, x_3) \text{ for } x_1, x_2, x_3 >0 \\
-         \max(x_1, x_2, x_3) \text{ for } x_1, x_2, x_3 <0 \\
-         0 \text{ else}
- \end{cases}
- \f$
- *
- * might be useful for flux limiter schemes
- * @tparam T value-type
+ * @brief
+ * \f$ f(x) = f(x,y) = f(x,y,z) = B + A(1 - \sin(k_xx )) \f$
  */
-template < class T>
-struct MinMod
+struct SinProfX
 {
     /**
-     * @brief Minmod of three numbers
-     *
-     * @param a1 a1
-     * @param a2 a2
-     * @param a3 a3
+     * @brief Construct with two coefficients
      *
-     * @return minmod(a1, a2, a3)
+     * @param amp amplitude A
+     * @param bamp backgroundamp B
+     * @param kx  kx
      */
+    SinProfX( double amp, double bamp, double kx):m_amp(amp), m_bamp(bamp),m_kx(kx){}
     DG_DEVICE
-    T operator() ( T a1, T a2, T a3)const
-    {
-        if( a1*a2 > 0)
-            if( a1*a3 > 0)
-            {
-                if( a1 > 0)
-                    return min( a1, a2, a3, +1.);
-                else
-                    return min( a1, a2, a3, -1.);
-            }
-        return 0.;
-
-
-    }
-    private:
-    T min( T a1, T a2, T a3, T sign)const
-    {
-        T temp = sign*a1;
-        if( sign*a2 < temp)
-            temp = sign*a2;
-        if( sign*a3 < temp)
-            temp = sign*a3;
-        return sign*temp;
-
-    }
+    double operator()( double x)const{ return m_bamp+m_amp*(1.-sin(x*m_kx));}
+    DG_DEVICE
+    double operator()( double x, double y)const{ return this->operator()(x);}
+    DG_DEVICE
+    double operator()( double x, double y, double z)const{ return this->operator()(x);}
+  private:
+    double m_amp, m_bamp, m_kx;
 };
-
 /**
  * @brief
- * \f$ f(x) = x + c\f$
- *
- * Add a constant value
- * @tparam T value type
+ * \f$ f(x) = f(x,y) = f(x,y,z) = B + A\exp(-x/L_n) \f$
  */
-template <class T = double>
-struct PLUS
+struct ExpProfX
 {
     /**
-     * @brief Construct
-     *
-     * @param value to be added
-     */
-    PLUS( T value): x_(value){}
-    /**
-     * @brief Add a constant value
-     *
-     * @param x  the input
+     * @brief Construct with two coefficients
      *
-     * @return  x + value
+     * @param amp amplitude B
+     * @param bamp backgroundamp A (choose zero for constant gradient length
+     * @param ln  ln (must be !=0)
      */
+    ExpProfX( double amp, double bamp, double ln):m_amp(amp),m_bamp(bamp),m_ln(ln){
+        assert( ln!=0 && "ln parameter must be != 0 in ExpProfX!");
+    }
     DG_DEVICE
-    T operator()( T x)const{ return x + x_;}
-    private:
-    T x_;
+    double operator()( double x)const{ return m_bamp+m_amp*exp(-x/m_ln);}
+    DG_DEVICE
+    double operator()( double x, double y)const{ return this->operator()(x);}
+    DG_DEVICE
+    double operator()( double x, double y, double z)const{ return this->operator()(x);}
+  private:
+    double m_amp, m_bamp, m_ln;
 };
 
+
 /**
  * @brief
- * \f$ f(x) = 1/x \f$
- *
- * %Invert the given value
- * @tparam T value type
+     \f$ f(\psi) = \begin{cases}
+ 1 \text{ if } \psi < \psi_{\max}\\
+ 0 \text{ if } \psi > (\psi_{\max} + 4\alpha) \\
+ \exp\left( - \frac{(\psi - \psi_{\max})^2}{2\alpha^2}\right), \text{ else}
+ \end{cases}
+   \f$
+
+   One up to \c psimax, then a %Gaussian down to zero
  */
-template <class T = double>
-struct INVERT
+struct GaussianDamping
 {
-    /**
-     * @brief Invert the given value
-     *
-     * @param x  the input
-     * @return  1/x
-     */
+    GaussianDamping( double psimax, double alpha):
+        m_psimax(psimax), m_alpha(alpha) {
+            assert( alpha!= 0 && "Damping width in GaussianDamping must not be zero");
+        }
     DG_DEVICE
-    T operator()( T x)const{ return 1./x;}
+    double operator()(double psi)const
+    {
+        if( psi > m_psimax + 4.*m_alpha) return 0.;
+        if( psi < m_psimax) return 1.;
+        return exp( -( psi-m_psimax)*( psi-m_psimax)/2./m_alpha/m_alpha);
+    }
+    private:
+    double m_psimax, m_alpha;
 };
-
 /**
- * @brief returns (positive) modulo
- * \f$ f(x) = x\mod m\f$
+ * @brief
+ * \f$ f(x) = B + 0.5 A(1+ \text{sign} \tanh((x-x_b)/\alpha ) ) \f$
  *
- * @tparam T value type
+ * An approximation to Heaviside using tanh
  */
-template <class T= double>
-struct MOD
-{
+struct TanhProfX {
     /**
-     * @brief Construct from modulo
+     * @brief Construct with xb, width and sign
      *
-     * @param m modulo basis
+     * @param xb boundary value
+     * @param width damping width \c alpha (must be !=0)
+     * @param sign sign of the Tanh, defines the damping direction
+     * @param bgamp background amplitude \c B
+     * @param profamp profile amplitude \c A
+     * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
      */
-    MOD( T m): x_(m){}
+    TanhProfX(double xb, double width, int sign =1,double bgamp = 0.,
+        double profamp = 1.) :
+        xb_(xb),w_(width), s_(sign),bga_(bgamp),profa_(profamp)  {
+            assert( width != 0&& "Width in TanhProfX must not be zero!");
+    }
+    DG_DEVICE
+    double operator() (double x)const
+    {
+        return profa_*0.5*(1.+s_*tanh((x-xb_)/w_))+bga_;
+    }
+    DG_DEVICE
+    double operator()( double x, double y)const{ return this->operator()(x);}
+    DG_DEVICE
+    double operator()( double x, double y, double z)const{ return this->operator()(x);}
+    private:
+    double xb_;
+    double w_;
+    int s_;
+    double bga_;
+    double profa_;
+};
+
+/**
+ * @brief \f$ f(x) = \begin{cases}
+     0 \text{ if } x < x_b-a \\
+        ((16 a^3 - 29 a^2 (x - x_b) + 20 a (x - x_b)^2 - 5 (x - x_b)^3) (a + x -
+   x_b)^4)/(32 a^7) \text{ if } |x-x_b| < a \\
+        1  \text{ if } x > x_b + a
+     \end{cases}\f$
 
+ An approximation to Heaviside using polynomials.
+     This function is 3 times continuously differentiable, takes the value 0.5 at xb and
+     has a transition width a on both sides of xb.
+ */
+struct PolynomialHeaviside {
     /**
-     * @brief Compute mod(x, value), positively defined
-     *
-     * @param x
+     * @brief Construct with xb, width and sign
      *
-     * @return
+     * @param xb boundary value
+     * @param a transition width (must be != 0)
+     * @param sign either +1 (original Heaviside) or -1 (the function is mirrored at the \c x=xb axis: f(2xb-x))
+     * @note When sign is positive the function leaves the positive and damps the negative and vice versa when sign is negative the function leaves the negative and damps the positive.
      */
+    PolynomialHeaviside(double xb, double a, int sign = +1) :
+        x0(xb), a(a), m_s(sign){
+            assert( a!=0 && "PolynomialHeaviside width must not be zero");
+    }
     DG_DEVICE
-    T operator()( T x)const{
-        return (fmod(x,x_) < 0 ) ? (x_ + fmod(x,x_)) : fmod(x,x_);
+    double operator() (double x)const
+    {
+        if( m_s == -1) x = 2*x0-x; //mirror
+        if ( x < x0-a) return 0;
+        if ( x > x0+a) return 1;
+        return ((16.*a*a*a - 29.*a*a*(x - x0)
+               + 20.*a*(x - x0)*(x - x0)
+               - 5.*(x - x0)*(x-x0)*(x-x0))
+               *(a + x - x0)*(a + x - x0)
+               *(a + x - x0)*(a + x - x0))/(32.*a*a*a * a*a*a*a);
     }
     private:
-    T x_;
-
+    double x0, a;
+    int m_s;
 };
 /**
- * @brief absolute value
- * \f$ f(x) = |x|\f$
- *
- * @tparam T value type
+ * @brief
+     \f$ f(x) = \begin{cases}
+     0 \text{ if } x < x_l-a_l \\
+        ((16 a_l^3 - 29 a_l^2 (x - x_l) + 20 a_l (x - x_l)^2 - 5 (x - x_l)^3) (a_l + x -
+   x_l)^4)/(32 a_l^7) \text{ if } |x-x_l| < a_l \\
+        1  \text{ if } x_l + a_l < x < x_r-a_r \\
+        ((16 a_r^3 - 29 a_r^2 (x - x_r) + 20 a_r (x - x_r)^2 - 5 (x - x_r)^3) (a_r + x -
+   x_l)^4)/(32 a_r^7) \text{ if } |x-x_r| < a_r \\
+   0 \text{ if } x > x_r + a_r
+     \end{cases}\f$
+
+ An approximation to the Rectangle function using polynomials
+     Basically just the product of two PolynomialHeaviside functions
+
+     This function is 3 times continuously differentiable, takes the value 0.5 at xl and xr and
+     has a transition width a_l on both sides of xl and a width a_r on both sides of xr.
  */
-template <class T = double>
-struct ABS
-{
+struct PolynomialRectangle {
     /**
-     * @brief The absolute value
-     *
-     * @param x of x
+     * @brief Construct with xb, width and sign
      *
-     * @return  abs(x)
+     * @param xl left boundary value
+     * @param al left transition width (must be != 0)
+     * @param xr right boundary value
+     * @param ar right transition width (must be != 0)
      */
+    PolynomialRectangle(double xl, double al, double xr, double ar) :
+        m_hl( xl, al, +1), m_hr( xr, ar, -1) {
+        assert( xl < xr && "left boundary must be left of right boundary");
+    }
     DG_DEVICE
-    T operator()(T x)const{ return fabs(x);}
+    double operator() (double x)const
+    {
+        return m_hl(x)*m_hr(x);
+    }
+    private:
+    PolynomialHeaviside m_hl, m_hr;
 };
+
 /**
- * @brief returns positive values
- \f$ f(x) = \begin{cases}
-         x \text{ for } x>0 \\
-         0 \text{ else}
- \end{cases}
- \f$
- *
- * @tparam T value type
+ * @brief \f$ f(x) = \begin{cases}
+     x_b \text{ if } x < x_b-a \\
+     x_b + ((35 a^3 - 47 a^2 (x - x_b) + 25 a (x - x_b)^2 - 5 (x - x_b)^3) (a + x - x_b)^5)/(256 a^7)
+        \text{ if } |x-x_b| < a \\
+        x  \text{ if } x > x_b + a
+     \end{cases}\f$
+ The integral of PolynomialHeaviside approximates xH(x)
+
+     This function is 4 times continuously differentiable,
+     has a transition width \c a on both sides of \c xb, where it transitions from the
+     constant \c xb to the linear function \c x.
  */
-template <class T = double>
-struct POSVALUE
-{
+struct IPolynomialHeaviside {
     /**
-     * @brief Returns positive values of x
-     *
-     * @param x of x
+     * @brief Construct with xb, width and sign
      *
-     * @return  x*0.5*(1+sign(x))
+     * @param xb boundary value
+     * @param a transition width (must be != 0)
+     * @param sign either +1 (original) or -1 (the function is point mirrored at \c x=xb: 2*xb-f(2xb-x))
      */
+    IPolynomialHeaviside(double xb, double a, int sign = +1) :
+        x0(xb), a(a), m_s(sign){
+            assert( a!=0 && "IPolynomialHeaviside width must not be zero");
+        }
     DG_DEVICE
-    T operator()( T x)const{
-        if (x >= 0.0) return x;
-        return 0.0;
+    double operator() (double x)const
+    {
+        if( m_s == -1) x = 2*x0-x; //mirror
+        double result;
+        if ( x < x0-a) result =  x0;
+        else if ( x > x0+a) result =  x;
+        else
+            result =  x0 + ((35.* a*a*a - 47.* a*a*(x - x0) + 25.*a*(x - x0)*(x-x0)
+                - 5.*(x - x0)*(x-x0)*(x-x0))
+                *(a+x-x0)*(a+x-x0)*(a+x-x0)*(a+x-x0)*(a+x-x0))
+            /(256.*a*a*a * a*a*a*a);
+        if ( m_s == +1) return result;
+        return 2*x0 - result;
+
     }
+    private:
+    double x0, a;
+    int m_s;
 };
 
 /**
- * @brief \f$ f(x) = c\f$
+ * @brief \f$ f(x) = \begin{cases}
+     0 \text{ if } x < x_b-a || x > x_b+a \\
+     (35 (a + x - x_b)^3 (a - x + x_b)^3)/(32 a^7)
+        \text{ if } |x-x_b| < a
+     \end{cases}\f$
+     The derivative of PolynomialHeaviside approximates delta(x)
+
+     This function is 2 times continuously differentiable, is symmetric around \c xb
+     and has a width \c a on both sides of \c x0.
+     The integral over this function yields 1.
  */
-struct CONSTANT
-{
+struct DPolynomialHeaviside {
     /**
-     * @brief Construct with a value
+     * @brief Construct with xb, width and sign
      *
-     * @param cte the constant value
+     * @param xb boundary value
+     * @param a transition width ( must be !=0)
+     * @param sign either +1 (original) or -1 (the function is mirrored at \c x=x0)
+     * (since this function is symmetric this parameter is ignored, it's there to be
+     * consistent with PolynomialHeaviside)
      */
-    CONSTANT( double cte): m_value(cte){}
-
-    DG_DEVICE
-    double operator()(double x)const{return m_value;}
-    DG_DEVICE
-    double operator()(double x, double y)const{return m_value;}
+    DPolynomialHeaviside(double xb, double a, int sign = +1) :
+        x0(xb), a(a){
+            assert( a!=0 && "DPolynomialHeaviside width must not be zero");
+    }
     DG_DEVICE
-    double operator()(double x, double y, double z)const{return m_value;}
+    double operator() (double x)const
+    {
+        if ( (x < x0-a) || (x > x0+a)) return 0;
+        return (35.*(a+x-x0)*(a+x-x0)*(a+x-x0)*(a-x+x0)*(a-x+x0)*(a-x+x0))
+            /(32.*a*a*a * a*a*a*a);
+    }
     private:
-    double m_value;
+    double x0, a;
 };
 
 /**
- * @brief \f$ f(x) = 1\f$
- */
-struct ONE
-{
-    DG_DEVICE
-    double operator()(double x)const{return 1.;}
-    DG_DEVICE
-    double operator()(double x, double y)const{return 1.;}
-    DG_DEVICE
-    double operator()(double x, double y, double z)const{return 1.;}
-};
-/**
- * @brief \f$ f(x) = 0\f$ Return 0
+ * @brief \f$ f(i) = \begin{cases}
+    1 \text{ if } \eta < \eta_c \\
+    \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
+    0 \text{ else} \\
+    \eta=\frac{i}{1-n}
+    \end{cases}\f$
+
+    where n is the number of polynomial coefficients
+
+    This function is s times continuously differentiable everywhere
+    @sa Its main use comes from the application in dg::ModalFilter
  */
-struct ZERO
+struct ExponentialFilter
 {
-    DG_DEVICE
-    double operator()(double x)const{return 0.;}
-    DG_DEVICE
-    double operator()(double x, double y)const{return 0.;}
-    DG_DEVICE
-    double operator()(double x, double y, double z)const{return 0.;}
+    /**
+     * @brief Create exponential filter \f$ \begin{cases}
+    1 \text{ if } \eta < \eta_c \\
+    \exp\left( -\alpha  \left(\frac{\eta-\eta_c}{1-\eta_c} \right)^{2s}\right) \text { if } \eta \geq \eta_c \\
+    0 \text{ else} \\
+    \eta := \frac{i}{n-1}
+    \end{cases}\f$
+     *
+     * @param alpha damping for the highest mode is \c exp( -alpha)
+     * @param eta_c cutoff frequency (0<eta_c<1), 0.5 or 0 are good starting values
+     * @param order 8 or 16 are good values
+     * @param n The number of polynomial coefficients
+     */
+    ExponentialFilter( double alpha, double eta_c, unsigned order, unsigned n):
+        m_alpha(alpha), m_etac(eta_c), m_s(order), m_n(n) {}
+    double operator()( unsigned i) const
+    {
+        double eta = (double)i/(double)(m_n-1);
+        if( m_n == 1) eta = 0.;
+        if( eta < m_etac)
+            return 1.;
+        if( eta <= 1.)
+            return exp( -m_alpha*pow( (eta-m_etac)/(1.-m_etac), 2*m_s));
+        return 0;
+    }
+    private:
+    double m_alpha, m_etac;
+    unsigned m_s, m_n;
 };
 
 /**
- * @brief Functor returning a Lamb dipole
+ * @brief
  \f$ f(x,y) = \begin{cases} 2\lambda U J_1(\lambda r) / J_0(\gamma)\cos(\theta) \text{ for } r<R \\
          0 \text{ else}
          \end{cases}
@@ -1459,6 +1439,7 @@ struct ZERO
  \f$J_0, J_1\f$ are
  Bessel functions of the first kind of order 0 and 1 and
  \f$\lambda = \gamma/R\f$ with \f$ \gamma = 3.83170597020751231561\f$
+ This is the Lamb dipole
  */
 struct Lamb
 {
@@ -1837,6 +1818,44 @@ struct BathRZ{
     std::vector<double> unif1, unif2, normal1,normal2,alpha,theta;
 };
 
+/**
+ * @brief \f$ f(x,y) = \sum_{i=0}^{M-1} \sum_{j=0}^{N-1} c_{iN+j} x^i y^j  \f$
+ *
+ * Evaluated using [Horner's method](https://en.wikipedia.org/wiki/Horner%27s_method)
+ */
+struct Horner2d
+{
+    ///Initialize 1 coefficient to 1
+    Horner2d(): m_c( 1, 1), m_M(1), m_N(1){}
+
+    /**
+     * @brief Initialize coefficients and dimensions
+     *
+     * @param c vector of size MN containing coefficientc c (accessed as c[i*N+j] i.e. y-direction is contiguous)
+     * @param M number of polynomials in x
+     * @param N number of polynomials in y
+     */
+    Horner2d( std::vector<double> c, unsigned M, unsigned N): m_c(c), m_M(M), m_N(N){}
+    double operator()( double x, double y) const
+    {
+        std::vector<double> cx( m_M);
+        for( unsigned i=0; i<m_M; i++)
+            cx[i] = horner( &m_c[i*m_N], m_N, y);
+        return horner( &cx[0], m_M, x);
+    }
+    private:
+    double horner( const double * c, unsigned M, double x) const
+    {
+        double b = c[M-1];
+        for( unsigned i=0; i<M-1; i++)
+            b = c[M-2-i] + b*x;
+        return b;
+    }
+    std::vector<double> m_c;
+    unsigned m_M, m_N;
+};
+
+
 /**
  * @brief Compute a histogram on a 1D grid
  * @tparam container
@@ -1963,19 +1982,6 @@ struct Histogram2D
 };
 
 
-/**
- * @brief Check for NaN: \f$ f(x) = std::isnan(x) \f$
- */
-template <class T>
-struct ISNAN
-{
-#ifdef __CUDACC__
-    DG_DEVICE bool operator()(T x){return isnan(x);}
-#else
-    bool operator()( T x){ return std::isnan(x);}
-#endif
-};
-
 
 /**
  * @brief Shortest Distance to a collection of vertical and horizontal lines
@@ -2009,60 +2015,8 @@ struct WallDistance
     std::vector<double> m_vertical;
     std::vector<double> m_horizontal;
 };
-///@cond
-namespace detail
-{
-template<class F, class G>
-struct Compose
-{
-    Compose( F f, G g):m_f(f), m_g(g){}
-    template<class ...Xs>
-    auto operator() ( Xs&& ... xs){
-        return m_f(m_g(std::forward<Xs>(xs)...));
-    }
-    template<class ...Xs>
-    auto operator() ( Xs&& ... xs) const {
-        return m_f(m_g(std::forward<Xs>(xs)...));
-    }
-    private:
-    F m_f;
-    G m_g;
-};
-}//namespace detail
-///@endcond
 
-/**
- * @brief Function composition \f$ f(g(x_0,x_1,...)) \f$
- *
- * @code{.cpp}
- * dg::Grid2d grid2d( -1., 1., -1., 1., 3, 40, 40);
- * //Mark everything above 2 with 1s and below with 0s
- * dg::HVec fg = dg::evaluate( dg::compose( dg::Heaviside( 2.), dg::Gaussian( 0., 0., 2., 2., 4.)), grid2d);
- * @endcode
- * @tparam F Outer functor
- * @tparam G Inner functor
- * @param f outer functor
- * @param g inner functor
- * @note only works for host functions. The rationale is that this function is
- * intended to work with lambda functions and is to be used in the \c dg::evaluate function.
- * If a version for device functions is ever needed
- * it can be easily provided but the lambda support for CUDA is rather poor.
- *
- * @return a function object that forwards all parameters to g and returns the
- * return value of f, which is \f$ f(g(x_0,x_1,...)) \f$
- */
-template <class F, class G>
-auto compose( F f, G g) {
-    return detail::Compose<F,G>( f, g);
-    //a C++-14 way of generating a generic lambda with a parameter pack. Taken from:
-    //https://stackoverflow.com/questions/19071268/function-composition-in-c-c11
-    //return [f,g](auto&&... xs){ return f(g(std::forward<decltype(xs)>(xs)...));};
-}
-///Composition of an arbitrary number of functions \f$ f_0(f_1(f_2( ... (x_0, x_1, ...)))\f$
-template <class F, typename... Fs>
-auto compose(F f, Fs... fs) {
-    return compose( f , compose(fs...));
-}
+
 ///@}
 } //namespace dg
 
diff --git a/inc/dg/subroutines.h b/inc/dg/subroutines.h
index 47ed17692..a1dd79552 100644
--- a/inc/dg/subroutines.h
+++ b/inc/dg/subroutines.h
@@ -4,7 +4,7 @@
 
 namespace dg{
 
-///@addtogroup functions
+///@addtogroup binary_operators
 ///@{
 
 ///\f$ x=y\f$
@@ -52,8 +52,12 @@ DG_DEVICE void operator()( T1& out, T2 in) const
         out /= in;
     }
 };
+///@}
+
+///@addtogroup variadic_evaluates
+///@{
 
-///@brief \f[ \sum_i x_i \f]
+///@brief \f$ y = \sum_i x_i \f$
 struct Sum
 {
     ///@brief \f[ \sum_i x_i \f]
@@ -79,15 +83,15 @@ DG_DEVICE void sum( T& tmp, T x) const
     }
 };
 
-///@brief \f[ \sum_i \alpha_i x_i \f]
+///@brief \f$ y = \sum_i a_i x_i \f$
 struct PairSum
 {
-    ///@brief \f[ \sum_i \alpha_i x_i \f]
+    ///@brief \f[ \sum_i a_i x_i \f]
     template< class T, class ...Ts>
-DG_DEVICE T operator()( T alpha, T x, Ts... rest) const
+DG_DEVICE T operator()( T a, T x, Ts... rest) const
     {
         T tmp = T{0};
-        sum( tmp, alpha, x, rest...);
+        sum( tmp, a, x, rest...);
         return tmp;
     }
     private:
@@ -104,7 +108,38 @@ DG_DEVICE void sum( T& tmp, T alpha, T x) const
         tmp = DG_FMA(alpha, x, tmp);
     }
 };
-///@brief \f[ y = \sum_i a_i x_i,\quad \tilde y = \sum_i \tilde a_i x_i \f]
+///@brief \f$ y = \sum_i a_i x_i y_i \f$
+struct TripletSum
+{
+    ///@brief \f[ \sum_i \alpha_i x_i y_i \f]
+    template< class T1, class ...Ts>
+DG_DEVICE T1 operator()( T1 a, T1 x1, T1 y1, Ts... rest) const
+    {
+        T1 tmp = T1{0};
+        sum( tmp, a, x1, y1, rest...);
+        return tmp;
+    }
+    private:
+    template<class T, class ...Ts>
+DG_DEVICE void sum( T& tmp, T alpha, T x, T y, Ts... rest) const
+    {
+        tmp = DG_FMA( alpha*x, y, tmp);
+        sum( tmp, rest...);
+    }
+
+    template<class T>
+DG_DEVICE void sum( T& tmp, T alpha, T x, T y) const
+    {
+        tmp = DG_FMA(alpha*x, y, tmp);
+    }
+};
+
+///@}
+
+///@addtogroup variadic_subroutines
+///@{
+
+///@brief \f$ y = \sum_i a_i x_i,\quad \tilde y = \sum_i \tilde a_i x_i \f$
 struct EmbeddedPairSum
 {
     ///@brief \f[ \sum_i \alpha_i x_i \f]
@@ -131,34 +166,8 @@ DG_DEVICE void sum( T1& y_1, T1& yt_1, T1 b, T1 bt, T1 k) const
         yt_1 = DG_FMA( bt, k, yt_1);
     }
 };
-///@brief \f[ \sum_i \alpha_i x_i y_i \f]
-struct TripletSum
-{
-    ///@brief \f[ \sum_i \alpha_i x_i y_i \f]
-    template< class T1, class ...Ts>
-DG_DEVICE T1 operator()( T1 alpha, T1 x1, T1 y1, Ts... rest) const
-    {
-        T1 tmp = T1{0};
-        sum( tmp, alpha, x1, y1, rest...);
-        return tmp;
-    }
-    private:
-    template<class T, class ...Ts>
-DG_DEVICE void sum( T& tmp, T alpha, T x, T y, Ts... rest) const
-    {
-        tmp = DG_FMA( alpha*x, y, tmp);
-        sum( tmp, rest...);
-    }
-
-    template<class T>
-DG_DEVICE void sum( T& tmp, T alpha, T x, T y) const
-    {
-        tmp = DG_FMA(alpha*x, y, tmp);
-    }
-};
 
-///@}
-///@cond
+/// \f$ f( y, g(x_0, ..., x_s)) \f$
 template<class BinarySub, class Functor>
 struct Evaluate
 {
@@ -174,6 +183,7 @@ DG_DEVICE void operator() ( T& y, Ts... xs){
     Functor m_g;
 };
 
+/// \f$ y\leftarrow ay \f$
 template<class T>
 struct Scal
 {
@@ -186,6 +196,7 @@ DG_DEVICE
     T m_a;
 };
 
+/// \f$ y\leftarrow y+a \f$
 template<class T>
 struct Plus
 {
@@ -198,6 +209,7 @@ DG_DEVICE
     T m_a;
 };
 
+/// \f$ y\leftarrow ax+by \f$
 template<class T>
 struct Axpby
 {
@@ -211,6 +223,7 @@ DG_DEVICE
     T m_a, m_b;
 };
 
+/// \f$ z\leftarrow ax+by+gz \f$
 template<class T>
 struct Axpbypgz
 {
@@ -226,6 +239,7 @@ DG_DEVICE
     T m_a, m_b, m_g;
 };
 
+/// \f$ z\leftarrow ax_1y_1+bx_2y_2+gz \f$
 template<class T>
 struct PointwiseDot
 {
@@ -256,6 +270,7 @@ DG_DEVICE
     T m_a, m_b, m_g;
 };
 
+/// \f$ z\leftarrow ax/y + bz \f$
 template<class T>
 struct PointwiseDivide
 {
@@ -273,7 +288,68 @@ DG_DEVICE
     private:
     T m_a, m_b;
 };
+///@}
+
+///@cond
+namespace detail
+{
+template<class F, class G>
+struct Compose
+{
+    Compose( F f, G g):m_f(f), m_g(g){}
+    template<class ...Xs>
+    auto operator() ( Xs&& ... xs){
+        return m_f(m_g(std::forward<Xs>(xs)...));
+    }
+    template<class ...Xs>
+    auto operator() ( Xs&& ... xs) const {
+        return m_f(m_g(std::forward<Xs>(xs)...));
+    }
+    private:
+    F m_f;
+    G m_g;
+};
+}//namespace detail
 ///@endcond
 
+///@addtogroup composition
+///@{
+/**
+ * @brief Create Composition functor \f$ f(g(x_0,x_1,...)) \f$
+ *
+ * @code{.cpp}
+ * dg::Grid2d grid2d( -1., 1., -1., 1., 3, 40, 40);
+ * //Mark everything above 2 with 1s and below with 0s
+ * dg::HVec fg = dg::evaluate( dg::compose( dg::Heaviside( 2.), dg::Gaussian( 0., 0., 2., 2., 4.)), grid2d);
+ * @endcode
+ * @tparam UnaryOp Model of Unary Function taking the return type of g \c return_type_f \c f(return_type_g)
+ * @tparam Functor Inner functor, takes an arbitrary number of parameters and returns one \c return_type_g \c g(value_type0,value_type1,...)
+ * @param f outer functor
+ * @param g inner functor
+ * @attention only works for host functions. The rationale is that this function is
+ * intended to work with lambda functions and is to be used in the \c dg::evaluate function.
+ * If a version for device functions is ever needed
+ * it can be easily provided but the lambda support for CUDA is rather poor.
+ *
+ * @return a function object that forwards all parameters to g and returns the
+ * return value of f, which is \f$ f(g(x_0,x_1,...)) \f$
+ */
+template <class UnaryOp, class Functor>
+auto compose( UnaryOp f, Functor g) {
+    return detail::Compose<UnaryOp,Functor>( f, g);
+    //a C++-14 way of generating a generic lambda with a parameter pack. Taken from:
+    //https://stackoverflow.com/questions/19071268/function-composition-in-c-c11
+    //return [f,g](auto&&... xs){ return f(g(std::forward<decltype(xs)>(xs)...));};
+}
+/**@brief Create Composition funtor of an arbitrary number of functions \f$ f_0(f_1(f_2( ... f_s(x_0, x_1, ...)))\f$
+ *
+ * @tparam UnaryOp Model of Unary Function taking the return type of f_1 \c return_type_f0 \c f0(return_type_f1)
+ * @tparam Fs UnaryOps except the innermost functor, which takes an arbitrary number of parameters and returns one \c return_type_fs \c f_s(value_type0,value_type1,...)
+ */
+template <class UnaryOp, typename... Functors>
+auto compose(UnaryOp f0, Functors... fs) {
+    return compose( f0 , compose(fs...));
+}
+///@}
 
 }//namespace dg
diff --git a/inc/dg/topology/evaluation.h b/inc/dg/topology/evaluation.h
index 6624a98cd..1656f366e 100644
--- a/inc/dg/topology/evaluation.h
+++ b/inc/dg/topology/evaluation.h
@@ -55,7 +55,7 @@ thrust::host_vector<real_type> abscissas( const RealGrid1d<real_type>& g)
  * .
  * @copydoc hide_code_evaluate1d
  * @tparam UnaryOp Model of Unary Function \c real_type \c f(real_type)
- * @param f The function to evaluate
+ * @param f The function to evaluate, see @ref functions for a host of predefined functors to evaluate
  * @param g The grid that defines the computational space on which to evaluate f
  *
  * @return The output vector \c v as a host vector
@@ -92,7 +92,7 @@ thrust::host_vector<real_type> evaluate( real_type (f)(real_type), const RealGri
  *.
  * @copydoc hide_code_evaluate2d
  * @copydoc hide_binary
- * @param f The function to evaluate: f = f(x,y)
+ * @param f The function to evaluate: f = f(x,y), see @ref functions for a host of predefined functors to evaluate
  * @param g The 2d grid on which to evaluate \c f
  *
  * @return The output vector \c v as a host vector
@@ -136,7 +136,7 @@ thrust::host_vector<real_type> evaluate( real_type(f)(real_type, real_type), con
  *.
  * @copydoc hide_code_evaluate3d
  * @copydoc hide_ternary
- * @param f The function to evaluate: f = f(x,y,z)
+ * @param f The function to evaluate: f = f(x,y,z), see @ref functions for a host of predefined functors to evaluate
  * @param g The 3d grid on which to evaluate \c f
  *
  * @return The output vector \c v as a host vector
diff --git a/inc/dg/topology/functions.h b/inc/dg/topology/functions.h
index 330007061..5ea135c81 100644
--- a/inc/dg/topology/functions.h
+++ b/inc/dg/topology/functions.h
@@ -13,65 +13,65 @@
 #endif
 
 namespace dg{
-///@addtogroup functions
+///@addtogroup basics
 ///@{
 
-///@brief \f[ f(x) = 1\f]
+///@brief \f$ f(x) = 1\f$
 DG_DEVICE static inline double one( double x) {return 1;}
 
-///@brief \f[ f(x,y) = 1\f]
+///@brief \f$ f(x,y) = 1\f$
 DG_DEVICE static inline double one( double x, double y) {return 1;}
 
-///@brief \f[ f(x,y,z) = 1\f]
+///@brief \f$ f(x,y,z) = 1\f$
 DG_DEVICE static inline double one( double x, double y, double z) {return 1;}
 
-///@brief \f[ f(x) = 0\f]
+///@brief \f$ f(x) = 0\f$
 DG_DEVICE static inline double zero( double x) {return 0.;}
 
-///@brief \f[ f(x,y) = 0\f]
+///@brief \f$ f(x,y) = 0\f$
 DG_DEVICE static inline double zero( double x, double y) {return 0.;}
 
-///@brief \f[ f(x,y,z) = 0\f]
+///@brief \f$ f(x,y,z) = 0\f$
 DG_DEVICE static inline double zero( double x, double y, double z) {return 0.;}
 
-///@brief \f[ f(x) = x\f]
+///@brief \f$ f(x) = x\f$
 DG_DEVICE static inline double cooX1d( double x) {return x;}
-///@brief \f[ f(x,y) = x\f]
+///@brief \f$ f(x,y) = x\f$
 DG_DEVICE static inline double cooX2d( double x, double y) {return x;}
-///@brief \f[ f(x,y,z) = x\f]
+///@brief \f$ f(x,y,z) = x\f$
 DG_DEVICE static inline double cooX3d( double x, double y, double z) {return x;}
 
-///@brief \f[ f(x,y) = y\f]
+///@brief \f$ f(x,y) = y\f$
 DG_DEVICE static inline double cooY2d( double x, double y) {return y;}
-///@brief \f[ f(x,y,z) = y\f]
+///@brief \f$ f(x,y,z) = y\f$
 DG_DEVICE static inline double cooY3d( double x, double y, double z) {return y;}
-///@brief \f[ f(x,y,z) = z\f]
+///@brief \f$ f(x,y,z) = z\f$
 DG_DEVICE static inline double cooZ3d( double x, double y, double z) {return z;}
 
 
-///@brief \f[ R\sin(\varphi)\f]
+///@brief \f$ x = R\sin(\varphi)\f$
 DG_DEVICE static inline double cooRZP2X( double R, double Z, double P){ return R*sin(P);}
-///@brief \f[ R\cos(\varphi)\f]
+///@brief \f$ y = R\cos(\varphi)\f$
 DG_DEVICE static inline double cooRZP2Y( double R, double Z, double P){ return R*cos(P);}
-///@brief \f[ Z\f]
+///@brief \f$ z = Z\f$
 DG_DEVICE static inline double cooRZP2Z( double R, double Z, double P){ return Z;}
 
-///@brief \f[ f(x) = 1\f]
+///@brief \f$ f(x) = 1\f$
 DG_DEVICE static inline float one( float x) {return 1;}
 
-///@brief \f[ f(x,y) = 1\f]
+///@brief \f$ f(x,y) = 1\f$
 DG_DEVICE static inline float one( float x, float y) {return 1;}
 
-///@brief \f[ f(x,y,z) = 1\f]
+///@brief \f$ f(x,y,z) = 1\f$
 DG_DEVICE static inline float one( float x, float y, float z) {return 1;}
 
-///@brief \f[ f(x) = 0\f]
+///@brief \f$ f(x) = 0\f$
 DG_DEVICE static inline float zero( float x) {return 0.;}
 
-///@brief \f[ f(x,y) = 0\f]
+///@brief \f$ f(x,y) = 0\f$
 DG_DEVICE static inline float zero( float x, float y) {return 0.;}
 
-///@brief \f[ f(x,y,z) = 0\f]
+///@brief \f$ f(x,y,z) = 0\f$
 DG_DEVICE static inline float zero( float x, float y, float z) {return 0.;}
 
 } //namespace dg
-- 
GitLab


From 95e4baa1000a00be1e8adf770321f9d45c94ecb9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 5 Feb 2021 12:35:43 +0100
Subject: [PATCH 461/540] Add new dg::Upwind to shu

---
 src/lamb_dipole/shu.cuh | 50 ++++++++++++-----------------------------
 1 file changed, 14 insertions(+), 36 deletions(-)

diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index e7d756f0c..1e9e0b184 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -12,26 +12,6 @@
 namespace shu
 {
 
-struct Upwind{
-    DG_DEVICE
-    void operator()( double& result, double fw, double bw, double v)
-    {
-        if( v < 0)
-            result -= fw; // yp = - Div( F)
-        else
-            result -= bw;
-    }
-};
-struct UpwindAdvection{
-    DG_DEVICE
-    void operator()( double& result, double fw, double bw, double v)
-    {
-        if( v < 0)
-            result -= v*fw; // yp = - v Grad f
-        else
-            result -= v*bw;
-    }
-};
 template< class Geometry, class Matrix, class Container>
 struct Diffusion
 {
@@ -216,26 +196,26 @@ void Shu<Geometry, Matrix, Container>::operator()(double t, const Container& y,
         else if( "upwind" == m_advection)
         {
             dg::blas1::copy( 0., yp);
-            //dx ( nv_x)
+            // - dx ( nv_x)
             dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
             dg::blas1::pointwiseDot( y, m_v, m_temp[0]); //f_x
             dg::blas2::symv( m_forward[0], m_temp[0], m_temp[1]);
             dg::blas2::symv( m_backward[0], m_temp[0], m_temp[2]);
-            dg::blas1::subroutine( shu::Upwind(), yp, m_temp[1], m_temp[2], m_v);
-            //dy ( nv_y)
+            dg::blas1::evaluate( yp, dg::minus_equals(), dg::Upwind(), m_v, m_temp[2], m_temp[1]);
+            // - dy ( nv_y)
             dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_v); //v_y
             dg::blas1::pointwiseDot( y, m_v, m_temp[0]); //f_y
             dg::blas2::symv( m_forward[1], m_temp[0], m_temp[1]);
             dg::blas2::symv( m_backward[1], m_temp[0], m_temp[2]);
-            dg::blas1::subroutine( shu::Upwind(), yp, m_temp[1], m_temp[2], m_v);
+            dg::blas1::evaluate( yp, dg::minus_equals(), dg::Upwind(), m_v, m_temp[2], m_temp[1]);
         }
         else if( "centered" == m_advection)
         {
-            //dx ( nv_x)
+            // - dx ( nv_x)
             dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
             dg::blas1::pointwiseDot( y, m_v, m_temp[0]); //f_x
             dg::blas2::symv( -1., m_centered[0], m_temp[0], 0., yp);
-            //dy ( nv_y)
+            // - dy ( nv_y)
             dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_v); //v_y
             dg::blas1::pointwiseDot( y, m_v, m_temp[0]); //f_y
             dg::blas2::symv( -1., m_centered[1], m_temp[0], 1., yp);
@@ -243,16 +223,16 @@ void Shu<Geometry, Matrix, Container>::operator()(double t, const Container& y,
         else if( "upwind-advection" == m_advection)
         {
             dg::blas1::copy( 0., yp);
-            // v_x dx n
+            //  - v_x dx n
             dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
             dg::blas2::symv( m_forward[0], y, m_temp[1]);
             dg::blas2::symv( m_backward[0], y, m_temp[2]);
-            dg::blas1::subroutine( shu::UpwindAdvection(), yp, m_temp[1], m_temp[2], m_v);
-            // v_y dy n
+            dg::blas1::evaluate( yp, dg::minus_equals(), dg::UpwindProduct(), m_v, m_temp[2], m_temp[1]);
+            //  - v_y dy n
             dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_v); //v_y
             dg::blas2::symv( m_forward[1], y, m_temp[1]);
             dg::blas2::symv( m_backward[1], y, m_temp[2]);
-            dg::blas1::subroutine( shu::UpwindAdvection(), yp, m_temp[1], m_temp[2], m_v);
+            dg::blas1::evaluate( yp, dg::minus_equals(), dg::UpwindProduct(), m_v, m_temp[2], m_temp[1]);
         }
         else if( "centered-advection" == m_advection)
         {
@@ -284,13 +264,13 @@ void Shu<Geometry, Matrix, Container>::operator()(double t, const Container& y,
             dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_x
             dg::blas2::symv( m_fine_forward[0], m_fine_temp[0], m_fine_temp[1]);
             dg::blas2::symv( m_fine_backward[0], m_fine_temp[0], m_fine_temp[2]);
-            dg::blas1::subroutine( shu::Upwind(), m_fine_yp, m_fine_temp[1], m_fine_temp[2], m_fine_v);
+            dg::blas1::evaluate( m_fine_yp, dg::minus_equals(), dg::Upwind(), m_fine_v, m_fine_temp[2], m_fine_temp[1]);
             //dy ( nv_y)
             dg::blas2::symv( 1., m_fine_centered[0], m_fine_psi, 0., m_fine_v); //v_y
             dg::blas1::pointwiseDot( m_fine_y, m_fine_v, m_fine_temp[0]); //f_y
             dg::blas2::symv( m_fine_forward[1], m_fine_temp[0], m_fine_temp[1]);
             dg::blas2::symv( m_fine_backward[1], m_fine_temp[0], m_fine_temp[2]);
-            dg::blas1::subroutine( shu::Upwind(), m_fine_yp, m_fine_temp[1], m_fine_temp[2], m_fine_v);
+            dg::blas1::evaluate( m_fine_yp, dg::minus_equals(), dg::Upwind(), m_fine_v, m_fine_temp[2], m_fine_temp[1]);
         }
         else if( "centered" == m_advection)
         {
@@ -315,8 +295,7 @@ void Shu<Geometry, Matrix, Container>::operator()(double t, const Container& y,
             dg::blas2::symv( m_inter, m_v, m_fine_v);
             dg::blas2::symv( m_inter, m_temp[1], m_fine_temp[1]);
             dg::blas2::symv( m_inter, m_temp[2], m_fine_temp[2]);
-            dg::blas1::subroutine( shu::UpwindAdvection(), m_fine_yp,
-                    m_fine_temp[1], m_fine_temp[2], m_fine_v);
+            dg::blas1::evaluate( m_fine_yp, dg::minus_equals(), dg::UpwindProduct(), m_fine_v, m_fine_temp[2], m_fine_temp[1]);
             // v_y dy n
             dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_v); //v_y
             dg::blas2::symv( m_forward[1], y, m_temp[1]);
@@ -324,8 +303,7 @@ void Shu<Geometry, Matrix, Container>::operator()(double t, const Container& y,
             dg::blas2::symv( m_inter, m_temp[1], m_fine_temp[1]);
             dg::blas2::symv( m_inter, m_temp[2], m_fine_temp[2]);
             dg::blas2::symv( m_inter, m_v, m_fine_v);
-            dg::blas1::subroutine( shu::UpwindAdvection(), m_fine_yp,
-                    m_fine_temp[1], m_fine_temp[2], m_fine_v);
+            dg::blas1::evaluate( m_fine_yp, dg::minus_equals(), dg::UpwindProduct(), m_fine_v, m_fine_temp[2], m_fine_temp[1]);
         }
         else if( "centered-advection" == m_advection)
         {
-- 
GitLab


From ea62d837e6ed0469b40d87855e559980be2d66e5 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 5 Feb 2021 15:59:35 +0100
Subject: [PATCH 462/540] Move n parameter in fast_* to first argument

In order to make it consistent with the grid constructor
---
 inc/dg/blas_b.cu                        |  4 +-
 inc/dg/blas_mpib.cu                     |  4 +-
 inc/dg/multigrid.h                      |  6 +--
 inc/dg/topology/ell_interpolation_b.cu  |  2 +-
 inc/dg/topology/fast_interpolation.h    | 60 ++++++++++++-------------
 inc/dg/topology/projection_mpit.cu      |  6 +--
 inc/dg/topology/projection_t.cu         | 12 ++---
 inc/geometries/conformalX_elliptic_b.cu |  2 +-
 src/feltor/feltor_hpc.cu                |  4 +-
 src/lamb_dipole/shu.cuh                 |  7 +--
 10 files changed, 53 insertions(+), 54 deletions(-)

diff --git a/inc/dg/blas_b.cu b/inc/dg/blas_b.cu
index 6f81ba11f..5f9962677 100644
--- a/inc/dg/blas_b.cu
+++ b/inc/dg/blas_b.cu
@@ -76,8 +76,8 @@ int main()
     value_type gbytes=(value_type)x.size()*x[0].size()*sizeof(value_type)/1e9;
     std::cout << "Size of vectors is "<<gbytes<<" GB\n";
     dg::MultiMatrix<Matrix, ArrayVec> inter, project;
-    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2,1), inter);
-    dg::blas2::transfer(dg::create::fast_projection( grid, 2,2,1), project);
+    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 1,2,2), inter);
+    dg::blas2::transfer(dg::create::fast_projection( grid, 1,2,2), project);
     //dg::IDMatrix inter = dg::create::interpolation( grid, grid_half);
     //dg::IDMatrix project = dg::create::projection( grid_half, grid);
     int multi=100;
diff --git a/inc/dg/blas_mpib.cu b/inc/dg/blas_mpib.cu
index 2c6f610a4..16a8000ac 100644
--- a/inc/dg/blas_mpib.cu
+++ b/inc/dg/blas_mpib.cu
@@ -84,8 +84,8 @@ int main( int argc, char* argv[])
     value_type gbytes=(value_type)x.size()*grid.size()*sizeof(value_type)/1e9;
     if(rank==0)std::cout << "Sizeof vectors is "<<gbytes<<" GB\n";
     dg::MultiMatrix<Matrix, ArrayVec> inter, project;
-    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 2,2,1), inter);
-    dg::blas2::transfer(dg::create::fast_projection( grid, 2,2,1), project);
+    dg::blas2::transfer(dg::create::fast_interpolation( grid_half, 1,2,2), inter);
+    dg::blas2::transfer(dg::create::fast_projection( grid, 1,2,2), project);
 
     int multi=100;
     if(rank==0)std::cout<<"\nNo communication\n";
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 155ca9f8d..565dfa2df 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -91,9 +91,9 @@ struct MultigridCG2d
         {
             // Projecting from one grid to the next is the same as
             // projecting from the original grid to the coarse grids
-            m_project[u].construct( dg::create::fast_projection(*m_grids[u], 2, 2, 1, dg::normed), std::forward<Params>(ps)...);
-            m_inter[u].construct( dg::create::fast_interpolation(*m_grids[u+1], 2, 2, 1), std::forward<Params>(ps)...);
-            m_interT[u].construct( dg::create::fast_projection(*m_grids[u], 2, 2, 1, dg::not_normed), std::forward<Params>(ps)...);
+            m_project[u].construct( dg::create::fast_projection(*m_grids[u], 1, 2, 2, dg::normed), std::forward<Params>(ps)...);
+            m_inter[u].construct( dg::create::fast_interpolation(*m_grids[u+1], 1, 2, 2), std::forward<Params>(ps)...);
+            m_interT[u].construct( dg::create::fast_projection(*m_grids[u], 1, 2, 2, dg::not_normed), std::forward<Params>(ps)...);
         }
 
         m_x.resize( m_stages);
diff --git a/inc/dg/topology/ell_interpolation_b.cu b/inc/dg/topology/ell_interpolation_b.cu
index 0aede885d..9329f5e58 100644
--- a/inc/dg/topology/ell_interpolation_b.cu
+++ b/inc/dg/topology/ell_interpolation_b.cu
@@ -9,7 +9,7 @@ int main(){
 #include <cusp/print.h>
 #include "dg/backend/timer.h"
 #include "xspacelib.h"
-#include "ell_interpolation.h"
+#include "ell_interpolation.cuh"
 #include "interpolation.h"
 
 double sinus( double x, double y) {return sin(x)*sin(y);}
diff --git a/inc/dg/topology/fast_interpolation.h b/inc/dg/topology/fast_interpolation.h
index 033b24e76..29a62ef6c 100644
--- a/inc/dg/topology/fast_interpolation.h
+++ b/inc/dg/topology/fast_interpolation.h
@@ -109,16 +109,15 @@ namespace create
  * the general purpose interpolation function, especially since it requires
  * no communication from neighboring cells
  * @sa dg::create::interpolation
- * @sa For a derivation of the coefficients consult the dg manual <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @sa For a derivation of the coefficients consult the %dg manual <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * @tparam real_type a floating point type
+ * @return a matrix that when applied to vectors on the old grid produces a vector on the new grid
  * @param t The existing (old/coarse) grid
- * @param multiplyNx integer multiplier, the new grid has \c Nx*multiplyNx points
  * @param multiplyn integer multiplier, the new grid has \c n*multiplyn polynomial coefficients
- *
- * @return a matrix that when applied to vectors on the old grid produces a vector on the new grid
+ * @param multiplyNx integer multiplier, the new grid has \c Nx*multiplyNx points
  */
 template<class real_type>
-MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolation( const RealGrid1d<real_type>& t, unsigned multiplyNx, unsigned multiplyn)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolation( const RealGrid1d<real_type>& t, unsigned multiplyn, unsigned multiplyNx)
 {
     unsigned n=t.n();
     dg::RealGrid1d<real_type> g_old( -1., 1., n, 1);
@@ -149,20 +148,19 @@ MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolatio
  * the general purpose projection function, especially since it requires
  * no communication from neighboring cells
  * @sa dg::create::projection dg::create::interpolationT
- * @sa For a derivation of the coefficients consult the dg manual <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
+ * @sa For a derivation of the coefficients consult the %dg manual <a href="./dg_introduction.pdf" target="_blank">Introduction to dg methods</a>
  * @tparam real_type a floating point type
+ * @return a matrix that when applied to vectors on the old grid produces a vector on the new grid
  * @param t The existing (old/fine) grid
- * @param divideNx integer divisor, the new grid has \c Nx/multiplyNx points
  * @param dividen integer divisor, the new grid has \c n/multiplyn polynomial coefficients
+ * @param divideNx integer divisor, the new grid has \c Nx/multiplyNx points
  * @param no if dg::normed than a projection matrix is returned, if dg::not_normed the interpolationT is computed
- *
- * @return a matrix that when applied to vectors on the old grid produces a vector on the new grid
  */
 template<class real_type>
-MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection( const RealGrid1d<real_type>& t, unsigned divideNx, unsigned dividen, enum dg::norm no = normed)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection( const RealGrid1d<real_type>& t, unsigned dividen, unsigned divideNx, enum dg::norm no = normed)
 {
     if( t.N()%divideNx != 0) throw Error( Message(_ping_)<< "Nx and divideNx don't match: Nx: " << t.N()<< " divideNx "<< (unsigned)divideNx);
-    if( t.n()%dividen != 0) throw Error( Message(_ping_)<< "n and dividen don't match: Nx: " << t.n()<< " dividen "<< (unsigned)dividen);
+    if( t.n()%dividen != 0) throw Error( Message(_ping_)<< "n and dividen don't match: n: " << t.n()<< " dividen "<< (unsigned)dividen);
     unsigned n=t.n()/dividen;
     dg::RealGrid1d<real_type> g_old( -1., 1., n*dividen, divideNx);
     dg::RealGrid1d<real_type> g_new( -1., 1., n, 1);
@@ -195,14 +193,14 @@ MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection(
 ///@copydoc fast_interpolation(const RealGrid1d<real_type>&,unsigned,unsigned)
 ///@param multiplyNy integer multiplier, the new grid has \c Ny*multiplyNy points
 template<class real_type>
-MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolation( const aRealTopology2d<real_type>& t, unsigned multiplyNx, unsigned multiplyNy, unsigned multiplyn)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolation( const aRealTopology2d<real_type>& t, unsigned multiplyn, unsigned multiplyNx, unsigned multiplyNy)
 {
     dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_interpolation( gx, multiplyNx, multiplyn);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_interpolation( gx, multiplyn,multiplyNx);
     interX.get_matrices()[0].left_size = t.n()*t.Ny();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_interpolation( gy, multiplyNy, multiplyn);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_interpolation( gy, multiplyn,multiplyNy);
     interY.get_matrices()[0].right_size = t.n()*t.Nx()*multiplyNx*multiplyn;
     interY.get_matrices()[0].set_default_range();
 
@@ -217,14 +215,14 @@ MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolatio
 ///@copydoc fast_projection(const RealGrid1d<real_type>&,unsigned,unsigned,enum dg::norm)
 ///@param divideNy integer multiplier, the new grid has \c Ny/divideNy points
 template<class real_type>
-MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection( const aRealTopology2d<real_type>& t, unsigned divideNx, unsigned divideNy, unsigned dividen, enum dg::norm no = normed)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection( const aRealTopology2d<real_type>& t, unsigned dividen, unsigned divideNx, unsigned divideNy, enum dg::norm no = normed)
 {
     dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_projection( gx, divideNx, dividen, no);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_projection( gx, dividen, divideNx, no);
     interX.get_matrices()[0].left_size = t.n()*t.Ny();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_projection( gy, divideNy, dividen, no);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_projection( gy, dividen, divideNy, no);
     interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideNx/dividen;
     interY.get_matrices()[0].set_default_range();
 
@@ -239,14 +237,14 @@ MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection(
 ///@copydoc fast_interpolation(const RealGrid1d<real_type>&,unsigned,unsigned)
 ///@param multiplyNy integer multiplier, the new grid has \c Ny*multiplyNy points
 template<class real_type>
-MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolation( const aRealTopology3d<real_type>& t, unsigned multiplyNx, unsigned multiplyNy, unsigned multiplyn)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolation( const aRealTopology3d<real_type>& t, unsigned multiplyn, unsigned multiplyNx, unsigned multiplyNy)
 {
     dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_interpolation( gx, multiplyNx,multiplyn);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_interpolation( gx, multiplyn, multiplyNx);
     interX.get_matrices()[0].left_size = t.n()*t.Ny()*t.Nz();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_interpolation( gy, multiplyNy,multiplyn);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_interpolation( gy, multiplyn, multiplyNy);
     interY.get_matrices()[0].right_size = t.n()*t.Nx()*multiplyNx*multiplyn;
     interY.get_matrices()[0].left_size = t.Nz();
     interY.get_matrices()[0].set_default_range();
@@ -262,14 +260,14 @@ MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_interpolatio
 ///@copydoc fast_projection(const RealGrid1d<real_type>&,unsigned,unsigned,enum dg::norm)
 ///@param divideNy integer multiplier, the new grid has \c Ny/divideNy points
 template<class real_type>
-MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection( const aRealTopology3d<real_type>& t, unsigned divideNx, unsigned divideNy, unsigned dividen, enum dg::norm no = normed)
+MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection( const aRealTopology3d<real_type>& t, unsigned dividen, unsigned divideNx, unsigned divideNy, enum dg::norm no = normed)
 {
     dg::RealGrid1d<real_type> gx(t.x0(), t.x1(), t.n(), t.Nx());
     dg::RealGrid1d<real_type> gy(t.y0(), t.y1(), t.n(), t.Ny());
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_projection( gx, divideNx, dividen, no);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interX = dg::create::fast_projection( gx, dividen,divideNx, no);
     interX.get_matrices()[0].left_size = t.n()*t.Ny()*t.Nz();
     interX.get_matrices()[0].set_default_range();
-    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_projection( gy, divideNy, dividen, no);
+    MultiMatrix < EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > interY = dg::create::fast_projection( gy, dividen, divideNy, no);
     interY.get_matrices()[0].right_size = t.n()*t.Nx()/divideNx/dividen;
     interY.get_matrices()[0].left_size = t.Nz();
     interY.get_matrices()[0].set_default_range();
@@ -287,11 +285,11 @@ MultiMatrix< dg::HMatrix_t<real_type>, dg::HVec_t<real_type> > fast_projection(
 ///@copydoc fast_interpolation(const RealGrid1d<real_type>&,unsigned,unsigned)
 ///@param multiplyNy integer multiplier, the new grid has \c Ny*multiplyNy points
 template<class real_type>
-MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_interpolation( const aRealMPITopology2d<real_type>& t, unsigned multiplyNx, unsigned multiplyNy, unsigned multiplyn)
+MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_interpolation( const aRealMPITopology2d<real_type>& t, unsigned multiplyn, unsigned multiplyNx, unsigned multiplyNy)
 {
     typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
     typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
-    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_interpolation( t.local(), multiplyNx, multiplyNy, multiplyn);
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_interpolation( t.local(), multiplyn,multiplyNx, multiplyNy);
     MultiMatrix< Matrix, Vector > inter(2);
     inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
     inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
@@ -302,11 +300,11 @@ MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_interpolation( con
 ///@copydoc fast_projection(const RealGrid1d<real_type>&,unsigned,unsigned,enum dg::norm)
 ///@param divideNy integer multiplier, the new grid has \c Ny/divideNy points
 template<class real_type>
-MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_projection( const aRealMPITopology2d<real_type>& t, unsigned divideNx, unsigned divideNy, unsigned dividen, enum dg::norm no = normed)
+MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_projection( const aRealMPITopology2d<real_type>& t, unsigned dividen, unsigned divideNx, unsigned divideNy, enum dg::norm no = normed)
 {
     typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
     typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
-    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_projection( t.local(), divideNx, divideNy, dividen, no);
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_projection( t.local(), dividen,divideNx, divideNy, no);
     MultiMatrix< Matrix, Vector > inter(2);
     inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
     inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
@@ -317,11 +315,11 @@ MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_projection( const
 ///@copydoc fast_interpolation(const RealGrid1d<real_type>&,unsigned,unsigned)
 ///@param multiplyNy integer multiplier, the new grid has \c Ny*multiplyNy points
 template<class real_type>
-MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_interpolation( const aRealMPITopology3d<real_type>& t, unsigned multiplyNx, unsigned multiplyNy, unsigned multiplyn)
+MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_interpolation( const aRealMPITopology3d<real_type>& t, unsigned multiplyn, unsigned multiplyNx, unsigned multiplyNy)
 {
     typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
     typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
-    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_interpolation( t.local(), multiplyNx, multiplyNy, multiplyn);
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_interpolation( t.local(), multiplyn,multiplyNx, multiplyNy);
     MultiMatrix< Matrix, Vector > inter(2);
     inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
     inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
@@ -332,11 +330,11 @@ MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_interpolation( con
 ///@copydoc fast_projection(const RealGrid1d<real_type>&,unsigned,unsigned,enum dg::norm)
 ///@param divideNy integer multiplier, the new grid has \c Ny/divideNy points
 template<class real_type>
-MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_projection( const aRealMPITopology3d<real_type>& t, unsigned divideNx, unsigned divideNy, unsigned dividen, enum dg::norm no = normed)
+MultiMatrix< MHMatrix_t<real_type>, MHVec_t<real_type> > fast_projection( const aRealMPITopology3d<real_type>& t, unsigned dividen, unsigned divideNx, unsigned divideNy, enum dg::norm no = normed)
 {
     typedef RowColDistMat<EllSparseBlockMat<real_type>, CooSparseBlockMat<real_type>, NNCH<real_type>> Matrix;
     typedef MPI_Vector<thrust::host_vector<real_type> > Vector;
-    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_projection( t.local(), divideNx, divideNy, dividen, no);
+    MultiMatrix<EllSparseBlockMat<real_type>, thrust::host_vector<real_type> > temp = dg::create::fast_projection( t.local(), dividen,divideNx, divideNy, no);
     MultiMatrix< Matrix, Vector > inter(2);
     inter.get_matrices()[0] = Matrix( temp.get_matrices()[0], CooSparseBlockMat<real_type>(), NNCH<real_type>());
     inter.get_matrices()[1] = Matrix( temp.get_matrices()[1], CooSparseBlockMat<real_type>(), NNCH<real_type>());
diff --git a/inc/dg/topology/projection_mpit.cu b/inc/dg/topology/projection_mpit.cu
index edf9fd054..68d07df8a 100644
--- a/inc/dg/topology/projection_mpit.cu
+++ b/inc/dg/topology/projection_mpit.cu
@@ -35,14 +35,14 @@ int main(int argc, char* argv[])
 
     ///%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%//
     if(rank==0)std::cout << "TEST 2D and 3D\n";
-    unsigned n_old = 6, n_new = 3, N_old = 40, N_new = 20;
+    unsigned n_old = 9, n_new = 3, N_old = 40, N_new = 20;
     if(rank==0)std::cout << "Fine   Grid "<< n_old << " x "<<N_old <<"\n";
     if(rank==0)std::cout << "Coarse Grid "<< n_new << " x "<<N_new <<"\n";
     dg::MPIGrid3d g2o (0, M_PI, 0, M_PI, 0,1, n_old, N_old, N_old, 4, comm);
     dg::MPIGrid3d g2n (0, M_PI, 0, M_PI, 0,1, n_new, N_new, N_new, 4, comm);
     dg::MIHMatrix inte2d = dg::create::interpolation( g2n, g2o);
-    dg::MultiMatrix< dg::MHMatrix, dg::MHVec > proj2d = dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new, n_old/n_new);
-    dg::MultiMatrix< dg::MHMatrix, dg::MHVec > fast_inte2d = dg::create::fast_interpolation( g2n, N_old/N_new, N_old/N_new, n_old/n_new);
+    dg::MultiMatrix< dg::MHMatrix, dg::MHVec > proj2d = dg::create::fast_projection( g2o, n_old/n_new, N_old/N_new, N_old/N_new);
+    dg::MultiMatrix< dg::MHMatrix, dg::MHVec > fast_inte2d = dg::create::fast_interpolation( g2n, n_old/n_new, N_old/N_new, N_old/N_new);
     const dg::MHVec sinO( dg::evaluate( sine, g2o)),
                                 sinN( dg::evaluate( sine, g2n));
     dg::MHVec w2do = dg::create::weights( g2o);
diff --git a/inc/dg/topology/projection_t.cu b/inc/dg/topology/projection_t.cu
index aa1da28a7..c98675e3f 100644
--- a/inc/dg/topology/projection_t.cu
+++ b/inc/dg/topology/projection_t.cu
@@ -13,7 +13,7 @@ double sine( double x, double y, double z){return sin(x)*sin(y);}
 int main()
 {
     std::cout << "TEST 1D\n";
-    unsigned n_old = 6, n_new = 3, N_old = 40, N_new = 20;
+    unsigned n_old = 9, n_new = 3, N_old = 40, N_new = 20;
     //std::cout << "Type n and N of old (fine) grid!\n";
     //std::cin >> n_old >> N_old;
     //std::cout << "Type n and N of new (coarser) grid!\n";
@@ -24,8 +24,8 @@ int main()
     dg::Grid1d gn ( 0, M_PI/2., n_new, N_new);
     //cusp::coo_matrix<int, double, cusp::host_memory> proj = dg::create::projection( gn, go);
     //cusp::coo_matrix<int, double, cusp::host_memory> inte = dg::create::interpolation( go, gn);
-    dg::MultiMatrix< dg::DMatrix, dg::DVec > proj = dg::create::fast_projection( go,  N_old/N_new, n_old/n_new);
-    dg::MultiMatrix< dg::DMatrix, dg::DVec > inte = dg::create::fast_interpolation( gn, N_old/N_new, n_old/n_new);
+    dg::MultiMatrix< dg::DMatrix, dg::DVec > proj = dg::create::fast_projection( go, n_old/n_new,  N_old/N_new);
+    dg::MultiMatrix< dg::DMatrix, dg::DVec > inte = dg::create::fast_interpolation( gn, n_old/n_new, N_old/N_new);
     dg::DVec v = dg::evaluate( sine, go);
     dg::DVec w1do = dg::create::weights( go);
     dg::DVec w1dn = dg::create::weights( gn);
@@ -49,10 +49,10 @@ int main()
     //cusp::coo_matrix<int, double, cusp::host_memory> proj2d = dg::create::transformation( g2n, g2o);
     cusp::coo_matrix<int, double, cusp::host_memory> inte2d = dg::create::interpolation( g2n, g2o);
     //dg::MultiMatrix< dg::HMatrix, std::vector<thrust::host_vector<double>> > proj2d;
-    //proj2d.construct( dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new, n_old/n_new), 2);
+    //proj2d.construct( dg::create::fast_projection( g2o, n_old/n_new, N_old/N_new, N_old/N_new), 2);
     //dg::IHMatrix proj2d = dg::create::projection( g2n, g2o);
-    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > proj2d = dg::create::fast_projection( g2o, N_old/N_new, N_old/N_new, n_old/n_new);
-    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > fast_inte2d = dg::create::fast_interpolation( g2n, N_old/N_new, N_old/N_new, n_old/n_new);
+    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > proj2d = dg::create::fast_projection( g2o, n_old/n_new, N_old/N_new, N_old/N_new);
+    dg::MultiMatrix< dg::HMatrix, thrust::host_vector<double> > fast_inte2d = dg::create::fast_interpolation( g2n, n_old/n_new, N_old/N_new, N_old/N_new);
     const dg::HVec sinO( dg::evaluate( sine, g2o)),
                                 sinN( dg::evaluate( sine, g2n));
     dg::HVec w2do = dg::create::weights( g2o);
diff --git a/inc/geometries/conformalX_elliptic_b.cu b/inc/geometries/conformalX_elliptic_b.cu
index 1041d8be9..0b0183228 100644
--- a/inc/geometries/conformalX_elliptic_b.cu
+++ b/inc/geometries/conformalX_elliptic_b.cu
@@ -127,7 +127,7 @@ int main(int argc, char**argv)
     for( unsigned i=1; i<nIter; i++)
     {
         Nx*=2; Ny*=2;
-        dg::MultiMatrix<dg::DMatrix, dg::DVec >  inter = dg::create::fast_interpolation(g2d.grid(), 2, 2, 1);
+        dg::MultiMatrix<dg::DMatrix, dg::DVec >  inter = dg::create::fast_interpolation(g2d.grid(), 1, 2, 2);
         dg::geo::CurvilinearGridX2d g2d_new( generator, 0.25, 1./22., n, Nx, Ny, dg::DIR, dg::DIR);
         std::cout << "Computing on "<<n<<" x "<<Nx<<" x "<<Ny<<"\n";
         dg::DVec x_new = dg::evaluate( dg::zero, g2d_new);
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index 9201ada7d..eb1b1d169 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -208,8 +208,8 @@ int main( int argc, char* argv[])
     // helper variables for output computations
     std::map<std::string, dg::Simpsons<HVec>> time_integrals;
     dg::Average<HVec> toroidal_average( g3d_out, dg::coo3d::z, "simple");
-    dg::MultiMatrix<HMatrix,HVec> projectH = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
-    dg::MultiMatrix<DMatrix,DVec> projectD = dg::create::fast_projection( grid, p.cx, p.cy, dg::normed);
+    dg::MultiMatrix<HMatrix,HVec> projectH = dg::create::fast_projection( grid, 1, p.cx, p.cy, dg::normed);
+    dg::MultiMatrix<DMatrix,DVec> projectD = dg::create::fast_projection( grid, 1, p.cx, p.cy, dg::normed);
     HVec transferH( dg::evaluate(dg::zero, g3d_out));
     DVec transferD( dg::evaluate(dg::zero, g3d_out));
     HVec transferH2d = dg::evaluate( dg::zero, *g2d_out_ptr);
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index 1e9e0b184..94066880b 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -124,8 +124,8 @@ Shu< Geometry, Matrix, Container>::Shu(
         Geometry fine_grid = g;
         fine_grid.set( 2*g.n(), g.Nx(), g.Ny());
         //theoretically we only need 2n-1 but it isn't wrong to take more
-        m_inter = dg::create::fast_interpolation( g, 1, 1, 2);
-        m_project = dg::create::fast_projection( fine_grid, 1, 1, 2);
+        m_inter = dg::create::fast_interpolation( g, 2, 1, 1);
+        m_project = dg::create::fast_projection( fine_grid, 2, 1, 1);
 
         m_fine_centered[0] = dg::create::dx( fine_grid, g.bcx(), dg::centered);
         m_fine_centered[1] = dg::create::dy( fine_grid, g.bcy(), dg::centered);
@@ -142,13 +142,14 @@ Shu< Geometry, Matrix, Container>::Shu(
         m_fine_temp[2] = dg::evaluate( dg::zero, fine_grid);
         m_arakawa.construct( fine_grid);
     }
+    else
+        m_arakawa.construct( g);
     m_centered[0] = dg::create::dx( g, g.bcx(), dg::centered);
     m_centered[1] = dg::create::dy( g, g.bcy(), dg::centered);
     m_forward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::forward);
     m_forward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::forward);
     m_backward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::backward);
     m_backward[1] = dg::create::dy( g, dg::inverse( g.bcy()), dg::backward);
-    m_arakawa.construct( g);
 
     unsigned stages = dg::file::get( mode, js, "elliptic", "stages", 3).asUInt();
     m_eps.resize(stages);
-- 
GitLab


From 841f594104eb0458c068f9f1c535713b9cf251e9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 8 Feb 2021 12:16:00 +0100
Subject: [PATCH 463/540] Bump default compute capability to sm_61

---
 config/default.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/default.mk b/config/default.mk
index 496f54704..384cf825e 100644
--- a/config/default.mk
+++ b/config/default.mk
@@ -6,7 +6,7 @@ CC=g++ #C++ compiler
 MPICC=mpic++  #mpi compiler
 CFLAGS=-Wall -std=c++14 -mavx -mfma #flags for CC
 NVCC=nvcc #CUDA compiler
-NVCCARCH=-arch sm_35 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
+NVCCARCH=-arch sm_61 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
 NVCCFLAGS= -std=c++14 -Xcompiler "-Wall -mavx -mfma" #flags for NVCC
 OPT=-O2 # optimization flags for host code (it is O2 and not O3 because g++-7 up to g++-8.0 have a bug with fma in -O3, fixed in g++-8.1)
 OMPFLAG=-fopenmp #openmp flag for CC and MPICC
-- 
GitLab


From b02500b4c7e45374dbee74477d92a20577da8bea Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 8 Feb 2021 12:27:17 +0100
Subject: [PATCH 464/540] Fix missing if bug in shu_hpc

---
 src/lamb_dipole/shu_b.cu | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index de7fc2f2b..86f533f04 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -177,8 +177,8 @@ int main( int argc, char* argv[])
         }
         glfwTerminate();
     }
-    else if( "netcdf" == output)
 #endif //WITHOUT_GLFW
+    if( "netcdf" == output)
     {
         std::string inputfile = js.toStyledString(); //save input without comments, which is important if netcdf file is later read by another parser
         std::string outputfile;
-- 
GitLab


From 624aa8b8ce5448bdcf2ee66193d2a5079c2b6528 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 8 Feb 2021 15:10:57 +0100
Subject: [PATCH 465/540] Fix feltor codes with timesteppers

---
 inc/dg/functors.h        | 24 ++++++++++++------------
 src/feltor/feltor.cu     |  2 +-
 src/feltor/feltor.h      |  2 +-
 src/feltor/feltor_hpc.cu |  2 +-
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index e9d8a2cfe..61aab4544 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -942,7 +942,7 @@ struct IslandXY
 struct SinXSinY
 {
     /**
-     * @brief Construct with two coefficients
+     * @brief Construct
      *
      * @param amp amplitude A
      * @param bamp backgroundamp B
@@ -970,7 +970,7 @@ struct SinXSinY
 struct CosXCosY
 {
     /**
-     * @brief Construct with two coefficients
+     * @brief Construct
      *
      * @param amp amplitude A
      * @param bamp backgroundamp B
@@ -998,7 +998,7 @@ struct CosXCosY
 struct SinXCosY
 {
     /**
-     * @brief Construct with two coefficients
+     * @brief Construct
      *
      * @param amp amplitude
      * @param bamp backgroundamp
@@ -1026,7 +1026,7 @@ struct SinXCosY
 struct SinX
 {
     /**
-     * @brief Construct with two coefficients
+     * @brief Construct
      *
      * @param amp amplitude A
      * @param bamp backgroundamp B
@@ -1049,7 +1049,7 @@ struct SinX
 struct SinY
 {
     /**
-     * @brief Construct with two coefficients
+     * @brief Construct
      *
      * @param amp amplitude
      * @param bamp backgroundamp
@@ -1068,7 +1068,7 @@ struct SinY
 struct CosY
 {
     /**
-     * @brief Construct with two coefficients
+     * @brief Construct
      *
      * @param amp amplitude A
      * @param bamp backgroundamp B
@@ -1109,7 +1109,7 @@ struct InvCoshXsq
 struct SinProfX
 {
     /**
-     * @brief Construct with two coefficients
+     * @brief Construct
      *
      * @param amp amplitude A
      * @param bamp backgroundamp B
@@ -1127,16 +1127,16 @@ struct SinProfX
 };
 /**
  * @brief
- * \f$ f(x) = f(x,y) = f(x,y,z) = B + A\exp(-x/L_n) \f$
+ * \f$ f(x) = f(x,y) = f(x,y,z) = A\exp(-x/L_n) + B \f$
  */
 struct ExpProfX
 {
     /**
-     * @brief Construct with two coefficients
+     * @brief Construct with three coefficients
      *
-     * @param amp amplitude B
-     * @param bamp backgroundamp A (choose zero for constant gradient length
-     * @param ln  ln (must be !=0)
+     * @param amp amplitude A
+     * @param bamp background amplitude B (choose zero for constant gradient length
+     * @param ln  gradient lenght L_n (must be !=0)
      */
     ExpProfX( double amp, double bamp, double ln):m_amp(amp),m_bamp(bamp),m_ln(ln){
         assert( ln!=0 && "ln parameter must be != 0 in ExpProfX!");
diff --git a/src/feltor/feltor.cu b/src/feltor/feltor.cu
index ea2fc2c16..f921eb867 100644
--- a/src/feltor/feltor.cu
+++ b/src/feltor/feltor.cu
@@ -140,7 +140,7 @@ int main( int argc, char* argv[])
     //    feltor::FeltorSpecialSolver<
     //        Geometry, IDMatrix, DMatrix, DVec>
     //    > karniadakis( grid, p, mag);
-    dg::ExplicitMultistep< std::array<std::array<dg::DVec,2>,2 > > mp("TVB", 3, y0);
+    dg::ExplicitMultistep< std::array<std::array<dg::DVec,2>,2 > > mp("TVB-3-3", y0);
     {
     HVec h_wall = dg::pullback( wall, grid);
     HVec h_sheath = dg::pullback( sheath, grid);
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index b52aaf97b..49e7131d4 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -1130,7 +1130,7 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         else // "bohm" == m_p.sheath_bc
         {
             //exp(-phi)
-            dg::blas1::transform( m_phi[0], m_temp0, dg::EXP<double>(1., -1.));
+            dg::blas1::transform( m_phi[0], m_temp0, dg::ExpProfX(1., 0., 1.));
             dg::blas1::pointwiseDot( m_sheath_forcing*sqrt(1+m_p.tau[1]), m_U_sheath, m_temp0, 1.,  yp[1][0]);
         }
         // u_i = +- sqrt(1+tau)
diff --git a/src/feltor/feltor_hpc.cu b/src/feltor/feltor_hpc.cu
index eb1b1d169..31d81ff33 100644
--- a/src/feltor/feltor_hpc.cu
+++ b/src/feltor/feltor_hpc.cu
@@ -467,7 +467,7 @@ int main( int argc, char* argv[])
     //    feltor::FeltorSpecialSolver<
     //        Geometry, IDMatrix, DMatrix, DVec>
     //    > karniadakis( grid, p, mag);
-    dg::ExplicitMultistep< std::array<std::array<DVec,2>,2 > > mp( "TVB", 3, y0);
+    dg::ExplicitMultistep< std::array<std::array<DVec,2>,2 > > mp( "TVB-3-3", y0);
     {
     HVec h_wall = dg::pullback( wall, grid);
     HVec h_sheath = dg::pullback( sheath, grid);
-- 
GitLab


From a0a40ca082f3e6806b69b207e418dbaaa99a0eba Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 8 Feb 2021 15:11:41 +0100
Subject: [PATCH 466/540] Fix Shu Makefile and bug in output check

---
 src/lamb_dipole/Makefile | 2 +-
 src/lamb_dipole/shu_b.cu | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lamb_dipole/Makefile b/src/lamb_dipole/Makefile
index 5a43bc1bf..703eb53ac 100644
--- a/src/lamb_dipole/Makefile
+++ b/src/lamb_dipole/Makefile
@@ -19,4 +19,4 @@ shu_hpc: shu_b.cu shu.cuh init.h
 .PHONY: clean
 
 clean:
-	rm -f shu_b
+	rm -f shu_b shu_hpc
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 86f533f04..4894e5c14 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -334,7 +334,7 @@ int main( int argc, char* argv[])
         }
         err = nc_close(ncid);
     }
-    else
+    if( !("netcdf" == output) && !("glfw" == output))
     {
         throw dg::Error(dg::Message(_ping_)<<"Error: Wrong value for output type "<<output<<" Must be glfw or netcdf! Exit now!");
 
-- 
GitLab


From 7fca963fd01af2d5bc88e8727eeafc994acedf2a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 8 Feb 2021 15:12:34 +0100
Subject: [PATCH 467/540] Add --extended-lambda flag in default config

- this enables us to use device lambdas. Very nice!!
- we should start to review the code regarding its usage
---
 config/default.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/default.mk b/config/default.mk
index 384cf825e..ae4e5d2c1 100644
--- a/config/default.mk
+++ b/config/default.mk
@@ -7,7 +7,7 @@ MPICC=mpic++  #mpi compiler
 CFLAGS=-Wall -std=c++14 -mavx -mfma #flags for CC
 NVCC=nvcc #CUDA compiler
 NVCCARCH=-arch sm_61 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
-NVCCFLAGS= -std=c++14 -Xcompiler "-Wall -mavx -mfma" #flags for NVCC
+NVCCFLAGS= -std=c++14 -Xcompiler "-Wall -mavx -mfma" --extended-lambda #flags for NVCC
 OPT=-O2 # optimization flags for host code (it is O2 and not O3 because g++-7 up to g++-8.0 have a bug with fma in -O3, fixed in g++-8.1)
 OMPFLAG=-fopenmp #openmp flag for CC and MPICC
 
-- 
GitLab


From 06992818af88afa6bb015e25c54d3712a7a9ddb7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 9 Feb 2021 15:40:20 +0100
Subject: [PATCH 468/540] HOTFIX: remove exdot_cuda race condition

This bug manifested in rare crashes of simulations that were hard to
reproduce and happened seemingly without reason
---
 inc/dg/backend/exblas/exdot_cuda.cuh | 29 ++++++++++++++++++++--------
 1 file changed, 21 insertions(+), 8 deletions(-)

diff --git a/inc/dg/backend/exblas/exdot_cuda.cuh b/inc/dg/backend/exblas/exdot_cuda.cuh
index cb124a8fa..04fbe2493 100644
--- a/inc/dg/backend/exblas/exdot_cuda.cuh
+++ b/inc/dg/backend/exblas/exdot_cuda.cuh
@@ -62,7 +62,7 @@ __global__ void ExDOT(
     //Initialize superaccs
     for (uint i = 0; i < BIN_COUNT; i++)
         l_workingBase[i * WARP_COUNT] = 0;
-    __syncthreads();
+    __syncthreads(); //syncs all threads in a block (but not across blocks)
 
     //Read data from global memory and scatter it to sub-superaccs
     double a[NBFPE] = {0.0};
@@ -153,7 +153,7 @@ __global__ void ExDOT(
     //Initialize superaccs
     for (uint i = 0; i < BIN_COUNT; i++)
         l_workingBase[i * WARP_COUNT] = 0;
-    __syncthreads();
+    __syncthreads(); //syncs all threads in a block (but not across blocks)
 
     //Read data from global memory and scatter it to sub-superaccs
     double a[NBFPE] = {0.0};
@@ -301,18 +301,27 @@ void ExDOTComplete(
         d_PartialSuperaccs[gid * BIN_COUNT * MERGE_SIZE + lid] = sum;
     }
 
-    __syncthreads();
+    __syncthreads(); //syncs all threads in a block (but not across blocks)
     if (lid == 0) { //every block normalize its summed superacc
         int imin = IMIN, imax = IMAX;
         Normalize(&d_PartialSuperaccs[gid * BIN_COUNT * MERGE_SIZE], imin, imax);
     }
-
-    //MW: don't we need a global synchronization here??
-    __syncthreads();
+}
+//MW: we need a global synchronization here!!
+//one block of threads with at least 39 threads
+template<uint MERGE_SIZE>
+__global__
+void ExDOTCompleteFinal(
+     int64_t *d_PartialSuperaccs,
+     int64_t *d_superacc
+) {
+    uint lid = threadIdx.x;
+    uint gid = blockIdx.x;
+    uint blocks = gpu::PARTIAL_SUPERACCS_COUNT/gpu::MERGE_SUPERACCS_SIZE;
     if ((lid < BIN_COUNT) && (gid == 0)) {
         int64_t sum = 0;
 
-        for(uint i = 0; i < gridDim.x; i++)
+        for(uint i = 0; i < blocks; i++)
             sum += d_PartialSuperaccs[i * BIN_COUNT * MERGE_SIZE + lid];
 
         d_superacc[lid] = sum;
@@ -341,12 +350,14 @@ void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
 {
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
     static_assert( has_floating_value<PointerOrValue2>::value, "PointerOrValue2 needs to be T or T* with T one of (const) float or (const) double");
-    static thrust::device_vector<int64_t> d_PartialSuperaccsV( gpu::PARTIAL_SUPERACCS_COUNT*BIN_COUNT, 0.0); //39 columns and PSC rows
+    static thrust::device_vector<int64_t> d_PartialSuperaccsV( gpu::PARTIAL_SUPERACCS_COUNT*BIN_COUNT, 0); //39 columns and PSC rows
     int64_t *d_PartialSuperaccs = thrust::raw_pointer_cast( d_PartialSuperaccsV.data());
     thrust::device_vector<bool> d_errorV(1, false);
     bool *d_error = thrust::raw_pointer_cast( d_errorV.data());
     gpu::ExDOT<NBFPE, gpu::WARP_COUNT><<<gpu::PARTIAL_SUPERACCS_COUNT, gpu::WORKGROUP_SIZE>>>( d_PartialSuperaccs, x1_ptr, x2_ptr,size, d_error);
     gpu::ExDOTComplete<gpu::MERGE_SUPERACCS_SIZE><<<gpu::PARTIAL_SUPERACCS_COUNT/gpu::MERGE_SUPERACCS_SIZE, gpu::MERGE_WORKGROUP_SIZE>>>( d_PartialSuperaccs, d_superacc );
+    //# blocks, # threads per block
+    gpu::ExDOTCompleteFinal<gpu::MERGE_SUPERACCS_SIZE><<<1, 64>>>( d_PartialSuperaccs, d_superacc );
     *status = 0;
     if( d_errorV[0] ) *status = 1;
 }
@@ -378,6 +389,8 @@ void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, Po
     bool *d_error = thrust::raw_pointer_cast( d_errorV.data());
     gpu::ExDOT<NBFPE, gpu::WARP_COUNT><<<gpu::PARTIAL_SUPERACCS_COUNT, gpu::WORKGROUP_SIZE>>>( d_PartialSuperaccs, x1_ptr, x2_ptr, x3_ptr,size,d_error);
     gpu::ExDOTComplete<gpu::MERGE_SUPERACCS_SIZE><<<gpu::PARTIAL_SUPERACCS_COUNT/gpu::MERGE_SUPERACCS_SIZE, gpu::MERGE_WORKGROUP_SIZE>>>( d_PartialSuperaccs, d_superacc );
+    //# blocks, # threads per block
+    gpu::ExDOTCompleteFinal<gpu::MERGE_SUPERACCS_SIZE><<<1, 64>>>( d_PartialSuperaccs, d_superacc );
     *status = 0;
     if( d_errorV[0] ) *status = 1;
 }
-- 
GitLab


From 77c9251aaaf2c91f9fea81832302b3af5a6636f2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 9 Feb 2021 16:25:56 +0100
Subject: [PATCH 469/540] Bump README to newest vcl version1 of vectorclass

- Once this is in master we can remove feltor-dev/vcl
---
 README.adoc      | 17 +++++++++--------
 config/README.md |  4 ++--
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/README.adoc b/README.adoc
index d2996fc55..8694fe4a2 100644
--- a/README.adoc
+++ b/README.adoc
@@ -48,18 +48,19 @@ git clone https://www.github.com/feltor-dev/feltor
 
 You also need to clone https://github.com/thrust/thrust[thrust] and
 https://github.com/cusplibrary/cusplibrary[cusp] distributed under the
-Apache-2.0 license. Also, we need Agner Fog's https://github.com/feltor-dev/vcl[vcl] library (GPL-3.0). So again in a folder of your choice
+Apache-2.0 license. Also, we need Agner Fog's https://github.com/vectorclass/version1[vcl] library (Apache 2.0). So again in a folder of your choice
 
 [source,sh]
 ----
 git clone https://www.github.com/thrust/thrust
 git clone https://www.github.com/cusplibrary/cusplibrary
-git clone https://www.github.com/feltor-dev/vcl
+git clone https://www.github.com/vectorclass/version1 vcl
 ----
 
 ____
 Our code only depends on external libraries that are themselves openly
 available.
+ We use version1 of the vectorclass library, as version2 requires C{plus}{plus}-17 and does not work with the intel compiler.
 ____
 
 .System requirements [[tab_requirements]]
@@ -68,7 +69,7 @@ ____
 |=======================================================================
 |    | Minimum system requirements  | Recommended system requirements
 | *CPU*     | Any         |support for AVX and FMA instruction set
-| *Compiler*| gcc-4.9 or msvc-15 or icc-15.0 (C{plus}{plus}-11 standard)| OpenMP-4 support, avx, fma instruction set flags
+| *Compiler*| gcc-4.9 or msvc-15 or icc-15.0 (C{plus}{plus}-14 standard)| OpenMP-4 support, avx, fma instruction set flags
 | *GPU*     | - | NVidia GPU with compute-capability > 6 and nvcc-8.0
 | *MPI*     | - | mpi installation compatible with compiler (must be cuda-aware in case hybrid MPI+GPU is the target system)
 |=======================================================================
@@ -76,7 +77,7 @@ ____
 Our GPU backend uses the
 https://developer.nvidia.com/cuda-zone[Nvidia-CUDA] programming
 environment and in order to compile and run a program for a GPU a user
-needs at least the nvcc-7.5 compiler (available free of charge) and a NVidia
+needs at least the nvcc-8.0 compiler (available free of charge) and a NVidia
 GPU. However, we explicitly note here that due to the modular design of
 our software a user does not have to possess a GPU nor the nvcc
 compiler. The CPU version of the backend is equally valid and provides
@@ -203,7 +204,7 @@ To compile and run this code for a GPU use (assuming the external libraries are
 
 [source,sh]
 ----
-nvcc -x cu -std=c++11 -Ipath/to/feltor/inc -Ipath/to/include test.cpp -o test
+nvcc -x cu -std=c++14 -Ipath/to/feltor/inc -Ipath/to/include test.cpp -o test
 ./test
 ----
 
@@ -212,7 +213,7 @@ functions you can also use
 
 [source,sh]
 ----
-g++ -std=c++11 -fopenmp -mavx -mfma -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/include test.cpp -o test
+g++ -std=c++14 -fopenmp -mavx -mfma -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/include test.cpp -o test
 export OMP_NUM_THREADS=4
 ./test
 ----
@@ -255,7 +256,7 @@ Compile e.g. for a hybrid MPI {plus} OpenMP hardware platform with
 
 [source,sh]
 ----
-mpic++ -std=c++11 -mavx -mfma -fopenmp -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/include test_mpi.cpp -o test_mpi
+mpic++ -std=c++14 -mavx -mfma -fopenmp -DTHRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_OMP -Ipath/to/feltor/inc -Ipath/to/include test_mpi.cpp -o test_mpi
 export OMP_NUM_THREADS=2
 mpirun -n 4 ./test_mpi
 ----
@@ -356,7 +357,7 @@ In Visual Studio we suggest to create a Property Sheet for FELTOR.
 The Property Sheet can then be conveniently added to any project that includes
 the FELTOR library headers `dg/algorithm.h` and/or `dg/geometries/geometries.h`
 
-* Open an existing solution in Visual Studio or create a new project with `File -> New -> Projet ...` selecting `Empty Project` in Visual C++.
+* Open an existing solution in Visual Studio or create a new project with `File -> New -> Projet ...` selecting `Empty Project` in Visual C{plus}{plus}.
 * In the Solution Explorer change to the `Property Manager` tab, then click on `Add New Project Property Sheet`, name it `FeltorPropertySheet.props` and save it
 to a convenient location.
 * Double click on `FeltorPropertySheet` (expand your solution and any of the Debug or Release tabs to find it)
diff --git a/config/README.md b/config/README.md
index a79c862e4..952226d93 100644
--- a/config/README.md
+++ b/config/README.md
@@ -22,7 +22,7 @@ Your machine specific config file (e.g. feltor/config/your-machine.mk) should ha
 | NVCCFLAGS | -std=c++14  -Xcompiler "-Wall -mavx -mfma"                             | flags for nvcc  and underlying host compiler, (minimum instruction set is sse4.1, avx and fma are recommended)                         |
 | NVCCARCH  | -arch sm_35                              | specify the **gpu** compute capability  https://developer.nvidia.com/cuda-gpus (note: can be overwritten on the command line) |
 |                                          |                                          |     |
-|  INCLUDE  | -I$(HOME)/include                        | cusp, thrust, json, vcl and the draw libraries. The default expects to find (symbolic links to ) these libraries in your home folder |
+|  INCLUDE  | -I$(HOME)/include                        | cusp, thrust, json, vcl and the draw (if needed) libraries. The default expects to find (symbolic links to ) these libraries in your home folder |
 |   LIBS    | -lnetcdf -lhdf5 -ldhf5_hl                | netcdf library                           |
 |  JSONLIB  | -L$(HOME)/include/json/../../src/lib_json -ljsoncpp | the JSONCPP library                      |
 |  GLFLAGS  | $$(pkg-config --static --libs glfw3)     | glfw3 installation (if glfw3 was installed correctly the default should work) |
@@ -60,6 +60,6 @@ make blas_mpib device=gpu NVCCARCH='-arch sm_60' OPT=-O2
  - If MPI is used in connection with the gpu backend, the mpi installation needs to be **cuda-aware**
  - If `icc` is used as the C++ compiler the `-restrict` option has to be used to enable the recognition of the restrict keyword
  - Support for OpenMP-4 is recommended (at least gcc-4.9 or icc-15), but not mandatory
- - The library headers are compliant with the c++11 standard but we reserve the right to upgrade that in future updates
+ - The library headers are compliant with the c++14 standard but we reserve the right to upgrade that in future updates
 
 
-- 
GitLab


From e5b8d12fc271fd6a2fc13aa173658d8acb2094ab Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 9 Feb 2021 17:11:26 +0100
Subject: [PATCH 470/540] Update 2d evaluation tests

with the one that caught the race condition in exdot_cuda (if executed
with cuda-memcheck)
---
 inc/dg/topology/evaluation_mpit.cu | 35 ++++++++++++++------------
 inc/dg/topology/evaluation_t.cu    | 40 ++++++++++++++++--------------
 2 files changed, 41 insertions(+), 34 deletions(-)

diff --git a/inc/dg/topology/evaluation_mpit.cu b/inc/dg/topology/evaluation_mpit.cu
index 85a91645f..75d41da34 100644
--- a/inc/dg/topology/evaluation_mpit.cu
+++ b/inc/dg/topology/evaluation_mpit.cu
@@ -13,11 +13,14 @@ double function( double x)
 {
     return exp(x);
 }
-
 template<class T>
-T function( T x, T y)
+T function(T x, T y)
 {
-        return exp(x)*exp(y);
+    T rho = 0.20943951023931953; //pi/15
+    T delta = 0.050000000000000003;
+    if( y<= M_PI)
+        return delta*cos(x) - 1./rho/cosh( (y-M_PI/2.)/rho)/cosh( (y-M_PI/2.)/rho);
+    return delta*cos(x) + 1./rho/cosh( (3.*M_PI/2.-y)/rho)/cosh( (3.*M_PI/2.-y)/rho);
 }
 double function( double x, double y, double z)
 {
@@ -35,14 +38,14 @@ int main(int argc, char** argv)
     if(rank==0)std::cout << "On Grid "<<n<<" x "<<Nx<<" x "<<Ny<<" x "<<Nz<<"\n";
     MPI_Comm comm2d, comm3d;
     mpi_init2d( dg::PER, dg::PER, comm2d);
-    dg::MPIGrid2d g2d( 1, 2., 3, 4, n, Nx, Ny, dg::PER, dg::PER, comm2d);
-    dg::RealMPIGrid2d<float> gf2d( 1, 2., 3, 4, n, Nx, Ny, dg::PER, dg::PER, comm2d);
+    dg::MPIGrid2d g2d( 0.0, 6.2831853071795862, 0.0, 6.2831853071795862, 3, 48, 48, dg::PER, dg::PER, comm2d);
+    dg::RealMPIGrid2d<float> gf2d( 0.0, 6.2831853071795862, 0.0, 6.2831853071795862, 3, 48, 48, dg::PER, dg::PER, comm2d);
     mpi_init3d( dg::PER, dg::PER, dg::PER, comm3d);
     dg::MPIGrid3d g3d( 1, 2, 3, 4, 5, 6, n, Nx, Ny, Nz, dg::PER, dg::PER, dg::PER, comm3d);
 
     //test evaluation and expand functions
     dg::MDVec func2d = dg::construct<dg::MDVec>(dg::evaluate( function<double>, g2d));
-    dg::MDVec funcf2d = dg::construct<dg::MDVec>(dg::evaluate( function<float>, g2d));
+    dg::fMDVec funcf2d = dg::construct<dg::fMDVec>(dg::evaluate( function<float>, gf2d));
     dg::MDVec func3d = dg::construct<dg::MDVec>(dg::evaluate( function, g3d));
     //test weights
     const dg::MDVec w2d = dg::construct<dg::MDVec>(dg::create::weights(g2d));
@@ -51,31 +54,31 @@ int main(int argc, char** argv)
     exblas::udouble res;
 
     double integral2d = dg::blas1::dot( w2d, func2d); res.d = integral2d;
-    if(rank==0)std::cout << "2D integral               "<<std::setw(6)<<integral2d <<"\t" << res.i - 4639875759346476257 << "\n";
-    double sol2d = (exp(2.)-exp(1))*(exp(4.)-exp(3));
+    if(rank==0)std::cout << "2D integral               "<<std::setw(6)<<integral2d <<"\t" << res.i + 4823280491526356992<< "\n";
+    double sol2d = 0.;
     if(rank==0)std::cout << "Correct integral is       "<<std::setw(6)<<sol2d<<std::endl;
-    if(rank==0)std::cout << "Relative 2d error is      "<<(integral2d-sol2d)/sol2d<<"\n\n";
+    if(rank==0)std::cout << "2d error is               "<<(integral2d-sol2d)<<"\n\n";
     float integralf2d = dg::blas1::dot( wf2d, funcf2d); res.d = integralf2d;
-    if(rank==0)std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\t" << res.i - 4639875760323035136<< "\n";
-    float solf2d = (exp(2.)-exp(1))*(exp(4.)-exp(3));
+    if(rank==0)std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\t" << res.i - 4525606114229747712<< "\n";
+    float solf2d = 0.;
     if(rank==0)std::cout << "Correct integral is       "<<std::setw(6)<<solf2d<<std::endl;
-    if(rank==0)std::cout << "Relative 2d error (float) "<<(integralf2d-solf2d)/solf2d<<"\n\n";
+    if(rank==0)std::cout << "2d error (float)          "<<(integralf2d-solf2d)<<"\n\n";
 
     double integral3d = dg::blas1::dot( w3d, func3d); res.d = integral3d;
     if(rank==0)std::cout << "3D integral               "<<std::setw(6)<<integral3d <<"\t" << res.i - 4675882723962622631<< "\n";
-    double sol3d = sol2d*(exp(6.)-exp(5));
+    double sol3d = (exp(2.)-exp(1))*(exp(4.)-exp(3))*(exp(6.)-exp(5));
     if(rank==0)std::cout << "Correct integral is       "<<std::setw(6)<<sol3d<<std::endl;
     if(rank==0)std::cout << "Relative 3d error is      "<<(integral3d-sol3d)/sol3d<<"\n\n";
 
     double norm2d = dg::blas2::dot( w2d, func2d); res.d = norm2d;
-    if(rank==0)std::cout << "Square normalized 2D norm "<<std::setw(6)<<norm2d<<"\t" << res.i - 4674091193523851724<<"\n";
-    double solution2d = (exp(4.)-exp(2))/2.*(exp(8.) -exp(6))/2.;
+    if(rank==0)std::cout << "Square normalized 2D norm "<<std::setw(6)<<norm2d<<"\t" << res.i - 4635333359953759707<<"\n";
+    double solution2d = 80.0489;
     if(rank==0)std::cout << "Correct square norm is    "<<std::setw(6)<<solution2d<<std::endl;
     if(rank==0)std::cout << "Relative 2d error is      "<<(norm2d-solution2d)/solution2d<<"\n\n";
 
     double norm3d = dg::blas2::dot( func3d, w3d, func3d); res.d = norm3d;
     if(rank==0)std::cout << "Square normalized 3D norm "<<std::setw(6)<<norm3d<<"\t" << res.i - 4746764681002108278<<"\n";
-    double solution3d = solution2d*(exp(12.)-exp(10))/2.;
+    double solution3d = (exp(4.)-exp(2))/2.*(exp(8.)-exp(6.))/2.*(exp(12.)-exp(10))/2.;
     if(rank==0)std::cout << "Correct square norm is    "<<std::setw(6)<<solution3d<<std::endl;
     if(rank==0)std::cout << "Relative 3d error is      "<<(norm3d-solution3d)/solution3d<<"\n";
     if(rank==0)std::cout << "\nFINISHED! Continue with topology/derivatives_mpit.cu !\n\n";
diff --git a/inc/dg/topology/evaluation_t.cu b/inc/dg/topology/evaluation_t.cu
index a2e7d1966..44eb23d89 100644
--- a/inc/dg/topology/evaluation_t.cu
+++ b/inc/dg/topology/evaluation_t.cu
@@ -25,17 +25,21 @@ double operator()( double x)
     return sin(x);
 }
 };
-
 template<class T>
-T function( T x, T y)
+T function(T x, T y)
 {
-        return exp(x)*exp(y);
+    T rho = 0.20943951023931953; //pi/15
+    T delta = 0.050000000000000003;
+    if( y<= M_PI)
+        return delta*cos(x) - 1./rho/cosh( (y-M_PI/2.)/rho)/cosh( (y-M_PI/2.)/rho);
+    return delta*cos(x) + 1./rho/cosh( (3.*M_PI/2.-y)/rho)/cosh( (3.*M_PI/2.-y)/rho);
 }
 double function3d( double x, double y, double z)
 {
         return exp(x)*exp(y)*exp(z);
 }
 
+
 int main()
 {
     std::cout << "This program tests the exblas::dot function. The tests succeed only if the evaluation and grid functions but also the weights and especially the exblas::dot function are correctly implemented and compiled. Furthermore, the compiler implementation of the exp function in the math library must be consistent across platforms to get reproducible results\n";
@@ -43,9 +47,9 @@ int main()
     unsigned n = 3, Nx = 12, Ny = 28, Nz = 100;
     std::cout << "On Grid "<<n<<" x "<<Nx<<" x "<<Ny<<" x "<<Nz<<"\n";
 
-    dg::Grid1d g1d( 1, 2, n, Nx);
-    dg::Grid2d g2d( 1, 2, 3, 4, n, Nx, Ny);
-    dg::RealGrid2d<float> gf2d( 1, 2, 3, 4, n, Nx, Ny);
+    dg::Grid1d g1d( 1, 2, n, 12);
+    dg::Grid2d g2d( 0.0, 6.2831853071795862, 0.0, 6.2831853071795862, 3, 48, 48);
+    dg::RealGrid2d<float> gf2d( 0.0, 6.2831853071795862, 0.0, 6.2831853071795862, 3, 48, 48);
     dg::Grid3d g3d( 1, 2, 3, 4, 5, 6, n, Nx, Ny, Nz,dg::PER,dg::PER,dg::PER);
 
     //test evaluation functions
@@ -66,20 +70,20 @@ int main()
     std::cout << "Relative 1d error is      "<<(integral-sol)/sol<<"\n\n";
 
     double integral2d = dg::blas1::dot( w2d, func2d); res.d = integral2d;
-    std::cout << "2D integral               "<<std::setw(6)<<integral2d <<"\t" << res.i - 4639875759346476257<< "\n";
-    double sol2d = (exp(2.)-exp(1))*(exp(4.)-exp(3));
+    std::cout << "2D integral               "<<std::setw(6)<<integral2d <<"\t" << res.i + 4823280491526356992<< "\n";
+    double sol2d = 0;
     std::cout << "Correct integral is       "<<std::setw(6)<<sol2d<<std::endl;
-    std::cout << "Relative 2d error is      "<<(integral2d-sol2d)/sol2d<<"\n\n";
+    std::cout << "2d error is               "<<(integral2d-sol2d)<<"\n\n";
 
-    float integralf2d = dg::blas1::dot( wf2d, func2d); res.d = integralf2d;
-    std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\t" << res.i - 4639875760323035136<< "\n";
-    float solf2d = (exp(2.)-exp(1))*(exp(4.)-exp(3));
+    float integralf2d = dg::blas1::dot( wf2d, funcf2d); res.d = integralf2d;
+    std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\t" << res.i - 4525606114229747712<< "\n";
+    float solf2d = 0;
     std::cout << "Correct integral is       "<<std::setw(6)<<solf2d<<std::endl;
-    std::cout << "Relative 2d error (float) "<<(integralf2d-solf2d)/solf2d<<"\n\n";
+    std::cout << "2d error (float)          "<<(integralf2d-solf2d)<<"\n\n";
 
     double integral3d = dg::blas1::dot( w3d, func3d); res.d = integral3d;
     std::cout << "3D integral               "<<std::setw(6)<<integral3d <<"\t" << res.i - 4675882723962622631<< "\n";
-    double sol3d = sol2d*(exp(6.)-exp(5.));
+    double sol3d = (exp(2.)-exp(1))*(exp(4.)-exp(3))*(exp(6.)-exp(5));
     std::cout << "Correct integral is       "<<std::setw(6)<<sol3d<<std::endl;
     std::cout << "Relative 3d error is      "<<(integral3d-sol3d)/sol3d<<"\n\n";
 
@@ -90,14 +94,14 @@ int main()
     std::cout << "Relative 1d error is      "<<(norm-solution)/solution<<"\n\n";
 
     double norm2d = dg::blas2::dot( w2d, func2d); res.d = norm2d;
-    std::cout << "Square normalized 2D norm "<<std::setw(6)<<norm2d<<"\t" << res.i - 4674091193523851724<<"\n";
-    double solution2d = (exp(4.)-exp(2))/2.*(exp(8.) -exp(6.))/2.;
+    std::cout << "Square normalized 2D norm "<<std::setw(6)<<norm2d<<"\t" << res.i - 4635333359953759707<<"\n";
+    double solution2d = 80.0489;
     std::cout << "Correct square norm is    "<<std::setw(6)<<solution2d<<std::endl;
     std::cout << "Relative 2d error is      "<<(norm2d-solution2d)/solution2d<<"\n\n";
 
     double norm3d = dg::blas2::dot( func3d, w3d, func3d); res.d = norm3d;
     std::cout << "Square normalized 3D norm "<<std::setw(6)<<norm3d<<"\t" << res.i - 4746764681002108278<<"\n";
-    double solution3d = solution2d*(exp(12.) -exp(10.))/2.;
+    double solution3d = (exp(4.)-exp(2))/2.*(exp(8.)-exp(6.))/2.*(exp(12.)-exp(10))/2.;
     std::cout << "Correct square norm is    "<<std::setw(6)<<solution3d<<std::endl;
     std::cout << "Relative 3d error is      "<<(norm3d-solution3d)/solution3d<<"\n\n";
 
@@ -127,7 +131,7 @@ int main()
     }catch ( std::exception& e)
     {
         std::cerr << "Error thrown as expected\n";
-        std::cerr << e.what() << std::endl;
+        //std::cerr << e.what() << std::endl;
     }
 
     std::cout << "\nFINISHED! Continue with topology/derivatives_t.cu !\n\n";
-- 
GitLab


From 175f2abdcc352a07c2e3b7051f9f41ab09f56599 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Feb 2021 10:42:40 +0100
Subject: [PATCH 471/540] Add remark about invert ni in feltor

---
 src/feltor/feltor.h | 2 ++
 src/feltor/init.h   | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 49e7131d4..e09d51ddd 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -696,6 +696,8 @@ template<class Geometry, class IMatrix, class Matrix, class Container>
 void Explicit<Geometry, IMatrix, Matrix, Container>::initializeni(
     const Container& src, Container& target, std::string initphi)
 {
+    //According to Markus we should actually always invert
+    //so we should reconsider this function
     // Ni = ne
     dg::blas1::copy( src, target);
     if (m_p.tau[1] != 0.) {
diff --git a/src/feltor/init.h b/src/feltor/init.h
index cd6243fee..3a4ef8628 100644
--- a/src/feltor/init.h
+++ b/src/feltor/init.h
@@ -127,6 +127,8 @@ HVec turbulent_bath(const Geometry& grid,
 }
 
 
+//actually we should always invert according to Markus
+//because the dG direct application is supraconvergent
 void init_ni(
     std::array<std::array<DVec,2>,2>& y0,
     Explicit<Geometry, IDMatrix, DMatrix, DVec>& feltor,
-- 
GitLab


From bac869396d3cd7e1400df0f7c50e56802dbbabc2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Feb 2021 10:45:42 +0100
Subject: [PATCH 472/540] Add standard construct method to multigrid

---
 inc/dg/multigrid.h | 41 +++++++++++++++++++++++------------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 565dfa2df..28406912f 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -49,12 +49,6 @@ struct MultigridCG2d
     using value_type = get_value_type<Container>;
     ///@brief Allocate nothing, Call \c construct method before usage
     MultigridCG2d(){}
-    ///@copydoc construct()
-    template<class ...Params>
-    MultigridCG2d( const Geometry& grid, const unsigned stages, Params&& ... ps)
-    {
-        construct( grid, stages, std::forward<Params>(ps)...);
-    }
     /**
      * @brief Construct the grids and the interpolation/projection operators
      *
@@ -64,14 +58,18 @@ struct MultigridCG2d
      * @param ps parameters necessary for \c dg::construct to construct a \c Container from a \c dg::HVec
     */
     template<class ...Params>
-    void construct( const Geometry& grid, const unsigned stages, Params&& ... ps)
+    MultigridCG2d( const Geometry& grid, const unsigned stages, Params&& ... ps):
+        m_stages(stages),
+        m_grids( stages),
+        m_inter(    stages-1),
+        m_interT(   stages-1),
+        m_project(  stages-1),
+        m_cg(    stages),
+        m_cheby( stages),
+        m_x( stages)
     {
-        m_stages = stages;
-        if(stages < 2 ) throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
-
-		m_grids.resize(stages);
-        m_cg.resize(stages);
-        m_cheby.resize(stages);
+        if(stages < 2 )
+            throw Error( Message(_ping_)<<" There must be minimum 2 stages in a multigrid solver! You gave " << stages);
 
         m_grids[0].reset( grid);
         //m_grids[0].get().display();
@@ -83,10 +81,6 @@ struct MultigridCG2d
             //m_grids[u]->display();
         }
 
-		m_inter.resize(stages-1);
-		m_interT.resize(stages-1);
-        m_project.resize( stages-1);
-
 		for(unsigned u=0; u<stages-1; u++)
         {
             // Projecting from one grid to the next is the same as
@@ -96,7 +90,6 @@ struct MultigridCG2d
             m_interT[u].construct( dg::create::fast_projection(*m_grids[u], 1, 2, 2, dg::not_normed), std::forward<Params>(ps)...);
         }
 
-        m_x.resize( m_stages);
         for( unsigned u=0; u<m_stages; u++)
             m_x[u] = dg::construct<Container>( dg::evaluate( dg::zero, *m_grids[u]), std::forward<Params>(ps)...);
         m_r = m_b = m_x;
@@ -109,6 +102,18 @@ struct MultigridCG2d
         }
     }
 
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        //construct and swap
+        *this = MultigridCG2d( std::forward<Params>( ps)...);
+    }
 
     /**
     * @brief Project vector to all involved grids
-- 
GitLab


From 91d30441443d12d0978892bb269a64e58d9ecf29 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Feb 2021 10:47:47 +0100
Subject: [PATCH 473/540] Change default to gpu in Makefile

---
 inc/dg/Makefile          | 2 +-
 src/lamb_dipole/Makefile | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/dg/Makefile b/inc/dg/Makefile
index 1d6b1a310..058deb17a 100644
--- a/inc/dg/Makefile
+++ b/inc/dg/Makefile
@@ -1,4 +1,4 @@
-device=omp
+device=gpu
 
 #configure machine
 include ../../config/default.mk
diff --git a/src/lamb_dipole/Makefile b/src/lamb_dipole/Makefile
index 703eb53ac..ec7bfc220 100644
--- a/src/lamb_dipole/Makefile
+++ b/src/lamb_dipole/Makefile
@@ -1,4 +1,4 @@
-device=omp
+device=gpu
 
 #configure machine
 include ../../config/default.mk
-- 
GitLab


From 8d104c18a107adf4a92316ad6106d87c31772166 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Feb 2021 11:10:30 +0100
Subject: [PATCH 474/540] Add ISNFINITE and ISNSANE utility functors

---
 inc/dg/blas1.h                  | 10 ++++--
 inc/dg/blas2.h                  |  5 ++-
 inc/dg/functors.h               | 57 ++++++++++++++++++++++++++++++---
 inc/dg/topology/evaluation_t.cu |  5 +++
 4 files changed, 68 insertions(+), 9 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 2aabff4e0..2f8dc552d 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -55,7 +55,10 @@ For example
 dg::DVec two( 100,2), three(100,3);
 double result = dg::blas1::dot( two, three); // result = 600 (100*(2*3))
 @endcode
- * @attention if one of the input vectors contains \c NaN then the behaviour is undefined and the function may throw
+ * @attention if one of the input vectors contains \c Inf or \c NaN or the
+ * product of the input numbers reaches \c Inf or \c Nan then the behaviour
+ * is undefined and the function may throw. See @ref dg::ISNFINITE and @ref
+ * dg::ISNSANE in that case
  * @note Our implementation guarantees binary reproducible results.
  * The sum is computed with infinite precision and the result is rounded
  * to the nearest double precision number.
@@ -88,11 +91,12 @@ inline get_value_type<ContainerType1> dot( const ContainerType1& x, const Contai
 
 For example
 @code
-//Check if a vector contains NaN
+//Check if a vector contains Inf or NaN
 thrust::device_vector<double> x( 100);
 thrust::device_vector<bool> boolvec ( 100, false);
-dg::blas1::transform( x, boolvec, dg::ISNAN<double>());
+dg::blas1::transform( x, boolvec, dg::ISNFINITE<double>());
 bool hasnan = dg::blas1::reduce( boolvec, false, thrust::logical_or<bool>());
+std::cout << "x contains Inf or NaN "<<std::boolalpha<<hasnan<<"\n";
 @endcode
  * @param x Left Container
  * @param init initial value of the reduction
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index 76097fd94..828368f18 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -60,7 +60,10 @@ inline std::vector<int64_t> doDot_superacc( const ContainerType1& x, const Matri
  * matrix M \f[ x^T M y = \sum_{i,j=0}^{N-1} x_i M_{ij} y_j \f]
  *
  * @copydoc hide_code_evaluate2d
- * @attention if one of the input vectors contains \c NaN then the behaviour is undefined and the function may throw
+ * @attention if one of the input vectors contains \c Inf or \c NaN or the
+ * product of the input numbers reaches \c Inf or \c Nan then the behaviour
+ * is undefined and the function may throw. See @ref dg::ISNFINITE and @ref
+ * dg::ISNSANE in that case
  * @note Our implementation guarantees binary reproducible results up to and excluding the last mantissa bit of the result.
  * Furthermore, the sum is computed with infinite precision and the result is then rounded
  * to the nearest double precision number. Although the products are not computed with
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index 61aab4544..d6eb446aa 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -271,18 +271,65 @@ struct MOD
     T m_m;
 
 };
+
+/**
+ * @brief \f$ f(x) =\f$ \c !std::isfinite(x)
+ *
+ * return true if \c x is \c NaN or \c Inf
+@code
+//Check if a vector contains Inf or NaN
+thrust::device_vector<double> x( 100);
+thrust::device_vector<bool> boolvec ( 100, false);
+dg::blas1::transform( x, boolvec, dg::ISNFINITE<double>());
+bool hasnan = dg::blas1::reduce( boolvec, false, thrust::logical_or<bool>());
+std::cout << "x contains Inf or NaN "<<std::boolalpha<<hasnan<<"\n";
+@endcode
+ */
+template <class T>
+struct ISNFINITE
+{
+#ifdef __CUDACC__
+    DG_DEVICE bool operator()(T x){ return !isfinite(x);}
+#else
+    bool operator()( T x){ return !std::isfinite(x);}
+#endif
+};
 /**
- * @brief \f$ f(x) =\f$ \c std::isnan(x)
+ * @brief \f$ f(x) =\begin{cases} \mathrm{true\ if}\ |x| > 10^{100}\\
+ * \mathrm{false\ else}
+ * \end{cases}\f$
  *
- * Check for NaN
+ * Also return true if \c x is \c NaN or \c Inf.
+ * The intention is to use this in the reduce function to debug code if
+ * you get an error message of Inf or Nan from the \c dot function
+@code
+//Check if a vector contains is sane
+thrust::device_vector<double> x( 100);
+thrust::device_vector<bool> boolvec ( 100, false);
+dg::blas1::transform( x, boolvec, dg::ISNSANE<double>());
+bool hasnan = dg::blas1::reduce( boolvec, false, thrust::logical_or<bool>());
+std::cout << "x contains insane numbers "<<std::boolalpha<<hasnan<<"\n";
+@endcode
  */
 template <class T>
-struct ISNAN
+struct ISNSANE
 {
 #ifdef __CUDACC__
-    DG_DEVICE bool operator()(T x){return isnan(x);}
+    DG_DEVICE bool operator()(T x){
+        if( !isfinite(x))
+            return true;
+        if( x > 1e100 || x < -1e100)
+            return true;
+        return false;
+    }
 #else
-    bool operator()( T x){ return std::isnan(x);}
+    bool operator()( T x){
+        if( !std::isfinite(x))
+            return true;
+        if( x > 1e100 || x < -1e100)
+            return true;
+        return false;
+    }
 #endif
 };
 
diff --git a/inc/dg/topology/evaluation_t.cu b/inc/dg/topology/evaluation_t.cu
index 44eb23d89..06d5cf685 100644
--- a/inc/dg/topology/evaluation_t.cu
+++ b/inc/dg/topology/evaluation_t.cu
@@ -125,7 +125,12 @@ int main()
     norm = dg::blas2::dot( integral_num, dg::create::weights( g1d), integral_num);
     std::cout << " Error norm of  1d integral function "<<norm<<"\n";
     // TEST if dot throws on NaN
+    std::cout << "TEST if dot throws on Inf or Nan:\n";
     dg::blas1::transform( x,x, dg::LN<double>());
+    thrust::device_vector<bool> boolvec ( 100, false);
+    dg::blas1::transform( x, boolvec, dg::ISNFINITE<double>());
+    bool hasnan = dg::blas1::reduce( boolvec, false, thrust::logical_or<bool>());
+    std::cout << "x contains Inf or Nan numbers "<<std::boolalpha<<hasnan<<"\n";
     try{
         dg::blas1::dot( x,x);
     }catch ( std::exception& e)
-- 
GitLab


From cef625744c4fb2b08a4e134a41de5fbba7c6932f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Feb 2021 16:36:29 +0100
Subject: [PATCH 475/540] Add 2 more parameters to tensor::multiply

this forms the basis on which to remove the duplicate
code in elliptic
---
 inc/dg/Doxyfile               |  15 ++
 inc/dg/functors.h             |  24 +--
 inc/dg/topology/functions.h   |   4 +-
 inc/dg/topology/multiply.h    | 345 +++++++++++++++++++++++-----------
 inc/dg/topology/multiply_b.cu |   8 +-
 inc/dg/topology/multiply_t.cu |  12 +-
 inc/dg/topology/tensor.h      |   5 +
 7 files changed, 280 insertions(+), 133 deletions(-)

diff --git a/inc/dg/Doxyfile b/inc/dg/Doxyfile
index 1653863ab..159f86c06 100644
--- a/inc/dg/Doxyfile
+++ b/inc/dg/Doxyfile
@@ -881,6 +881,21 @@ EXCLUDE                = topology/ell_interpolation.cuh \
                          backend/average_cpu.h \
                          backend/average_gpu.cuh \
                          backend/average_omp.h \
+                         backend/blas1_array.h \
+                         backend/blas1_cuda.cuh \
+                         backend/blas1_omp.h \
+                         backend/blas1_serial.h \
+                         backend/blas1_dispatch_mpi.h \
+                         backend/blas1_dispatch_scalar.h \
+                         backend/blas1_dispatch_shared.h \
+                         backend/blas1_dispatch_vector.h \
+                         backend/blas2_cusp.h \
+                         backend/blas2_dispatch_mpi.h \
+                         backend/blas2_dispatch_scalar.h \
+                         backend/blas2_dispatch_shared.h \
+                         backend/blas2_dispatch_vector.h \
+                         backend/blas2_selfmade.h \
+                         backend/blas2_sparseblockmat.h \
                          backend/config.h \
                          backend/predicate.h \
                          backend/sparseblockmat_gpu_kernels.cuh \
diff --git a/inc/dg/functors.h b/inc/dg/functors.h
index d6eb446aa..251a66836 100644
--- a/inc/dg/functors.h
+++ b/inc/dg/functors.h
@@ -134,30 +134,26 @@ struct LN
 template < class T = double>
 struct SQRT
 {
-    /**
-     * @brief Square root
-     *
-     * @param x of x
-     *
-     * @return sqrt(x)
-     */
     DG_DEVICE T operator() (T x) const
     {
         return sqrt(x);
     }
 };
+///@brief \f$ f(x) = \frac{1}{\sqrt{x}}\f$
+template < class T = double>
+struct InvSqrt
+{
+    DG_DEVICE T operator() (T x) const
+    {
+        return 1./sqrt(x);
+    }
+};
 
 
 ///@brief \f$ f(x) = 1/x \f$
 template <class T = double>
 struct INVERT
 {
-    /**
-     * @brief Invert the given value
-     *
-     * @param x  the input
-     * @return  1/x
-     */
     DG_DEVICE T operator()( T x)const{ return 1./x;}
 };
 
@@ -273,7 +269,7 @@ struct MOD
 };
 
 /**
- * @brief \f$ f(x) =\f$ \c !std::isfinite(x)
+ * @brief \f$ f(x) = \mathrm{!std::isfinite(x)}\f$
  *
  * return true if \c x is \c NaN or \c Inf
 @code
diff --git a/inc/dg/topology/functions.h b/inc/dg/topology/functions.h
index 5ea135c81..2cc9ba536 100644
--- a/inc/dg/topology/functions.h
+++ b/inc/dg/topology/functions.h
@@ -74,7 +74,5 @@ DG_DEVICE static inline float zero( float x, float y) {return 0.;}
 ///@brief \f$ f(x,y,z) = 0\f$
 DG_DEVICE static inline float zero( float x, float y, float z) {return 0.;}
 
-} //namespace dg
-
 ///@}
-
+} //namespace dg
diff --git a/inc/dg/topology/multiply.h b/inc/dg/topology/multiply.h
index 42418d9f4..fe5ebffed 100644
--- a/inc/dg/topology/multiply.h
+++ b/inc/dg/topology/multiply.h
@@ -5,104 +5,88 @@
 #include "dg/blas1.h"
 #include "tensor.h"
 
-namespace dg
-{
-///@brief Utility functions used in connection with the SparseTensor class
-namespace tensor
-{
-
-///@addtogroup tensor
-///@{
-
-/**
- * @brief \f$ t^{ij} = \mu t^{ij} \ \forall i,j \f$
+/*!@file
  *
- * Scale tensor with a Scalar or a Vector
- * @param t input (contains result on output)
- * @param mu all elements in t are scaled with mu
- * @copydoc hide_ContainerType
+ * Basic tensor functions (functions that involve sparse tensors)
  */
-template<class ContainerType0, class ContainerType1>
-void scal( SparseTensor<ContainerType0>& t, const ContainerType1& mu)
-{
-    unsigned size=t.values().size();
-    for( unsigned i=0; i<size; i++)
-        dg::blas1::pointwiseDot( mu, t.values()[i], t.values()[i]);
-}
 
-///@cond
-namespace detail
+namespace dg
 {
+///@addtogroup variadic_subroutines
+///@{
+
+/// \f$ y_i \leftarrow \lambda T_{ij} x_i + \mu y_i\f$
 template<class value_type>
-struct Multiply{
+struct TensorMultiply2d{
     DG_DEVICE
-    void operator() ( value_type t00, value_type t01,
-                      value_type t10, value_type t11,
-                      value_type in0, value_type in1,
-                      value_type& out0, value_type& out1) const
+    void operator() (
+              value_type lambda,
+              value_type t00, value_type t01,
+              value_type t10, value_type t11,
+              value_type in0, value_type in1,
+              value_type mu,
+              value_type& out0, value_type& out1) const
     {
         value_type tmp0 = DG_FMA(t00,in0 , t01*in1);
         value_type tmp1 = DG_FMA(t10,in0 , t11*in1);
-        out1 = tmp1;
-        out0 = tmp0;
+        value_type temp = out1*mu;
+        out1 = DG_FMA( lambda, tmp1, temp);
+        temp = out0*mu;
+        out0 = DG_FMA( lambda, tmp0, temp);
     }
+};
+/// \f$ y_i \leftarrow \lambda T_{ij} x_i + \mu y_i\f$
+template<class value_type>
+struct TensorMultiply3d{
     DG_DEVICE
-    void operator() ( value_type t00, value_type t01, value_type t02,
+    void operator() ( value_type lambda,
+                      value_type t00, value_type t01, value_type t02,
                       value_type t10, value_type t11, value_type t12,
                       value_type t20, value_type t21, value_type t22,
                       value_type in0, value_type in1, value_type in2,
+                      value_type mu,
                       value_type& out0, value_type& out1, value_type& out2) const
     {
         value_type tmp0 = DG_FMA( t00,in0 , (DG_FMA( t01,in1 , t02*in2)));
         value_type tmp1 = DG_FMA( t10,in0 , (DG_FMA( t11,in1 , t12*in2)));
         value_type tmp2 = DG_FMA( t20,in0 , (DG_FMA( t21,in1 , t22*in2)));
-        out2 = tmp2;
-        out1 = tmp1;
-        out0 = tmp0;
-    }
-};
-template<class value_type>
-struct Determinant
-{
-    DG_DEVICE
-    value_type operator()( value_type in) const{
-        return 1./sqrt(in);
-    }
-    DG_DEVICE
-    value_type operator() ( value_type t00, value_type t01,
-                            value_type t10, value_type t11) const
-    {
-        return DG_FMA( t00,t11 , (-t10*t01));
-    }
-    DG_DEVICE
-    value_type operator() ( value_type t00, value_type t01, value_type t02,
-                            value_type t10, value_type t11, value_type t12,
-                            value_type t20, value_type t21, value_type t22) const
-    {
-        return t00*this->operator()(t11, t12, t21, t22)
-              -t01*this->operator()(t10, t12, t20, t22)
-              +t02*this->operator()(t10, t11, t20, t21);
+        value_type temp = out2*mu;
+        out2 = DG_FMA( lambda, tmp2, temp);
+        temp = out1*mu;
+        out1 = DG_FMA( lambda, tmp1, temp);
+        temp = out0*mu;
+        out0 = DG_FMA( lambda, tmp0, temp);
     }
 };
+/// \f$ y_i \leftarrow \lambda T^{-1}_{ij} x_i + \mu y_i\f$
 template<class value_type>
-struct InverseMultiply{
+struct InverseTensorMultiply2d{
     DG_DEVICE
-    void operator() ( value_type t00, value_type t01,
-                      value_type t10, value_type t11,
-                      value_type in0, value_type in1,
-                      value_type& out0, value_type& out1) const
+    void operator() (  value_type lambda,
+                       value_type t00, value_type t01,
+                       value_type t10, value_type t11,
+                       value_type in0, value_type in1,
+        value_type mu, value_type& out0, value_type& out1) const
     {
         value_type dett = DG_FMA( t00,t11 , (-t10*t01));
         value_type tmp0 = DG_FMA( in0,t11 , (-in1*t01));
         value_type tmp1 = DG_FMA( t00,in1 , (-t10*in0));
-        out1 = tmp1/dett;
-        out0 = tmp0/dett;
+        value_type temp = out1*mu;
+        out1 = DG_FMA( lambda, tmp1/dett, temp);
+        temp = out0*mu;
+        out0 = DG_FMA( lambda, tmp0/dett, temp);
     }
+};
+/// \f$ y_i \leftarrow \lambda T^{-1}_{ij} x_i + \mu y_i\f$
+template<class value_type>
+struct InverseTensorMultiply3d{
     DG_DEVICE
-    void operator() ( value_type t00, value_type t01, value_type t02,
+    void operator() ( value_type lambda,
+                      value_type t00, value_type t01, value_type t02,
                       value_type t10, value_type t11, value_type t12,
                       value_type t20, value_type t21, value_type t22,
                       value_type in0, value_type in1, value_type in2,
+                      value_type mu,
                       value_type& out0, value_type& out1, value_type& out2) const
     {
         value_type dett = det( t00,t01,t02, t10,t11,t12, t20,t21,t22);
@@ -110,9 +94,12 @@ struct InverseMultiply{
         value_type tmp0 = det( in0,t01,t02, in1,t11,t12, in2,t21,t22);
         value_type tmp1 = det( t00,in0,t02, t10,in1,t12, t20,in2,t22);
         value_type tmp2 = det( t00,t01,in0, t10,t11,in1, t20,t21,in2);
-        out2 = tmp2/dett;
-        out1 = tmp1/dett;
-        out0 = tmp0/dett;
+        value_type temp = out2*mu;
+        out2 = DG_FMA( lambda, tmp2/dett, temp);
+        temp = out1*mu;
+        out1 = DG_FMA( lambda, tmp1/dett, temp);
+        temp = out0*mu;
+        out0 = DG_FMA( lambda, tmp0/dett, temp);
     }
     private:
     DG_DEVICE
@@ -125,102 +112,170 @@ struct InverseMultiply{
               +t02*DG_FMA(t10, t21, (-t20*t11));
     }
 };
-}//namespace detail
-///@endcond
+///@}
+
+///@addtogroup variadic_evaluates
+///@{
+
+///\f$ y = t_{00} t_{11} - t_{10}t_{01} \f$
+template<class value_type>
+struct TensorDeterminant2d
+{
+    DG_DEVICE
+    value_type operator() ( value_type t00, value_type t01,
+                            value_type t10, value_type t11) const
+    {
+        return DG_FMA( t00,t11 , (-t10*t01));
+    }
+};
+///\f$ y = t_{00} t_{11}t_{22} + t_{01}t_{12}t_{20} + t_{02}t_{10}t_{21} - t_{02}t_{11}t_{20} - t_{01}t_{10}t_{22} - t_{00}t_{12}t_{21} \f$
+template<class value_type>
+struct TensorDeterminant3d
+{
+    DG_DEVICE
+    value_type operator() ( value_type t00, value_type t01, value_type t02,
+                            value_type t10, value_type t11, value_type t12,
+                            value_type t20, value_type t21, value_type t22) const
+    {
+        return t00*m_t(t11, t12, t21, t22)
+              -t01*m_t(t10, t12, t20, t22)
+              +t02*m_t(t10, t11, t20, t21);
+    }
+    private:
+    TensorDeterminant2d<value_type> m_t;
+};
+///@}
+
+/**
+ * @namespace dg::tensor
+ * @brief Utility functions used in connection with the SparseTensor class
+ */
+
+namespace tensor
+{
+
+///@addtogroup tensor
+///@{
+
 /**
- * @brief \f$ w^i = \sum_{i=0}^1t^{ij}v_j \text{ for } i\in \{0,1\}\f$
+ * @brief \f$ t^{ij} = \mu t^{ij} \ \forall i,j \f$
+ *
+ * Scale tensor with a Scalar or a Vector
+ * @param t input (contains result on output)
+ * @param mu all elements in t are scaled with mu
+ * @copydoc hide_ContainerType
+ */
+template<class ContainerType0, class ContainerType1>
+void scal( SparseTensor<ContainerType0>& t, const ContainerType1& mu)
+{
+    unsigned size=t.values().size();
+    for( unsigned i=0; i<size; i++)
+        dg::blas1::pointwiseDot( mu, t.values()[i], t.values()[i]);
+}
+
+/**
+ * @brief \f$ w^i = \sum_{i=0}^1 \lambda t^{ij}v_j + \mu w^i \text{ for } i\in \{0,1\}\f$
  *
  * Multiply a tensor with a vector in 2d.
  * Ignore the 3rd dimension in \c t.
  * @param t input Tensor
+ * @param lambda (input)
  * @param in0 (input) first component  of \c v  (may alias out0)
  * @param in1 (input) second component of \c v  (may alias out1)
+ * @param mu (input)
  * @param out0 (output) first component  of \c w (may alias in0)
  * @param out1 (output) second component of \c w (may alias in1)
- * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with the appropriate functor
+ * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with \c dg::TensorMultiply2d
  * @copydoc hide_ContainerType
  */
-template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4>
-void multiply2d( const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, ContainerType3& out0, ContainerType4& out1)
+template<class ContainerTypeL, class ContainerType0, class ContainerType1, class ContainerType2, class ContainerTypeM, class ContainerType3, class ContainerType4>
+void multiply2d( const ContainerTypeL& lambda, const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, const ContainerTypeM& mu, ContainerType3& out0, ContainerType4& out1)
 {
-    dg::blas1::subroutine( detail::Multiply<get_value_type<ContainerType0>>(),
-                         t.value(0,0), t.value(0,1),
+    dg::blas1::subroutine( dg::TensorMultiply2d<get_value_type<ContainerType0>>(),
+            lambda,      t.value(0,0), t.value(0,1),
                          t.value(1,0), t.value(1,1),
                          in0,  in1,
-                         out0, out1);
+            mu,          out0, out1);
 }
 
 /**
- * @brief \f$ w^i = \sum_{i=0}^2t^{ij}v_j \text{ for } i\in \{0,1,2\}\f$
+ * @brief \f$ w^i = \sum_{i=0}^2\lambda t^{ij}v_j + \mu w^i \text{ for } i\in \{0,1,2\}\f$
  *
  * Multiply a tensor with a vector in 3d.
  * @param t input Tensor
+ * @param lambda (input) (may be a vector or an actual number like 0 or 1)
  * @param in0 (input)  first component of \c v  (may alias out0)
  * @param in1 (input)  second component of \c v (may alias out1)
  * @param in2 (input)  third component of \c v  (may alias out2)
+ * @param mu  (input) (may be a vector or an actual number like 0 or 1)
  * @param out0 (output)  first component of \c w  (may alias in0)
  * @param out1 (output)  second component of \c w (may alias in1)
  * @param out2 (output)  third component of \c w  (may alias in2)
- * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with the appropriate functor
+ * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with \c dg::TensorMultiply3d
  * @copydoc hide_ContainerType
  */
-template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4, class ContainerType5, class ContainerType6>
-void multiply3d( const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, const ContainerType3& in2, ContainerType4& out0, ContainerType5& out1, ContainerType6& out2)
+template<class ContainerTypeL, class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerTypeM, class ContainerType4, class ContainerType5, class ContainerType6>
+void multiply3d( const ContainerTypeL& lambda, const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, const ContainerType3& in2, const ContainerTypeM& mu, ContainerType4& out0, ContainerType5& out1, ContainerType6& out2)
 {
-    dg::blas1::subroutine( detail::Multiply<get_value_type<ContainerType0>>(),
-                         t.value(0,0), t.value(0,1), t.value(0,2),
+    dg::blas1::subroutine( dg::TensorMultiply3d<get_value_type<ContainerType0>>(),
+            lambda,      t.value(0,0), t.value(0,1), t.value(0,2),
                          t.value(1,0), t.value(1,1), t.value(1,2),
                          t.value(2,0), t.value(2,1), t.value(2,2),
                          in0, in1, in2,
-                         out0, out1, out2);
+            mu,          out0, out1, out2);
 }
 
 /**
- * @brief \f$ v_j = \sum_{i=0}^1(t^{-1})_{ji}w^i \text{ for } i\in \{0,1\}\f$
+ * @brief \f$ v_j = \sum_{i=0}^1\lambda (t^{-1})_{ji}w^i + \mu v_j \text{ for } i\in \{0,1\}\f$
  *
  * Multiply the inverse of a tensor \c t with a vector in 2d.
  * Ignore the 3rd dimension in \c t. The inverse of \c t is computed inplace.
  * @param t input Tensor
+ * @param lambda (input) (may be a vector or an actual number like 0 or 1)
  * @param in0 (input) first component of \c w    (may alias out0)
  * @param in1 (input) second component of \c w   (may alias out1)
+ * @param mu  (input) (may be a vector or an actual number like 0 or 1)
  * @param out0 (output) first component of \c v  (may alias in0)
  * @param out1 (output) second component of \c v (may alias in1)
+ * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with \c dg::InverseTensorMultiply2d
  * @copydoc hide_ContainerType
  */
-template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4>
-void inv_multiply2d( const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, ContainerType3& out0, ContainerType4& out1)
+template<class ContainerTypeL, class ContainerType0, class ContainerType1, class ContainerType2, class ContainerTypeM, class ContainerType3, class ContainerType4>
+void inv_multiply2d( const ContainerTypeL& lambda, const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, const ContainerTypeM& mu, ContainerType3& out0, ContainerType4& out1)
 {
-    dg::blas1::subroutine( detail::InverseMultiply<get_value_type<ContainerType0>>(),
-                         t.value(0,0), t.value(0,1),
+    dg::blas1::subroutine( dg::InverseTensorMultiply2d<get_value_type<ContainerType0>>(),
+              lambda,    t.value(0,0), t.value(0,1),
                          t.value(1,0), t.value(1,1),
                          in0,  in1,
-                         out0, out1);
+              mu,        out0, out1);
 }
 
 /**
- * @brief \f$ v_j = \sum_{i=0}^2(t^{-1})_{ji}w^i \text{ for } i\in \{0,1,2\}\f$i
+ * @brief \f$ v_j = \sum_{i=0}^2\lambda(t^{-1})_{ji}w^i + \mu v_j \text{ for } i\in \{0,1,2\}\f$i
  *
  * Multiply the inverse of a tensor with a vector in 3d.
  * The inverse of \c t is computed inplace.
  * @param t input Tensor
+ * @param lambda (input) (may be a vector or an actual number like 0 or 1)
  * @param in0 (input)  first component  of \c w (may alias out0)
  * @param in1 (input)  second component of \c w (may alias out1)
  * @param in2 (input)  third component  of \c w (may alias out2)
+ * @param mu  (input) (may be a vector or an actual number like 0 or 1)
  * @param out0 (output)  first component  of \c v (may alias in0)
  * @param out1 (output)  second component of \c v (may alias in1)
  * @param out2 (output)  third component  of \c v (may alias in2)
- * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with the appropriate functor
+ * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with \c dg::InverseTensorMultiply3d
  * @copydoc hide_ContainerType
  */
-template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4, class ContainerType5, class ContainerType6>
-void inv_multiply3d( const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, const ContainerType3& in2, ContainerType4& out0, ContainerType5& out1, ContainerType6& out2)
+template<class ContainerTypeL, class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerTypeM, class ContainerType4, class ContainerType5, class ContainerType6>
+void inv_multiply3d( const ContainerTypeL& lambda, const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, const ContainerType3& in2, const ContainerTypeM& mu, ContainerType4& out0, ContainerType5& out1, ContainerType6& out2)
 {
-    dg::blas1::subroutine( detail::InverseMultiply<get_value_type<ContainerType0>>(),
-                         t.value(0,0), t.value(0,1), t.value(0,2),
+    dg::blas1::subroutine( dg::InverseTensorMultiply3d<get_value_type<ContainerType0>>(),
+           lambda,       t.value(0,0), t.value(0,1), t.value(0,2),
                          t.value(1,0), t.value(1,1), t.value(1,2),
                          t.value(2,0), t.value(2,1), t.value(2,2),
                          in0, in1, in2,
-                         out0, out1, out2);
+           mu,           out0, out1, out2);
 }
 
 /**
@@ -229,13 +284,14 @@ void inv_multiply3d( const SparseTensor<ContainerType0>& t, const ContainerType1
 * Compute the minor determinant of a tensor \f$ \det_{2d}(t) := t_{00}t_{01}-t_{10}t_{11}\f$.
 * @param t the input tensor
 * @return the upper left minor determinant of \c t
+ * @note This function is just a shortcut for a call to \c dg::blas1::evaluate with \c dg::TensorDeterminant2d
 * @copydoc hide_ContainerType
 */
 template<class ContainerType>
 ContainerType determinant2d( const SparseTensor<ContainerType>& t)
 {
     ContainerType det = t.value(0,0);
-    dg::blas1::evaluate( det, dg::equals(), detail::Determinant<get_value_type<ContainerType>>(),
+    dg::blas1::evaluate( det, dg::equals(), dg::TensorDeterminant2d<get_value_type<ContainerType>>(),
                            t.value(0,0), t.value(0,1),
                            t.value(1,0), t.value(1,1));
     return det;
@@ -248,13 +304,14 @@ ContainerType determinant2d( const SparseTensor<ContainerType>& t)
 * \f$ \det(t) := t_{00}t_{11}t_{22} + t_{01}t_{12}t_{20} + \ldots - t_{22}t_{10}t_{01}\f$.
 * @param t the input tensor
 * @return the determinant of t
+ * @note This function is just a shortcut for a call to \c dg::blas1::evaluate with \c dg::TensorDeterminant3d
 * @copydoc hide_ContainerType
 */
 template<class ContainerType>
 ContainerType determinant( const SparseTensor<ContainerType>& t)
 {
     ContainerType det = t.value(0,0);
-    dg::blas1::evaluate( det, dg::equals(), detail::Determinant<get_value_type<ContainerType>>(),
+    dg::blas1::evaluate( det, dg::equals(), dg::TensorDeterminant3d<get_value_type<ContainerType>>(),
                            t.value(0,0), t.value(0,1), t.value(0,2),
                            t.value(1,0), t.value(1,1), t.value(1,2),
                            t.value(2,0), t.value(2,1), t.value(2,2));
@@ -268,8 +325,7 @@ ContainerType determinant( const SparseTensor<ContainerType>& t)
  * This is a convenience function that is equivalent to
  * @code
     ContainerType vol=determinant2d(t);
-    dg::blas1::transform(vol, vol, dg::INVERT<>());
-    dg::blas1::transform(vol, vol, dg::SQRT<>());
+    dg::blas1::transform(vol, vol, dg::InvSqrt<>());
     @endcode
  * @param t the input tensor
  * @return the inverse square root of the determinant of \c t
@@ -279,7 +335,7 @@ template<class ContainerType>
 ContainerType volume2d( const SparseTensor<ContainerType>& t)
 {
     ContainerType vol=determinant2d(t);
-    dg::blas1::transform(vol, vol, detail::Determinant<get_value_type<ContainerType>>());
+    dg::blas1::transform(vol, vol, dg::InvSqrt<get_value_type<ContainerType>>());
     return vol;
 }
 
@@ -290,8 +346,7 @@ ContainerType volume2d( const SparseTensor<ContainerType>& t)
  * This is a convenience function that is equivalent to
  * @code
     ContainerType vol=determinant(t);
-    dg::blas1::transform(vol, vol, dg::INVERT<>());
-    dg::blas1::transform(vol, vol, dg::SQRT<>());
+    dg::blas1::transform(vol, vol, dg::InvSqrt<>());
     @endcode
  * @param t the input tensor
  * @return the inverse square root of the determinant of \c t
@@ -301,10 +356,88 @@ template<class ContainerType>
 ContainerType volume( const SparseTensor<ContainerType>& t)
 {
     ContainerType vol=determinant(t);
-    dg::blas1::transform(vol, vol, detail::Determinant<get_value_type<ContainerType>>());
+    dg::blas1::transform(vol, vol, dg::InvSqrt<get_value_type<ContainerType>>());
     return vol;
 }
 
+//For convenience
+/**
+ * @brief \f$ w^i = \sum_{i=0}^1 t^{ij}v_j  \text{ for } i\in \{0,1\}\f$
+ *
+ * Multiply a tensor with a vector in 2d.
+ * Ignore the 3rd dimension in \c t.
+ * @param t input Tensor
+ * @param in0 (input) first component  of \c v  (may alias out0)
+ * @param in1 (input) second component of \c v  (may alias out1)
+ * @param out0 (output) first component  of \c w (may alias in0)
+ * @param out1 (output) second component of \c w (may alias in1)
+ * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with the appropriate functor
+ * @copydoc hide_ContainerType
+ */
+template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4>
+void multiply2d( const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, ContainerType3& out0, ContainerType4& out1)
+{
+    multiply2d( 1, t, in0, in1, 0., out0, out1);
+}
+
+/**
+ * @brief \f$ w^i = \sum_{i=0}^2 t^{ij}v_j \text{ for } i\in \{0,1,2\}\f$
+ *
+ * Multiply a tensor with a vector in 3d.
+ * @param t input Tensor
+ * @param in0 (input)  first component of \c v  (may alias out0)
+ * @param in1 (input)  second component of \c v (may alias out1)
+ * @param in2 (input)  third component of \c v  (may alias out2)
+ * @param out0 (output)  first component of \c w  (may alias in0)
+ * @param out1 (output)  second component of \c w (may alias in1)
+ * @param out2 (output)  third component of \c w  (may alias in2)
+ * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with the appropriate functor
+ * @copydoc hide_ContainerType
+ */
+template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4, class ContainerType5, class ContainerType6>
+void multiply3d( const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, const ContainerType3& in2, ContainerType4& out0, ContainerType5& out1, ContainerType6& out2)
+{
+    multiply3d( 1., t, in0, in1, in2, 0., out0, out1, out2);
+}
+
+/**
+ * @brief \f$ v_j = \sum_{i=0}^1(t^{-1})_{ji}w^i \text{ for } i\in \{0,1\}\f$
+ *
+ * Multiply the inverse of a tensor \c t with a vector in 2d.
+ * Ignore the 3rd dimension in \c t. The inverse of \c t is computed inplace.
+ * @param t input Tensor
+ * @param in0 (input) first component of \c w    (may alias out0)
+ * @param in1 (input) second component of \c w   (may alias out1)
+ * @param out0 (output) first component of \c v  (may alias in0)
+ * @param out1 (output) second component of \c v (may alias in1)
+ * @copydoc hide_ContainerType
+ */
+template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4>
+void inv_multiply2d( const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, ContainerType3& out0, ContainerType4& out1)
+{
+    inv_multiply2d( 1., t, in0, in1, out0, out1);
+}
+
+/**
+ * @brief \f$ v_j = \sum_{i=0}^2(t^{-1})_{ji}w^i \text{ for } i\in \{0,1,2\}\f$i
+ *
+ * Multiply the inverse of a tensor with a vector in 3d.
+ * The inverse of \c t is computed inplace.
+ * @param t input Tensor
+ * @param in0 (input)  first component  of \c w (may alias out0)
+ * @param in1 (input)  second component of \c w (may alias out1)
+ * @param in2 (input)  third component  of \c w (may alias out2)
+ * @param out0 (output)  first component  of \c v (may alias in0)
+ * @param out1 (output)  second component of \c v (may alias in1)
+ * @param out2 (output)  third component  of \c v (may alias in2)
+ * @note This function is just a shortcut for a call to \c dg::blas1::subroutine with the appropriate functor
+ * @copydoc hide_ContainerType
+ */
+template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4, class ContainerType5, class ContainerType6>
+void inv_multiply3d( const SparseTensor<ContainerType0>& t, const ContainerType1& in0, const ContainerType2& in1, const ContainerType3& in2, ContainerType4& out0, ContainerType5& out1, ContainerType6& out2)
+{
+    inv_multiply3d( 1., t, in0, in1, in2, 0., out0, out1, out2);
+}
 ///@}
 
 }//namespace tensor
diff --git a/inc/dg/topology/multiply_b.cu b/inc/dg/topology/multiply_b.cu
index 1a1cb9a76..d20aa71bf 100644
--- a/inc/dg/topology/multiply_b.cu
+++ b/inc/dg/topology/multiply_b.cu
@@ -24,10 +24,10 @@ int main()
     Vector v_y = dg::evaluate( dg::CONSTANT(5), grid), w_y(v_y);
     dg::SparseTensor<Vector> g(grid);
     int multi=20;
-    dg::tensor::multiply2d( g, v_x, v_y, v_x, v_y);
+    dg::tensor::multiply2d( 1., g, v_x, v_y, 0., v_x, v_y);
     t.tic();
     for( int i=0; i<multi; i++)
-        dg::tensor::multiply2d( g, v_x, v_y, v_x, v_y);
+        dg::tensor::multiply2d( 1., g, v_x, v_y, 0., v_x, v_y);
     t.toc();
     std::cout<<"Multiply2d Unit tensor inplace took   "<<t.diff()/multi<<"s\t"<<6*gbytes*multi/t.diff()<<"GB/s\n";
 
@@ -36,12 +36,12 @@ int main()
     g.values()[0] = g.values()[1] = g.values()[2] = w2d;
     t.tic();
     for( int i=0; i<multi; i++)
-        dg::tensor::multiply2d( g, v_x, v_y, v_x, v_y);
+        dg::tensor::multiply2d( 1., g, v_x, v_y, 0., v_x, v_y);
     t.toc();
     std::cout<<"multiply_inplace(g,v_x,v_y) took      "<<t.diff()/multi<<"s\t"<<7*gbytes*multi/t.diff()<<"GB/s\n";
     t.tic();
     for( int i=0; i<multi; i++)
-        dg::tensor::multiply2d( g, v_x, v_y, w_x, w_y);
+        dg::tensor::multiply2d( 1., g, v_x, v_y, 0., w_x, w_y);
     t.toc();
     std::cout<<"multiply2d(g,v_x,v_y,w_x,w_y) took    "<<t.diff()/multi<<"s\t"<<9*gbytes*multi/t.diff()<<"GB/s\n";
     return 0;
diff --git a/inc/dg/topology/multiply_t.cu b/inc/dg/topology/multiply_t.cu
index b5ba5366d..0067610c6 100644
--- a/inc/dg/topology/multiply_t.cu
+++ b/inc/dg/topology/multiply_t.cu
@@ -44,22 +44,22 @@ int main()
     std::cout << "Test Tensor multiplies \n";
     print(t);
     std::cout << "Multiply T with [8,9]\n";
-    dg::tensor::multiply2d(t, eight, nine, work0, work1);
+    dg::tensor::multiply2d(1., t, eight, nine, 0., work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"] ([86 120])\n";
     std::cout << "Multiply T^{-1} with [86,120]\n";
-    dg::tensor::inv_multiply2d(t, work0, work1, work0, work1);
+    dg::tensor::inv_multiply2d(1., t, work0, work1, 0., work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"] ([8 9])\n";
-    dg::tensor::multiply2d(t, inout0, inout1, work0, inout1);
+    dg::tensor::multiply2d(1., t, inout0, inout1, 0., work0, inout1);
     std::cout << "Result inplace is ["<<work0[0]<<" "<<inout1[0]<<"] ([86 120])\n T is \n";
     t.idx(0,2) = 4; std::swap( t.idx(1,1), t.idx(2,1)); print(t);
     std::cout << "Multiply T with [8,9,2]\n";
-    dg::tensor::multiply3d(t, eight, nine,two, work0, work1, work2);
+    dg::tensor::multiply3d(1., t, eight, nine,two, 0., work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] ([102 48 76])\n";
     std::cout << "Multiply T^{-1} with [102,48,76]\n";
-    dg::tensor::inv_multiply3d(t, work0, work1, work2, work0, work1, work2);
+    dg::tensor::inv_multiply3d(1., t, work0, work1, work2, 0., work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] ([8 9 2])\n";
     inout0=eight, inout1=nine, inout2=two;
-    dg::tensor::multiply3d(t, inout0, inout1, inout2, inout0, inout1, inout2);
+    dg::tensor::multiply3d(1., t, inout0, inout1, inout2, 0., inout0, inout1, inout2);
     std::cout << "Result inplace is ["<<inout0[0]<<" "<<inout1[0]<<" "<<inout2[0]<<"] ([102 48 76])\n";
     std::cout << "Determinant3d of T: "<<dg::tensor::determinant(t)[0]<<" (312)\n";
     std::cout << "Determinant2d of T: "<<dg::tensor::determinant2d(t)[0]<<" (-36)\n";
diff --git a/inc/dg/topology/tensor.h b/inc/dg/topology/tensor.h
index ff0fbe420..32ea0eae6 100644
--- a/inc/dg/topology/tensor.h
+++ b/inc/dg/topology/tensor.h
@@ -6,6 +6,11 @@
 #include "dg/functors.h"
 #include "dg/blas1.h"
 
+/*!@file
+ *
+ * SparseTensor
+ */
+
 namespace dg
 {
     //separate algorithms from interface!!
-- 
GitLab


From 24ec659d6da9a4e022f81a3bbbb5700f85b06098 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Feb 2021 16:47:06 +0100
Subject: [PATCH 476/540] Remove duplicate code in symv and multiply_sigma

making use of the fact that we can insert numbers into blas1 functions
---
 inc/dg/elliptic.h | 96 ++++++-----------------------------------------
 1 file changed, 11 insertions(+), 85 deletions(-)

diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index b717aceda..3480c970a 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -244,35 +244,7 @@ class Elliptic
     template<class ContainerType0, class ContainerType1>
     void symv( value_type alpha, const ContainerType0& x, value_type beta, ContainerType1& y)
     {
-        //compute gradient
-        dg::blas2::gemv( m_rightx, x, m_tempx); //R_x*f
-        dg::blas2::gemv( m_righty, x, m_tempy); //R_y*f
-
-        //multiply with tensor (note the alias)
-        dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
-
-        //now take divergence
-        dg::blas2::symv( m_lefty, m_tempy, m_temp);
-        dg::blas2::symv( -1., m_leftx, m_tempx, -1., m_temp);
-
-        //add jump terms
-        if(m_chi_weight_jump)
-        {
-            dg::blas2::symv( m_jfactor, m_jumpX, x, 0., m_tempx);
-            dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
-            dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
-            dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
-        } 
-        else
-        {
-            dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
-            dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
-        }
-        
-        if( m_no == normed)
-            dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
-        if( m_no == not_normed)//multiply weights without volume
-            dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
+        multiply_sigma( alpha, 1., x, beta, y);
     }
 
     /**
@@ -298,27 +270,24 @@ class Elliptic
         dg::blas2::gemv( m_righty, x, m_tempy); //R_y*f
 
         //multiply with tensor (note the alias)
-        dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
-        //sigma is possibly zero so we don't multiply it to m_chi
-        dg::blas1::pointwiseDot( m_tempx, sigma, m_tempx); ///////
-        dg::blas1::pointwiseDot( m_tempy, sigma, m_tempy); ///////
+        dg::tensor::multiply2d(sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
 
         //now take divergence
         dg::blas2::symv( m_lefty, m_tempy, m_temp);
         dg::blas2::symv( -1., m_leftx, m_tempx, -1., m_temp);
 
         //add jump terms
-        if( 0 != m_jfactor )
+        if( 0.0 != m_jfactor )
         {
             if(m_chi_weight_jump)
             {
                 dg::blas2::symv( m_jfactor, m_jumpX, x, 0., m_tempx);
                 dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
-                dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
+                dg::tensor::multiply2d(sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
                 dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
-            } 
+            }
             else
-            {   
+            {
                 dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
                 dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
             }
@@ -544,43 +513,7 @@ class Elliptic3d
     template<class ContainerType0, class ContainerType1>
     void symv( value_type alpha, const ContainerType0& x, value_type beta, ContainerType1& y)
     {
-        //compute gradient
-        dg::blas2::gemv( m_rightx, x, m_tempx); //R_x*f
-        dg::blas2::gemv( m_righty, x, m_tempy); //R_y*f
-        if( m_multiplyZ )
-        {
-            dg::blas2::gemv( m_rightz, x, m_tempz); //R_z*f
-
-            //multiply with tensor (note the alias)
-            dg::tensor::multiply3d(m_chi, m_tempx, m_tempy, m_tempz, m_tempx, m_tempy, m_tempz);
-            //now take divergence
-            dg::blas2::symv( -1., m_leftz, m_tempz, 0., m_temp);
-            dg::blas2::symv( -1., m_lefty, m_tempy, 1., m_temp);
-        }
-        else
-        {
-            dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
-            dg::blas2::symv( -1.,m_lefty, m_tempy, 0., m_temp);
-        }
-        dg::blas2::symv( -1., m_leftx, m_tempx, 1., m_temp);
-
-        //add jump terms
-        if(m_chi_weight_jump)
-        {
-            dg::blas2::symv( m_jfactor, m_jumpX, x, 0., m_tempx);
-            dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
-            dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
-            dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
-        } 
-        else
-        {
-            dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
-            dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
-        }
-        if( m_no == normed)
-            dg::blas1::pointwiseDivide( alpha, m_temp, m_vol, beta, y);
-        if( m_no == not_normed)//multiply weights without volume
-            dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
+        multiply_sigma( alpha, 1., x, beta, y);
     }
     ///@copydoc Elliptic::multiply_sigma(value_type,const ContainerType2&,const ContainerType0&,value_type,ContainerType1&)
     template<class ContainerType0, class ContainerType1, class ContainerType2>
@@ -594,20 +527,14 @@ class Elliptic3d
             dg::blas2::gemv( m_rightz, x, m_tempz); //R_z*f
 
             //multiply with tensor (note the alias)
-            dg::tensor::multiply3d(m_chi, m_tempx, m_tempy, m_tempz, m_tempx, m_tempy, m_tempz);
-            //sigma is possibly zero so we don't multiply it to m_chi
-            dg::blas1::pointwiseDot( m_tempx, sigma, m_tempx); ///////
-            dg::blas1::pointwiseDot( m_tempy, sigma, m_tempy); ///////
-            dg::blas1::pointwiseDot( m_tempz, sigma, m_tempz); ///////
+            dg::tensor::multiply3d(sigma, m_chi, m_tempx, m_tempy, m_tempz, 0., m_tempx, m_tempy, m_tempz);
             //now take divergence
             dg::blas2::symv( -1., m_leftz, m_tempz, 0., m_temp);
             dg::blas2::symv( -1., m_lefty, m_tempy, 1., m_temp);
         }
         else
         {
-            dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
-            dg::blas1::pointwiseDot( m_tempx, sigma, m_tempx); ///////
-            dg::blas1::pointwiseDot( m_tempy, sigma, m_tempy); ///////
+            dg::tensor::multiply2d(sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
             dg::blas2::symv( -1.,m_lefty, m_tempy, 0., m_temp);
         }
         dg::blas2::symv( -1., m_leftx, m_tempx, 1., m_temp);
@@ -621,9 +548,9 @@ class Elliptic3d
                 dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
                 dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
                 dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
-            } 
+            }
             else
-            {   
+            {
                 dg::blas2::symv( m_jfactor, m_jumpX, x, 1., m_temp);
                 dg::blas2::symv( m_jfactor, m_jumpY, x, 1., m_temp);
             }
@@ -642,7 +569,6 @@ class Elliptic3d
     void set_norm( dg::norm new_norm) {
         m_no = new_norm;
     }
-    
     private:
     Matrix m_leftx, m_lefty, m_leftz, m_rightx, m_righty, m_rightz, m_jumpX, m_jumpY;
     Container m_weights, m_inv_weights, m_precond, m_weights_wo_vol;
-- 
GitLab


From e5a37759826e5a66d8276c278d0226baf839ac16 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Feb 2021 17:54:46 +0100
Subject: [PATCH 477/540] Add references to Arakawa and Poisson doku

---
 inc/dg/arakawa.h | 4 +++-
 inc/dg/poisson.h | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 542bd00c5..9487c7f00 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -17,7 +17,6 @@
   */
 namespace dg
 {
-//citation missing in documentation
 /**
  * @brief X-space generalized version of Arakawa's scheme
  *
@@ -26,6 +25,9 @@ namespace dg
  * If \f$ \chi=1\f$, then the discretization conserves, mass, energy and enstrophy.
  * @snippet arakawa_t.cu function
  * @snippet arakawa_t.cu doxygen
+ * @note This is the algorithm published in
+ * <a href="https://doi.org/10.1016/j.cpc.2014.07.007">L. Einkemmer, M. Wiesenberger A conservative discontinuous Galerkin scheme for the 2D incompressible Navier-Stokes equations Computer Physics Communications 185, 2865-2873 (2014)</a>
+ * @sa A discussion of this and other advection schemes can also be found here https://mwiesenberger.github.io/advection
  * @copydoc hide_geometry_matrix_container
  * @ingroup arakawa
  */
diff --git a/inc/dg/poisson.h b/inc/dg/poisson.h
index 7960b976e..4462caea7 100644
--- a/inc/dg/poisson.h
+++ b/inc/dg/poisson.h
@@ -25,6 +25,7 @@ namespace dg
  * where \f$ g_{2d} = g/g_{zz}\f$ is the two-dimensional volume element of the plane in 2x1 product space and \f$ \chi\f$ is an optional factor.
  * Has the possitility to use mixed boundary conditions
  * @snippet poisson_t.cu doxygen
+ * @sa A discussion of this and other advection schemes can also be found here https://mwiesenberger.github.io/advection
  * @ingroup arakawa
  * @copydoc hide_geometry_matrix_container
  */
-- 
GitLab


From 908bbeb44702e8b2a71cdc90a6384c8d76d9d7aa Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 10 Feb 2021 17:57:18 +0100
Subject: [PATCH 478/540] Exchange input and output in evaluate binary

- this is in order to make the Axpby and other existing subroutines work
that have the output in the second argument
---
 inc/dg/blas1.h       | 14 ++++++-------
 inc/dg/subroutines.h | 48 ++++++++++++++++++++++++++------------------
 2 files changed, 36 insertions(+), 26 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 2f8dc552d..3a1c36749 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -131,7 +131,7 @@ template<class ContainerTypeIn, class ContainerTypeOut>
 inline void copy( const ContainerTypeIn& source, ContainerTypeOut& target){
     if( std::is_same<ContainerTypeIn, ContainerTypeOut>::value && &source==(const ContainerTypeIn*)&target)
         return;
-    dg::blas1::subroutine( dg::equals(), target, source );
+    dg::blas1::subroutine( dg::equals(), source, target);
 }
 
 /*! @brief \f$ x = \alpha x\f$
@@ -301,12 +301,12 @@ inline void pointwiseDot( get_value_type<ContainerType> alpha, const ContainerTy
         return;
     }
     if( std::is_same<ContainerType, ContainerType1>::value && &x1==(const ContainerType1*)&y){
-        dg::blas1::subroutine( dg::PointwiseDot<get_value_type<ContainerType>>(alpha,beta), x2, y );
+        dg::blas1::subroutine( dg::AxyPby<get_value_type<ContainerType>>(alpha,beta), x2, y );
 
         return;
     }
     if( std::is_same<ContainerType, ContainerType2>::value && &x2==(const ContainerType2*)&y){
-        dg::blas1::subroutine( dg::PointwiseDot<get_value_type<ContainerType>>(alpha,beta), x1, y );
+        dg::blas1::subroutine( dg::AxyPby<get_value_type<ContainerType>>(alpha,beta), x1, y );
 
         return;
     }
@@ -419,7 +419,7 @@ inline void pointwiseDivide( const ContainerType1& x1, const ContainerType2& x2,
 }
 
 /**
-* @brief \f$ z = \alpha x_1x_2 + \beta x_2y_2 + \gamma z\f$
+* @brief \f$ z = \alpha x_1y_1 + \beta x_2y_2 + \gamma z\f$
 *
 * Multiplies and adds vectors element by element: \f[ z_i = \alpha x_{1i}y_{1i} + \beta x_{2i}y_{2i} + \gamma z_i \f]
 * @copydoc hide_iterations
@@ -480,9 +480,9 @@ inline void transform( const ContainerType1& x, ContainerType& y, UnaryOp op )
     dg::blas1::subroutine( dg::Evaluate<dg::equals, UnaryOp>(dg::equals(),op), y, x);
 }
 
-/*! @brief \f$ f(y, g(x_0,x_1,...))\f$
+/*! @brief \f$ f(g(x_0,x_1,...), y)\f$
  *
- * This routine elementwise evaluates \f[ f(y_i , g(x_{0i}, x_{1i}, ...)) \f]
+ * This routine elementwise evaluates \f[ f(g(x_{0i}, x_{1i}, ...), y_i) \f]
  * @copydoc hide_iterations
  *
 @code
@@ -493,7 +493,7 @@ dg::HVec pi2(20, M_PI/2.), pi3( 20, 3*M_PI/2.), result(20, 0);
 dg::blas1::evaluate( result, dg::equals(), function, pi2, pi3);
 // result[i] = sin(M_PI/2.)*sin(3*M_PI/2.) = -1
 @endcode
- * @tparam BinarySubroutine Functor with signature: <tt> void operator()( value_type_y&, value_type_g) </tt> i.e. it writes into the first and reads from its (first and) second argument
+ * @tparam BinarySubroutine Functor with signature: <tt> void ( value_type_g, value_type_y&) </tt> i.e. it reads the first (and second) and writes into the second argument
  * @tparam Functor signature: <tt> value_type_g operator()( value_type_x0, value_type_x1, ...) </tt>
  * @attention Both \c BinarySubroutine and \c Functor must be callable on the device in use. In particular, with CUDA they must be functor tpyes (@b not functions) and their signatures must contain the \__device__ specifier. (s.a. \ref DG_DEVICE)
  * @param y contains result
diff --git a/inc/dg/subroutines.h b/inc/dg/subroutines.h
index a1dd79552..0cf1270c1 100644
--- a/inc/dg/subroutines.h
+++ b/inc/dg/subroutines.h
@@ -7,47 +7,47 @@ namespace dg{
 ///@addtogroup binary_operators
 ///@{
 
-///\f$ x=y\f$
+///\f$ y=x\f$
 struct equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1& out, T2 in) const
+DG_DEVICE void operator()( T1 in, T2& out) const
     {
         out = in;
     }
 };
-///\f$ x+=y\f$
+///\f$ y=y+x\f$
 struct plus_equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1& out, T2 in) const
+DG_DEVICE void operator()( T1 in, T2& out) const
     {
         out += in;
     }
 };
-///\f$ x-=y\f$
+///\f$ y=y-x\f$
 struct minus_equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1& out, T2 in) const
+DG_DEVICE void operator()( T1 in, T2& out) const
     {
         out -= in;
     }
 };
-///\f$ x*=y\f$
+///\f$ y=xy\f$
 struct times_equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1& out, T2 in) const
+DG_DEVICE void operator()( T1 in, T2& out) const
     {
         out *= in;
     }
 };
-///\f$ x/=y\f$
+///\f$ y = y/x\f$
 struct divides_equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1& out, T2 in) const
+DG_DEVICE void operator()( T1 in, T2& out) const
     {
         out /= in;
     }
@@ -176,7 +176,7 @@ struct Evaluate
         m_g( g) {}
     template< class T, class... Ts>
 DG_DEVICE void operator() ( T& y, Ts... xs){
-        m_f(y, m_g(xs...));
+        m_f(m_g(xs...), y);
     }
     private:
     BinarySub m_f;
@@ -209,7 +209,8 @@ DG_DEVICE
     T m_a;
 };
 
-/// \f$ y\leftarrow ax+by \f$
+///@brief \f$ y\leftarrow ax+by \f$
+///@ingroup binary_operators
 template<class T>
 struct Axpby
 {
@@ -222,6 +223,20 @@ DG_DEVICE
     private:
     T m_a, m_b;
 };
+///@brief \f$ y\leftarrow axy+by \f$
+///@ingroup binary_operators
+template<class T>
+struct AxyPby
+{
+    AxyPby( T a, T b): m_a(a), m_b(b){}
+DG_DEVICE
+    void operator()( T x, T& y)const {
+        T temp = y*m_b;
+        y = DG_FMA( m_a*x, y, temp);
+    }
+    private:
+    T m_a, m_b;
+};
 
 /// \f$ z\leftarrow ax+by+gz \f$
 template<class T>
@@ -244,13 +259,8 @@ template<class T>
 struct PointwiseDot
 {
     PointwiseDot( T a, T b, T g = (T)0): m_a(a), m_b(b), m_g(g) {}
-DG_DEVICE
-    void operator()( T x, T &y)const{
-        T temp = y*m_b;
-        y = DG_FMA( m_a*x, y, temp);
-    }
-DG_DEVICE
-    void operator()( T x, T y, T& z)const{
+    ///\f$ z = axy+bz \f$
+DG_DEVICE void operator()( T x, T y, T& z)const{
         T temp = z*m_b;
         z = DG_FMA( m_a*x, y, temp);
     }
-- 
GitLab


From 52e995bad4c69c53ead84ba09088e8e7c0d6c95b Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 11 Feb 2021 11:13:59 +0100
Subject: [PATCH 479/540] Add Advection class tested in shu

---
 inc/dg/advection.h      | 114 ++++++++++++++++++++++++++++++++++++++++
 inc/dg/algorithm.h      |   1 +
 src/lamb_dipole/shu.cuh |  18 +++----
 3 files changed, 124 insertions(+), 9 deletions(-)
 create mode 100644 inc/dg/advection.h

diff --git a/inc/dg/advection.h b/inc/dg/advection.h
new file mode 100644
index 000000000..c8512fb0d
--- /dev/null
+++ b/inc/dg/advection.h
@@ -0,0 +1,114 @@
+#ifndef _DG_ADVECTION_H
+#define _DG_ADVECTION_H
+#include "blas.h"
+#include "topology/geometry.h"
+#include "enums.h"
+#include "topology/evaluation.h"
+#include "topology/derivatives.h"
+#ifdef MPI_VERSION
+#include "topology/mpi_derivatives.h"
+#include "topology/mpi_evaluation.h"
+#endif
+/*! @file
+
+  @brief Computation of advection, gradients and divergences
+  */
+namespace dg
+{
+
+/**
+ * @brief The upwind advection operator \f$ y =  \alpha \vec v\cdot\nabla f + \beta y \f$
+ *
+ * This is the upwind scheme where a backward derivative is used if v is
+ * positive and a forward derivative else
+ * @note This scheme brings its own numerical diffusion and thus does not need any other artificial viscosity mechanisms. The only places where the scheme might run into oscillations is if there is a stagnation point with v==0 at a fixed position
+ * @sa A discussion of this and other advection schemes can be found here https://mwiesenberger.github.io/advection
+ * @copydoc hide_geometry_matrix_container
+ * @ingroup arakawa
+ */
+template< class Geometry, class Matrix, class Container >
+struct Advection
+{
+    using geometry_type = Geometry;
+    using matrix_type = Matrix;
+    using container_type = Container;
+    using value_type = get_value_type<Container>;
+    Advection(){}
+    /**
+     * @brief Create Arakawa on a grid
+     * @param g The grid
+     */
+    Advection( const Geometry& g);
+    /**
+     * @brief Create Advection on a grid using different boundary conditions
+     * @param g The grid
+     * @param bcx The boundary condition in x
+     * @param bcy The boundary condition in y
+     */
+    Advection( const Geometry& g, bc bcx, bc bcy);
+
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        //construct and swap
+        *this = Advection( std::forward<Params>( ps)...);
+    }
+
+    /**
+     * @brief Compute Advection term \f$ y =  \alpha \vec v\cdot\nabla f + \beta y \f$
+     *
+     * This uses the upwind advection mechanism, i.e. a backward derivative for positive v and a forward derivative else
+     * @param alpha constant
+     * @param vx Velocity in x direction
+     * @param vy Velocity in y direction
+     * @param f function
+     * @param beta constant
+     * @param result Result
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3>
+    void upwind( value_type alpha, const ContainerType0& vx, const ContainerType1& vy, const ContainerType2& f, value_type beta, ContainerType3& result);
+
+  private:
+    Container m_temp0, m_temp1;
+    Matrix m_dxf, m_dyf, m_dxb, m_dyb;
+};
+
+///@cond
+template<class Geometry, class Matrix, class Container>
+Advection<Geometry, Matrix, Container>::Advection( const Geometry& g ):
+    Advection( g, g.bcx(), g.bcy()) { }
+
+template<class Geometry, class Matrix, class Container>
+Advection<Geometry, Matrix, Container>::Advection( const Geometry& g, bc bcx, bc bcy):
+    m_temp0( dg::construct<Container>(dg::evaluate( one, g)) ), m_temp1(m_temp0),
+    m_dxf(dg::create::dx( g, bcx, dg::forward)),
+    m_dyf(dg::create::dy( g, bcy, dg::forward)),
+    m_dxb(dg::create::dx( g, bcx, dg::backward)),
+    m_dyb(dg::create::dy( g, bcy, dg::backward))
+{
+}
+
+template< class Geometry, class Matrix, class Container>
+template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3>
+void Advection<Geometry, Matrix, Container>::upwind( value_type alpha, const ContainerType0& vx, const ContainerType1& vy, const ContainerType2& f, value_type beta, ContainerType3& result)
+{
+    blas2::symv( m_dxb, f, m_temp0);
+    blas2::symv( m_dxf, f, m_temp1);
+    blas1::evaluate( result, dg::Axpby<value_type>( alpha, beta), dg::UpwindProduct(), vx, m_temp0, m_temp1);
+    blas2::symv( m_dyb, f, m_temp0);
+    blas2::symv( m_dyf, f, m_temp1);
+    blas1::evaluate( result, dg::Axpby<value_type>( alpha, 1.), dg::UpwindProduct(), vy, m_temp0, m_temp1);
+}
+///@endcond
+
+}//namespace dg
+
+
+#endif //_DG_ADVECTION_H
diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 5e301f065..5ad99a7a7 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -28,6 +28,7 @@
 #include "multigrid.h"
 #include "refined_elliptic.h"
 #include "arakawa.h"
+#include "advection.h"
 #include "poisson.h"
 #include "simpsons.h"
 #include "topology/average.h"
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index 94066880b..9896d7378 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -82,6 +82,7 @@ struct Shu
     std::vector<dg::Elliptic<Geometry, Matrix, Container>> m_multi_laplaceM;
     dg::Elliptic<Geometry, Matrix,Container> m_LaplacianM;
     dg::ArakawaX<Geometry, Matrix, Container> m_arakawa;
+    dg::Advection<Geometry, Matrix, Container> m_adv;
     dg::Extrapolation<Container> m_old_psi;
     dg::MultigridCG2d<Geometry, Matrix, Container> m_multigrid;
     dg::MultiMatrix<Matrix,Container> m_inter, m_project;
@@ -141,9 +142,13 @@ Shu< Geometry, Matrix, Container>::Shu(
         m_fine_temp[1] = dg::evaluate( dg::zero, fine_grid);
         m_fine_temp[2] = dg::evaluate( dg::zero, fine_grid);
         m_arakawa.construct( fine_grid);
+        m_adv.construct( fine_grid);
     }
     else
+    {
         m_arakawa.construct( g);
+        m_adv.construct(g);
+    }
     m_centered[0] = dg::create::dx( g, g.bcx(), dg::centered);
     m_centered[1] = dg::create::dy( g, g.bcy(), dg::centered);
     m_forward[0] = dg::create::dx( g, dg::inverse( g.bcx()), dg::forward);
@@ -223,17 +228,12 @@ void Shu<Geometry, Matrix, Container>::operator()(double t, const Container& y,
         }
         else if( "upwind-advection" == m_advection)
         {
-            dg::blas1::copy( 0., yp);
             //  - v_x dx n
-            dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_v); //v_x
-            dg::blas2::symv( m_forward[0], y, m_temp[1]);
-            dg::blas2::symv( m_backward[0], y, m_temp[2]);
-            dg::blas1::evaluate( yp, dg::minus_equals(), dg::UpwindProduct(), m_v, m_temp[2], m_temp[1]);
+            dg::blas2::symv( -1., m_centered[1], m_psi, 0., m_temp[0]); //v_x
+
             //  - v_y dy n
-            dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_v); //v_y
-            dg::blas2::symv( m_forward[1], y, m_temp[1]);
-            dg::blas2::symv( m_backward[1], y, m_temp[2]);
-            dg::blas1::evaluate( yp, dg::minus_equals(), dg::UpwindProduct(), m_v, m_temp[2], m_temp[1]);
+            dg::blas2::symv( 1., m_centered[0], m_psi, 0., m_temp[1]); //v_y
+            m_adv.upwind( -1., m_temp[0], m_temp[1], y, 0., yp);
         }
         else if( "centered-advection" == m_advection)
         {
-- 
GitLab


From a4a4ea4931454e2562256db95e0870660ef2b43c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 11 Feb 2021 15:01:59 +0100
Subject: [PATCH 480/540] Add Scalar Product function in tensor

---
 inc/dg/gradient.h             | 330 ++++++++++++++++++++++++++++++++++
 inc/dg/gradient_t.cu          | 128 +++++++++++++
 inc/dg/topology/multiply.h    | 108 +++++++++++
 inc/dg/topology/multiply_t.cu |  11 +-
 4 files changed, 575 insertions(+), 2 deletions(-)
 create mode 100644 inc/dg/gradient.h
 create mode 100644 inc/dg/gradient_t.cu

diff --git a/inc/dg/gradient.h b/inc/dg/gradient.h
new file mode 100644
index 000000000..c8708e06c
--- /dev/null
+++ b/inc/dg/gradient.h
@@ -0,0 +1,330 @@
+#pragma once
+
+#include "blas.h"
+#include "enums.h"
+#include "backend/memory.h"
+#include "topology/evaluation.h"
+#include "topology/derivatives.h"
+#ifdef MPI_VERSION
+#include "topology/mpi_derivatives.h"
+#include "topology/mpi_evaluation.h"
+#endif
+#include "topology/geometry.h"
+
+/*! @file
+
+  @brief General gradient operator
+  */
+namespace dg
+{
+
+/**
+ * @brief A 2d gradient operator
+ *
+ * @ingroup matrixoperators
+ *
+ * The term discretized is \f[ \chi \nabla  \f]
+ * where \f$ \nabla \f$ is the two-dimensional nabla and \f$\chi\f$ is a
+ * tensor (usually the metric).
+ *
+ * In general coordinates that means
+ * \f[\left(\chi^{xx}\partial_x + \chi^{xy}\partial_y \right)\right)
+ + \left(\chi^{yx}\partial_x + \chi^{yy}\partial_y \right) \f]
+ is discretized.
+ Per default, \f$ \chi\f$ is the metric tensor but you can set it to any tensor
+ you like.
+
+ * @copydoc hide_geometry_matrix_container
+ * @note The constructors initialize \f$ \chi=g\f$ so that a traditional gradient results
+ */
+template <class Geometry, class Matrix, class Container>
+class Gradient
+{
+    public:
+    using geometry_type = Geometry;
+    using matrix_type = Matrix;
+    using container_type = Container;
+    using value_type = get_value_type<Container>;
+    ///@brief empty object ( no memory allocation)
+    Gradient(){}
+    /**
+     * @brief Construct from Grid
+     *
+     * @param g The Grid, boundary conditions are taken from here
+     * @param dir Direction of the right first derivative in x and y
+     *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
+     * @note chi is assumed the metric per default
+     */
+    Gradient( const Geometry& g, direction dir = forward):
+        Gradient( g, g.bcx(), g.bcy(), dir)
+    {
+    }
+
+    /**
+     * @brief Construct from grid and boundary conditions
+     * @param g The Grid
+     * @param bcx boundary condition in x
+     * @param bcy boundary contition in y
+     * @param dir Direction of the right first derivative in x and y
+     *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
+     * @note chi is assumed the metric per default
+     */
+    Gradient( const Geometry& g, bc bcx, bc bcy, direction dir = forward)
+    {
+        dg::blas2::transfer( dg::create::dx( g, bcx, dir), m_rightx);
+        dg::blas2::transfer( dg::create::dy( g, bcy, dir), m_righty);
+        m_chi=g.metric();
+        m_tempx = m_tempy = dg::construct<Container>( dg::evaluate( dg::zero, g));
+    }
+
+    /**
+    * @brief Perfect forward parameters to one of the constructors
+    *
+    * @tparam Params deduced by the compiler
+    * @param ps parameters forwarded to constructors
+    */
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        //construct and swap
+        *this = Gradient( std::forward<Params>( ps)...);
+    }
+
+    /**
+     * @copydoc Gradient3d::set_chi(const SparseTensor<ContainerType0>&)
+     * @note the 3d parts in \c tau will be ignored
+     */
+    template<class ContainerType0>
+    void set_chi( const SparseTensor<ContainerType0>& tau)
+    {
+        m_chi = SparseTensor<Container>(tau);
+    }
+
+    /**
+     * @brief \f$ \vec v=\chi \nabla f \f$
+     *
+     * @param f the vector to take the gradient of
+     * @param vx (output) x-component
+     * @param vy (output) y-component
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1, class ContainerType2>
+    void gradient( const ContainerType0& f, ContainerType1& vx, ContainerType2& vy){
+        dg::blas2::gemv( m_rightx, f, vx); //R_x*f
+        dg::blas2::gemv( m_righty, f, vy); //R_y*f
+        dg::tensor::multiply2d(1., m_chi, vx, vy, 0., vx, vy);
+    }
+
+    /**
+     * @brief \f$ \vec v = \lambda \chi\nabla f + \mu \vec v \f$
+     *
+     * @param lambda a prefactor
+     * @param f the vector to take the gradient of
+     * @param mu the output prefactor
+     * @param vx (inout) x-component
+     * @param vy (inout) y-component
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1, class ContainerType2,
+        class ContainerType3, class ContainerType4>
+    void gradient(const ContainerType0& lambda, const ContainerType1& f, const
+            ContainerType2& mu, ContainerType3& vx, ContainerType4& vy)
+    {
+        //compute gradient
+        dg::blas2::gemv( m_rightx, f, m_tempx); //R_x*f
+        dg::blas2::gemv( m_righty, f, m_tempy); //R_y*f
+        dg::tensor::multiply2d(lambda, m_chi, m_tempx, m_tempy, mu, vx, vy);
+    }
+    /**
+     * @brief \f$ \sigma = \alpha (\nabla\phi\chi\nabla \phi) + \beta \sigma\f$
+     *
+     * @param alpha input prefactor
+     * @param phi the vector to take the variation of
+     * @param beta the output prefactor
+     * @param sigma (inout) the variation
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1>
+    void variation(value_type alpha, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
+    {
+        dg::blas2::gemv( m_rightx, phi, m_tempx); //R_x*f
+        dg::blas2::gemv( m_righty, phi, m_tempy); //R_y*f
+        dg::tensor::scalar_product2d(alpha,  m_tempx, m_tempy, m_chi, m_tempx, m_tempy, beta, sigma);
+    }
+    private:
+    Matrix m_rightx, m_righty;
+    Container m_tempx, m_tempy;
+    SparseTensor<Container> m_chi;
+};
+
+///@copydoc Gradient
+///@ingroup matrixoperators
+template <class Geometry, class Matrix, class Container>
+using Gradient2d = Gradient<Geometry, Matrix, Container>;
+
+/**
+ * @brief A 3d gradient operator
+ *
+ * @ingroup matrixoperators
+ *
+ * The term discretized is \f[ \chi\cdot \nabla  \f]
+ * where \f$ \mathbf \chi \f$ is a tensor (usually the metric).
+ * In general coordinates that means
+ * \f[
+ * \left(\chi^{xx}\partial_x + \chi^{xy}\partial_y + \chi^{xz}\partial_z \right)
+ + \left(\chi^{yx}\partial_x + \chi^{yy}\partial_y + \chi^{yz}\partial_z \right)
+ + \left(\chi^{zx}\partial_x + \chi^{zy}\partial_y + \chi^{zz}\partial_z \right)
+ \f]
+ Per default, \f$ \chi\f$ is the metric tensor but you can set it to any tensor
+ you like.
+ * @copydoc hide_geometry_matrix_container
+ * @note The constructors initialize \f$ \chi=g\f$ so that a traditional gradient
+ * results
+ */
+template <class Geometry, class Matrix, class Container>
+class Gradient3d
+{
+    public:
+    using geometry_type = Geometry;
+    using matrix_type = Matrix;
+    using container_type = Container;
+    using value_type = get_value_type<Container>;
+    ///@brief empty object ( no memory allocation)
+    Gradient3d(){}
+    /**
+     * @brief Construct from Grid
+     *
+     * @param g The Grid; boundary conditions are taken from here
+     * @param dir Direction of the right first derivative in x and y
+     *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
+     * the direction of the z derivative is always \c dg::centered
+     * @note chi is assumed the metric per default
+     */
+    Gradient3d( const Geometry& g, direction dir = forward):
+        Gradient3d( g, g.bcx(), g.bcy(), g.bcz(), dir)
+    {
+    }
+
+    /**
+     * @brief Construct from grid and boundary conditions
+     * @param g The Grid
+     * @param bcx boundary condition in x
+     * @param bcy boundary contition in y
+     * @param bcz boundary contition in z
+     * @param dir Direction of the right first derivative in x and y
+     *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
+     * the direction of the z derivative is always \c dg::centered
+     * @note chi is assumed the metric per default
+     */
+    Gradient3d( const Geometry& g, bc bcx, bc bcy, bc bcz, direction dir = forward)
+    {
+        dg::blas2::transfer( dg::create::dx( g, bcx, dir), m_rightx);
+        dg::blas2::transfer( dg::create::dy( g, bcy, dir), m_righty);
+        dg::blas2::transfer( dg::create::dz( g, bcz, dg::centered), m_rightz);
+        m_chi=g.metric();
+        m_tempx = m_tempy = m_tempz = dg::construct<Container>( dg::evaluate( dg::zero, g));
+    }
+    ///@copydoc Gradient::construct()
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        //construct and swap
+        *this = Gradient3d( std::forward<Params>( ps)...);
+    }
+
+    /**
+     * @brief Change the Chi tensor
+     *
+     * @param tau The new tensor \f$\chi\f$
+     * @tparam ContainerType0 must be usable in \c dg::assign to \c Container
+     */
+    template<class ContainerType0>
+    void set_chi( const SparseTensor<ContainerType0>& tau)
+    {
+        m_chi = SparseTensor<Container>(tau);
+    }
+
+    /**
+     * @brief Restrict the problem to the first 2 dimensions
+     *
+     * This effectively makes the behaviour of dg::Gradient3d
+     * identical to the dg::Gradient class.
+     * @param compute_in_2d if true, the gradient and variaton functions replace all derivatives in z with 0, false reverts to the original behaviour.
+     */
+    void set_compute_in_2d( bool compute_in_2d ) {
+        m_multiplyZ = !compute_in_2d;
+    }
+    /**
+     * @brief \f$ \vec v=\chi \nabla f \f$
+     *
+     * @param f the vector to take the gradient of
+     * @param vx (output) x-component
+     * @param vy (output) y-component
+     * @param vz (output) z-component (0, if set_compute_in_2d(true) was set)
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3>
+    void gradient( const ContainerType0& f, ContainerType1& vx, ContainerType2& vy, ContainerType3& vz){
+        //compute gradient
+        dg::blas2::gemv( m_rightx, f, vx); //R_x*f
+        dg::blas2::gemv( m_righty, f, vy); //R_y*f
+        if( m_multiplyZ)
+            dg::blas2::gemv( m_rightz, f, vz); //R_y*f
+        else
+            dg::blas1::scal( vz, 0.);
+        dg::tensor::multiply3d(1., m_chi, vx, vy, vz, 0., vx, vy, vz);
+    }
+
+    /**
+     * @brief \f$ \vec v = \lambda \chi\nabla f + \mu \vec v \f$
+     *
+     * @param lambda a prefactor
+     * @param f the vector to take the gradient of
+     * @param mu the output prefactor
+     * @param vx (inout) x-component
+     * @param vy (inout) y-component
+     * @param vz (inout) z-component
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1, class ContainerType2,
+        class ContainerType3, class ContainerType4, class ContainerType5>
+    void gradient(const ContainerType0& lambda, const ContainerType1& f, const
+            ContainerType2& mu, ContainerType3& vx, ContainerType4& vy, ContainerType5& vz)
+    {
+        //compute gradient
+        dg::blas2::gemv( m_rightx, f, m_tempx); //R_x*f
+        dg::blas2::gemv( m_righty, f, m_tempy); //R_y*f
+        if( m_multiplyZ)
+            dg::blas2::gemv( m_rightz, f, m_tempz); //R_y*f
+        else
+            dg::blas1::scal( m_tempz, 0.);
+        dg::tensor::multiply3d(lambda, m_chi, m_tempx, m_tempy, m_tempz, mu, vx, vy, vz);
+    }
+    /**
+     * @brief \f$ \sigma = \alpha (\nabla\phi\chi\nabla \phi) + \beta \sigma\f$
+     *
+     * @param alpha input prefactor
+     * @param phi the vector to take the variation of
+     * @param beta the output prefactor
+     * @param sigma (inout) the variation
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1>
+    void variation(value_type alpha, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
+    {
+        dg::blas2::gemv( m_rightx, phi, m_tempx); //R_x*f
+        dg::blas2::gemv( m_righty, phi, m_tempy); //R_y*f
+        if( m_multiplyZ)
+            dg::blas2::gemv( m_rightz, phi, m_tempz); //R_y*f
+        else
+            dg::blas1::scal( m_tempz, 0.);
+        dg::tensor::scalar_product3d(alpha,  m_tempx, m_tempy, m_tempz, m_chi, m_tempx, m_tempy, m_tempz, beta, sigma);
+    }
+    private:
+    Matrix m_rightx, m_righty, m_rightz;
+    Container m_tempx, m_tempy, m_tempz;
+    SparseTensor<Container> m_chi;
+    bool m_multiplyZ = true;
+};
+
+} //namespace dg
diff --git a/inc/dg/gradient_t.cu b/inc/dg/gradient_t.cu
new file mode 100644
index 000000000..7107e8436
--- /dev/null
+++ b/inc/dg/gradient_t.cu
@@ -0,0 +1,128 @@
+#include <iostream>
+#include <iomanip>
+
+#include "gradient.h"
+
+//![function]
+const double lx = 2*M_PI;
+const double ly = 2*M_PI;
+dg::bc bcx = dg::PER;
+dg::bc bcy = dg::PER;
+
+double phi( double x, double y) {
+    return sin(x)*cos(y);
+}
+double dxphi( double x, double y) {
+    return cos(x)*cos(y);
+}
+double dyphi( double x, double y) {
+    return -sin(x)*sin(y);
+}
+double variation( double x, double y) {
+    return dxphi(x,y)*dxphi(x,y) + dyphi(x,y)*dyphi(x,y);
+}
+double phi3d( double x, double y,double z) {
+    return sin(x)*cos(y)*cos(z);
+}
+double dxphi3d( double x, double y, double z) {
+    return cos(x)*cos(y)*cos(z);
+}
+double dyphi3d( double x, double y, double z) {
+    return -sin(x)*sin(y)*sin(z);
+}
+double dzphi3d( double x, double y, double z) {
+    return -x*x*sin(x)*cos(y)*sin(z);
+}
+double variation3d( double x, double y, double z) {
+    return dxphi3d(x,y,z)*dxphi3d(x,y,z)
+        + dyphi3d(x,y,z)*dyphi3d(x,y,z)
+        + dzphi3d(x,y,z)*dzphi3d(x,y,z)/x/x;
+}
+
+int main()
+{
+    std::cout<<"This program tests the execution of the gradient and variation scheme! A test is passed if the number in the second column shows exactly zero!\n";
+    unsigned n = 5, Nx = 32, Ny = 48;
+    std::cout << "TEST 2D\n";
+    std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<std::endl;
+
+    // create a Cartesian grid on the domain [0,lx]x[0,ly]
+    const dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
+
+    // evaluate left and right hand side on the grid
+    const dg::DVec ph = dg::construct<dg::DVec>( dg::evaluate( phi, grid));
+    const dg::DVec phx = dg::construct<dg::DVec>( dg::evaluate( dxphi, grid));
+    const dg::DVec phy = dg::construct<dg::DVec>( dg::evaluate( dyphi, grid));
+    const dg::DVec var = dg::construct<dg::DVec>( dg::evaluate( variation, grid));
+    dg::DVec dxph(ph), dyph(ph), va(ph);
+
+    // create a Gradient object
+    dg::Gradient<dg::aGeometry2d, dg::DMatrix, dg::DVec> gradient( grid);
+
+    //apply arakawa scheme
+    gradient.gradient( ph, dxph, dyph);
+    gradient.variation( 1.,ph, 0.,va);
+
+    int64_t binary[] = {4517228146715811314,4506390577922324430,4519650974219167728};
+    exblas::udouble res;
+    dg::DVec w2d = dg::create::weights( grid);
+
+    dg::blas1::axpby( 1., phx, -1., dxph);
+    res.d = sqrt(dg::blas2::dot( w2d, dxph)); //don't forget sqrt when computing errors
+    std::cout << "Gx Distance to solution "<<res.d<<"\t\t"<<res.i-binary[0]<<std::endl;
+    dg::blas1::axpby( 1., phy, -1., dyph);
+    res.d = sqrt(dg::blas2::dot( w2d, dyph)); //don't forget sqrt when computing errors
+    std::cout << "Gy Distance to solution "<<res.d<<"\t\t"<<res.i-binary[1]<<std::endl;
+    dg::blas1::axpby( 1., var, -1., va);
+    res.d = sqrt(dg::blas2::dot( w2d, va)); //don't forget sqrt when computing errors
+    std::cout << "V  Distance to solution "<<res.d<<"\t\t"<<res.i-binary[2]<<std::endl;
+    //periocid bc       |  dirichlet bc
+    //n = 1 -> p = 2    |
+    //n = 2 -> p = 1    |
+    //n = 3 -> p = 3    |        3
+    //n = 4 -> p = 3    |
+    //n = 5 -> p = 5    |
+    std::cout << "TEST 3D\n";
+    unsigned Nz = 100;
+    std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<" x "<<Nz<<std::endl;
+
+    // create a Cylindrical grid on the domain [0,lx]x[0,ly]
+    const dg::CylindricalGrid3d grid3d( M_PI, 3*M_PI, -M_PI, M_PI, 0., 2*M_PI, n, Nx, Ny, Nz, bcx, bcy, dg::PER);
+
+    // evaluate left and right hand side on the grid
+    const dg::DVec ph3d = dg::construct<dg::DVec>( dg::evaluate( phi3d, grid3d));
+    const dg::DVec phx3d = dg::construct<dg::DVec>( dg::evaluate( dxphi3d, grid3d));
+    const dg::DVec phy3d = dg::construct<dg::DVec>( dg::evaluate( dyphi3d, grid3d));
+    const dg::DVec phz3d = dg::construct<dg::DVec>( dg::evaluate( dyphi3d, grid3d));
+    const dg::DVec var3d = dg::construct<dg::DVec>( dg::evaluate( variation3d, grid3d));
+    dg::DVec dxph3d(ph3d), dyph3d(ph3d), va3d(ph3d);
+
+    // create a Gradient object
+    dg::Gradient<dg::aGeometry3d, dg::DMatrix, dg::DVec> gradient3d( grid3d);
+
+    //apply arakawa scheme
+    gradient3d.gradient( ph3d, dxph3d, dyph3d);
+    gradient3d.variation( 1.,ph3d, 0.,va3d);
+
+    int64_t binary3d[] = {4520797903162066895,4520797903162066895,4520797903162066895};
+    exblas::udouble res3d;
+    dg::DVec w3d = dg::create::weights( grid3d);
+
+    dg::blas1::axpby( 1., phx3d, -1., dxph3d);
+    res3d.d = sqrt(dg::blas2::dot( w3d, dxph3d)); //don't forget sqrt when computing errors
+    std::cout << "Gx Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[0]<<std::endl;
+    dg::blas1::axpby( 1., phy3d, -1., dyph3d);
+    res.d = sqrt(dg::blas2::dot( w3d, dyph3d)); //don't forget sqrt when computing errors
+    std::cout << "Gy Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[1]<<std::endl;
+    dg::blas1::axpby( 1., var3d, -1., va3d);
+    res.d = sqrt(dg::blas2::dot( w3d, va3d)); //don't forget sqrt when computing errors
+    std::cout << "V  Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[2]<<std::endl;
+    //periocid bc       |  dirichlet bc
+    //n = 1 -> p = 2    |
+    //n = 2 -> p = 1    |
+    //n = 3 -> p = 3    |        3
+    //n = 4 -> p = 3    |
+    //n = 5 -> p = 5    |
+    std::cout << "\nContinue with topology/average_t.cu !\n\n";
+    return 0;
+}
diff --git a/inc/dg/topology/multiply.h b/inc/dg/topology/multiply.h
index fe5ebffed..21d4b811d 100644
--- a/inc/dg/topology/multiply.h
+++ b/inc/dg/topology/multiply.h
@@ -112,6 +112,40 @@ struct InverseTensorMultiply3d{
               +t02*DG_FMA(t10, t21, (-t20*t11));
     }
 };
+
+/// \f$ y \leftarrow \alpha v_i\lambda T_{ij} w_j + \beta y\f$
+template<class value_type>
+struct TensorDot2d{
+    DG_DEVICE
+    value_type operator() (
+              value_type v0,  value_type v1,
+              value_type t00, value_type t01,
+              value_type t10, value_type t11,
+              value_type w0, value_type w1
+              ) const
+    {
+        value_type tmp0 = DG_FMA(t00,w0 , t01*w1);
+        value_type tmp1 = DG_FMA(t10,w0 , t11*w1);
+        return DG_FMA(v0,tmp0  , v1*tmp1);
+    }
+};
+/// \f$ y \leftarrow \alpha v_i\lambda T_{ij} w_j + \beta y\f$
+template<class value_type>
+struct TensorDot3d{
+    DG_DEVICE
+    value_type operator() (
+                      value_type v0,  value_type v1,  value_type v2,
+                      value_type t00, value_type t01, value_type t02,
+                      value_type t10, value_type t11, value_type t12,
+                      value_type t20, value_type t21, value_type t22,
+                      value_type w0, value_type w1, value_type w2) const
+    {
+        value_type tmp0 = DG_FMA( t00,w0 , (DG_FMA( t01,w1 , t02*w2)));
+        value_type tmp1 = DG_FMA( t10,w0 , (DG_FMA( t11,w1 , t12*w2)));
+        value_type tmp2 = DG_FMA( t20,w0 , (DG_FMA( t21,w1 , t22*w2)));
+        return DG_FMA(v0,tmp0 , DG_FMA(v1,tmp1 , v2*tmp2));
+    }
+};
 ///@}
 
 ///@addtogroup variadic_evaluates
@@ -438,6 +472,80 @@ void inv_multiply3d( const SparseTensor<ContainerType0>& t, const ContainerType1
 {
     inv_multiply3d( 1., t, in0, in1, in2, 0., out0, out1, out2);
 }
+
+/**
+ * @brief \f$ y = \alpha \sum_{i=0}^1 v_it^{ij}w_j + \beta y \text{ for } i\in \{0,1\}\f$
+ *
+ * Ignore the 3rd dimension in \c t.
+ * @param alpha input prefactor
+ * @param v0 (input) first component  of \c v  (may alias w0)
+ * @param v1 (input) second component of \c v  (may alias w1)
+ * @param t input Tensor
+ * @param w0 (input) first component  of \c w  (may alias v0)
+ * @param w1 (input) second component of \c w  (may alias v1)
+ * @param beta output prefactor
+ * @param y (output)
+ * @note This function is just a shortcut for a call to \c dg::blas1::evaluate with \c dg::TensorDot2d
+ * @copydoc hide_ContainerType
+ */
+template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4, class ContainerType5>
+void scalar_product2d(
+        get_value_type<ContainerType0> alpha,
+        const ContainerType0& v0,
+        const ContainerType1& v1,
+        const SparseTensor<ContainerType2>& t,
+        const ContainerType3& w0,
+        const ContainerType4& w1,
+        get_value_type<ContainerType0> beta,
+        ContainerType5& y)
+{
+    dg::blas1::evaluate( y,
+             dg::Axpby<get_value_type<ContainerType0>>( alpha, beta),
+             dg::TensorDot2d<get_value_type<ContainerType0>>(),
+             v0, v1,
+             t.value(0,0), t.value(0,1),
+             t.value(1,0), t.value(1,1),
+             w0, w1);
+}
+
+/**
+ * @brief \f$ y = \alpha \sum_{i=0}^2 v_it^{ij}w_j + \beta y \text{ for } i\in \{0,1,2\}\f$
+ *
+ * @param alpha input prefactor
+ * @param v0 (input) first component  of \c v  (may alias w0)
+ * @param v1 (input) second component of \c v  (may alias w1)
+ * @param v2 (input) third component of \c v  (may alias w1)
+ * @param t input Tensor
+ * @param w0 (input) first component  of \c w  (may alias v0)
+ * @param w1 (input) second component of \c w  (may alias v1)
+ * @param w2 (input) third component of \c w  (may alias v1)
+ * @param beta output prefactor
+ * @param y (output)
+ * @note This function is just a shortcut for a call to \c dg::blas1::evaluate with \c dg::TensorDot3d
+ * @copydoc hide_ContainerType
+ */
+template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4, class ContainerType5, class ContainerType6, class ContainerType7>
+void scalar_product3d(
+        get_value_type<ContainerType0> alpha,
+        const ContainerType0& v0,
+        const ContainerType1& v1,
+        const ContainerType2& v2,
+        const SparseTensor<ContainerType3>& t,
+        const ContainerType4& w0,
+        const ContainerType5& w1,
+        const ContainerType6& w2,
+        get_value_type<ContainerType0> beta,
+        ContainerType7& y)
+{
+    dg::blas1::evaluate( y,
+            dg::Axpby<get_value_type<ContainerType0>>( alpha, beta),
+            dg::TensorDot3d<get_value_type<ContainerType0>>(),
+             v0, v1, v2,
+             t.value(0,0), t.value(0,1), t.value(0,2),
+             t.value(1,0), t.value(1,1), t.value(1,2),
+             t.value(2,0), t.value(2,1), t.value(2,2),
+             w0, w1, w2);
+}
 ///@}
 
 }//namespace tensor
diff --git a/inc/dg/topology/multiply_t.cu b/inc/dg/topology/multiply_t.cu
index 0067610c6..630fd0d0c 100644
--- a/inc/dg/topology/multiply_t.cu
+++ b/inc/dg/topology/multiply_t.cu
@@ -44,17 +44,24 @@ int main()
     std::cout << "Test Tensor multiplies \n";
     print(t);
     std::cout << "Multiply T with [8,9]\n";
-    dg::tensor::multiply2d(1., t, eight, nine, 0., work0, work1);
+    dg::tensor::multiply2d( t, eight, nine, work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"] ([86 120])\n";
+    std::cout << "Scalar product [1,2] T [8,9]\n";
+    dg::tensor::scalar_product2d( 1., one, two, t, eight, nine, 0., inout0);
+    std::cout << "Result         is "<<inout0[0]<<" (326)\n";
     std::cout << "Multiply T^{-1} with [86,120]\n";
     dg::tensor::inv_multiply2d(1., t, work0, work1, 0., work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"] ([8 9])\n";
+    inout0=eight, inout1=nine, inout2=two;
     dg::tensor::multiply2d(1., t, inout0, inout1, 0., work0, inout1);
     std::cout << "Result inplace is ["<<work0[0]<<" "<<inout1[0]<<"] ([86 120])\n T is \n";
     t.idx(0,2) = 4; std::swap( t.idx(1,1), t.idx(2,1)); print(t);
     std::cout << "Multiply T with [8,9,2]\n";
-    dg::tensor::multiply3d(1., t, eight, nine,two, 0., work0, work1, work2);
+    dg::tensor::multiply3d(t, eight, nine,two, work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] ([102 48 76])\n";
+    std::cout << "Scalar product [1,2,3] T [8,9,2]\n";
+    dg::tensor::scalar_product3d( 1., 1.,2.,3., t, 8.,9.,2., 0., inout0);
+    std::cout << "Result         is "<<inout0[0]<<" (426)\n";
     std::cout << "Multiply T^{-1} with [102,48,76]\n";
     dg::tensor::inv_multiply3d(1., t, work0, work1, work2, 0., work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] ([8 9 2])\n";
-- 
GitLab


From b97077b278e08d99750579a84b291895945d6425 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 11 Feb 2021 15:02:56 +0100
Subject: [PATCH 481/540] Add gradient with gradient and variation

actually the files got accidentally commmited in the previous commit
---
 inc/dg/arakawa_t.cu |  2 +-
 inc/dg/elliptic.h   | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/inc/dg/arakawa_t.cu b/inc/dg/arakawa_t.cu
index 1a83a3321..be57c63a4 100644
--- a/inc/dg/arakawa_t.cu
+++ b/inc/dg/arakawa_t.cu
@@ -103,6 +103,6 @@ int main()
     //n = 5 -> p = 5    |
     // quantities are all conserved to 1e-15 for periodic bc
     // for dirichlet bc these are not better conserved than normal jacobian
-    std::cout << "\nContinue with topology/average_t.cu !\n\n";
+    std::cout << "\nContinue with gradient_t.cu !\n\n";
     return 0;
 }
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 3480c970a..15f085511 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -51,7 +51,7 @@ namespace dg
  * @copydoc hide_geometry_matrix_container
  * This class has the \c SelfMadeMatrixTag so it can be used in blas2::symv functions
  * and thus in a conjugate gradient solver.
- * @note The constructors initialize \f$ \chi=1\f$ so that a negative laplacian operator
+ * @note The constructors initialize \f$ \chi=g\f$ so that a negative laplacian operator
  * results
  * @note The inverse of \f$ \chi\f$ makes a good general purpose preconditioner
  * @note the jump term \f$ \alpha J\f$  adds artificial numerical diffusion as discussed above
@@ -78,7 +78,7 @@ class Elliptic
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      * @param chi_weight_jump If true, the Jump terms are multiplied with the Chi matrix, else it is ignored
-     * @note chi is assumed 1 per default
+     * @note chi is assumed the metric per default
      */
     Elliptic( const Geometry& g, norm no = not_normed,
         direction dir = forward, value_type jfactor=1., bool chi_weight_jump = false):
@@ -97,7 +97,7 @@ class Elliptic
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      * @param chi_weight_jump If true, the Jump terms are multiplied with the Chi matrix, else it is ignored
-     * @note chi is assumed 1 per default
+     * @note chi is assumed the metric per default
      */
     Elliptic( const Geometry& g, bc bcx, bc bcy,
         norm no = not_normed, direction dir = forward,
@@ -353,7 +353,7 @@ using Elliptic2d = Elliptic<Geometry, Matrix, Container>;
  * @copydoc hide_geometry_matrix_container
  * This class has the \c SelfMadeMatrixTag so it can be used in \c blas2::symv functions
  * and thus in a conjugate gradient solver.
- * @note The constructors initialize \f$ \chi=1\f$ so that a negative laplacian operator
+ * @note The constructors initialize \f$ \chi=g\f$ so that a negative laplacian operator
  * results
  * @note the jump term \f$ \alpha J\f$  adds artificial numerical diffusion as discussed above
  * @attention Pay attention to the negative sign which is necessary to make the matrix @b positive @b definite
@@ -380,7 +380,7 @@ class Elliptic3d
      * the direction of the z derivative is always \c dg::centered
      * @param jfactor (\f$ = \alpha \f$ ) scale jump terms (1 is a good value but in some cases 0.1 or 0.01 might be better)
      * @param chi_weight_jump If true, the Jump terms are multiplied with the Chi matrix, else it is ignored
-     * @note chi is assumed 1 per default
+     * @note chi is assumed the metric per default
      */
     Elliptic3d( const Geometry& g, norm no = not_normed, direction dir = forward, value_type jfactor=1., bool chi_weight_jump = false):
         Elliptic3d( g, g.bcx(), g.bcy(), g.bcz(), no, dir, jfactor, chi_weight_jump)
-- 
GitLab


From 912c464d36031dc24c8c8c36e2d5288f9e47a0b2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 11 Feb 2021 15:43:54 +0100
Subject: [PATCH 482/540] More variation tests in geometries

---
 inc/dg/algorithm.h                        |  1 +
 inc/dg/gradient.h                         | 28 ++++++-------------
 inc/dg/gradient_t.cu                      |  1 +
 inc/geometries/geometry_advection_b.cu    | 33 ++++++++++++++---------
 inc/geometries/geometry_advection_mpib.cu |  9 ++++++-
 5 files changed, 38 insertions(+), 34 deletions(-)

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 5ad99a7a7..4d776f32d 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -29,6 +29,7 @@
 #include "refined_elliptic.h"
 #include "arakawa.h"
 #include "advection.h"
+#include "gradient.h"
 #include "poisson.h"
 #include "simpsons.h"
 #include "topology/average.h"
diff --git a/inc/dg/gradient.h b/inc/dg/gradient.h
index c8708e06c..12a0fba73 100644
--- a/inc/dg/gradient.h
+++ b/inc/dg/gradient.h
@@ -90,15 +90,10 @@ class Gradient
         *this = Gradient( std::forward<Params>( ps)...);
     }
 
-    /**
-     * @copydoc Gradient3d::set_chi(const SparseTensor<ContainerType0>&)
-     * @note the 3d parts in \c tau will be ignored
-     */
-    template<class ContainerType0>
-    void set_chi( const SparseTensor<ContainerType0>& tau)
-    {
-        m_chi = SparseTensor<Container>(tau);
-    }
+    ///@copydoc Gradient3d::chi()
+    SparseTensor<Container>& chi( ){return m_chi;}
+    ///@brief Access the Chi tensor
+    const SparseTensor<Container>& chi( ) const{return m_chi;}
 
     /**
      * @brief \f$ \vec v=\chi \nabla f \f$
@@ -232,17 +227,10 @@ class Gradient3d
         *this = Gradient3d( std::forward<Params>( ps)...);
     }
 
-    /**
-     * @brief Change the Chi tensor
-     *
-     * @param tau The new tensor \f$\chi\f$
-     * @tparam ContainerType0 must be usable in \c dg::assign to \c Container
-     */
-    template<class ContainerType0>
-    void set_chi( const SparseTensor<ContainerType0>& tau)
-    {
-        m_chi = SparseTensor<Container>(tau);
-    }
+    ///@brief Access the Chi tensor
+    SparseTensor<Container>& chi( ){return m_chi;}
+    ///@brief Access the Chi tensor
+    const SparseTensor<Container>& chi( ) const{return m_chi;}
 
     /**
      * @brief Restrict the problem to the first 2 dimensions
diff --git a/inc/dg/gradient_t.cu b/inc/dg/gradient_t.cu
index 7107e8436..326e45663 100644
--- a/inc/dg/gradient_t.cu
+++ b/inc/dg/gradient_t.cu
@@ -39,6 +39,7 @@ double variation3d( double x, double y, double z) {
         + dzphi3d(x,y,z)*dzphi3d(x,y,z)/x/x;
 }
 
+// There are more tests in geometries/geometry_advection_(mpi)b.cu
 int main()
 {
     std::cout<<"This program tests the execution of the gradient and variation scheme! A test is passed if the number in the second column shows exactly zero!\n";
diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index dffe33b49..05d259305 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -103,13 +103,13 @@ struct CurvatureDirPer
 
 int main(int argc, char** argv)
 {
-    std::cout << "Type n, Nx, Ny\n";
+    std::cout << "Type n (5), Nx (10), Ny (80)\n";
     unsigned n, Nx, Ny;
     std::cin >> n>> Nx>>Ny;
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
@@ -118,9 +118,9 @@ int main(int argc, char** argv)
         is >> js;
     }
     dg::geo::solovev::Parameters gp(js);
-    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField( gp);
 
-    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
+    std::cout << "Psi min "<<mag.psip()(gp.R_0, 0)<<"\n";
     std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
@@ -130,9 +130,9 @@ int main(int argc, char** argv)
     //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing grid ... \n";
     t.tic();
-    //dg::geo::RibeiroFluxGenerator ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::geo::FluxGenerator ribeiro( c.get_psip(), c.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
-    //dg::geo::SimpleOrthogonal ribeiro( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::RibeiroFluxGenerator ribeiro( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::FluxGenerator ribeiro( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::SimpleOrthogonal ribeiro( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearGrid2d grid(ribeiro, n, Nx, Ny, dg::DIR); //2d
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
@@ -142,10 +142,10 @@ int main(int argc, char** argv)
     std::cout <<std::fixed<< std::setprecision(6)<<std::endl;
 
 
-    dg::geo::FuncDirPer left(c, psi_0, psi_1, 4);
-    FuncDirPer2 right( c, psi_0, psi_1);
-    ArakawaDirPer jacobian( c, psi_0, psi_1);
-    VariationDirPer variationLHS(c, psi_0, psi_1);
+    dg::geo::FuncDirPer left(mag, psi_0, psi_1, 4);
+    FuncDirPer2 right( mag, psi_0, psi_1);
+    ArakawaDirPer jacobian( mag, psi_0, psi_1);
+    VariationDirPer variationLHS(mag, psi_0, psi_1);
 
     const dg::DVec lhs = dg::pullback( left, grid);
     dg::DVec jac(lhs);
@@ -189,12 +189,19 @@ int main(int argc, char** argv)
     dg::blas1::axpby( 1., sol, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     std::cout << "          Rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
+    ///////////////////////////////////////////////////////////////////////
+    std::cout << "TESTING VARIATION\n";
+    dg::Gradient<dg::aGeometry2d, dg::DMatrix, dg::DVec> gradient( grid);
+    gradient.variation( 1., lhs, 0., jac);
+    dg::blas1::axpby( 1., variation, -1., jac);
+    result = dg::blas2::dot( jac, vol, jac);
+    std::cout << "               distance to solution "<<sqrt( result)<<std::endl; //don't forget sqrt when comuting errors
 
     ////////////////////////////transform curvature components////////
     std::cout << "TESTING CURVATURE 3D\n";
     dg::DVec curvX, curvY;
     dg::HVec tempX, tempY;
-    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(c,+1), dg::geo::CurvatureNablaBZ(c,+1), tempX, tempY, grid);
+    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(mag,+1), dg::geo::CurvatureNablaBZ(mag,+1), tempX, tempY, grid);
     dg::blas1::transfer(  tempX, curvX);
     dg::blas1::transfer(  tempY, curvY);
     dg::DMatrix dx, dy;
@@ -207,7 +214,7 @@ int main(int argc, char** argv)
     dg::blas1::pointwiseDot( 1., tempy, curvY, 1.,  tempx);
     const double normCurv = dg::blas2::dot( tempx, vol, tempx);
 
-    CurvatureDirPer curv(c, psi_0, psi_1);
+    CurvatureDirPer curv(mag, psi_0, psi_1);
     dg::DVec curvature;
     dg::blas1::transfer( dg::pullback(curv, grid), curvature);
 
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index 21f246d08..24711e32f 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -112,7 +112,7 @@ int main(int argc, char** argv)
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
@@ -186,6 +186,13 @@ int main(int argc, char** argv)
     dg::blas1::axpby( 1., sol, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     if(rank==0)std::cout << "          Rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
+    ///////////////////////////////////////////////////////////////////////
+    if(rank==0)std::cout << "TESTING VARIATION 3D\n";
+    dg::Gradient<Geometry, dg::MDMatrix, dg::MDVec> gradient( grid);
+    gradient.variation( 1., lhs, 0., jac);
+    dg::blas1::axpby( 1., variation, -1., jac);
+    result = dg::blas2::dot( jac, vol, jac);
+    if(rank==0)std::cout << "               distance to solution "<<sqrt( result)<<std::endl; //don't forget sqrt when comuting errors
 
     ////////////////////////////transform curvature components////////
     if(rank==0)std::cout << "TESTING CURVATURE 3D\n";
-- 
GitLab


From 523a47b54327523b3ca0d08d98179e825204fa88 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 11 Feb 2021 16:09:07 +0100
Subject: [PATCH 483/540] Write formulas in quick description of doku

---
 inc/dg/advection.h  |  2 +-
 inc/dg/arakawa.h    |  4 ++--
 inc/dg/dg_doc.h     | 20 ++++++++++++++------
 inc/dg/nullstelle.h |  7 ++++---
 inc/dg/poisson.h    |  4 ++--
 5 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/inc/dg/advection.h b/inc/dg/advection.h
index c8512fb0d..a2475dfff 100644
--- a/inc/dg/advection.h
+++ b/inc/dg/advection.h
@@ -17,7 +17,7 @@ namespace dg
 {
 
 /**
- * @brief The upwind advection operator \f$ y =  \alpha \vec v\cdot\nabla f + \beta y \f$
+ * @brief Upwind discretization of advection operator \f$ \vec v\cdot\nabla f\f$
  *
  * This is the upwind scheme where a backward derivative is used if v is
  * positive and a forward derivative else
diff --git a/inc/dg/arakawa.h b/inc/dg/arakawa.h
index 9487c7f00..97c5b729f 100644
--- a/inc/dg/arakawa.h
+++ b/inc/dg/arakawa.h
@@ -18,9 +18,9 @@
 namespace dg
 {
 /**
- * @brief X-space generalized version of Arakawa's scheme
+ * @brief Arakawa's scheme for %Poisson bracket \f$ \{ f,g\} \f$
  *
- * Computes \f[ [f,g] := \chi/\sqrt{g_{2d}}\left(\partial_x f\partial_y g - \partial_y f\partial_x g\right) \f]
+ * Computes \f[ \{f,g\} := \chi/\sqrt{g_{2d}}\left(\partial_x f\partial_y g - \partial_y f\partial_x g\right) \f]
  * where \f$ g_{2d} = g/g_{zz}\f$ is the two-dimensional volume element of the plane in 2x1 product space and \f$ \chi\f$ is an optional factor.
  * If \f$ \chi=1\f$, then the discretization conserves, mass, energy and enstrophy.
  * @snippet arakawa_t.cu function
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 58f3c4191..52e6a3748 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -13,18 +13,21 @@
  *         time integrators.
  *     @{
  *         @defgroup blas1 BLAS level 1 routines: Vector-Vector
+ *              \f$ f( x_{0i}, x_{1i}, x_{2i}, ...) \f$ and \f$ x\cdot y\f$
  *
  *             Successive calls to blas routines are executed sequentially.
  *             A manual synchronization of threads or devices is never needed
  *             in an application using these functions. All functions returning
  *             a value block until the value is ready.
  *         @defgroup blas2 BLAS level 2 routines: Matrix-Vector
+ *              \f$ \alpha M \cdot x + \beta y\f$ and \f$ x^T M \cdot y \f$
  *
  *             Successive calls to blas routines are executed sequentially.
  *             A manual synchronization of threads or devices is never needed
  *             in an application using these functions. All functions returning
  *             a value block until the value is ready.
  *         @defgroup tensor Tensor-Vector operations
+ *              \f$ v^i = T^{ij} w_j\f$
  *
  *              Although a tensor needs a topology to be well-defined mathematically,
  *              we do not need a grid to perform basic operations computationally.
@@ -45,17 +48,18 @@
  *             the chapter \ref mpi_backend in the introduction for more
  *             details.
  *     @defgroup dispatch The tag dispatch system
- *           Read the chapter \ref dispatch in the introduction.
+ *           Implementation details of \ref dispatch
  * @}
  * @defgroup numerical0 Level 2: Basic numerical algorithms
- *      Algorithms that make use only of blas level 1 and 2 functions
+ *      Based on blas1 and blas2 functions
  * @{
  *     @defgroup time Time integrators
+ *      \f$ \dot y = f(y,t) \f$
  *     @{
  *          @defgroup time_utils Utilities for time integration
  *     @}
  *     @defgroup invert Linear and nonlinear solvers
- *     @defgroup root Root finding
+ *     Linear \f$ Ax = b\f$ and non-linear \f$ f(x) = b\f$
  * @}
  * @defgroup geo Level 3: Topology and Geometry
  * @{
@@ -103,13 +107,17 @@
  *      These routines make use of both the basic operations as well as the
  *      interfaces defined in the Geometry section.
  * @{
- *     @defgroup arakawa Discretization of Poisson bracket
- *     @defgroup matrixoperators Elliptic and Helmholtz operators
- *     @defgroup multigrid Advanced matrix inversion
+ *     @defgroup arakawa Advection terms
+ *          \f$ \vec v \cdot \nabla u\f$ and \f$ \{ f,g\} \f$
+ *     @defgroup matrixoperators Matrix operators
+ *     Gradient \f$ \chi\cdot\nabla f\f$, Elliptic \f$ -\nabla\cdot (\chi \nabla f)\f$ and Helmholtz \f$ (\chi + \alpha \Delta) f\f$
+ *     @defgroup multigrid Multigrid matrix inversion
+ *     \f$ A x = b\f$
  * @}
  * @defgroup misc Level 0: Miscellaneous additions
  * @{
  *     @defgroup timer Timer class
+ *          t.tic() and T.toc()
  *     @defgroup blas1_helpers Functions and functors for subroutine and evaluate
  *     @{
  *          @defgroup basics Simple
diff --git a/inc/dg/nullstelle.h b/inc/dg/nullstelle.h
index 24653713f..7b9bbd7a6 100644
--- a/inc/dg/nullstelle.h
+++ b/inc/dg/nullstelle.h
@@ -14,7 +14,7 @@ namespace dg{
 
 /*! @brief Exception class, that stores boundaries for 1D root finding
  *
- * @ingroup root
+ * @ingroup misc
  */
 class NoRoot1d: public std::exception
 {
@@ -41,9 +41,10 @@ class NoRoot1d: public std::exception
     char const* what() const throw(){ return "There is no root!";}
 };
 
-/*! @brief Find a root of a 1d function in given boundaries using bisection
+/*! @brief Find a root of a 1d function \f$ f(x) = 0\f$
  *
- * @ingroup root
+ * in given boundaries using bisection
+ * @ingroup invert
  * It is assumed that a sign change occurs at the root.
  * Function jumps closer to the root by checking the sign.
  * \tparam UnaryOp unary function operator
diff --git a/inc/dg/poisson.h b/inc/dg/poisson.h
index 4462caea7..95a5ce8ac 100644
--- a/inc/dg/poisson.h
+++ b/inc/dg/poisson.h
@@ -19,9 +19,9 @@ namespace dg
 {
 
 /**
- * @brief Poisson bracket scheme
+ * @brief Direct discretization of %Poisson bracket \f$ \{ f,g\} \f$
  *
- * Computes \f[ [f,g] := \chi/\sqrt{g_{2d}}\left(\partial_x f\partial_y g - \partial_y f\partial_x g\right) \f]
+ * Computes \f[ \{f,g\} := \chi/\sqrt{g_{2d}}\left(\partial_x f\partial_y g - \partial_y f\partial_x g\right) \f]
  * where \f$ g_{2d} = g/g_{zz}\f$ is the two-dimensional volume element of the plane in 2x1 product space and \f$ \chi\f$ is an optional factor.
  * Has the possitility to use mixed boundary conditions
  * @snippet poisson_t.cu doxygen
-- 
GitLab


From 76df29e0040053ae25f408206dc64fc3edde3401 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 11 Feb 2021 16:13:33 +0100
Subject: [PATCH 484/540] Remove --extended lambda from default config

---
 config/default.mk | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/config/default.mk b/config/default.mk
index ae4e5d2c1..384cf825e 100644
--- a/config/default.mk
+++ b/config/default.mk
@@ -7,7 +7,7 @@ MPICC=mpic++  #mpi compiler
 CFLAGS=-Wall -std=c++14 -mavx -mfma #flags for CC
 NVCC=nvcc #CUDA compiler
 NVCCARCH=-arch sm_61 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
-NVCCFLAGS= -std=c++14 -Xcompiler "-Wall -mavx -mfma" --extended-lambda #flags for NVCC
+NVCCFLAGS= -std=c++14 -Xcompiler "-Wall -mavx -mfma" #flags for NVCC
 OPT=-O2 # optimization flags for host code (it is O2 and not O3 because g++-7 up to g++-8.0 have a bug with fma in -O3, fixed in g++-8.1)
 OMPFLAG=-fopenmp #openmp flag for CC and MPICC
 
-- 
GitLab


From 9740c9c131fa65c80d5a6e552e342e8f478ab934 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 11 Feb 2021 16:45:35 +0100
Subject: [PATCH 485/540] Modify feltor with new tensor::scalar_product3d

this shortens the computation of Psi a bit
---
 src/feltor/feltor.h | 24 +++++++-----------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index e09d51ddd..16aeefee1 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -193,21 +193,11 @@ struct ComputeChi{
 };
 struct ComputePsi{
     DG_DEVICE
-    void operator()( double& gradPhi2, double dxPhi, double dyPhi,
-        double dzPhi, double& HdxPhi, double HdyPhi, double HdzPhi
-        ) const{
-        gradPhi2 = (dxPhi*HdxPhi + dyPhi*HdyPhi + dzPhi*HdzPhi);
-    }
-    DG_DEVICE
-    void operator()( double& GammaPhi, double dxPhi, double dyPhi,
-        double dzPhi, double& HdxPhi, double HdyPhi, double HdzPhi,
-        double binv) const{
+    void operator()( double& GammaPhi, double& uE2, double binv) const{
         //u_E^2
-        this->operator()(
-            HdxPhi, dxPhi, dyPhi, dzPhi, HdxPhi, HdyPhi , HdzPhi);
-        HdxPhi   = binv*binv*HdxPhi;
+        uE2   = binv*binv*uE2;
         //Psi
-        GammaPhi = GammaPhi - 0.5*HdxPhi;
+        GammaPhi = GammaPhi - 0.5*uE2;
     }
 };
 //struct ComputeLogN{
@@ -805,10 +795,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
     dg::blas2::symv( m_dx_P, m_phi[0], m_dP[0][0]);
     dg::blas2::symv( m_dy_P, m_phi[0], m_dP[0][1]);
     if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[0], m_dP[0][2]);
-    dg::tensor::multiply3d( m_hh, //grad_perp
-        m_dP[0][0], m_dP[0][1], m_dP[0][2], m_UE2, m_temp0, m_temp1);
-    dg::blas1::subroutine( routines::ComputePsi(), m_phi[1],
-        m_dP[0][0], m_dP[0][1], m_dP[0][2], m_UE2, m_temp0, m_temp1, m_binv);
+    dg::tensor::scalar_product3d( 1.,
+        m_dP[0][0], m_dP[0][1], m_dP[0][2], m_hh, //grad_perp
+        m_dP[0][0], m_dP[0][1], m_dP[0][2], 0., m_UE2);
+    dg::blas1::subroutine( routines::ComputePsi(), m_phi[1], m_UE2, m_binv);
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( m_phi[1], dg::plus_equals(), manufactured::SPhii{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
-- 
GitLab


From dc9c7713567fa4033c8e982ace910c2c517f9072 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 11 Feb 2021 18:35:47 +0100
Subject: [PATCH 486/540] Fix bug of swapped evaluate call in mpi_collective

---
 inc/dg/backend/mpi_collective.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 9ac1443bb..14fc9b83f 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -521,8 +521,8 @@ struct GeneralComm : public aCommunicator<Vector>
             get_execution_policy<Vector>(),
             this->local_size(),
             dg::equals(),
-            values,
-            0
+            0,
+            values
         );
         thrust::scatter( m_store.data().begin(), m_store.data().end(), m_scatterMap.begin(), values_ptr);
     }
-- 
GitLab


From 91d0b95ec9b3d2cb495775d9341cbb6eb94f6395 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 11:28:19 +0100
Subject: [PATCH 487/540] Fix probes.h

added std namespace in front
---
 diag/probes.h | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/diag/probes.h b/diag/probes.h
index eb77dda2c..954a05e5e 100644
--- a/diag/probes.h
+++ b/diag/probes.h
@@ -13,9 +13,8 @@
 #include <fstream>
 #include <sstream>
 
-//using namespace std;
-/* 
- * Class that takes care of probe output
+/**
+ * @brief Class that takes care of probe output
  * 
  * Upon instantiation, generate N probesm positioned in equidistant radial
  * positions at the vertical center of the simulation domain:
@@ -25,7 +24,6 @@
  * Creates files probe_[0-9][0-9][0-9].dat in which to write output
  *
  */
-
 template<class IMatrix, class Matrix, class container = thrust::device_vector<double> >
 struct probes
 {
@@ -70,8 +68,8 @@ probes<IMatrix, Matrix, container> :: probes (container x_c, container y_c, cons
     dg::blas1::transfer( y_c, t2);
     dg::blas2::transfer( dg::create::interpolation( t1, t2, g, dg::NEU), probe_interp);
     assert(x_coords.size () == y_coords.size());
-    ofstream of;
-    stringstream fn;
+    std::ofstream of;
+    std::stringstream fn;
 
     /* Create datafiles for probe data */
     for(int n = 0; n < num_probes; n++)
@@ -83,7 +81,7 @@ probes<IMatrix, Matrix, container> :: probes (container x_c, container y_c, cons
         of << x_coords[n] << "\t" << y_coords[n] << "\n";
         of.close();
 
-        fn.str(string(""));
+        fn.str(std::string(""));
     }
 
     /* Create datafiles for radial profiles */
@@ -105,19 +103,19 @@ probes<IMatrix, Matrix, container> :: probes (container x_c, container y_c, cons
 template<class IMatrix, class Matrix, class container>
 void probes<IMatrix, Matrix, container> :: profiles(double time, container& npe, container& phi)
 {
-    cout << "Computing profiles " << std::endl;
+    std::cout << "Computing profiles " << std::endl;
 
     static container prof_phi(Nx);
     static container prof_ne(Nx);
 
-    ofstream of_ne;
-    ofstream of_phi;
+    std::ofstream of_ne;
+    std::ofstream of_phi;
 
     pol_avg(phi, prof_phi,false);
     pol_avg(npe, prof_ne,false);
 
-    of_ne.open("ne_prof.dat", ios::trunc);
-    of_phi.open("phi_prof.dat", ios::trunc);
+    of_ne.open("ne_prof.dat", std::ios::trunc);
+    of_phi.open("phi_prof.dat", std::ios::trunc);
     of_ne << time << "\n";
     of_phi << time << "\n";
 
@@ -157,7 +155,7 @@ void probes<IMatrix, Matrix, container> :: fluxes(double time, container& npe, c
     // Compute radial flux
     dg::blas2::gemv(probe_interp, phi_y, ip_phi_y);
 
-    ofstream of;
+    std::ofstream of;
 
     for(int n = 0; n < num_probes; n++)
     {
@@ -170,7 +168,7 @@ void probes<IMatrix, Matrix, container> :: fluxes(double time, container& npe, c
         of << ip_n[n] << "\t";
         of << ip_phi[n] << "\t";
         of << ip_gamma_n;
-        of << endl;
+        of << std::endl;
         of.close();
     }
 
-- 
GitLab


From 0bd66dd1a58b29ff5158c5e77d6fe51ad4fcb22d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 11:29:32 +0100
Subject: [PATCH 488/540] Modify scalar_product2d and scalar_product3d

such that it can compute psi or uE2 in one go
---
 inc/dg/gradient.h             | 111 ++++++++++++++++++++++++++--------
 inc/dg/gradient_t.cu          |  31 +++++-----
 inc/dg/topology/multiply.h    |  60 +++++++++++-------
 inc/dg/topology/multiply_t.cu |  14 +++--
 4 files changed, 149 insertions(+), 67 deletions(-)

diff --git a/inc/dg/gradient.h b/inc/dg/gradient.h
index 12a0fba73..28bb0f7e6 100644
--- a/inc/dg/gradient.h
+++ b/inc/dg/gradient.h
@@ -19,23 +19,30 @@ namespace dg
 {
 
 /**
- * @brief A 2d gradient operator
+ * @brief A 2d gradient \f$\chi\cdot\nabla\f$ and variation \f$ \nabla\phi \cdot \chi \nabla\phi\f$ operator
  *
  * @ingroup matrixoperators
  *
- * The term discretized is \f[ \chi \nabla  \f]
+ * The terms discretized are the gradient \f[\chi\cdot\nabla\f] and the variation \f[ \nabla\phi \cdot \chi \nabla\phi\f]
  * where \f$ \nabla \f$ is the two-dimensional nabla and \f$\chi\f$ is a
  * tensor (usually the metric).
  *
  * In general coordinates that means
- * \f[\left(\chi^{xx}\partial_x + \chi^{xy}\partial_y \right)\right)
+ * \f[\chi\cdot\nabla = \left(\chi^{xx}\partial_x + \chi^{xy}\partial_y \right)
  + \left(\chi^{yx}\partial_x + \chi^{yy}\partial_y \right) \f]
  is discretized.
  Per default, \f$ \chi\f$ is the metric tensor but you can set it to any tensor
  you like.
 
  * @copydoc hide_geometry_matrix_container
- * @note The constructors initialize \f$ \chi=g\f$ so that a traditional gradient results
+ * @note The constructors initialize \f$ \chi=g\f$ so that a traditional
+ * gradient results
+ * @attention This a convenience class. It is often more
+ * efficient to compute the simple derivatives of a vector yourself, because
+ * you can re-use them in other places; the same goes for the storage of the
+ * metric tensor, it often can be re-used at other places.  To compute the above
+ * expressions you then simply use the relevant tensor functions
+ * \c dg::tensor::multiply2d and \c dg::tensor::scalar_product2d
  */
 template <class Geometry, class Matrix, class Container>
 class Gradient
@@ -55,7 +62,7 @@ class Gradient
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * @note chi is assumed the metric per default
      */
-    Gradient( const Geometry& g, direction dir = forward):
+    Gradient( const Geometry& g, direction dir = centered):
         Gradient( g, g.bcx(), g.bcy(), dir)
     {
     }
@@ -69,7 +76,7 @@ class Gradient
      *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
      * @note chi is assumed the metric per default
      */
-    Gradient( const Geometry& g, bc bcx, bc bcy, direction dir = forward)
+    Gradient( const Geometry& g, bc bcx, bc bcy, direction dir = centered)
     {
         dg::blas2::transfer( dg::create::dx( g, bcx, dir), m_rightx);
         dg::blas2::transfer( dg::create::dy( g, bcy, dir), m_righty);
@@ -96,7 +103,7 @@ class Gradient
     const SparseTensor<Container>& chi( ) const{return m_chi;}
 
     /**
-     * @brief \f$ \vec v=\chi \nabla f \f$
+     * @brief \f$ \vec v=\chi \cdot\nabla f \f$
      *
      * @param f the vector to take the gradient of
      * @param vx (output) x-component
@@ -111,7 +118,7 @@ class Gradient
     }
 
     /**
-     * @brief \f$ \vec v = \lambda \chi\nabla f + \mu \vec v \f$
+     * @brief \f$ \vec v = \lambda \chi\cdot\nabla f + \mu \vec v \f$
      *
      * @param lambda a prefactor
      * @param f the vector to take the gradient of
@@ -131,20 +138,44 @@ class Gradient
         dg::tensor::multiply2d(lambda, m_chi, m_tempx, m_tempy, mu, vx, vy);
     }
     /**
-     * @brief \f$ \sigma = \alpha (\nabla\phi\chi\nabla \phi) + \beta \sigma\f$
+     * @brief \f$ \sigma = (\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
      *
-     * @param alpha input prefactor
      * @param phi the vector to take the variation of
-     * @param beta the output prefactor
      * @param sigma (inout) the variation
      * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
      */
     template<class ContainerType0, class ContainerType1>
-    void variation(value_type alpha, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
+    void variation(const ContainerType0& phi, ContainerType1& sigma){
+        variation(1., 1., phi, 0., sigma);
+    }
+    /**
+     * @brief \f$ \sigma = \lambda^2(\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
+     *
+     * @param lambda input prefactor
+     * @param phi the vector to take the variation of
+     * @param sigma (out) the variation
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
+    void variation(const ContainerTypeL& lambda, const ContainerType0& phi, ContainerType1& sigma){
+        variation(1.,lambda, phi, 0., sigma);
+    }
+    /**
+     * @brief \f$ \sigma = \alpha \lambda^2 (\nabla\phi\cdot\chi\cdot\nabla \phi) + \beta \sigma\f$
+     *
+     * @param alpha scalar input prefactor
+     * @param lambda input prefactor
+     * @param phi the vector to take the variation of
+     * @param beta the output prefactor
+     * @param sigma (inout) the variation
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
+    void variation(value_type alpha, const ContainerTypeL& lambda, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
     {
         dg::blas2::gemv( m_rightx, phi, m_tempx); //R_x*f
         dg::blas2::gemv( m_righty, phi, m_tempy); //R_y*f
-        dg::tensor::scalar_product2d(alpha,  m_tempx, m_tempy, m_chi, m_tempx, m_tempy, beta, sigma);
+        dg::tensor::scalar_product2d(alpha, lambda, m_tempx, m_tempy, m_chi, lambda, m_tempx, m_tempy, beta, sigma);
     }
     private:
     Matrix m_rightx, m_righty;
@@ -158,14 +189,14 @@ template <class Geometry, class Matrix, class Container>
 using Gradient2d = Gradient<Geometry, Matrix, Container>;
 
 /**
- * @brief A 3d gradient operator
+ * @brief A 3d gradient \f$\chi\cdot\nabla\f$ and variation \f$ \nabla\phi \cdot \chi \nabla\phi\f$ operator
  *
  * @ingroup matrixoperators
  *
- * The term discretized is \f[ \chi\cdot \nabla  \f]
+ * The terms discretized are the gradient \f$\chi\cdot\nabla\f$ and the variation \f[ \nabla\phi \cdot \chi \nabla\phi\f]
  * where \f$ \mathbf \chi \f$ is a tensor (usually the metric).
  * In general coordinates that means
- * \f[
+ * \f[ \chi\cdot\nabla =
  * \left(\chi^{xx}\partial_x + \chi^{xy}\partial_y + \chi^{xz}\partial_z \right)
  + \left(\chi^{yx}\partial_x + \chi^{yy}\partial_y + \chi^{yz}\partial_z \right)
  + \left(\chi^{zx}\partial_x + \chi^{zy}\partial_y + \chi^{zz}\partial_z \right)
@@ -175,6 +206,12 @@ using Gradient2d = Gradient<Geometry, Matrix, Container>;
  * @copydoc hide_geometry_matrix_container
  * @note The constructors initialize \f$ \chi=g\f$ so that a traditional gradient
  * results
+ * @attention This a convenience class. It is often more
+ * efficient to compute the simple derivatives of a vector yourself, because
+ * you can re-use them in other places; the same goes for the storage of the
+ * metric tensor, it often can be re-used at other places.  To compute the above
+ * expressions you then simply use the relevant tensor functions
+ * \c dg::tensor::multiply3d and \c dg::tensor::scalar_product3d
  */
 template <class Geometry, class Matrix, class Container>
 class Gradient3d
@@ -195,7 +232,7 @@ class Gradient3d
      * the direction of the z derivative is always \c dg::centered
      * @note chi is assumed the metric per default
      */
-    Gradient3d( const Geometry& g, direction dir = forward):
+    Gradient3d( const Geometry& g, direction dir = centered):
         Gradient3d( g, g.bcx(), g.bcy(), g.bcz(), dir)
     {
     }
@@ -211,7 +248,7 @@ class Gradient3d
      * the direction of the z derivative is always \c dg::centered
      * @note chi is assumed the metric per default
      */
-    Gradient3d( const Geometry& g, bc bcx, bc bcy, bc bcz, direction dir = forward)
+    Gradient3d( const Geometry& g, bc bcx, bc bcy, bc bcz, direction dir = centered)
     {
         dg::blas2::transfer( dg::create::dx( g, bcx, dir), m_rightx);
         dg::blas2::transfer( dg::create::dy( g, bcy, dir), m_righty);
@@ -243,7 +280,7 @@ class Gradient3d
         m_multiplyZ = !compute_in_2d;
     }
     /**
-     * @brief \f$ \vec v=\chi \nabla f \f$
+     * @brief \f$ \vec v=\chi \cdot\nabla f \f$
      *
      * @param f the vector to take the gradient of
      * @param vx (output) x-component
@@ -264,7 +301,7 @@ class Gradient3d
     }
 
     /**
-     * @brief \f$ \vec v = \lambda \chi\nabla f + \mu \vec v \f$
+     * @brief \f$ \vec v = \lambda \chi\cdot\nabla f + \mu \vec v \f$
      *
      * @param lambda a prefactor
      * @param f the vector to take the gradient of
@@ -289,16 +326,40 @@ class Gradient3d
         dg::tensor::multiply3d(lambda, m_chi, m_tempx, m_tempy, m_tempz, mu, vx, vy, vz);
     }
     /**
-     * @brief \f$ \sigma = \alpha (\nabla\phi\chi\nabla \phi) + \beta \sigma\f$
+     * @brief \f$ \sigma = (\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
      *
-     * @param alpha input prefactor
+     * @param phi the vector to take the variation of
+     * @param sigma (out) the variation
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1>
+    void variation(const ContainerType0& phi, ContainerType1& sigma){
+        variation(1.,1., phi, 0., sigma);
+    }
+    /**
+     * @brief \f$ \sigma = \lambda^2(\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
+     *
+     * @param lambda input prefactor
+     * @param phi the vector to take the variation of
+     * @param sigma (out) the variation
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
+    void variation(const ContainerTypeL& lambda, const ContainerType0& phi, ContainerType1& sigma){
+        variation(1.,lambda, phi, 0., sigma);
+    }
+    /**
+     * @brief \f$ \sigma = \alpha\lambda^2 (\nabla\phi\cdot\chi\cdot\nabla \phi) + \beta \sigma\f$
+     *
+     * @param alpha scalar input prefactor
+     * @param lambda input prefactor
      * @param phi the vector to take the variation of
      * @param beta the output prefactor
      * @param sigma (inout) the variation
      * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
      */
-    template<class ContainerType0, class ContainerType1>
-    void variation(value_type alpha, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
+    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
+    void variation(value_type alpha, const ContainerTypeL& lambda, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
     {
         dg::blas2::gemv( m_rightx, phi, m_tempx); //R_x*f
         dg::blas2::gemv( m_righty, phi, m_tempy); //R_y*f
@@ -306,7 +367,7 @@ class Gradient3d
             dg::blas2::gemv( m_rightz, phi, m_tempz); //R_y*f
         else
             dg::blas1::scal( m_tempz, 0.);
-        dg::tensor::scalar_product3d(alpha,  m_tempx, m_tempy, m_tempz, m_chi, m_tempx, m_tempy, m_tempz, beta, sigma);
+        dg::tensor::scalar_product3d(alpha, lambda,  m_tempx, m_tempy, m_tempz, m_chi, lambda, m_tempx, m_tempy, m_tempz, beta, sigma);
     }
     private:
     Matrix m_rightx, m_righty, m_rightz;
diff --git a/inc/dg/gradient_t.cu b/inc/dg/gradient_t.cu
index 326e45663..2ee3e1037 100644
--- a/inc/dg/gradient_t.cu
+++ b/inc/dg/gradient_t.cu
@@ -28,15 +28,15 @@ double dxphi3d( double x, double y, double z) {
     return cos(x)*cos(y)*cos(z);
 }
 double dyphi3d( double x, double y, double z) {
-    return -sin(x)*sin(y)*sin(z);
+    return -sin(x)*sin(y)*cos(z);
 }
 double dzphi3d( double x, double y, double z) {
-    return -x*x*sin(x)*cos(y)*sin(z);
+    return -sin(x)*cos(y)*sin(z)/x/x;
 }
 double variation3d( double x, double y, double z) {
     return dxphi3d(x,y,z)*dxphi3d(x,y,z)
         + dyphi3d(x,y,z)*dyphi3d(x,y,z)
-        + dzphi3d(x,y,z)*dzphi3d(x,y,z)/x/x;
+        + dzphi3d(x,y,z)*dzphi3d(x,y,z)*x*x;
 }
 
 // There are more tests in geometries/geometry_advection_(mpi)b.cu
@@ -62,9 +62,9 @@ int main()
 
     //apply arakawa scheme
     gradient.gradient( ph, dxph, dyph);
-    gradient.variation( 1.,ph, 0.,va);
+    gradient.variation( ph, va);
 
-    int64_t binary[] = {4517228146715811314,4506390577922324430,4519650974219167728};
+    int64_t binary[] = {4500635718861276907,4487444521638156650,4499885861996435701};
     exblas::udouble res;
     dg::DVec w2d = dg::create::weights( grid);
 
@@ -94,18 +94,18 @@ int main()
     const dg::DVec ph3d = dg::construct<dg::DVec>( dg::evaluate( phi3d, grid3d));
     const dg::DVec phx3d = dg::construct<dg::DVec>( dg::evaluate( dxphi3d, grid3d));
     const dg::DVec phy3d = dg::construct<dg::DVec>( dg::evaluate( dyphi3d, grid3d));
-    const dg::DVec phz3d = dg::construct<dg::DVec>( dg::evaluate( dyphi3d, grid3d));
+    const dg::DVec phz3d = dg::construct<dg::DVec>( dg::evaluate( dzphi3d, grid3d));
     const dg::DVec var3d = dg::construct<dg::DVec>( dg::evaluate( variation3d, grid3d));
-    dg::DVec dxph3d(ph3d), dyph3d(ph3d), va3d(ph3d);
+    dg::DVec dxph3d(ph3d), dyph3d(ph3d), dzph3d(ph3d), va3d(ph3d);
 
     // create a Gradient object
-    dg::Gradient<dg::aGeometry3d, dg::DMatrix, dg::DVec> gradient3d( grid3d);
+    dg::Gradient3d<dg::aGeometry3d, dg::DMatrix, dg::DVec> gradient3d( grid3d);
 
     //apply arakawa scheme
-    gradient3d.gradient( ph3d, dxph3d, dyph3d);
-    gradient3d.variation( 1.,ph3d, 0.,va3d);
+    gradient3d.gradient( ph3d, dxph3d, dyph3d, dzph3d);
+    gradient3d.variation( ph3d, va3d);
 
-    int64_t binary3d[] = {4520797903162066895,4520797903162066895,4520797903162066895};
+    int64_t binary3d[] = {4504451755369532568,4491224193368827475,4549042274897523598,4550331496568322612};
     exblas::udouble res3d;
     dg::DVec w3d = dg::create::weights( grid3d);
 
@@ -113,11 +113,14 @@ int main()
     res3d.d = sqrt(dg::blas2::dot( w3d, dxph3d)); //don't forget sqrt when computing errors
     std::cout << "Gx Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[0]<<std::endl;
     dg::blas1::axpby( 1., phy3d, -1., dyph3d);
-    res.d = sqrt(dg::blas2::dot( w3d, dyph3d)); //don't forget sqrt when computing errors
+    res3d.d = sqrt(dg::blas2::dot( w3d, dyph3d)); //don't forget sqrt when computing errors
     std::cout << "Gy Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[1]<<std::endl;
+    dg::blas1::axpby( 1., phz3d, -1., dzph3d);
+    res3d.d = sqrt(dg::blas2::dot( w3d, dzph3d)); //don't forget sqrt when computing errors
+    std::cout << "Gz Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[2]<<std::endl;
     dg::blas1::axpby( 1., var3d, -1., va3d);
-    res.d = sqrt(dg::blas2::dot( w3d, va3d)); //don't forget sqrt when computing errors
-    std::cout << "V  Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[2]<<std::endl;
+    res3d.d = sqrt(dg::blas2::dot( w3d, va3d)); //don't forget sqrt when computing errors
+    std::cout << "V  Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[3]<<std::endl;
     //periocid bc       |  dirichlet bc
     //n = 1 -> p = 2    |
     //n = 2 -> p = 1    |
diff --git a/inc/dg/topology/multiply.h b/inc/dg/topology/multiply.h
index 21d4b811d..0100aeeb2 100644
--- a/inc/dg/topology/multiply.h
+++ b/inc/dg/topology/multiply.h
@@ -113,37 +113,41 @@ struct InverseTensorMultiply3d{
     }
 };
 
-/// \f$ y \leftarrow \alpha v_i\lambda T_{ij} w_j + \beta y\f$
+/// \f$ y = \lambda\mu v_i T_{ij} w_j \f$
 template<class value_type>
 struct TensorDot2d{
     DG_DEVICE
     value_type operator() (
+              value_type lambda,
               value_type v0,  value_type v1,
               value_type t00, value_type t01,
               value_type t10, value_type t11,
+              value_type mu,
               value_type w0, value_type w1
               ) const
     {
         value_type tmp0 = DG_FMA(t00,w0 , t01*w1);
         value_type tmp1 = DG_FMA(t10,w0 , t11*w1);
-        return DG_FMA(v0,tmp0  , v1*tmp1);
+        return lambda*mu*DG_FMA(v0,tmp0  , v1*tmp1);
     }
 };
-/// \f$ y \leftarrow \alpha v_i\lambda T_{ij} w_j + \beta y\f$
+/// \f$ y = \lambda \mu v_i T_{ij} w_j \f$
 template<class value_type>
 struct TensorDot3d{
     DG_DEVICE
     value_type operator() (
-                      value_type v0,  value_type v1,  value_type v2,
-                      value_type t00, value_type t01, value_type t02,
-                      value_type t10, value_type t11, value_type t12,
-                      value_type t20, value_type t21, value_type t22,
-                      value_type w0, value_type w1, value_type w2) const
+              value_type lambda,
+              value_type v0,  value_type v1,  value_type v2,
+              value_type t00, value_type t01, value_type t02,
+              value_type t10, value_type t11, value_type t12,
+              value_type t20, value_type t21, value_type t22,
+              value_type mu,
+              value_type w0, value_type w1, value_type w2) const
     {
         value_type tmp0 = DG_FMA( t00,w0 , (DG_FMA( t01,w1 , t02*w2)));
         value_type tmp1 = DG_FMA( t10,w0 , (DG_FMA( t11,w1 , t12*w2)));
         value_type tmp2 = DG_FMA( t20,w0 , (DG_FMA( t21,w1 , t22*w2)));
-        return DG_FMA(v0,tmp0 , DG_FMA(v1,tmp1 , v2*tmp2));
+        return lambda*mu*DG_FMA(v0,tmp0 , DG_FMA(v1,tmp1 , v2*tmp2));
     }
 };
 ///@}
@@ -474,26 +478,30 @@ void inv_multiply3d( const SparseTensor<ContainerType0>& t, const ContainerType1
 }
 
 /**
- * @brief \f$ y = \alpha \sum_{i=0}^1 v_it^{ij}w_j + \beta y \text{ for } i\in \{0,1\}\f$
+ * @brief \f$ y = \alpha \lambda\mu \sum_{i=0}^1 v_it^{ij}w_j + \beta y \text{ for } i\in \{0,1\}\f$
  *
  * Ignore the 3rd dimension in \c t.
- * @param alpha input prefactor
+ * @param alpha scalar input prefactor
+ * @param lambda second input prefactor
  * @param v0 (input) first component  of \c v  (may alias w0)
  * @param v1 (input) second component of \c v  (may alias w1)
  * @param t input Tensor
+ * @param mu third input prefactor
  * @param w0 (input) first component  of \c w  (may alias v0)
  * @param w1 (input) second component of \c w  (may alias v1)
- * @param beta output prefactor
+ * @param beta scalar output prefactor
  * @param y (output)
  * @note This function is just a shortcut for a call to \c dg::blas1::evaluate with \c dg::TensorDot2d
  * @copydoc hide_ContainerType
  */
-template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4, class ContainerType5>
+template<class ContainerTypeL, class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerTypeM, class ContainerType4, class ContainerType5>
 void scalar_product2d(
         get_value_type<ContainerType0> alpha,
+        const ContainerTypeL& lambda,
         const ContainerType0& v0,
         const ContainerType1& v1,
         const SparseTensor<ContainerType2>& t,
+        const ContainerTypeM& mu,
         const ContainerType3& w0,
         const ContainerType4& w1,
         get_value_type<ContainerType0> beta,
@@ -502,35 +510,41 @@ void scalar_product2d(
     dg::blas1::evaluate( y,
              dg::Axpby<get_value_type<ContainerType0>>( alpha, beta),
              dg::TensorDot2d<get_value_type<ContainerType0>>(),
+             lambda,
              v0, v1,
              t.value(0,0), t.value(0,1),
              t.value(1,0), t.value(1,1),
+             mu,
              w0, w1);
 }
 
 /**
- * @brief \f$ y = \alpha \sum_{i=0}^2 v_it^{ij}w_j + \beta y \text{ for } i\in \{0,1,2\}\f$
+ * @brief \f$ y = \alpha \lambda\mu \sum_{i=0}^2 v_it^{ij}w_j + \beta y \text{ for } i\in \{0,1,2\}\f$
  *
- * @param alpha input prefactor
+ * @param alpha scalar input prefactor
+ * @param lambda second input prefactor
  * @param v0 (input) first component  of \c v  (may alias w0)
  * @param v1 (input) second component of \c v  (may alias w1)
  * @param v2 (input) third component of \c v  (may alias w1)
  * @param t input Tensor
+ * @param mu third input prefactor
  * @param w0 (input) first component  of \c w  (may alias v0)
  * @param w1 (input) second component of \c w  (may alias v1)
  * @param w2 (input) third component of \c w  (may alias v1)
- * @param beta output prefactor
+ * @param beta scalar output prefactor
  * @param y (output)
  * @note This function is just a shortcut for a call to \c dg::blas1::evaluate with \c dg::TensorDot3d
  * @copydoc hide_ContainerType
  */
-template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4, class ContainerType5, class ContainerType6, class ContainerType7>
+template<class ContainerTypeL, class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerTypeM, class ContainerType4, class ContainerType5, class ContainerType6, class ContainerType7>
 void scalar_product3d(
         get_value_type<ContainerType0> alpha,
+        const ContainerTypeL& lambda,
         const ContainerType0& v0,
         const ContainerType1& v1,
         const ContainerType2& v2,
         const SparseTensor<ContainerType3>& t,
+        const ContainerTypeM& mu,
         const ContainerType4& w0,
         const ContainerType5& w1,
         const ContainerType6& w2,
@@ -540,11 +554,13 @@ void scalar_product3d(
     dg::blas1::evaluate( y,
             dg::Axpby<get_value_type<ContainerType0>>( alpha, beta),
             dg::TensorDot3d<get_value_type<ContainerType0>>(),
-             v0, v1, v2,
-             t.value(0,0), t.value(0,1), t.value(0,2),
-             t.value(1,0), t.value(1,1), t.value(1,2),
-             t.value(2,0), t.value(2,1), t.value(2,2),
-             w0, w1, w2);
+            lambda,
+            v0, v1, v2,
+            t.value(0,0), t.value(0,1), t.value(0,2),
+            t.value(1,0), t.value(1,1), t.value(1,2),
+            t.value(2,0), t.value(2,1), t.value(2,2),
+            mu,
+            w0, w1, w2);
 }
 ///@}
 
diff --git a/inc/dg/topology/multiply_t.cu b/inc/dg/topology/multiply_t.cu
index 630fd0d0c..01f1d14ed 100644
--- a/inc/dg/topology/multiply_t.cu
+++ b/inc/dg/topology/multiply_t.cu
@@ -46,9 +46,10 @@ int main()
     std::cout << "Multiply T with [8,9]\n";
     dg::tensor::multiply2d( t, eight, nine, work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"] ([86 120])\n";
-    std::cout << "Scalar product [1,2] T [8,9]\n";
-    dg::tensor::scalar_product2d( 1., one, two, t, eight, nine, 0., inout0);
-    std::cout << "Result         is "<<inout0[0]<<" (326)\n";
+    std::cout << "Scalar product 2d\n";
+    inout0 = eight;
+    dg::tensor::scalar_product2d( 1., 2., one, two, t, 2., eight, nine, 1., inout0);
+    std::cout << "Result         is "<<inout0[0]<<" (1312)\n";
     std::cout << "Multiply T^{-1} with [86,120]\n";
     dg::tensor::inv_multiply2d(1., t, work0, work1, 0., work0, work1);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<"] ([8 9])\n";
@@ -59,9 +60,10 @@ int main()
     std::cout << "Multiply T with [8,9,2]\n";
     dg::tensor::multiply3d(t, eight, nine,two, work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] ([102 48 76])\n";
-    std::cout << "Scalar product [1,2,3] T [8,9,2]\n";
-    dg::tensor::scalar_product3d( 1., 1.,2.,3., t, 8.,9.,2., 0., inout0);
-    std::cout << "Result         is "<<inout0[0]<<" (426)\n";
+    std::cout << "Scalar product 3d\n";
+    inout0 = eight;
+    dg::tensor::scalar_product3d( 1., 3., one, two,three, t, 3., 8.,9.,2., -100., inout0);
+    std::cout << "Result         is "<<inout0[0]<<" (3034)\n";
     std::cout << "Multiply T^{-1} with [102,48,76]\n";
     dg::tensor::inv_multiply3d(1., t, work0, work1, work2, 0., work0, work1, work2);
     std::cout << "Result         is ["<<work0[0]<<" "<<work1[0]<<" "<<work2[0]<<"] ([8 9 2])\n";
-- 
GitLab


From f5473cbe97f79a509f8024160c9d526140bd84c4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 11:33:56 +0100
Subject: [PATCH 489/540] Improve Documentation

- improve modules with formulas
- improve pid_control documentation
- improve Gradient documentation on variation
---
 inc/dg/adaptive.h     | 23 ++++++++++++---
 inc/dg/advection.h    | 13 +++++++-
 inc/dg/dg_doc.h       | 14 +++++----
 inc/dg/elliptic.h     |  4 +--
 inc/dg/helmholtz.h    |  4 +--
 inc/dg/multistep.h    | 69 ++++++++++++++++++++++---------------------
 inc/dg/multistep_t.cu |  4 ---
 inc/dg/subroutines.h  | 20 ++++++-------
 8 files changed, 89 insertions(+), 62 deletions(-)

diff --git a/inc/dg/adaptive.h b/inc/dg/adaptive.h
index 62600af8a..f33e65324 100644
--- a/inc/dg/adaptive.h
+++ b/inc/dg/adaptive.h
@@ -20,7 +20,23 @@ get_value_type<ContainerType> l2norm( const ContainerType& x)
 {
     return sqrt( dg::blas1::dot( x,x));
 }
-///\f[ h'= h \epsilon_n^{-0.58/p}\epsilon_{n-1}^{0.21/p}\epsilon_{n-2}^{-0.1/p}\f]
+/**
+ * @brief \f[ h'= h \epsilon_n^{-0.58/p}\epsilon_{n-1}^{0.21/p}\epsilon_{n-2}^{-0.1/p}\f]
+ *
+ * PID stands for "Proportional" (the present error), "Integral" (the past error), "Derivative" (the future error). See a good tutorial here https://www.youtube.com/watch?v=UR0hOmjaHp0
+ * The PID controller is a good controller to start with, it does not overshoot
+ * too much, is smooth, has no systematic over- or under-estimation and
+ * converges very quickly to the desired timestep
+ * @tparam value_type
+ * @param dt_old the old (present) timestep
+ * @param eps_0 the error relative to the tolerance of the present timestep
+ * @param eps_1 the error relative to the tolerance of the previous timestep
+ * @param eps_2 the error relative to the tolerance of the second previous timestep
+ * @param embedded_order order of the embedded timestepper
+ * @param order order of the timestepper
+ *
+ * @return the new timestep
+ */
 template<class value_type>
 value_type pid_control( value_type dt_old, value_type eps_0, value_type eps_1, value_type eps_2, unsigned embedded_order, unsigned order)
 {
@@ -54,7 +70,6 @@ value_type i_control( value_type dt_old, value_type eps_0, value_type eps_1, val
 template<class value_type>
 struct PIDController
 {
-    //PID means proportional (present), integral (past), derivative (future)
     PIDController( ){}
     value_type operator()( value_type dt_old, value_type eps_n, value_type eps_n1, value_type eps_n2, unsigned embedded_order, unsigned order)const
     {
@@ -319,7 +334,7 @@ typename Adaptive<Stepper>::value_type Adaptive<Stepper>::guess_stepsize( Explic
  * @param u1 (write only) contains the updated result on output if the step was accepted, otherwise a copy of \c u0 (may alias \c u0)
  * @param dt on input: timestep to try out (see dg::Adaptive::guess_stepsize() for an initial stepsize).
  * On output: stepsize proposed by the controller that can be used to continue the integration in the next step.
- * @param control The control function. Usually \c dg::pid_control is a good choice
+ * @param control The control function. Usually \c dg::pid_control is a good choice. The task of the control function is to compute a new timestep size based on the old timestep size, the order of the method and the past error(s)
  * @param norm The error norm. Usually \c dg::l2norm is a good choice, but for
  * very small vector sizes the time for the binary reproducible dot product might become
  * a performance bottleneck. Then it's time for your own implementation.
@@ -340,7 +355,7 @@ typename Adaptive<Stepper>::value_type Adaptive<Stepper>::guess_stepsize( Explic
 
 /*!@class hide_control_error
  *
- * @tparam ControlFunction function or Functor called as dt' = control( dt, eps0, eps1, eps2, order, embedded_order), where all parameters are of type value_type except the last two, which are unsigned
+ * @tparam ControlFunction function or Functor called as dt' = control( dt, eps0, eps1, eps2, order, embedded_order), where all parameters are of type value_type except the last two, which are unsigned.
  * @tparam ErrorNorm function or Functor of type value_type( const ContainerType&)
  */
 
diff --git a/inc/dg/advection.h b/inc/dg/advection.h
index a2475dfff..654f770d2 100644
--- a/inc/dg/advection.h
+++ b/inc/dg/advection.h
@@ -17,10 +17,21 @@ namespace dg
 {
 
 /**
- * @brief Upwind discretization of advection operator \f$ \vec v\cdot\nabla f\f$
+ * @brief %Upwind discretization of advection operator \f$ \vec v\cdot\nabla f\f$
  *
  * This is the upwind scheme where a backward derivative is used if v is
  * positive and a forward derivative else
+ * For example
+ * @code
+// v_x  = -dy phi
+dg::blas2::symv( -1., dy, phi, 0., vx);
+// v_y = dx phi
+dg::blas2::symv( 1., dx, phi, 0., vy);
+// compute on Cartesian grid in 2d on device
+dg::Advection < dg::CartesianGrid2d, dg::DMatrix, dg::DVec> advection(grid);
+// df = - v Grad f
+advection.upwind( -1., vx, vy, f, 0., df);
+@endcode
  * @note This scheme brings its own numerical diffusion and thus does not need any other artificial viscosity mechanisms. The only places where the scheme might run into oscillations is if there is a stagnation point with v==0 at a fixed position
  * @sa A discussion of this and other advection schemes can be found here https://mwiesenberger.github.io/advection
  * @copydoc hide_geometry_matrix_container
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 52e6a3748..ab2227f26 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -13,14 +13,14 @@
  *         time integrators.
  *     @{
  *         @defgroup blas1 BLAS level 1 routines: Vector-Vector
- *              \f$ f( x_{0i}, x_{1i}, x_{2i}, ...) \f$ and \f$ x\cdot y\f$
+ *              \f$ f( x_{0i}, x_{1i}, x_{2i}, ...) \f$ and \f$ x^T y\f$
  *
  *             Successive calls to blas routines are executed sequentially.
  *             A manual synchronization of threads or devices is never needed
  *             in an application using these functions. All functions returning
  *             a value block until the value is ready.
  *         @defgroup blas2 BLAS level 2 routines: Matrix-Vector
- *              \f$ \alpha M \cdot x + \beta y\f$ and \f$ x^T M \cdot y \f$
+ *              \f$ \alpha M \cdot x + \beta y\f$ and \f$ x^T M y \f$
  *
  *             Successive calls to blas routines are executed sequentially.
  *             A manual synchronization of threads or devices is never needed
@@ -51,7 +51,6 @@
  *           Implementation details of \ref dispatch
  * @}
  * @defgroup numerical0 Level 2: Basic numerical algorithms
- *      Based on blas1 and blas2 functions
  * @{
  *     @defgroup time Time integrators
  *      \f$ \dot y = f(y,t) \f$
@@ -70,6 +69,7 @@
  *     @{
  *         @defgroup basictopology Topology base classes
  *         @defgroup evaluation evaluate
+ *          \f$ f_i = f(x_i) \f$
  *
  *             The function discretisation routines compute the DG discretisation
  *             of analytic functions on a given grid. In 1D the discretisation
@@ -84,9 +84,11 @@
  *              overloads for the \c dg::create::weights and \c dg::create::inv_weights functions for all
  *              available topologies
  *         @defgroup creation create derivatives
+ *           \f$ D_x\f$, \f$ D_y\f$ and \f$ D_z \f$
  *
  *             High level matrix creation functions
  *         @defgroup interpolation Interpolation and projection
+ *          \f$ I \f$ and \f$ P = I^\dagger\f$
  *         @defgroup utilities Averaging
  *         @defgroup scatter Scatter and Gather
  *     @}
@@ -97,7 +99,9 @@
  *     @{
  *         @defgroup basicgeometry Geometry base classes
  *         @defgroup pullback pullback and pushforward
+ *          \f$ f_i = f( x (\zeta_i,\eta_i), y(\zeta_i,\eta_i)) \f$
  *         @defgroup metric create volume
+ *           \f$ \sqrt{g} \f$
  *         @defgroup generators Grid Generator classes
  *     @}
  *     @defgroup gridtypes Useful Typedefs
@@ -132,10 +136,10 @@
  *              Binary subroutines for the dg::blas1::evaluate function
  *
  *          @defgroup variadic_evaluates blas1::evaluate variadic subroutines
- *              Functors with an arbitrary number of arguments to use in the dg::blas1::evaluate function
+ *              Functors to use in the dg::blas1::evaluate function
  *
  *          @defgroup variadic_subroutines blas1::subroutine subroutines
- *              Functors with a variable number of arguments for use in the dg::blas1::subroutine functions
+ *              Functors to use in the dg::blas1::subroutine functions
  *     @}
  *     @defgroup lowlevel Lowlevel helper functions and classes
  *
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 15f085511..e59327ae6 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -19,7 +19,7 @@ namespace dg
 {
 
 /**
- * @brief A 2d negative elliptic differential operator
+ * @brief A 2d negative elliptic differential operator \f$ -\nabla \cdot ( \chi \nabla ) \f$
  *
  * @ingroup matrixoperators
  *
@@ -323,7 +323,7 @@ using Elliptic2d = Elliptic<Geometry, Matrix, Container>;
 
 //Elliptic3d is tested in inc/geometries/elliptic3d_t.cu
 /**
- * @brief A 3d negative elliptic differential operator
+ * @brief A 3d negative elliptic differential operator \f$ -\nabla \cdot ( \chi \nabla ) \f$
  *
  * @ingroup matrixoperators
  *
diff --git a/inc/dg/helmholtz.h b/inc/dg/helmholtz.h
index c4c213f1e..caf5059b6 100644
--- a/inc/dg/helmholtz.h
+++ b/inc/dg/helmholtz.h
@@ -12,7 +12,7 @@
 namespace dg{
 
 /**
- * @brief Matrix class that represents a Helmholtz-type operator
+ * @brief Matrix class that represents a Helmholtz-type operator \f$ (\chi+\alpha\Delta) \f$
  *
  * @ingroup matrixoperators
  *
@@ -152,7 +152,7 @@ struct Helmholtz
 };
 
 /**
- * @brief Matrix class that represents a 3d Helmholtz-type operator
+ * @brief Matrix class that represents a 3d Helmholtz-type operator \f$ (\chi+\alpha\Delta) \f$
  *
  * @ingroup matrixoperators
  *
diff --git a/inc/dg/multistep.h b/inc/dg/multistep.h
index 61cd3cf4e..85c9e5d0c 100644
--- a/inc/dg/multistep.h
+++ b/inc/dg/multistep.h
@@ -245,41 +245,9 @@ void ImExMultistep<ContainerType, SolverType>::step( RHS& f, Diffusion& diff, va
 }
 ///@endcond
 
-/** @brief Deprecated  (use ImExMultistep and select "Karniadakis" from the multistep tableaus)
-* @ingroup time
-* @sa dg::ImExMultistep
-*/
-template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
-struct Karniadakis
-{
-    using value_type = get_value_type<ContainerType>;
-    using container_type = ContainerType;
-    Karniadakis(){}
-    template<class ...SolverParams>
-    Karniadakis( SolverParams&& ...ps): m_imex( "Karniadakis", std::forward<SolverParams> (ps)...) { }
-    template<class ...Params>
-    void construct( Params&& ...ps)
-    {
-        *this = Karniadakis( std::forward<Params>( ps)...);
-    }
-    const ContainerType& copyable()const{ return m_imex.copyable();}
-    SolverType& solver() { return m_imex.solver();}
-    const SolverType& solver() const { return m_imex.solver();}
-    template< class Explicit, class Implicit>
-    void init( Explicit& ex, Implicit& im, value_type t0, const ContainerType& u0, value_type dt){
-        m_imex.init( ex, im, t0, u0, dt);
-    }
-    template< class Explicit, class Implicit>
-    void step( Explicit& ex, Implicit& im, value_type& t, ContainerType& u){
-        m_imex.step( ex, im, t, u);
-    }
-  private:
-    ImExMultistep<ContainerType, SolverType> m_imex;
-};
-
 
 /**
-* @brief Implicit multistep time-integration with Limiter/Filter
+* @brief EXPERIMENTAL: Implicit multistep time-integration with Limiter/Filter
 * \f[
 * \begin{align}
     \tilde v &= \sum_{i=0}^{s-1} a_i v^{n-i} + \Delta t \sum_{i=1}^{s} c_i\hat I(t^{n+1-i}, v^{n+1-i}) + \Delta t c_{0} \hat I (t + \Delta t, \tilde v) \\
@@ -547,7 +515,7 @@ struct ImplicitMultistep
 
 
 /**
-* @brief General explicit linear multistep time-integration with Limiter / Filter
+* @brief EXPERIMENTAL: General explicit linear multistep time-integration with Limiter / Filter
 * \f[
 * \begin{align}
     \tilde v &= \sum_{j=0}^{s-1} a_j v^{n-j} + \Delta t\left(\sum_{j=0}^{s-1}b_j  \hat f\left(t^{n}-j\Delta t, v^{n-j}\right)\right) \\
@@ -794,4 +762,37 @@ struct ExplicitMultistep
     FilteredExplicitMultistep<ContainerType> m_fem;
 };
 
+/** @brief DEPRECATED  (use ImExMultistep and select "Karniadakis" from the multistep tableaus)
+* @ingroup time
+* @sa dg::ImExMultistep
+*/
+template<class ContainerType, class SolverType = dg::DefaultSolver<ContainerType>>
+struct Karniadakis
+{
+    using value_type = get_value_type<ContainerType>;
+    using container_type = ContainerType;
+    Karniadakis(){}
+    template<class ...SolverParams>
+    Karniadakis( SolverParams&& ...ps): m_imex( "Karniadakis", std::forward<SolverParams> (ps)...) { }
+    template<class ...Params>
+    void construct( Params&& ...ps)
+    {
+        *this = Karniadakis( std::forward<Params>( ps)...);
+    }
+    const ContainerType& copyable()const{ return m_imex.copyable();}
+    SolverType& solver() { return m_imex.solver();}
+    const SolverType& solver() const { return m_imex.solver();}
+    template< class Explicit, class Implicit>
+    void init( Explicit& ex, Implicit& im, value_type t0, const ContainerType& u0, value_type dt){
+        m_imex.init( ex, im, t0, u0, dt);
+    }
+    template< class Explicit, class Implicit>
+    void step( Explicit& ex, Implicit& im, value_type& t, ContainerType& u){
+        m_imex.step( ex, im, t, u);
+    }
+  private:
+    ImExMultistep<ContainerType, SolverType> m_imex;
+};
+
+
 } //namespace dg
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 4ba10b1c6..91d61b813 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -85,9 +85,6 @@ struct Full
 };
 
 
-const double lx = 2.*M_PI;
-const double ly = 2.*M_PI;
-
 //const unsigned NT = (unsigned)(nu*T*n*n*N*N/0.01/lx/lx);
 
 int main()
@@ -105,7 +102,6 @@ int main()
     const std::array<double,2> sol = solution(T,nu);
     const double norm_sol = dg::blas1::dot( sol, sol);
     double time = 0.;
-    std::array<double,2> error( sol);
     exblas::udouble res;
     std::cout << "### Test Explicit Multistep methods with "<<NT<<" steps\n";
     std::vector<std::string> ex_names{
diff --git a/inc/dg/subroutines.h b/inc/dg/subroutines.h
index 0cf1270c1..9e5fd983a 100644
--- a/inc/dg/subroutines.h
+++ b/inc/dg/subroutines.h
@@ -11,45 +11,45 @@ namespace dg{
 struct equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1 in, T2& out) const
+DG_DEVICE void operator()( T1 x, T2& y) const
     {
-        out = in;
+        y = x;
     }
 };
 ///\f$ y=y+x\f$
 struct plus_equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1 in, T2& out) const
+DG_DEVICE void operator()( T1 x, T2& y) const
     {
-        out += in;
+        y += x;
     }
 };
 ///\f$ y=y-x\f$
 struct minus_equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1 in, T2& out) const
+DG_DEVICE void operator()( T1 x, T2& y) const
     {
-        out -= in;
+        y -= x;
     }
 };
 ///\f$ y=xy\f$
 struct times_equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1 in, T2& out) const
+DG_DEVICE void operator()( T1 x, T2& y) const
     {
-        out *= in;
+        y *= x;
     }
 };
 ///\f$ y = y/x\f$
 struct divides_equals
 {
     template< class T1, class T2>
-DG_DEVICE void operator()( T1 in, T2& out) const
+DG_DEVICE void operator()( T1 x, T2& y) const
     {
-        out /= in;
+        y /= x;
     }
 };
 ///@}
-- 
GitLab


From 099046b865e81337724423f076918074b346eb18 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 11:39:09 +0100
Subject: [PATCH 490/540] Fix geometry_advection programs

with variation interface
---
 inc/geometries/geometry_advection_b.cu    | 2 +-
 inc/geometries/geometry_advection_mpib.cu | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index 05d259305..074b11f9a 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -192,7 +192,7 @@ int main(int argc, char** argv)
     ///////////////////////////////////////////////////////////////////////
     std::cout << "TESTING VARIATION\n";
     dg::Gradient<dg::aGeometry2d, dg::DMatrix, dg::DVec> gradient( grid);
-    gradient.variation( 1., lhs, 0., jac);
+    gradient.variation( lhs, jac);
     dg::blas1::axpby( 1., variation, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     std::cout << "               distance to solution "<<sqrt( result)<<std::endl; //don't forget sqrt when comuting errors
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index 24711e32f..b7df42838 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -189,7 +189,7 @@ int main(int argc, char** argv)
     ///////////////////////////////////////////////////////////////////////
     if(rank==0)std::cout << "TESTING VARIATION 3D\n";
     dg::Gradient<Geometry, dg::MDMatrix, dg::MDVec> gradient( grid);
-    gradient.variation( 1., lhs, 0., jac);
+    gradient.variation( lhs, jac);
     dg::blas1::axpby( 1., variation, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     if(rank==0)std::cout << "               distance to solution "<<sqrt( result)<<std::endl; //don't forget sqrt when comuting errors
-- 
GitLab


From 58bc0ea5c423b53a36c775e1a2c7a1dfb543b489 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 11:40:02 +0100
Subject: [PATCH 491/540] Change diag and src codes to Gradient.variation

everything should compile again now
---
 diag/feltorSesoldiag.cpp      |  5 +++--
 diag/feltorShwdiag.cpp        |  9 +++++----
 diag/impRdiag.cu              |  3 ++-
 diag/normdiag.cu              |  3 ++-
 diag/toeflEPdiag.cu           |  3 ++-
 diag/toeflRdiag.cu            |  3 ++-
 src/ep/toeflR.cuh             | 14 +++++++-------
 src/feltor/feltor.h           | 15 +++------------
 src/feltorSH/feltor.cuh       |  5 +++--
 src/feltorSHp/feltor.cuh      |  5 +++--
 src/feltorSesol/feltor.cuh    |  5 +++--
 src/feltorSesol/feltor_hpc.cu |  2 +-
 src/feltorShw/feltor.cuh      |  7 ++++---
 src/feltorShw/feltor_hpc.cu   |  2 +-
 src/hasegawa/mima.cuh         | 22 +++++++++++-----------
 src/impurities/toeflI.cuh     | 16 ++++++++--------
 src/lamb_dipole/shu.cuh       |  6 ------
 src/reco2D/reconnection.cuh   |  8 +++++---
 src/toefl/toeflR.cuh          |  5 +++--
 19 files changed, 68 insertions(+), 70 deletions(-)

diff --git a/diag/feltorSesoldiag.cpp b/diag/feltorSesoldiag.cpp
index ca409e9b6..251ebd544 100644
--- a/diag/feltorSesoldiag.cpp
+++ b/diag/feltorSesoldiag.cpp
@@ -62,6 +62,7 @@ int main( int argc, char* argv[])
     dg::IHMatrix interp(dg::create::interpolation(xcoo,y0coo,g2d));
     dg::IHMatrix interp_in = dg::create::interpolation(g2d,g2d_in);
     dg::Poisson<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> poisson(g2d,  p.bc_x, p.bc_y,  p.bc_x, p.bc_y);
+    dg::Gradient<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> gradient(g2d, p.bc_x, p.bc_y);
 
 
     //2d field
@@ -185,9 +186,9 @@ int main( int argc, char* argv[])
             polavg(logn[1],temp1d,false);
             err_out = nc_put_vara_double( ncid_out, dataIDs1d[3],   start1d, count1d, temp1d.data()); 
             polavg(phi,temp1d,false);
-            poisson.variationRHS(phi,temp2);
+            gradient.variation(phi,temp2);
             double T_perp = 0.5*dg::blas2::dot( npe[1], w2d, temp2);
-            poisson.variationRHS(temp,temp2);
+            gradient.variation(temp,temp2);
             double T_perp_zonal = 0.5*dg::blas2::dot( npe[1], w2d, temp2);   
             double T_perpratio = T_perp_zonal/T_perp;
             dg::blas2::gemv( poisson.dyrhs(), phi, temp2); 
diff --git a/diag/feltorShwdiag.cpp b/diag/feltorShwdiag.cpp
index a328e9ed4..eb603efdb 100644
--- a/diag/feltorShwdiag.cpp
+++ b/diag/feltorShwdiag.cpp
@@ -69,6 +69,7 @@ int main( int argc, char* argv[])
     dg::IHMatrix interp(dg::create::interpolation(xcoo,y0coo,g2d));
     dg::IHMatrix interp_in = dg::create::interpolation(g2d,g2d_in);
     dg::Poisson<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> poisson(g2d,  p.bc_x, p.bc_y,  p.bc_x_phi, p.bc_y);
+    dg::Gradient<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> gradient(g2d, p.bc_x_phi, p.bc_y);
     dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> pol(g2d,   p.bc_x_phi, p.bc_y, dg::normed, dg::centered);
     dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> lap(g2d,   p.bc_x, p.bc_y, dg::normed, dg::centered);
     
@@ -251,10 +252,10 @@ int main( int argc, char* argv[])
 		    
                 }
 
-                poisson.variationRHS(phi,temp2);
+                gradient.variation(phi,temp2);
                 Tperp = 0.5*dg::blas2::dot( one, w2d, temp2);   // 0.5   u_E^2            
                 polavg(phi,temp);      // <N u_E^2 > 
-                poisson.variationRHS(temp,temp2);
+                gradient.variation(temp,temp2);
                 Tperpz = 0.5*dg::blas2::dot( one, w2d, temp2);   //0.5 ( D_x <phi> )^2 
                 Tperpratio = Tperpz/Tperp;
                 dg::blas2::gemv( poisson.dyrhs(), phi, temp2); 
@@ -320,10 +321,10 @@ int main( int argc, char* argv[])
 		    dg::blas1::transform(navgtilde[i], navgtilde[i], dg::PLUS<>(-1.0));
                 }
                                     
-                poisson.variationRHS(phi,temp2);
+                gradient.variation(phi,temp2);
                 Tperp = 0.5*dg::blas2::dot( one, w2d, temp2);   // 0.5   u_E^2            
                 polavg(phi,temp);      // <N u_E^2 > 
-                poisson.variationRHS(temp,temp2);
+                gradient.variation(temp,temp2);
                 Tperpz = 0.5*dg::blas2::dot( one, w2d, temp2);   //0.5 ( D_x <phi> )^2 
                 Tperpratio = Tperpz/Tperp;
                 dg::blas2::gemv( poisson.dyrhs(), phi, temp2); 
diff --git a/diag/impRdiag.cu b/diag/impRdiag.cu
index 9688db9db..611ef8040 100644
--- a/diag/impRdiag.cu
+++ b/diag/impRdiag.cu
@@ -122,6 +122,7 @@ int main( int argc, char* argv[])
   std::vector<dg::HVec> npe_h(3, dg::evaluate(dg::zero, g2d));
   //eval field
   dg::ArakawaX< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa(g2d);
+  dg::Gradient< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient(g2d);
   //eval particle densities
   const dg::DVec binv( dg::evaluate(dg::LinearX(p.kappa, 1.), g2d));
   dg::DVec chi = dg::evaluate(dg::zero, g2d);
@@ -373,7 +374,7 @@ int main( int argc, char* argv[])
       m[j]++;
     }
     //field
-    arakawa.variation(field[0], helper);
+    gradient.variation(field[0], helper);
     double energy[5] = {};
     energy[0] = dg::blas2::dot(lnn[0], w2d, npe[0]);
     energy[1] = p.a[1]*p.tau[1]*dg::blas2::dot(npe[1], w2d, lnn[1]);
diff --git a/diag/normdiag.cu b/diag/normdiag.cu
index 9c439a4bc..5cccf3485 100644
--- a/diag/normdiag.cu
+++ b/diag/normdiag.cu
@@ -61,6 +61,7 @@ int main( int argc, char* argv[])
     dg::DVec nG(dg::evaluate(prof,g2d));
     dg::DVec w2d = dg::create::weights( g2d);
     dg::Poisson<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> poisson(g2d,  p.bc_x, p.bc_y,  p.bc_x_phi, p.bc_y);
+    dg::Gradient<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient(g2d, p.bc_x_phi, p.bc_y);
     //open netcdf files
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     //set min and max timesteps
@@ -162,7 +163,7 @@ int main( int argc, char* argv[])
         }
         
 
-        poisson.variationRHS(phi,uE2);
+        gradient.variation(phi,uE2);
         uE2norm= 0.5*dg::blas2::dot( one, w2d,uE2);   // 0.5   u_E^2    
         nlnnnorm = dg::blas2::dot(ne,w2d,logne);
         NiuE2norm = 0.5*dg::blas2::dot(Ni, w2d,uE2);
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index 7ad81ff30..b19943f74 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -81,6 +81,7 @@ int main( int argc, char* argv[])
     dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
     dg::Grid1d g1d( 0., p.lx, p.n_out, p.Nx_out, p.bc_x);
     dg::ArakawaX< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa( g2d); 
+    dg::Gradient< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient( g2d); 
     double time = 0.;
     //2d field
     size_t count2d[3]  = {1, g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
@@ -278,7 +279,7 @@ int main( int argc, char* argv[])
         double Ue, Ui, Uphi;
         for( unsigned j=0; j<2; j++)
             dg::blas1::transform( npe[j], lnn[j], dg::LN<double>()); 
-        arakawa.variation(phi, helper); 
+        gradient.variation(phi, helper); 
         Ue = p.z[0]*p.tau[0]*dg::blas2::dot( lnn[0], w2d, npe[0]);
         Ui = p.z[1]*p.tau[1]*dg::blas2::dot( lnn[1], w2d, npe[1]);
         Uphi = 0.5*dg::blas2::dot( npe[1], w2d, helper); 
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index a752c4515..787d87131 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -76,6 +76,7 @@ int main( int argc, char* argv[])
     dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
     dg::Grid1d g1d( 0., p.lx, p.n_out, p.Nx_out, p.bc_x);
     dg::ArakawaX< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa( g2d); 
+    dg::Gradient< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient( g2d); 
     double time = 0.;
     //2d field
     size_t count2d[3]  = {1, g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
@@ -273,7 +274,7 @@ int main( int argc, char* argv[])
         double Ue, Ui, Uphi;
         for( unsigned j=0; j<2; j++)
             dg::blas1::transform( npe[j], lnn[j], dg::LN<double>()); 
-        arakawa.variation(phi, helper); 
+        gradient.variation(phi, helper);
         if(p.equations == "global" || p.equations == "ralf_global")
         {
             Ue = dg::blas2::dot( lnn[0], w2d, npe[0]);
diff --git a/src/ep/toeflR.cuh b/src/ep/toeflR.cuh
index 342f2290c..8eadab7e8 100644
--- a/src/ep/toeflR.cuh
+++ b/src/ep/toeflR.cuh
@@ -125,6 +125,7 @@ struct ToeflR
     dg::Elliptic<Geometry, Matrix, container> pol, laplaceM; //contains normalized laplacian
     dg::Helmholtz<Geometry,  Matrix, container> gamma1;
     dg::ArakawaX< Geometry, Matrix, container> arakawa; 
+    dg::Gradient< Geometry, Matrix, container> gradient; 
 
     dg::Invert<container> invert_pol, invert_invgamma;
 
@@ -144,11 +145,12 @@ ToeflR< Geometry, M, container>::ToeflR( const Geometry& grid, const Parameters&
     chi( evaluate( dg::zero, grid)), omega(chi),
     binv( evaluate( dg::LinearX( p.kappa, 1.-p.kappa*p.posX*p.lx), grid)), 
     gamma_n(chi), potential_(chi), psi( 2, chi), dypsi( psi), ype(psi),
-    dyy(2,chi), lny( dyy), lapy(dyy), 
-    pol(     grid, dg::not_normed, dg::centered), 
+    dyy(2,chi), lny( dyy), lapy(dyy),
+    pol(     grid, dg::not_normed, dg::centered),
     laplaceM( grid, dg::normed, dg::centered),
     gamma1(  grid, 0., dg::centered),
-    arakawa( grid), 
+    arakawa( grid),
+    gradient( grid, dg::centered),
     invert_pol(      omega, omega.size(), p.eps_pol),
     invert_invgamma( omega, omega.size(), p.eps_gamma),
     w2d( dg::create::volume(grid)), v2d( dg::create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
@@ -169,9 +171,7 @@ void ToeflR<G, M, container>::compute_psi( const container& phi)
         if(  number == invert_invgamma.get_max())
             throw dg::Fail( eps_gamma);
 
-        arakawa.variation(phi, omega); //needed also in local energy theorem
-        dg::blas1::pointwiseDot( binv, omega, omega);
-        dg::blas1::pointwiseDot( binv, omega, omega);
+        gradient.variation(binv, phi, omega); //needed also in local energy theorem
         dg::blas1::axpby( 1., psi[i], -0.5*mu[i], omega, psi[i]);   //psi  Gamma phi - 0.5 u_E^2
     }
 }
@@ -233,7 +233,7 @@ void ToeflR<G, M, container>::operator()(double t, const std::vector<container>&
         double Ue = z[0]*tau[0]*dg::blas2::dot( lny[0], w2d, ype[0]);
         double Up = z[1]*tau[1]*dg::blas2::dot( lny[1], w2d, ype[1]);
         double Uphi = 0.5*dg::blas2::dot( ype[0], w2d, omega) + 0.5*dg::blas2::dot( ype[1], w2d, omega); 
-        arakawa.variation(potential_, omega); 
+        gradient.variation(potential_, omega); 
         double UE = debye_*dg::blas2::dot( one, w2d, omega);
         energy_ = Ue + Up + Uphi + UE;
         //std::cout << "Ue "<<Ue<< "Up "<<Up<< "Uphi "<<Uphi<< "UE "<<UE<<"\n";
diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 16aeefee1..783796cf1 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -191,15 +191,6 @@ struct ComputeChi{
         chi = mu_i*(tilde_Ni+1.)*binv*binv;
     }
 };
-struct ComputePsi{
-    DG_DEVICE
-    void operator()( double& GammaPhi, double& uE2, double binv) const{
-        //u_E^2
-        uE2   = binv*binv*uE2;
-        //Psi
-        GammaPhi = GammaPhi - 0.5*uE2;
-    }
-};
 //struct ComputeLogN{
 //    DG_DEVICE
 //    void operator()( double tilde_n, double& npe, double& logn) const{
@@ -795,10 +786,10 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::compute_psi(
     dg::blas2::symv( m_dx_P, m_phi[0], m_dP[0][0]);
     dg::blas2::symv( m_dy_P, m_phi[0], m_dP[0][1]);
     if( !m_p.symmetric) dg::blas2::symv( m_dz, m_phi[0], m_dP[0][2]);
-    dg::tensor::scalar_product3d( 1.,
-        m_dP[0][0], m_dP[0][1], m_dP[0][2], m_hh, //grad_perp
+    dg::tensor::scalar_product3d( 1., m_binv,
+        m_dP[0][0], m_dP[0][1], m_dP[0][2], m_hh, m_binv, //grad_perp
         m_dP[0][0], m_dP[0][1], m_dP[0][2], 0., m_UE2);
-    dg::blas1::subroutine( routines::ComputePsi(), m_phi[1], m_UE2, m_binv);
+    dg::blas1::axpby( -0.5, m_UE2, 1., m_phi[1]);
 #ifdef DG_MANUFACTURED
     dg::blas1::evaluate( m_phi[1], dg::plus_equals(), manufactured::SPhii{
         m_p.mu[0],m_p.mu[1],m_p.tau[0],m_p.tau[1],m_p.eta,
diff --git a/src/feltorSH/feltor.cuh b/src/feltorSH/feltor.cuh
index 53b2b8866..cb7c60afb 100644
--- a/src/feltorSH/feltor.cuh
+++ b/src/feltorSH/feltor.cuh
@@ -103,6 +103,7 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
+    dg::Gradient< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic<   Geometry, Matrix, container> lapperpM; 
     std::vector<container> multi_chi;
@@ -130,6 +131,7 @@ Explicit<Grid, Matrix, container>::Explicit( const Grid& g, eule::Parameters p):
     phi( 2, chi),chii(chi),uE2(chi),// (phi,psi), (chi_i), u_ExB
     ype(4,chi), logype(ype), // y+(bgamp+profamp) , log(ype)
     poisson(g, g.bcx(), g.bcy(), g.bcx(), g.bcy()), //first N/U then phi BCC
+    gradient(g, g.bcx(), g.bcy(), dg::centered),
     lapperpM ( g,g.bcx(), g.bcy(),     dg::normed,         dg::centered),
     invert_pol(         omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma(    omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
@@ -243,8 +245,7 @@ container& Explicit<G, Matrix,container>::compute_psi(const container& ti,contai
         }
     }
 
-    poisson.variationRHS(potential, omega); // (nabla_perp phi)^2
-    dg::blas1::pointwiseDot(1.0, binv, binv, omega, 0.0, uE2);           //  u_E^2   
+    gradient.variation(binv, potential, uE2); // (nabla_perp phi)^2
     dg::blas1::axpby( 1., phi[1], -0.5, uE2,phi[1]);             //psi  Gamma phi - 0.5 u_E^2
     return phi[1];    
 }
diff --git a/src/feltorSHp/feltor.cuh b/src/feltorSHp/feltor.cuh
index f1595deda..42cb8490a 100644
--- a/src/feltorSHp/feltor.cuh
+++ b/src/feltorSHp/feltor.cuh
@@ -125,6 +125,7 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
+    dg::Gradient< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic<   Geometry, Matrix, container> lapperpM; 
     std::vector<container> multi_chi;
@@ -152,6 +153,7 @@ Explicit<Grid, Matrix, container>::Explicit( const Grid& g, eule::Parameters p):
     phi( 2, chi),chii(chi),uE2(chi),// (phi,psi), (chi_i), u_ExB
     n(2,chi), logn(n), pr(n), logpr(n), te(n), logte(n), tetilde(n),
     poisson(g, g.bcx(), g.bcy(), g.bcx(), g.bcy()), //first  N,P then phi BC
+    gradient(g, g.bcx(), g.bcy(), dg::centered), //first  N,P then phi BC
     lapperpM ( g,g.bcx(), g.bcy(),     dg::normed,         dg::centered),
     invert_pol(         omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma(    omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
@@ -264,8 +266,7 @@ container& Explicit<G, Matrix,container>::compute_psi(const container& ti,contai
             old_psi.update( phi[1]);
         }
     }
-    poisson.variationRHS(potential, omega); // (nabla_perp phi)^2
-    dg::blas1::pointwiseDot(1.0, binv, binv, omega, 0.0, uE2);           //  u_E^2
+    gradient.variation(binv,potential, uE2); // (nabla_perp phi)^2
     dg::blas1::axpby( 1., phi[1], -0.5, uE2,phi[1]);             //psi  Gamma phi - 0.5 u_E^2        
     return phi[1];    
 }
diff --git a/src/feltorSesol/feltor.cuh b/src/feltorSesol/feltor.cuh
index 5322021b9..61141e521 100644
--- a/src/feltorSesol/feltor.cuh
+++ b/src/feltorSesol/feltor.cuh
@@ -101,6 +101,7 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
+    dg::Gradient< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic< Geometry, Matrix, container > lapperpM; 
     std::vector<container> multi_chi;
@@ -131,6 +132,7 @@ Explicit<Grid, Matrix, container>::Explicit( const Grid& g, eule::Parameters p):
     w2d( dg::create::weights(g)), v2d( dg::create::inv_weights(g)), 
     phi( 2, chi), npe(phi), logn(phi),
     poisson(g, g.bcx(), g.bcy(), p.bc_x_phi, g.bcy()), //first N/U then phi BCC
+    gradient(g, p.bc_x_phi, g.bcy(), dg::centered), //first N/U then phi BCC
     lapperpM ( g,g.bcx(), g.bcy(),       dg::normed,         dg::centered),
     invert_pol(         omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma(   omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
@@ -210,8 +212,7 @@ container& Explicit<G, Matrix,container>::compute_psi( container& potential)
         if(  number[0] == invert_invgamma.get_max())
             throw dg::Fail( p.eps_gamma);
     }
-    poisson.variationRHS(potential, omega);
-    dg::blas1::pointwiseDot(1.0, binv, binv, omega, 0.0, omega);        // omega = u_E^2
+    gradient.variation(binv, potential, omega);        // omega = u_E^2
     dg::blas1::axpby( 1., phi[1], -0.5, omega,phi[1]);             //psi  Gamma phi - 0.5 u_E^2
     return phi[1];    
 }
diff --git a/src/feltorSesol/feltor_hpc.cu b/src/feltorSesol/feltor_hpc.cu
index 8a45c47b7..b035e2c21 100644
--- a/src/feltorSesol/feltor_hpc.cu
+++ b/src/feltorSesol/feltor_hpc.cu
@@ -169,7 +169,7 @@ int main( int argc, char* argv[])
     int dim_ids_probe[2];
     dim_ids_probe[0] = EtimeID;
     //dim_ids_probe[1] = 
-    file :: define_dimension(ncid, &dim_ids_probe[1],  grid_probe, "X_probe" );
+    dg::file::define_dimension(ncid, &dim_ids_probe[1],  grid_probe, "X_probe" );
     for(unsigned i = 0; i < varname_probes.size(); i++)
     {
         err = nc_def_var(ncid, varname_probes[i].data(), NC_DOUBLE, 2, dim_ids_probe, &ID_probes[i]);
diff --git a/src/feltorShw/feltor.cuh b/src/feltorShw/feltor.cuh
index 3cc7d2e89..3cc809472 100644
--- a/src/feltorShw/feltor.cuh
+++ b/src/feltorShw/feltor.cuh
@@ -106,6 +106,7 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
+    dg::Gradient< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic< Geometry, Matrix, container > lapperp; 
     std::vector<container> multi_chi;
@@ -139,6 +140,7 @@ Explicit<Grid, Matrix, container>::Explicit( const Grid& g, eule::Parameters p):
     profne(dg::evaluate(dg::ExpProfX(p.nprofileamp, p.bgprofamp,p.invkappa),g)),
     profNi(profne),
     poisson(g, g.bcx(), g.bcy(), p.bc_x_phi, g.bcy()), //first N then phi BCC
+    gradient(g, p.bc_x_phi, g.bcy(), dg::centered),
     lapperp ( g,g.bcx(), g.bcy(),       dg::normed,          dg::centered),
     invert_pol(         omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma(   omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
@@ -177,8 +179,7 @@ container& Explicit<Grid, Matrix, container>::compute_psi( container& potential)
             std::vector<unsigned> number = multigrid.direct_solve( multi_gammaPhi, phi[1], potential, p.eps_gamma);
             old_psi.update( phi[1]);
         }
-        poisson.variationRHS(potential, omega); 
-        dg::blas1::pointwiseDot(1.0, binv, binv, omega, 0.0, omega);        // omega = u_E^2
+        gradient.variation(binv,potential, omega);         // omega = u_E^2
         dg::blas1::axpby( 1., phi[1], -0.5, omega, phi[1]);   //psi =  Gamma phi - 0.5 u_E^2
     }
     if (p.modelmode==2)
@@ -497,7 +498,7 @@ void Explicit<Grid, Matrix, container>::operator()(double ttt, const std::vector
             S[i]    = 0.5*z[i]*p.tau[i]*dg::blas2::dot( y[i], w2d, y[i]); // N_tilde^2
         }
         mass_ = dg::blas2::dot( one, w2d, y[0] ); //take real ion density which is electron density!!
-        poisson.variationRHS(phi[0], omega);
+        gradient.variation( phi[0], omega);
         double Tperp = 0.5*p.mu[1]*dg::blas2::dot( one, w2d, omega);   //= 0.5 mu_i u_E^2
         energy_ = S[0] + S[1]  + Tperp; 
         evec[0] = S[0], evec[1] = S[1], evec[2] = Tperp;
diff --git a/src/feltorShw/feltor_hpc.cu b/src/feltorShw/feltor_hpc.cu
index 1a19388bb..a7f4304bf 100644
--- a/src/feltorShw/feltor_hpc.cu
+++ b/src/feltorShw/feltor_hpc.cu
@@ -193,7 +193,7 @@ int main( int argc, char* argv[])
     int dim_ids_probe[2];
     dim_ids_probe[0] = EtimeID;
     //dim_ids_probe[1] = 
-    file :: define_dimension(ncid,  &dim_ids_probe[1], grid_probe, "X_probe");
+    dg::file::define_dimension(ncid,  &dim_ids_probe[1], grid_probe, "X_probe");
     for(unsigned i = 0; i < varname_probes.size(); i++)
     {
         err = nc_def_var(ncid, varname_probes[i].data(), NC_DOUBLE, 2, dim_ids_probe, &ID_probes[i]);
diff --git a/src/hasegawa/mima.cuh b/src/hasegawa/mima.cuh
index 1149a882f..6c1527a7a 100644
--- a/src/hasegawa/mima.cuh
+++ b/src/hasegawa/mima.cuh
@@ -88,13 +88,13 @@ struct Mima
 };
 
 template< class M, class container>
-Mima< M, container>::Mima( const dg::CartesianGrid2d& grid, double kappa, double alpha, double eps, bool global ): 
+Mima< M, container>::Mima( const dg::CartesianGrid2d& grid, double kappa, double alpha, double eps, bool global ):
     kappa( kappa), global(global),
     phi( grid.size(), 0.), dxphi( phi), dyphi( phi), omega(phi), lambda(phi), chi(phi),
     nGinv(dg::evaluate(dg::ExpProfX(1.0, 0.0,kappa),grid)),
     dxxphi( phi), dxyphi(phi),
     laplaceM( grid, dg::normed, dg::centered),
-    arakawa( grid), 
+    arakawa( grid),
     w2d( dg::create::weights(grid)), v2d( dg::create::inv_weights(grid)),
     invert( phi, grid.size(), eps),
     helmholtz( grid, -1)
@@ -117,24 +117,24 @@ void Mima< M, container>::operator()( double t, const container& y, container& y
     //gradient terms
     dg::blas1::axpby( -1, dyphi, 1., yp);
 
-    
+
     //full-F terms (NOB) correction terms
     if( global )
     {
-        arakawa.variation(phi,omega); //(nabla phi)^2
-        dg::blas1::scal(omega,0.5);   //0.5*(nabla phi)^2
+        //0.5*(nabla phi)^2
+        dg::blas1::pointwiseDot( 0.5, dxphi, dxphi, 0.5, dyphi, dyphi, 0., omega);
         dg::blas2::gemv( arakawa.dy(), omega, dyphi); //d_y 0.5*(nabla phi)^2
         dg::blas1::axpby( +kappa, dyphi, 1., yp);     //kappa* d_y 0.5*(nabla phi)^2
-        
+
         arakawa( omega,phi, lambda);        // [0.5*(nabla phi)^2, phi]
         dg::blas1::axpby( -kappa, lambda, 1., yp);  // -kappa* [0.5*(nabla phi)^2, phi]
-        
+
         arakawa( omega, dxphi, lambda);         // [0.5*(nabla phi)^2, d_x phi]
         dg::blas1::axpby( +kappa*kappa, lambda, 1., yp); //kappa^2* [0.5*(nabla phi)^2, d_x phi]
-        
-        dg::blas1::pointwiseDot(y,dxphi,omega);   //omega = (phi - lap phi) d_x phi 
-        dg::blas1::pointwiseDivide(omega,nGinv,omega);   //omega = e^(kappa*x)*(phi - lap phi)*d_x phi 
-        dg::blas1::axpby( -kappa*alpha, omega, 1., yp);  // -kappa*alpha*e^(kappa*x)*(phi - lap phi)*d_x phi 
+
+        dg::blas1::pointwiseDot(y,dxphi,omega);   //omega = (phi - lap phi) d_x phi
+        dg::blas1::pointwiseDivide(omega,nGinv,omega);   //omega = e^(kappa*x)*(phi - lap phi)*d_x phi
+        dg::blas1::axpby( -kappa*alpha, omega, 1., yp);  // -kappa*alpha*e^(kappa*x)*(phi - lap phi)*d_x phi
     }
 }
 
diff --git a/src/impurities/toeflI.cuh b/src/impurities/toeflI.cuh
index eb18a7546..304809edb 100644
--- a/src/impurities/toeflI.cuh
+++ b/src/impurities/toeflI.cuh
@@ -120,6 +120,7 @@ private:
     //matrices and solvers
     Helmholtz< Geometry, Matrix, container > gamma1;
     ArakawaX< Geometry, Matrix, container> arakawa; 
+    Gradient< Geometry, Matrix, container> gradient; 
     dg::Elliptic< Geometry, Matrix, container > pol, laplaceM; 
     dg::Invert<container> invert_pol, invert_invgamma;
 
@@ -132,19 +133,20 @@ private:
 
 template< class Geometry, class Matrix, class container>
 ToeflI< Geometry, Matrix, container>::ToeflI( const Geometry& grid, imp::Parameters p) :
-    chi( evaluate( dg::zero, grid )), omega(chi),  
+    chi( evaluate( dg::zero, grid )), omega(chi),
     binv( evaluate( LinearX( p.kappa, 1.), grid)),
     phi( 3, chi), dyphi( phi), ype(phi),
     dyy( 3, chi), lny(dyy), lapy( dyy),
     gamma_n( 2, chi),
     gamma1(  grid, -0.5*p.tau[1]),
-    arakawa( grid), 
-    pol(     grid, not_normed, centered), 
+    arakawa( grid),
+    gradient( grid),
+    pol(     grid, not_normed, centered),
     laplaceM( grid, normed, centered),
     invert_pol(      omega, omega.size(), p.eps_pol),
     invert_invgamma( omega, omega.size(), p.eps_gamma),
     w2d( create::volume(grid)), v2d( create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)), p(p)
-    { 
+    {
     }
 
 
@@ -155,12 +157,10 @@ const container& ToeflI<G, M, container>::compute_psi( const container& potentia
     gamma1.alpha() = -0.5*p.tau[idx]*p.mu[idx];
     invert_invgamma( gamma1, phi[idx], potential);
 
-    arakawa.variation(potential, omega);
-    dg::blas1::pointwiseDot( binv, omega, omega);
-    dg::blas1::pointwiseDot( binv, omega, omega);
+    gradient.variation(binv,potential, omega); // u_E^2
 
     dg::blas1::axpby( 1., phi[idx], -0.5*p.mu[idx], omega, phi[idx]);   //psi  Gamma phi - 0.5 u_E^2
-    return phi[idx];    
+    return phi[idx];
 }
 
 
diff --git a/src/lamb_dipole/shu.cuh b/src/lamb_dipole/shu.cuh
index 9896d7378..33b4aad49 100644
--- a/src/lamb_dipole/shu.cuh
+++ b/src/lamb_dipole/shu.cuh
@@ -67,12 +67,6 @@ struct Shu
 
     void operator()(double t, const Container& y, Container& yp);
 
-    void variation( const Container& phi, Container& variation_phi){
-        dg::blas2::symv( m_centered_phi[0], phi,  m_temp[0]);
-        dg::blas2::symv( m_centered_phi[1], phi,  m_temp[1]);
-        dg::tensor::multiply2d( m_metric, m_temp[0], m_temp[1], variation_phi, m_temp[2]);
-        dg::blas1::pointwiseDot( 1., m_temp[0], variation_phi, 1., m_temp[1], m_temp[2], 0., variation_phi);
-    }
     void set_mms_source( double sigma, double velocity, double ly) {
         m_mms = shu::MMSSource( sigma, velocity, ly);
         m_add_mms = true;
diff --git a/src/reco2D/reconnection.cuh b/src/reco2D/reconnection.cuh
index f1610fc89..37b9c9c01 100644
--- a/src/reco2D/reconnection.cuh
+++ b/src/reco2D/reconnection.cuh
@@ -202,6 +202,7 @@ struct Asela
 
     //matrices and solvers
     dg::ArakawaX< Geometry, Matrix, container > arakawa; 
+    dg::Gradient< Geometry, Matrix, container > gradient; 
     dg::Elliptic<  Geometry, Matrix, container  > lapperp; //note the host vector    
     
     std::vector<container> multi_chi;
@@ -223,7 +224,8 @@ struct Asela
 template<class Grid, class IMatrix, class Matrix, class container>
 Asela<Grid, IMatrix, Matrix, container>::Asela( const Grid& g, Parameters p): 
     //////////the arakawa operators ////////////////////////////////////////
-    arakawa(g, g.bcx(), g.bcy()), 
+    arakawa(g, g.bcx(), g.bcy()),
+    gradient(g, g.bcx(), g.bcy(), dg::centered ),
     //////////the elliptic and Helmholtz operators//////////////////////////
     lapperp (     g, g.bcx(), g.bcy(),   dg::normed,        dg::centered),
     multigrid( g, 3),
@@ -346,7 +348,7 @@ container& Asela<Geometry, IMatrix, Matrix,container>::compute_psi( container& p
         if(  number[0] == invert_invgamma.get_max())
         throw dg::Fail( p.eps_gamma); 
     }
-    arakawa.variation(potential, omega); 
+    gradient.variation(potential, omega); 
     dg::blas1::axpby( 1., phi[1], -0.5, omega,phi[1]);        
     return phi[1];  
 }
@@ -452,7 +454,7 @@ void Asela<Geometry, IMatrix, Matrix, container>::operator()( double time,  cons
     }
     mass_ = dg::blas2::dot( one, w2d, y[0] ); //take real ion density which is electron density!!
     double Tperp = 0.5*p.mu[1]*dg::blas2::dot( npe[1], w2d, omega);   // Tperp = 0.5 mu_i N_i u_E^2
-    arakawa.variation( apar[0], omega); // |nabla_\perp Aparallel|^2 
+    gradient.variation( apar[0], omega); // |nabla_\perp Aparallel|^2 
     double Uapar = 0.5*p.beta*dg::blas2::dot( one, w2d, omega); // Uapar = 0.5 beta |nabla_\perp Aparallel|^2
     energy_ = S[0] + S[1]  + Tperp + Tpar[0] + Tpar[1]; 
     evec[0] = S[0], evec[1] = S[1], evec[2] = Tperp, evec[3] = Tpar[0], evec[4] = Tpar[1]; evec[5] = Uapar;
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index 41712472d..cff8e67a2 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -126,6 +126,7 @@ struct Explicit
     std::vector<dg::Elliptic<Geometry, Matrix, container> > multi_pol;
     std::vector<dg::Helmholtz<Geometry,  Matrix, container> > multi_gamma1;
     dg::ArakawaX< Geometry, Matrix, container> arakawa;
+    dg::Gradient< Geometry, Matrix, container> gradient;
 
     dg::MultigridCG2d<Geometry, Matrix, container> multigrid;
     dg::Extrapolation<container> old_phi, old_psi, old_gammaN;
@@ -151,6 +152,7 @@ Explicit< Geometry, M, container>::Explicit( const Geometry& grid, const Paramet
     pol(     grid, dg::not_normed, dg::centered, p.jfactor),
     laplaceM( grid, dg::normed, dg::centered),
     arakawa( grid),
+    gradient( grid, dg::centered),
     multigrid( grid, 3),
     old_phi( 2, chi), old_psi( 2, chi), old_gammaN( 2, chi),
     w2d( dg::create::volume(grid)), one( dg::evaluate(dg::one, grid)),
@@ -185,11 +187,10 @@ const container& Explicit<G, M, container>::compute_psi( double t, const contain
         }
     }
     //compute (nabla phi)^2
-    arakawa.variation(potential, omega);
+    gradient.variation(potential, omega);
     //compute psi
     if(equations == "global")
     {
-
         dg::blas1::pointwiseDot( -0.5, binv, binv, omega, 1., phi[1]);
     }
     else if ( equations == "drift_global")
-- 
GitLab


From dec2620376c39627b9a5b8492ba1446bbbece52a Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 11:53:35 +0100
Subject: [PATCH 492/540] Fix static functions

so that Feltor can compile in different compilation units
---
 inc/dg/multistep_tableau.h      | 4 ++--
 inc/dg/tableau.h                | 4 ++--
 inc/geometries/magnetic_field.h | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/inc/dg/multistep_tableau.h b/inc/dg/multistep_tableau.h
index d1fc67ba9..27afa6fec 100644
--- a/inc/dg/multistep_tableau.h
+++ b/inc/dg/multistep_tableau.h
@@ -458,14 +458,14 @@ static std::unordered_map<std::string, enum multistep_identifier> str2lmsid{
     {"BDF-5-5", BDF_5_5},
     {"BDF-6-6", BDF_6_6},
 };
-enum multistep_identifier str2lmstableau( std::string name)
+static inline enum multistep_identifier str2lmstableau( std::string name)
 {
     if( str2lmsid.find(name) == str2lmsid.end())
         throw dg::Error(dg::Message(_ping_)<<"Multistep coefficients for "<<name<<" not found!");
     else
         return str2lmsid[name];
 }
-std::string lmstableau2str( enum multistep_identifier id)
+static inline std::string lmstableau2str( enum multistep_identifier id)
 {
     for( auto name: str2lmsid)
     {
diff --git a/inc/dg/tableau.h b/inc/dg/tableau.h
index 6640c481c..2d08acd92 100644
--- a/inc/dg/tableau.h
+++ b/inc/dg/tableau.h
@@ -1167,14 +1167,14 @@ static std::unordered_map<std::string, enum tableau_identifier> str2id{
     {"SSPRK-5-3", SSPRK_5_3},
     {"SSPRK-5-4", SSPRK_5_4},
 };
-enum tableau_identifier str2tableau( std::string name)
+static inline enum tableau_identifier str2tableau( std::string name)
 {
     if( str2id.find(name) == str2id.end())
         throw dg::Error(dg::Message(_ping_)<<"Tableau "<<name<<" not found!");
     else
         return str2id[name];
 }
-std::string tableau2str( enum tableau_identifier id)
+static inline std::string tableau2str( enum tableau_identifier id)
 {
     for( auto name: str2id)
     {
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 1dea7eb9e..9e5aba919 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -191,14 +191,14 @@ struct TokamakMagneticField
 };
 
 ///@cond
-CylindricalFunctorsLvl1 periodify( const CylindricalFunctorsLvl1& in, double R0, double R1, double Z0, double Z1, bc bcx, bc bcy)
+static inline CylindricalFunctorsLvl1 periodify( const CylindricalFunctorsLvl1& in, double R0, double R1, double Z0, double Z1, bc bcx, bc bcy)
 {
     return CylindricalFunctorsLvl1(
             Periodify( in.f(),   R0, R1, Z0, Z1, bcx, bcy),
             Periodify( in.dfx(), R0, R1, Z0, Z1, inverse(bcx), bcy),
             Periodify( in.dfy(), R0, R1, Z0, Z1, bcx, inverse(bcy)));
 }
-CylindricalFunctorsLvl2 periodify( const CylindricalFunctorsLvl2& in, double R0, double R1, double Z0, double Z1, bc bcx, bc bcy)
+static inline CylindricalFunctorsLvl2 periodify( const CylindricalFunctorsLvl2& in, double R0, double R1, double Z0, double Z1, bc bcx, bc bcy)
 {
     return CylindricalFunctorsLvl2(
             Periodify( in.f(),   R0, R1, Z0, Z1, bcx, bcy),
@@ -224,7 +224,7 @@ CylindricalFunctorsLvl2 periodify( const CylindricalFunctorsLvl2& in, double R0,
  *
  * @return new periodified magnetic field
  */
-TokamakMagneticField periodify( const TokamakMagneticField& mag, double R0, double R1, double Z0, double Z1, dg::bc bcx, dg::bc bcy)
+static inline TokamakMagneticField periodify( const TokamakMagneticField& mag, double R0, double R1, double Z0, double Z1, dg::bc bcx, dg::bc bcy)
 {
     return TokamakMagneticField( mag.R0(),
             periodify( mag.get_psip(), R0, R1, Z0, Z1, bcx, bcy),
-- 
GitLab


From c2c604175b4dda96199c6091eba80489b4b03f70 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 12:00:48 +0100
Subject: [PATCH 493/540] Bump minimum and recommended compilers versions

in README
---
 README.adoc      | 4 ++--
 config/README.md | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/README.adoc b/README.adoc
index 8694fe4a2..011d71fd1 100644
--- a/README.adoc
+++ b/README.adoc
@@ -69,8 +69,8 @@ ____
 |=======================================================================
 |    | Minimum system requirements  | Recommended system requirements
 | *CPU*     | Any         |support for AVX and FMA instruction set
-| *Compiler*| gcc-4.9 or msvc-15 or icc-15.0 (C{plus}{plus}-14 standard)| OpenMP-4 support, avx, fma instruction set flags
-| *GPU*     | - | NVidia GPU with compute-capability > 6 and nvcc-8.0
+| *Compiler*| gcc-5.1 or msvc-15 or icc-17.0 (C{plus}{plus}-14 standard)| gcc-9.3, OpenMP-4 support, avx, fma instruction set flags
+| *GPU*     | - | NVidia GPU with compute-capability > 6 and nvcc-11.0
 | *MPI*     | - | mpi installation compatible with compiler (must be cuda-aware in case hybrid MPI+GPU is the target system)
 |=======================================================================
 ____
diff --git a/config/README.md b/config/README.md
index 952226d93..955dccb9f 100644
--- a/config/README.md
+++ b/config/README.md
@@ -20,7 +20,7 @@ Your machine specific config file (e.g. feltor/config/your-machine.mk) should ha
 |  OMPFLAG  | -fopenmp                                 | The compiler flag activating the OpenMP support |
 |   NVCC    | nvcc                                     | CUDA compiler                            |
 | NVCCFLAGS | -std=c++14  -Xcompiler "-Wall -mavx -mfma"                             | flags for nvcc  and underlying host compiler, (minimum instruction set is sse4.1, avx and fma are recommended)                         |
-| NVCCARCH  | -arch sm_35                              | specify the **gpu** compute capability  https://developer.nvidia.com/cuda-gpus (note: can be overwritten on the command line) |
+| NVCCARCH  | -arch sm_61                              | specify the **gpu** compute capability  https://developer.nvidia.com/cuda-gpus (note: can be overwritten on the command line) |
 |                                          |                                          |     |
 |  INCLUDE  | -I$(HOME)/include                        | cusp, thrust, json, vcl and the draw (if needed) libraries. The default expects to find (symbolic links to ) these libraries in your home folder |
 |   LIBS    | -lnetcdf -lhdf5 -ldhf5_hl                | netcdf library                           |
@@ -40,17 +40,17 @@ The main purpose of the file `feltor/config/devices/devices.mk` is to configure
 
 ### Examples
 
-The **device** variable should be, the **OPT** and the **NVCCARCH** variables can be specified on the command line: 
+The **device** variable should be, the **OPT** and the **NVCCARCH** variables can be specified on the command line:
 
 ```shell
 #Compile using nvcc for a Tesla K40:
-make blas_b device=gpu NVCCARCH='-arch sm_35'
+make blas_b device=gpu NVCCARCH='-arch sm_61'
 
 #Compile for OpenMP using -O2:
 make blas_b device=omp OPT=-O2
 
 #Hybrid MPI+OpenMP program for the Xeon Phi architecture:
-make blas_mpib device=mic 
+make blas_mpib device=mic
 
 #Hybrid MPI+GPU program for the Tesla P100 GPU, host code with -O2:
 make blas_mpib device=gpu NVCCARCH='-arch sm_60' OPT=-O2
-- 
GitLab


From e006b62c867241c9abb62ddc9d3c7637af700137 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 13:35:07 +0100
Subject: [PATCH 494/540] Remove Davide config file (does not exist any more)

---
 config/davide.mk | 10 ----------
 1 file changed, 10 deletions(-)
 delete mode 100644 config/davide.mk

diff --git a/config/davide.mk b/config/davide.mk
deleted file mode 100644
index 7c7746b48..000000000
--- a/config/davide.mk
+++ /dev/null
@@ -1,10 +0,0 @@
-ifeq ($(strip $(HPC_SYSTEM)),davide)
-CFLAGS=-Wall -std=c++14 -DWITHOUT_VCL -mcpu=power8 # -mavx -mfma #flags for CC
-OPT=-O3 # optimization flags for host code
-NVCC=nvcc #CUDA compiler
-NVCCARCH=-arch sm_60 -Xcudafe "--diag_suppress=code_is_unreachable --diag_suppress=initialization_not_reachable" #nvcc gpu compute capability
-NVCCFLAGS= -std=c++14 -Xcompiler "-mcpu=power8 -Wall"# -mavx -mfma" #flags for NVCC
-INCLUDE += -I$(NETCDF_INC) -I$(HDF5_INC)
-LIBS    +=-L$(HDF5_LIB) -lhdf5 -lhdf5_hl
-LIBS    +=-L$(NETCDF_LIB) -lnetcdf -lcurl
-endif
-- 
GitLab


From 5ce9faac4070b00256d815fd28faca6fe1ca9795 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 14:31:46 +0100
Subject: [PATCH 495/540] Change layout of dg::file documentation

---
 doc/header.html           | 2 +-
 inc/file/Doxyfile         | 2 +-
 inc/file/easy_output.h    | 8 ++++++++
 inc/file/json_utilities.h | 8 ++++++++
 inc/file/nc_utilities.h   | 5 +++++
 5 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/doc/header.html b/doc/header.html
index f6c96f72a..595053156 100644
--- a/doc/header.html
+++ b/doc/header.html
@@ -26,7 +26,7 @@ $extrastylesheet
       <!-- <li><a href="../../../index.html">/</a><li>-->
     <li><a href="../../dg/html/modules.html">dG</a></li>
     <li><a href="../../geometries/html/modules.html">geometries</a></li>
-    <li><a href="../../file/html/namespacedg_1_1file.html">file</a></li>
+    <li><a href="../../file/html/modules.html">file</a></li>
     <li><a href="../../exblas/html/namespaceexblas.html">exblas</a></li>
   </ul>
   <!--End My own title area-->
diff --git a/inc/file/Doxyfile b/inc/file/Doxyfile
index db44d5d2a..d1fb0f19e 100644
--- a/inc/file/Doxyfile
+++ b/inc/file/Doxyfile
@@ -44,7 +44,7 @@ PROJECT_NUMBER         =
 # for a project that appears at the top of each page and should give viewer a
 # quick idea about the purpose of the project. Keep the description short.
 
-PROJECT_BRIEF          = "Facilitate the handling of Json and NetCDF files in \"dg/file/file.h\", which consists of \"dg/file/nc_utilities.h\" (-lnetcdf -lhdf5 -lhdf5_hl) and \"dg/file/json_utilities.h\" (-ljsoncpp) "
+PROJECT_BRIEF          = "Utilities in \"dg/file/file.h\" (includes both Json and NetCDF utilities)"
 
 # With the PROJECT_LOGO tag one can specify a logo or an icon that is included
 # in the documentation. The maximum height of the logo should not exceed 55
diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
index b0d42ed87..d9cd82ba4 100644
--- a/inc/file/easy_output.h
+++ b/inc/file/easy_output.h
@@ -17,6 +17,13 @@ namespace dg
 {
 namespace file
 {
+/**
+ * @defgroup netcdf NetCDF utilities
+ * \#include "dg/file/nc_utilities.h" (link -lnetcdf -lhdf5 -lhdf5_hl)
+ *
+ * @addtogroup netcdf
+ * @{
+ */
 
 /**
  * @brief Class thrown by the NC_Error_Handle
@@ -400,5 +407,6 @@ void put_vara_double(int ncid, int varid, unsigned slice,
 }
 #endif //MPI_VERSION
 
+///@}
 }//namespace file
 }//namespace dg
diff --git a/inc/file/json_utilities.h b/inc/file/json_utilities.h
index d5b22c514..393016902 100644
--- a/inc/file/json_utilities.h
+++ b/inc/file/json_utilities.h
@@ -17,6 +17,13 @@ namespace dg
 {
 namespace file
 {
+/**
+ * @defgroup json JsonCPP utilities
+ * \#include "dg/file/json_utilities.h" (link -ljsoncpp)
+ *
+ * @addtogroup json
+ * @{
+ */
 
 ///@brief Switch between how to handle errors in a Json utitlity functions
 enum class error{
@@ -309,5 +316,6 @@ static inline void string2Json(std::string input, Json::Value& js, enum comments
     }
 }
 
+///@}
 }//namespace file
 }//namespace dg
diff --git a/inc/file/nc_utilities.h b/inc/file/nc_utilities.h
index f91b83855..8747ed9b0 100644
--- a/inc/file/nc_utilities.h
+++ b/inc/file/nc_utilities.h
@@ -25,6 +25,7 @@ namespace dg
 /**
 * @brief Namespace for netcdf output related classes and functions following the
  <a href="http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/cf-conventions.html">CF-conventions</a>
+ @sa @ref json and @ref netcdf
 */
 namespace file
 {
@@ -52,6 +53,9 @@ inline int put_var_T<double>( int ncid, int varID, double* data){
 }
 ///@endcond
 
+///@addtogroup netcdf
+///@{
+
 ///@copydoc define_time
 template<class T>
 inline int define_real_time( int ncid, const char* name, int* dimID, int* tvarID)
@@ -310,5 +314,6 @@ inline int define_dimensions( int ncid, int* dimsIDs, int* tvarID, const dg::aRe
 }
 #endif //MPI_VERSION
 
+///@}
 } //namespace file
 } //namespace dg
-- 
GitLab


From 979840138c8b291c8d3d02ec797f3f9d95310bb7 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 15:43:59 +0100
Subject: [PATCH 496/540] Put exblas also under dg namespace

- if file has to go then exblas also has to go
- in principle exblas could be its own library but it is not very usable
outside the dg lib and we developped it specifically for ourselves
- furthermore repair the exblas documentation
---
 doc/header.html                        |  8 ++++----
 inc/dg/andersonacc_t.cu                |  2 +-
 inc/dg/arakawa_mpit.cu                 |  2 +-
 inc/dg/arakawa_t.cu                    |  2 +-
 inc/dg/backend/exblas/Doxyfile         |  7 ++++---
 inc/dg/backend/exblas/ExSUM.FPE.hpp    |  3 +++
 inc/dg/backend/exblas/accumulate.cuh   |  3 +++
 inc/dg/backend/exblas/accumulate.h     |  3 +++
 inc/dg/backend/exblas/config.h         | 17 ++++-------------
 inc/dg/backend/exblas/exblas.h         | 13 +++++++++++++
 inc/dg/backend/exblas/exblas_doc.h     |  6 ++++++
 inc/dg/backend/exblas/exdot_cuda.cuh   |  3 +++
 inc/dg/backend/exblas/exdot_omp.h      |  3 +++
 inc/dg/backend/exblas/exdot_serial.h   | 17 +++++++++++++++++
 inc/dg/backend/exblas/mpi_accumulate.h |  3 +++
 inc/dg/backend/exblas/mylibm.cuh       |  3 +++
 inc/dg/backend/exblas/mylibm.hpp       |  3 +++
 inc/dg/backend/sparseblockmat.h        |  6 +++---
 inc/dg/blas1_mpit.cu                   |  2 +-
 inc/dg/blas1_t.cu                      |  2 +-
 inc/dg/blas_t.cu                       |  2 +-
 inc/dg/cg2d_mpit.cu                    |  2 +-
 inc/dg/cg2d_t.cu                       |  2 +-
 inc/dg/cluster_mpib.cu                 |  2 +-
 inc/dg/elliptic2d_b.cu                 |  2 +-
 inc/dg/elliptic_b.cu                   |  2 +-
 inc/dg/elliptic_mpib.cu                |  2 +-
 inc/dg/exblas/exblas.h                 |  3 +++
 inc/dg/gradient_t.cu                   |  4 ++--
 inc/dg/helmholtz_t.cu                  |  2 +-
 inc/dg/helmholtzg2_t.cu                |  2 +-
 inc/dg/multistep_t.cu                  |  2 +-
 inc/dg/poisson_t.cu                    |  2 +-
 inc/dg/topology/average_mpit.cu        |  2 +-
 inc/dg/topology/average_t.cu           |  2 +-
 inc/dg/topology/derivatives_mpit.cu    |  2 +-
 inc/dg/topology/derivatives_t.cu       |  2 +-
 inc/dg/topology/evaluation_mpit.cu     |  2 +-
 inc/dg/topology/evaluation_t.cu        |  2 +-
 39 files changed, 102 insertions(+), 47 deletions(-)
 create mode 100644 inc/dg/backend/exblas/exblas.h
 create mode 100644 inc/dg/exblas/exblas.h

diff --git a/doc/header.html b/doc/header.html
index 595053156..b0a68524e 100644
--- a/doc/header.html
+++ b/doc/header.html
@@ -24,10 +24,10 @@ $extrastylesheet
     <!--My own title area-->
   <ul class="menubar">
       <!-- <li><a href="../../../index.html">/</a><li>-->
-    <li><a href="../../dg/html/modules.html">dG</a></li>
-    <li><a href="../../geometries/html/modules.html">geometries</a></li>
-    <li><a href="../../file/html/modules.html">file</a></li>
-    <li><a href="../../exblas/html/namespaceexblas.html">exblas</a></li>
+    <li><a href="../../dg/html/modules.html">dg</a></li>
+    <li><a href="../../geometries/html/modules.html">dg::geometries</a></li>
+    <li><a href="../../file/html/modules.html">dg::file</a></li>
+    <li><a href="../../exblas/html/namespacedg_1_1exblas.html">dg::exblas</a></li>
   </ul>
   <!--End My own title area-->
 <table cellspacing="0" cellpadding="0">
diff --git a/inc/dg/andersonacc_t.cu b/inc/dg/andersonacc_t.cu
index 238081e5c..5422c2d01 100644
--- a/inc/dg/andersonacc_t.cu
+++ b/inc/dg/andersonacc_t.cu
@@ -56,7 +56,7 @@ int main()
     dg::blas2::symv(  A, x, Ax);
     dg::blas1::axpby( 1.,Ax,-1.,resi);
 
-    exblas::udouble res;
+    dg::exblas::udouble res;
     res.d = sqrt(dg::blas2::dot( w2d, x));
     std::cout << "L2 Norm of x0 is              " << res.d<<"\t"<<res.i << std::endl;
     res.d = sqrt(dg::blas2::dot(w2d , solution));
diff --git a/inc/dg/arakawa_mpit.cu b/inc/dg/arakawa_mpit.cu
index dd307bd36..163d32070 100644
--- a/inc/dg/arakawa_mpit.cu
+++ b/inc/dg/arakawa_mpit.cu
@@ -78,7 +78,7 @@ int main(int argc, char* argv[])
     arakawa( lhs, rhs, jac);
 
     int64_t binary[] = {4358628400772939776,4360428067382886400,4362477496701026304,4562674804459845067,4552797036354693398};
-    exblas::udouble res;
+    dg::exblas::udouble res;
     dg::MHVec w2d = dg::create::weights( grid);
     //dg::MHVec eins = dg::evaluate( dg::one, grid);
     const dg::MHVec sol = dg::evaluate ( jacobian, grid);
diff --git a/inc/dg/arakawa_t.cu b/inc/dg/arakawa_t.cu
index be57c63a4..cd8d09b26 100644
--- a/inc/dg/arakawa_t.cu
+++ b/inc/dg/arakawa_t.cu
@@ -81,7 +81,7 @@ int main()
     //![doxygen]
 
     int64_t binary[] = {4358628400772939776,4360428067382886400,4362477496701026304,4562674804459845067,4552797036354693398};
-    exblas::udouble res;
+    dg::exblas::udouble res;
     dg::DVec w2d = dg::create::weights( grid);
     //dg::DVec eins = dg::evaluate( dg::one, grid);
     const dg::DVec sol = dg::evaluate ( jacobian, grid);
diff --git a/inc/dg/backend/exblas/Doxyfile b/inc/dg/backend/exblas/Doxyfile
index 6919fc84e..c89ef49fc 100644
--- a/inc/dg/backend/exblas/Doxyfile
+++ b/inc/dg/backend/exblas/Doxyfile
@@ -6,7 +6,7 @@
 DOXYFILE_ENCODING      = UTF-8
 PROJECT_NAME           = "Extension: ExBLAS"
 PROJECT_NUMBER         =
-PROJECT_BRIEF          = "Adaption of the exblas library (included in \"dg/algorithm.h\")"
+PROJECT_BRIEF          = "Adaption of the ExBLAS library (included in \"dg/algorithm.h\" by default or available as a standalone library as \"dg/exblas/exblas.h\")"
 PROJECT_LOGO           =
 OUTPUT_DIRECTORY       = ./doc
 CREATE_SUBDIRS         = NO
@@ -125,7 +125,8 @@ RECURSIVE              = NO
 EXCLUDE                = mylibm.hpp \
                          mylibm.cuh \
                          config.h \
-                         ExSUM.FPE.hpp
+                         ExSUM.FPE.hpp \
+                         exblas.h
 EXCLUDE_SYMLINKS       = NO
 EXCLUDE_PATTERNS       =
 EXCLUDE_SYMBOLS        =
@@ -207,7 +208,7 @@ EXT_LINKS_IN_WINDOW    = NO
 FORMULA_FONTSIZE       = 10
 FORMULA_TRANSPARENT    = YES
 FORMULA_MACROFILE      =
-USE_MATHJAX            = NO
+USE_MATHJAX            = YES
 MATHJAX_FORMAT         = HTML-CSS
 MATHJAX_RELPATH        = http://cdn.mathjax.org/mathjax/latest
 MATHJAX_EXTENSIONS     =
diff --git a/inc/dg/backend/exblas/ExSUM.FPE.hpp b/inc/dg/backend/exblas/ExSUM.FPE.hpp
index 7d831d261..fcdf64724 100644
--- a/inc/dg/backend/exblas/ExSUM.FPE.hpp
+++ b/inc/dg/backend/exblas/ExSUM.FPE.hpp
@@ -17,6 +17,8 @@
 #define EXSUM_FPE_HPP_
 #include "accumulate.h"
 
+namespace dg
+{
 namespace exblas
 {
 namespace cpu
@@ -269,4 +271,5 @@ void FPExpansionVect<T,N,TRAITS>::FlushVector(T x) const
 
 }//namespace cpu
 }//namespace exblas
+} //namespace dg
 #endif // EXSUM_FPE_HPP_
diff --git a/inc/dg/backend/exblas/accumulate.cuh b/inc/dg/backend/exblas/accumulate.cuh
index 75166926b..71387603b 100644
--- a/inc/dg/backend/exblas/accumulate.cuh
+++ b/inc/dg/backend/exblas/accumulate.cuh
@@ -20,6 +20,8 @@
 #include "mylibm.cuh"
 //this file has a direct correspondance to cpu code accumulate.h
 
+namespace dg
+{
 namespace exblas
 {
 namespace gpu
@@ -213,3 +215,4 @@ static inline double Round( int64_t * accumulator) {
 
 } //namespace gpu
 } //namespace exblas
+} //namespace dg
diff --git a/inc/dg/backend/exblas/accumulate.h b/inc/dg/backend/exblas/accumulate.h
index 3efa315a3..1f8964d1b 100644
--- a/inc/dg/backend/exblas/accumulate.h
+++ b/inc/dg/backend/exblas/accumulate.h
@@ -20,6 +20,8 @@
 #include "mylibm.hpp"
 //this file has a direct correspondance to gpu code accumulate.cuh
 
+namespace dg
+{
 namespace exblas {
 namespace cpu {
 ///@cond
@@ -247,3 +249,4 @@ static inline double Round( int64_t * accumulator) {
 
 }//namespace cpu
 } //namespace exblas
+} //namespace dg
diff --git a/inc/dg/backend/exblas/config.h b/inc/dg/backend/exblas/config.h
index b04f75984..4eb2cc204 100644
--- a/inc/dg/backend/exblas/config.h
+++ b/inc/dg/backend/exblas/config.h
@@ -65,6 +65,8 @@
 #define unlikely(x) (x)
 #endif
 
+namespace dg
+{
 namespace exblas
 {
 ////////////// parameters for superaccumulator operations //////////////////////
@@ -80,6 +82,7 @@ static constexpr double DELTASCALE = double(1ull << DIGITS); //!< Assumes KRX>0
 ///@brief Characterizes the result of summation
 enum Status
 {
+    //MW: not used anywhere but it would probably be useful to work it in?
     Exact, /*!< Reproducible and accurate */
     Inexact, /*!< non-accurate */
     MinusInfinity, /*!< minus infinity */
@@ -89,19 +92,6 @@ enum Status
     qNaN /*!< not-a-number */
 };
 
-/*! @brief Utility union to display all bits of a double (using "type-punning")
-@code
-double result; // = ...
-udouble res;
-res.d = result;
-std::cout << "Result as double "<<res.d<<"  as integer "<<res.i<<std::endl;
-@endcode
-*/
-union udouble{
-    double d; //!< a double
-    int64_t i; //!< a 64 bit integer
-};
-
 ///@cond
 template<class T>
 struct ValueTraits
@@ -118,3 +108,4 @@ using has_floating_value = typename std::conditional< std::is_floating_point<typ
 ///@endcond
 
 }//namespace exblas
+} //namespace dg
diff --git a/inc/dg/backend/exblas/exblas.h b/inc/dg/backend/exblas/exblas.h
new file mode 100644
index 000000000..d2f2ad357
--- /dev/null
+++ b/inc/dg/backend/exblas/exblas.h
@@ -0,0 +1,13 @@
+#pragma once
+
+#include "exdot_serial.h"
+#include "thrust/device_vector.h"
+#if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
+#include "exdot_cuda.cuh" // accumulate.cuh , config.h, mylibm.cuh
+#else
+#include "exdot_omp.h" //accumulate.h, mylibm.hpp
+#endif
+
+#ifdef MPI_VERSION
+#include "mpi_accumulate.h"
+#endif //MPI_VERSION
diff --git a/inc/dg/backend/exblas/exblas_doc.h b/inc/dg/backend/exblas/exblas_doc.h
index 0eca6ba1e..8f67470b5 100644
--- a/inc/dg/backend/exblas/exblas_doc.h
+++ b/inc/dg/backend/exblas/exblas_doc.h
@@ -5,9 +5,14 @@
  * The algorithm is described in the paper \n
  * Sylvain Collange, David Defour, Stef Graillat, Roman Iakymchuk. "Numerical Reproducibility for the Parallel Reduction on Multi- and Many-Core Architectures", 2015. https://hal.archives-ouvertes.fr/hal-00949355v3
  */
+namespace dg{
 /*!
  * @brief This is the namespace for all functions and
  *     classes defined and used in the exblas library
+ *
+ * In principle you can use this as a standalone library but it is much easier
+ * to just use the \c dg::blas1::dot and \c dg::blas2::dot functions for general
+ * purpose usage
  */
 namespace exblas{
 /*!
@@ -20,3 +25,4 @@ namespace  gpu{}
  */
 namespace cpu{}
 }
+}//namespace dg
diff --git a/inc/dg/backend/exblas/exdot_cuda.cuh b/inc/dg/backend/exblas/exdot_cuda.cuh
index 04fbe2493..13c34ef13 100644
--- a/inc/dg/backend/exblas/exdot_cuda.cuh
+++ b/inc/dg/backend/exblas/exdot_cuda.cuh
@@ -19,6 +19,8 @@
 #include "thrust/device_vector.h"
 #include "accumulate.cuh"
 
+namespace dg
+{
 namespace exblas{
 ///@cond
 namespace gpu{
@@ -396,3 +398,4 @@ void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, Po
 }
 
 }//namespace exblas
+} //namespace dg
diff --git a/inc/dg/backend/exblas/exdot_omp.h b/inc/dg/backend/exblas/exdot_omp.h
index 3e47e47e2..aefe0afe4 100644
--- a/inc/dg/backend/exblas/exdot_omp.h
+++ b/inc/dg/backend/exblas/exdot_omp.h
@@ -26,6 +26,8 @@
 #include "ExSUM.FPE.hpp"
 #include <omp.h>
 
+namespace dg
+{
 namespace exblas{
 ///@cond
 namespace cpu{
@@ -299,3 +301,4 @@ void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, Po
 }
 
 }//namespace exblas
+} //namespace dg
diff --git a/inc/dg/backend/exblas/exdot_serial.h b/inc/dg/backend/exblas/exdot_serial.h
index 5616ca4c2..468b54da9 100644
--- a/inc/dg/backend/exblas/exdot_serial.h
+++ b/inc/dg/backend/exblas/exdot_serial.h
@@ -25,7 +25,23 @@
 #include "accumulate.h"
 #include "ExSUM.FPE.hpp"
 
+namespace dg
+{
 namespace exblas{
+
+/*! @brief Utility union to display all bits of a double (using <a href="https://en.wikipedia.org/wiki/Type_punning">type-punning</a>)
+@code
+double result; // = ...
+udouble res;
+res.d = result;
+std::cout << "Result as double "<<res.d<<"  as integer "<<res.i<<std::endl;
+@endcode
+*/
+union udouble{
+    double d; //!< a double
+    int64_t i; //!< a 64 bit integer
+};
+
 ///@cond
 namespace cpu{
 
@@ -186,3 +202,4 @@ void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, Po
 
 
 }//namespace exblas
+} //namespace dg
diff --git a/inc/dg/backend/exblas/mpi_accumulate.h b/inc/dg/backend/exblas/mpi_accumulate.h
index 83635b17e..166863040 100644
--- a/inc/dg/backend/exblas/mpi_accumulate.h
+++ b/inc/dg/backend/exblas/mpi_accumulate.h
@@ -12,6 +12,8 @@
 #include <map>
 #include "accumulate.h"
 
+namespace dg
+{
 namespace exblas {
 
 ///@cond
@@ -99,3 +101,4 @@ static void reduce_mpi_cpu(  unsigned num_superacc, int64_t* in, int64_t* out, M
 }
 
 }//namespace exblas
+} //namespace dg
diff --git a/inc/dg/backend/exblas/mylibm.cuh b/inc/dg/backend/exblas/mylibm.cuh
index 2d9ffe58b..b784cfd5c 100644
--- a/inc/dg/backend/exblas/mylibm.cuh
+++ b/inc/dg/backend/exblas/mylibm.cuh
@@ -1,5 +1,7 @@
 #pragma once
 #include <cstdint>
+namespace dg
+{
 namespace exblas
 {
 namespace gpu
@@ -60,3 +62,4 @@ static inline double OddRoundSumNonnegative(double th, double tl) {
 }
 }//namespace gpu
 }//namespace exblas
+} //namespace dg
diff --git a/inc/dg/backend/exblas/mylibm.hpp b/inc/dg/backend/exblas/mylibm.hpp
index 906b87712..e445e198e 100644
--- a/inc/dg/backend/exblas/mylibm.hpp
+++ b/inc/dg/backend/exblas/mylibm.hpp
@@ -19,6 +19,8 @@
 
 #include "config.h"
 
+namespace dg
+{
 namespace exblas{
 namespace cpu{
 
@@ -216,5 +218,6 @@ inline static bool horizontal_or( const double & a){
 
 }//namespace cpu
 }//namespace exblas
+} // namespace dg
 
 #endif
diff --git a/inc/dg/backend/sparseblockmat.h b/inc/dg/backend/sparseblockmat.h
index 8e686b8d5..8427ce1df 100644
--- a/inc/dg/backend/sparseblockmat.h
+++ b/inc/dg/backend/sparseblockmat.h
@@ -2,7 +2,7 @@
 
 #include <cmath>
 #include <thrust/host_vector.h>
-#include "exblas/config.h"
+#include "exblas/exdot_serial.h"
 #include "config.h"
 #include "exceptions.h"
 #include "tensor_traits.h"
@@ -279,7 +279,7 @@ void EllSparseBlockMat<T>::display( std::ostream& os, bool show_data ) const
         for( unsigned i=0; i<data.size()/n/n; i++)
             for(unsigned k=0; k<n*n; k++)
             {
-                exblas::udouble res;
+                dg::exblas::udouble res;
                 res.d = data[i*n*n+k];
                 os << "idx "<<i<<" "<<res.d <<"\t"<<res.i<<"\n";
             }
@@ -306,7 +306,7 @@ void CooSparseBlockMat<value_type>::display( std::ostream& os, bool show_data) c
         for( unsigned i=0; i<data.size()/n/n; i++)
             for(unsigned k=0; k<n*n; k++)
             {
-                exblas::udouble res;
+                dg::exblas::udouble res;
                 res.d = data[i*n*n+k];
                 os << "idx "<<i<<" "<<res.d <<"\t"<<res.i<<"\n";
             }
diff --git a/inc/dg/blas1_mpit.cu b/inc/dg/blas1_mpit.cu
index 16408881b..5c99d04de 100644
--- a/inc/dg/blas1_mpit.cu
+++ b/inc/dg/blas1_mpit.cu
@@ -37,7 +37,7 @@ int main( int argc, char* argv[])
     MVec v3 = dg::evaluate( five5, g);
     MVec v4 = dg::evaluate( four4, g), v5(v4);
     if(rank==0)std::cout << "A TEST IS PASSED IF THE RESULT IS ZERO.\n";
-    exblas::udouble ud;
+    dg::exblas::udouble ud;
     dg::blas1::scal( v3, 3e-10); ud.d = v3.data()[0];
     if(rank==0)std::cout << "scal (x=ax)           "<<ud.i-4474825110624711575<<std::endl;
     dg::blas1::plus( v3, 3e-10); ud.d = v3.data()[0];
diff --git a/inc/dg/blas1_t.cu b/inc/dg/blas1_t.cu
index c91c41ec6..80cb6b4a0 100644
--- a/inc/dg/blas1_t.cu
+++ b/inc/dg/blas1_t.cu
@@ -22,7 +22,7 @@ int main()
     //Vector v1( {2,2.0002}), v2({3,3.00003}), v3({5,5.0005}), v4({4,4.00004}), v5(v4); //std::array
     thrust::device_vector<double> v1p( 500, 2.0002), v2p( 500, 3.00003), v3p(500,5.0005), v4p(500,4.00004);
     Vector v1(v1p), v2(v2p), v3(v3p), v4(v4p), v5(v4p);
-    exblas::udouble ud;
+    dg::exblas::udouble ud;
     dg::blas1::scal( v3, 3e-10); ud.d = v3[0];
     std::cout << "scal (x=ax)           "<<ud.i-4474825110624711575<<std::endl;
     dg::blas1::plus( v3, 3e-10); ud.d = v3[0];
diff --git a/inc/dg/blas_t.cu b/inc/dg/blas_t.cu
index 00730455b..923acc78d 100644
--- a/inc/dg/blas_t.cu
+++ b/inc/dg/blas_t.cu
@@ -40,7 +40,7 @@ int main()
     dg::blas1::axpby( 2., vec1 , 3, arrdvec1);
     std::cout << "Recursive Scalar/Vetor addition   "<< (arrdvec1[0][0] == 26 && arrdvec1[1][0]==46.)<<std::endl;
     // test the examples in the documentation
-    dg::blas1::subroutine( []__host__ __device__(double& v){ v+=1.;}, dvec1);
+    // dg::blas1::subroutine( []__host__ __device__(double& v){ v+=1.;}, dvec1);
     std::array<dg::DVec, 3> array_v{ dvec1, dvec1, dvec1}, array_w(array_v);
     std::array<double, 3> array_p{ 1,2,3};
     dg::blas1::subroutine( Expression(), dvec1, array_w[2], 3);
diff --git a/inc/dg/cg2d_mpit.cu b/inc/dg/cg2d_mpit.cu
index f8511b59b..b9437dbce 100644
--- a/inc/dg/cg2d_mpit.cu
+++ b/inc/dg/cg2d_mpit.cu
@@ -55,7 +55,7 @@ int main( int argc, char* argv[])
     dg::blas2::symv(  A, x, Ax);
     dg::blas1::axpby( 1.,Ax,-1.,resi);
 
-    exblas::udouble res;
+    dg::exblas::udouble res;
     res.d = sqrt(dg::blas2::dot( w2d, x));
     if(rank==0)std::cout << "L2 Norm of x0 is              " << res.d<<"\t"<<res.i << std::endl;
     res.d = sqrt(dg::blas2::dot(w2d , solution));
diff --git a/inc/dg/cg2d_t.cu b/inc/dg/cg2d_t.cu
index a8c29ad5a..d0adb4849 100644
--- a/inc/dg/cg2d_t.cu
+++ b/inc/dg/cg2d_t.cu
@@ -126,7 +126,7 @@ int main()
     dg::blas2::symv(  A, x, Ax);
     dg::blas1::axpby( 1.,Ax,-1.,resi);
 
-    exblas::udouble res;
+    dg::exblas::udouble res;
     res.d = sqrt(dg::blas2::dot( w2d, x));
     std::cout << "L2 Norm of x0 is              " << res.d<<"\t"<<res.i << std::endl;
     res.d = sqrt(dg::blas2::dot(w2d , solution));
diff --git a/inc/dg/cluster_mpib.cu b/inc/dg/cluster_mpib.cu
index 6f4765f57..ed7e0fc78 100644
--- a/inc/dg/cluster_mpib.cu
+++ b/inc/dg/cluster_mpib.cu
@@ -174,7 +174,7 @@ int main(int argc, char* argv[])
     t.toc();
     if(rank==0)std::cout<<" "<<t.diff()/(double)multi<<std::flush;
     //The Elliptic scheme
-    exblas::udouble res;
+    dg::exblas::udouble res;
     if( !(Nz > 2))
     {
         const Vector ellw3d = dg::create::volume(gridEll);
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index 3d9aef5c8..f8a0c24a7 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -66,7 +66,7 @@ int main()
     const dg::DVec derivati = dg::evaluate( der, grid);
     const double norm = dg::blas2::dot( w2d, solution);
     dg::DVec error( solution);
-    exblas::udouble res;
+    dg::exblas::udouble res;
 
     //std::cout << "Create Polarisation object and set chi!\n";
     {
diff --git a/inc/dg/elliptic_b.cu b/inc/dg/elliptic_b.cu
index 9cf9389eb..fdb302614 100644
--- a/inc/dg/elliptic_b.cu
+++ b/inc/dg/elliptic_b.cu
@@ -72,7 +72,7 @@ int main()
 
     double normerr = dg::blas2::dot( w3d, error);
     double norm = dg::blas2::dot( w3d, solution);
-    exblas::udouble res;
+    dg::exblas::udouble res;
     norm = sqrt(normerr/norm); res.d = norm;
     std::cout << "L2 Norm of relative error is:               " <<res.d<<"\t"<<res.i<<std::endl;
     const dg::DVec deriv = dg::evaluate( derivative, grid);
diff --git a/inc/dg/elliptic_mpib.cu b/inc/dg/elliptic_mpib.cu
index 1b10ce0c0..967886b43 100644
--- a/inc/dg/elliptic_mpib.cu
+++ b/inc/dg/elliptic_mpib.cu
@@ -81,7 +81,7 @@ int main( int argc, char* argv[])
 
     value_type normerr = dg::blas2::dot( w3d, error);
     value_type norm = dg::blas2::dot( w3d, solution);
-    exblas::udouble res;
+    dg::exblas::udouble res;
     norm = sqrt(normerr/norm); res.d = norm;
     if(rank==0)std::cout << "L2 Norm of relative error is:               " <<res.d<<"\t"<<res.i<<std::endl;
     dg::blas2::gemv( DX, x, error);
diff --git a/inc/dg/exblas/exblas.h b/inc/dg/exblas/exblas.h
new file mode 100644
index 000000000..12e1f6d00
--- /dev/null
+++ b/inc/dg/exblas/exblas.h
@@ -0,0 +1,3 @@
+#pragma once
+#define _EXBLAS_INCLUDED_BY_DG_
+#include "../backend/exblas/exblas.h"
diff --git a/inc/dg/gradient_t.cu b/inc/dg/gradient_t.cu
index 2ee3e1037..9d2c37b93 100644
--- a/inc/dg/gradient_t.cu
+++ b/inc/dg/gradient_t.cu
@@ -65,7 +65,7 @@ int main()
     gradient.variation( ph, va);
 
     int64_t binary[] = {4500635718861276907,4487444521638156650,4499885861996435701};
-    exblas::udouble res;
+    dg::exblas::udouble res;
     dg::DVec w2d = dg::create::weights( grid);
 
     dg::blas1::axpby( 1., phx, -1., dxph);
@@ -106,7 +106,7 @@ int main()
     gradient3d.variation( ph3d, va3d);
 
     int64_t binary3d[] = {4504451755369532568,4491224193368827475,4549042274897523598,4550331496568322612};
-    exblas::udouble res3d;
+    dg::exblas::udouble res3d;
     dg::DVec w3d = dg::create::weights( grid3d);
 
     dg::blas1::axpby( 1., phx3d, -1., dxph3d);
diff --git a/inc/dg/helmholtz_t.cu b/inc/dg/helmholtz_t.cu
index bb6218018..dc5f368fc 100644
--- a/inc/dg/helmholtz_t.cu
+++ b/inc/dg/helmholtz_t.cu
@@ -65,7 +65,7 @@ int main()
 
     std::cout << "number of iterations:  "<<number<<std::endl;
     std::cout << "ALL METHODS SHOULD DO THE SAME!\n";
-    exblas::udouble res;
+    dg::exblas::udouble res;
     res.d = sqrt( dg::blas2::dot( w2d, x));
     std::cout << "error1 " << res.d<<"\t"<<res.i<<std::endl;
     res.d = sqrt( dg::blas2::dot( w2d, x_));
diff --git a/inc/dg/helmholtzg2_t.cu b/inc/dg/helmholtzg2_t.cu
index 2aeeb028c..ef2412b9a 100644
--- a/inc/dg/helmholtzg2_t.cu
+++ b/inc/dg/helmholtzg2_t.cu
@@ -61,7 +61,7 @@ int main()
             throw dg::Fail( eps);
 
     dg::blas1::axpby( 1., sol, -1., x_);
-    exblas::udouble res;
+    dg::exblas::udouble res;
     res.d = sqrt( dg::blas2::dot( w2d, x_));
 
     std::cout << "number of iterations:  "<<number<<std::endl;
diff --git a/inc/dg/multistep_t.cu b/inc/dg/multistep_t.cu
index 91d61b813..3e4c50fc6 100644
--- a/inc/dg/multistep_t.cu
+++ b/inc/dg/multistep_t.cu
@@ -102,7 +102,7 @@ int main()
     const std::array<double,2> sol = solution(T,nu);
     const double norm_sol = dg::blas1::dot( sol, sol);
     double time = 0.;
-    exblas::udouble res;
+    dg::exblas::udouble res;
     std::cout << "### Test Explicit Multistep methods with "<<NT<<" steps\n";
     std::vector<std::string> ex_names{
     "AB-1-1", "AB-2-2", "AB-3-3", "AB-4-4", "AB-5-5",
diff --git a/inc/dg/poisson_t.cu b/inc/dg/poisson_t.cu
index 7440c9b8e..1236bc5e6 100644
--- a/inc/dg/poisson_t.cu
+++ b/inc/dg/poisson_t.cu
@@ -57,7 +57,7 @@ int main()
     const dg::DVec w2d = dg::create::weights( grid);
     const dg::DVec eins = dg::evaluate( dg::one, grid);
     const dg::DVec sol = dg::evaluate ( jacobian, grid);
-    exblas::udouble res;
+    dg::exblas::udouble res;
     std::cout << std::scientific;
     res.d = dg::blas2::dot( eins, w2d, jac);
     std::cout << "Mean     Jacobian is "<<res.d<<"\t"<<res.i<<"\n";
diff --git a/inc/dg/topology/average_mpit.cu b/inc/dg/topology/average_mpit.cu
index add08c1bd..6f6a6486b 100644
--- a/inc/dg/topology/average_mpit.cu
+++ b/inc/dg/topology/average_mpit.cu
@@ -39,7 +39,7 @@ int main(int argc, char* argv[])
     dg::MDVec solution = dg::evaluate( z_average, gxy);
     dg::blas1::axpby( 1., solution, -1., average_z);
     int64_t binary[] = {4406193765905047925,4395311848786989976};
-    exblas::udouble res;
+    dg::exblas::udouble res;
     res.d = sqrt( dg::blas2::dot( average_z, w2d, average_z));
     if(rank==0)std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i<<std::endl;
     if(rank==0)std::cout << "(Converges with 2nd order).\n";
diff --git a/inc/dg/topology/average_t.cu b/inc/dg/topology/average_t.cu
index f2eab250b..212415ea7 100644
--- a/inc/dg/topology/average_t.cu
+++ b/inc/dg/topology/average_t.cu
@@ -29,7 +29,7 @@ int main()
     dg::DVec solution = dg::evaluate( pol_average, gx);
     dg::blas1::axpby( 1., solution, -1., average_y);
     int64_t binary[] = {4406193765905047925,4395311848786989976};
-    exblas::udouble res;
+    dg::exblas::udouble res;
     res.d = sqrt( dg::blas2::dot( average_y, w1d, average_y));
     std::cout << "Distance to solution is: "<<res.d<<"\t"<<res.i<<std::endl;
     std::cout << "Averaging x ... \n";
diff --git a/inc/dg/topology/derivatives_mpit.cu b/inc/dg/topology/derivatives_mpit.cu
index 30c34454c..547668fc0 100644
--- a/inc/dg/topology/derivatives_mpit.cu
+++ b/inc/dg/topology/derivatives_mpit.cu
@@ -52,7 +52,7 @@ int main(int argc, char* argv[])
     Vector sol2[] = {dx2d, dy2d, null2, null2};
     int64_t binary2[] = {4562611930300281864,4553674328256556132,4567083257206218817,4574111364446550002};
 
-    exblas::udouble res;
+    dg::exblas::udouble res;
     if(rank==0)std::cout << "TEST 2D: DX, DY, JX, JY\n";
     for( unsigned i=0; i<4; i++)
     {
diff --git a/inc/dg/topology/derivatives_t.cu b/inc/dg/topology/derivatives_t.cu
index e02652d93..7c7ddfd3e 100644
--- a/inc/dg/topology/derivatives_t.cu
+++ b/inc/dg/topology/derivatives_t.cu
@@ -40,7 +40,7 @@ int main()
     Vector sol2[] = {dx2d, dy2d, null2, null2};
     int64_t binary2[] = {4562611930300281864,4553674328256556132,4567083257206218817,4574111364446550002};
 
-    exblas::udouble res;
+    dg::exblas::udouble res;
     std::cout << "TEST 2D: DX, DY, JX, JY\n";
     for( unsigned i=0; i<4; i++)
     {
diff --git a/inc/dg/topology/evaluation_mpit.cu b/inc/dg/topology/evaluation_mpit.cu
index 75d41da34..1572b24b6 100644
--- a/inc/dg/topology/evaluation_mpit.cu
+++ b/inc/dg/topology/evaluation_mpit.cu
@@ -51,7 +51,7 @@ int main(int argc, char** argv)
     const dg::MDVec w2d = dg::construct<dg::MDVec>(dg::create::weights(g2d));
     const dg::fMDVec wf2d = dg::construct<dg::fMDVec>(dg::create::weights(gf2d));
     const dg::MDVec w3d = dg::construct<dg::MDVec>(dg::create::weights(g3d));
-    exblas::udouble res;
+    dg::exblas::udouble res;
 
     double integral2d = dg::blas1::dot( w2d, func2d); res.d = integral2d;
     if(rank==0)std::cout << "2D integral               "<<std::setw(6)<<integral2d <<"\t" << res.i + 4823280491526356992<< "\n";
diff --git a/inc/dg/topology/evaluation_t.cu b/inc/dg/topology/evaluation_t.cu
index 06d5cf685..d60d473f8 100644
--- a/inc/dg/topology/evaluation_t.cu
+++ b/inc/dg/topology/evaluation_t.cu
@@ -61,7 +61,7 @@ int main()
     const dg::DVec w2d = dg::construct<dg::DVec>( dg::create::weights( g2d));
     const dg::fDVec wf2d = dg::construct<dg::fDVec>( dg::create::weights( gf2d));
     const dg::DVec w3d = dg::construct<dg::DVec>( dg::create::weights( g3d));
-    exblas::udouble res;
+    dg::exblas::udouble res;
 
     double integral = dg::blas1::dot( w1d, func1d); res.d = integral;
     std::cout << "1D integral               "<<std::setw(6)<<integral <<"\t" << res.i - 4616944842743393935  << "\n";
-- 
GitLab


From efcc03b447070a22a3036a14c4d6634d8432547c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 15:45:55 +0100
Subject: [PATCH 497/540] Fix bug in filter_mpit

---
 inc/dg/topology/filter_mpit.cu | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/dg/topology/filter_mpit.cu b/inc/dg/topology/filter_mpit.cu
index c79e4f3ab..294a3e84b 100644
--- a/inc/dg/topology/filter_mpit.cu
+++ b/inc/dg/topology/filter_mpit.cu
@@ -40,7 +40,7 @@ int main(int argc, char* argv[])
     const dg::MDVec vec = dg::evaluate( function, g3);
     const dg::MDVec weights = dg::create::weights( g3);
     dg::MDVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
-    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( 36, 0.5, 8, g3);
+    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( dg::ExponentialFilter(36, 0.5, 8, g3.n()), g3);
     dg::MIDMatrix project = dg::create::projection( g2,g3);
     dg::MIDMatrix interpo = dg::create::interpolation( g3,g2);
 
@@ -71,7 +71,7 @@ int main(int argc, char* argv[])
     const dg::MDVec vec = dg::evaluate( function, g3);
     const dg::MDVec weights = dg::create::weights( g3);
     dg::MDVec filtered_vec(vec), projected_vec(dg::evaluate( dg::zero, g2)), inter_vec( vec);
-    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( 36, 0.5, 8, g3);
+    dg::ModalFilter<dg::MDMatrix, dg::MDVec> filter( dg::ExponentialFilter(36, 0.5, 8, g3.n()), g3);
     dg::MIDMatrix project = dg::create::projection( g2,g3);
     dg::MIDMatrix interpo = dg::create::interpolation( g3,g2);
 
-- 
GitLab


From 5738fbd6b3dafe1de956ba77cc73285da01befcf Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 18:19:26 +0100
Subject: [PATCH 498/540] Improve exblas documentation

- and start updating CHANGELOG
---
 CHANGELOG.md                         | 32 ++++++++++++-
 inc/dg/backend/exblas/Doxyfile       |  2 +-
 inc/dg/backend/exblas/exblas_doc.h   |  2 +-
 inc/dg/backend/exblas/exdot_cuda.cuh | 33 +++----------
 inc/dg/backend/exblas/exdot_omp.h    | 33 +++----------
 inc/dg/backend/exblas/exdot_serial.h | 71 ++++++++++++++++++++--------
 inc/dg/blas1.h                       |  2 +-
 inc/dg/blas2.h                       |  2 +-
 8 files changed, 98 insertions(+), 79 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index ec9170a6f..d5e892fb8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,10 +3,38 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 We do not (yet) follow semantic versioning.
-Only code changes are reported here, we do not track changes in the
+Only changes in code are reported here, we do not track changes in the
 doxygen documentation, READMEs or tex writeups.
+## [v5.2]
+### Added
+ - M100 config file
+ - json utility functions get, get_idx in json_utilities.h which adds a small abstraction layer that gives a user more control over what happens if a variable is not found
+ - json utility functions file2Json, and string2Json in json_utilities.h which adds a small abstraction layer that gives a user more control over what happens if an error happens during the parsing of a file
+ - "easy output" netcdf utility functions that are particularly useful for MPI output: either write data in parallel or funnel through the master thread
+ - new include files dg/file/json_utilities.h and dg/exblas/exblas.h
+ - new class dg::Gradient for gradient and variation
+ - new tensor functions tensor::
+ - new class dg::Advection for the upwind scheme
+ - dg::blas1::dot and dg::blas2::dot and corresponding exblas functions now detect NaN and Inf errors
+### Changed
+ - namespace file changed to **dg::file** and exblas changed to **dg::exblas** (for consistency reasons, everything should go into the dg namespace, which in particular reduces the chance for name-clashes to just one, namely 'dg')
+ - changed file paths **dg/file/file.h**, **dg/geometries/geometries.h** , **dg/file/nc_utilities.h**
+ - std=c++14 We use the C++-14 standard now (previously 11)
+ - vectorclass dependency changed to vectorclass/version1 (previously we used a custom upload on feltor-dev repository)
+ - default cuda compute capability bumped to sm-61 (previously sm-35)
+ - Marconi config now uses jsoncpp module (previously manually installed)
+ - Moved variation memeber function into new class Gradient (previously in ArakawaX and Poisson)
+ - blas1::dot and blas2::dot now both do not accumulate rest of multiplication
+### Deprecated
+### Removed
+ - removed diag/feltordiag.cu
+### Fixed
+ - Fix bug: race condition in dg::blas1::dot and dg::blas2::dot on GPUs that led to hard to reproduce and seemingly unreasonable crashes
+ - Fix bugs: std namespace in diag/probes.h
+ - Fix bug: const in exblas::cpu::get_element 
+ - Fix bug: correct  indices in exblas::cpu::make_vcl_vec8d
 
-## [Unreleased]
+## [v5.1]
 ### Added
 - dg::Elliptic3d: a three-dimensional version of dg::Elliptic
 - Add 4 parameter symv member to dg::Elliptic class
diff --git a/inc/dg/backend/exblas/Doxyfile b/inc/dg/backend/exblas/Doxyfile
index c89ef49fc..85db7e184 100644
--- a/inc/dg/backend/exblas/Doxyfile
+++ b/inc/dg/backend/exblas/Doxyfile
@@ -129,7 +129,7 @@ EXCLUDE                = mylibm.hpp \
                          exblas.h
 EXCLUDE_SYMLINKS       = NO
 EXCLUDE_PATTERNS       =
-EXCLUDE_SYMBOLS        =
+EXCLUDE_SYMBOLS        = hide_*
 EXAMPLE_PATH           =
 EXAMPLE_PATTERNS       = *
 EXAMPLE_RECURSIVE      = NO
diff --git a/inc/dg/backend/exblas/exblas_doc.h b/inc/dg/backend/exblas/exblas_doc.h
index 8f67470b5..9b12a1abb 100644
--- a/inc/dg/backend/exblas/exblas_doc.h
+++ b/inc/dg/backend/exblas/exblas_doc.h
@@ -3,7 +3,7 @@
  *
  * This version of exblas is an adaption of the original exblas library for FELTOR.
  * The algorithm is described in the paper \n
- * Sylvain Collange, David Defour, Stef Graillat, Roman Iakymchuk. "Numerical Reproducibility for the Parallel Reduction on Multi- and Many-Core Architectures", 2015. https://hal.archives-ouvertes.fr/hal-00949355v3
+ <a href = "https://hal.archives-ouvertes.fr/hal-00949355v3">Sylvain Collange, David Defour, Stef Graillat, Roman Iakymchuk. "Numerical Reproducibility for the Parallel Reduction on Multi- and Many-Core Architectures", 2015. </a>
  */
 namespace dg{
 /*!
diff --git a/inc/dg/backend/exblas/exdot_cuda.cuh b/inc/dg/backend/exblas/exdot_cuda.cuh
index 13c34ef13..fddf30c7e 100644
--- a/inc/dg/backend/exblas/exdot_cuda.cuh
+++ b/inc/dg/backend/exblas/exdot_cuda.cuh
@@ -333,19 +333,9 @@ void ExDOTCompleteFinal(
 }//namespace gpu
 ///@endcond
 
-/*!@brief gpu version of exact dot product
- *
- * Computes the exact sum \f[ \sum_{i=0}^{N-1} x_i y_i \f]
- * @ingroup highlevel
- * @tparam NBFPE size of the floating point expansion (should be between 3 and 8)
- * @tparam PointerOrValue must be one of <tt> T, T&&, T&, const T&, T* or const T* </tt>, where \c T is either \c float or \c double. If it is a pointer type, then we iterate through the pointed data from 0 to \c size, else we consider the value constant in every iteration.
- * @param size size N of the arrays to sum
- * @param x1_ptr first array
- * @param x2_ptr second array
- * @param d_superacc pointer to an array of 64 bit integers (the superaccumulator) in device memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
- * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
- * @sa \c exblas::gpu::Round to convert the superaccumulator into a double precision number
-*/
+///@brief GPU version of exact dot product
+///@copydoc hide_exdot2
+///@copydoc hide_deviceacc
 template<class PointerOrValue1, class PointerOrValue2, size_t NBFPE=3>
 __host__
 void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, int64_t* d_superacc, int* status)
@@ -364,20 +354,9 @@ void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
     if( d_errorV[0] ) *status = 1;
 }
 
-/*!@brief gpu version of exact triple dot product
- *
- * Computes the exact sum \f[ \sum_{i=0}^{N-1} x_i w_i y_i \f]
- * @ingroup highlevel
- * @tparam NBFPE size of the floating point expansion (should be between 3 and 8)
- * @tparam PointerOrValue must be one of <tt> T, T&&, T&, const T&, T* or const T* </tt>, where \c T is either \c float or \c double. If it is a pointer type, then we iterate through the pointed data from 0 to \c size, else we consider the value constant in every iteration.
- * @param size size N of the arrays to sum
- * @param x1_ptr first array
- * @param x2_ptr second array
- * @param x3_ptr third array
- * @param d_superacc pointer to an array of 64 bit integegers (the superaccumulator) in device memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
- * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
- * @sa \c exblas::gpu::Round to convert the superaccumulator into a double precision number
- */
+///@brief GPU version of exact dot product
+///@copydoc hide_exdot3
+///@copydoc hide_deviceacc
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3, size_t NBFPE=3>
 __host__
 void exdot_gpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, PointerOrValue3 x3_ptr, int64_t* d_superacc, int* status)
diff --git a/inc/dg/backend/exblas/exdot_omp.h b/inc/dg/backend/exblas/exdot_omp.h
index aefe0afe4..71c1666df 100644
--- a/inc/dg/backend/exblas/exdot_omp.h
+++ b/inc/dg/backend/exblas/exdot_omp.h
@@ -245,19 +245,9 @@ void ExDOTFPE(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c, in
 }//namespace cpu
 ///@endcond
 
-/*!@brief OpenMP parallel version of exact dot product
- *
- * Computes the exact sum \f[ \sum_{i=0}^{N-1} x_i y_i \f]
- * @ingroup highlevel
- * @tparam NBFPE size of the floating point expansion (should be between 3 and 8)
- * @tparam PointerOrValue must be one of <tt> T, T&&, T&, const T&, T* or const T* </tt>, where \c T is either \c float or \c double. If it is a pointer type, then we iterate through the pointed data from 0 to \c size, else we consider the value constant in every iteration.
- * @param size size N of the arrays to sum
- * @param x1_ptr first array
- * @param x2_ptr second array
- * @param h_superacc pointer to an array of 64 bit integers (the superaccumulator) in host memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
- * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
- * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
-*/
+///@brief OpenMP parallel version of exact triple dot product
+///@copydoc hide_exdot2
+///@copydoc hide_hostacc
 template<class PointerOrValue1, class PointerOrValue2, size_t NBFPE=8>
 void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, int64_t* h_superacc, int* status){
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
@@ -271,20 +261,9 @@ void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
     *status = 0;
     if( error ) *status = 1;
 }
-/*!@brief OpenMP parallel version of exact triple dot product
- *
- * Computes the exact sum \f[ \sum_{i=0}^{N-1} x_i w_i y_i \f]
- * @ingroup highlevel
- * @tparam NBFPE size of the floating point expansion (should be between 3 and 8)
- * @tparam PointerOrValue must be one of <tt> T, T&&, T&, const T&, T* or const T* </tt>, where \c T is either \c float or \c double. If it is a pointer type, then we iterate through the pointed data from 0 to \c size, else we consider the value constant in every iteration.
- * @param size size N of the arrays to sum
- * @param x1_ptr first array
- * @param x2_ptr second array
- * @param x3_ptr third array
- * @param h_superacc pointer to an array of 64 bit integegers (the superaccumulator) in host memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
- * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
- * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
- */
+///@brief OpenMP parallel version of exact triple dot product
+///@copydoc hide_exdot3
+///@copydoc hide_hostacc
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3, size_t NBFPE=8>
 void exdot_omp(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, PointerOrValue3 x3_ptr, int64_t* h_superacc, int* status) {
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
diff --git a/inc/dg/backend/exblas/exdot_serial.h b/inc/dg/backend/exblas/exdot_serial.h
index 468b54da9..e636a9f42 100644
--- a/inc/dg/backend/exblas/exdot_serial.h
+++ b/inc/dg/backend/exblas/exdot_serial.h
@@ -139,19 +139,63 @@ void ExDOTFPE_cpu(int N, PointerOrValue1 a, PointerOrValue2 b, PointerOrValue3 c
 }//namespace cpu
 ///@endcond
 
-/*!@brief serial version of exact dot product
+/*!@class hide_exdot2
  *
- * Computes the exact sum \f[ \sum_{i=0}^{N-1} x_i y_i \f]
+ * Accumulate the exact sum \f[ \sum_{i=0}^{N-1} x_i y_i \f] into a superaccumulator.
+ * The superaccumulator is an array of \c exblas::BIN_COUNT (39) 64 bit integers that
+ * represents a large fixed point number such that the summation is computed with virtually infinite precision and is thus bitwise reproducible even in a parallel environment.
+ * @note The superaccumulator can be converted to
+ * a double precision number using the Round function:
+ * \ref dg::exblas::cpu::Round for cpu and omp version or \ref dg::exblas::gpu::Round for gpu version
+ * @attention the product \f$ x_iy_i\f$ of numbers is **not** computed with infinite precision only the sum is (this does not break the reproducibility)
+ * @note The algorithm is described in the paper
+ <a href = "https://hal.archives-ouvertes.fr/hal-00949355v3">Sylvain Collange, David Defour, Stef Graillat, Roman Iakymchuk. "Numerical Reproducibility for the Parallel Reduction on Multi- and Many-Core Architectures", 2015. </a>
  * @ingroup highlevel
  * @tparam NBFPE size of the floating point expansion (should be between 3 and 8)
  * @tparam PointerOrValue must be one of <tt> T, T&&, T&, const T&, T* or const T* </tt>, where \c T is either \c float or \c double. If it is a pointer type, then we iterate through the pointed data from 0 to \c size, else we consider the value constant in every iteration.
  * @param size size N of the arrays to sum
  * @param x1_ptr first array
  * @param x2_ptr second array
- * @param h_superacc pointer to an array of 64 bit integers (the superaccumulator) in host memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
+ */
+
+/*!@class hide_exdot3
+ *
+ * Accumulate the exact sum \f[ \sum_{i=0}^{N-1} x_i w_i y_i \f] into a superaccumulator.
+ * The superaccumulator is an array of \c exblas::BIN_COUNT (39) 64 bit integers that
+ * represents a large fixed point number such that the summation is computed with virtually infinite precision and is thus bitwise reproducible even in a parallel environment.
+ * @note The superaccumulator can be converted to
+ * a double precision number using the Round function:
+ * \ref dg::exblas::cpu::Round for cpu and omp version or \ref dg::exblas::gpu::Round for gpu version
+ * @attention the product \f$ x_iw_iy_i\f$ of numbers is **not** computed with infinite precision only the sum is (this does not break the reproducibility)
+ * @note The algorithm is described in the paper
+ <a href = "https://hal.archives-ouvertes.fr/hal-00949355v3">Sylvain Collange, David Defour, Stef Graillat, Roman Iakymchuk. "Numerical Reproducibility for the Parallel Reduction on Multi- and Many-Core Architectures", 2015. </a>
+ * @ingroup highlevel
+ * @tparam NBFPE size of the floating point expansion (should be between 3 and 8)
+ * @tparam PointerOrValue must be one of <tt> T, T&&, T&, const T&, T* or const T* </tt>, where \c T is either \c float or \c double. If it is a pointer type, then we iterate through the pointed data from 0 to \c size, else we consider the value constant in every iteration.
+ * @param size size N of the arrays to sum
+ * @param x1_ptr first array
+ * @param x2_ptr second array
+ * @param x3_ptr third array
+ */
+/*!@class hide_hostacc
+ * @param h_superacc pointer to an array of 64 bit integegers (the
+ * superaccumulator) in **host memory** with size at least \c exblas::BIN_COUNT
+ * (39) (contents are overwritten, the function does not allocate memory i.e.
+ * the memory needs to be allocated **before** calling the function)
  * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
- * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
-*/
+ */
+/*!@class hide_deviceacc
+ * @param d_superacc pointer to an array of 64 bit integegers (the
+ * superaccumulator) in **device memory** with size at least \c
+ * exblas::BIN_COUNT (39) (contents are overwritten, the function does not
+ * allocate memory i.e. the memory needs to be allocated **before** calling the
+ * function)
+ * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
+ */
+
+///@brief Serial version of exact dot product
+///@copydoc hide_exdot2
+///@copydoc hide_hostacc
 template<class PointerOrValue1, class PointerOrValue2, size_t NBFPE=8>
 void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, int64_t* h_superacc, int* status){
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
@@ -168,20 +212,9 @@ void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, in
     if( error ) *status = 1;
 }
 
-/*!@brief gpu version of exact triple dot product
- *
- * Computes the exact sum \f[ \sum_{i=0}^{N-1} x_i w_i y_i \f]
- * @ingroup highlevel
- * @tparam NBFPE size of the floating point expansion (should be between 3 and 8)
- * @tparam PointerOrValue must be one of <tt> T, T&&, T&, const T&, T* or const T* </tt>, where \c T is either \c float or \c double. If it is a pointer type, then we iterate through the pointed data from 0 to \c size, else we consider the value constant in every iteration.
- * @param size size N of the arrays to sum
- * @param x1_ptr first array
- * @param x2_ptr second array
- * @param x3_ptr third array
- * @param h_superacc pointer to an array of 64 bit integegers (the superaccumulator) in host memory with size at least \c exblas::BIN_COUNT (39) (contents are overwritten)
- * @param status 0 indicates success, 1 indicates an input value was NaN or Inf
- * @sa \c exblas::cpu::Round  to convert the superaccumulator into a double precision number
- */
+///@brief Serial version of exact dot product
+///@copydoc hide_exdot3
+///@copydoc hide_hostacc
 template<class PointerOrValue1, class PointerOrValue2, class PointerOrValue3, size_t NBFPE=8>
 void exdot_cpu(unsigned size, PointerOrValue1 x1_ptr, PointerOrValue2 x2_ptr, PointerOrValue3 x3_ptr, int64_t* h_superacc, int* status) {
     static_assert( has_floating_value<PointerOrValue1>::value, "PointerOrValue1 needs to be T or T* with T one of (const) float or (const) double");
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 3a1c36749..1190e3f42 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -62,7 +62,7 @@ double result = dg::blas1::dot( two, three); // result = 600 (100*(2*3))
  * @note Our implementation guarantees binary reproducible results.
  * The sum is computed with infinite precision and the result is rounded
  * to the nearest double precision number.
- * This is possible with the help of an adapted version of the \c ::exblas library and
+ * This is possible with the help of an adapted version of the \c dg::exblas library and
 * works for single and double precision.
 
  * @param x Left Container
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index 828368f18..acf81883d 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -68,7 +68,7 @@ inline std::vector<int64_t> doDot_superacc( const ContainerType1& x, const Matri
  * Furthermore, the sum is computed with infinite precision and the result is then rounded
  * to the nearest double precision number. Although the products are not computed with
  * infinite precision, the order of multiplication is guaranteed.
- * This is possible with the help of an adapted version of the \c ::exblas library and
+ * This is possible with the help of an adapted version of the \c dg::exblas library and
 * works for single and double precision.
  *
  * @param x Left input
-- 
GitLab


From 6ece03e7b02c307580bae931ba5374ff7d05dbb3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 12 Feb 2021 23:47:55 +0100
Subject: [PATCH 499/540] Improve documentation

in preparation of the new release
---
 CHANGELOG.md                    | 71 ++++++++++++++++++++++++++++-----
 README.adoc                     |  2 +-
 config/README.md                |  2 +-
 inc/dg/blas1.h                  |  4 +-
 inc/dg/blas2.h                  |  4 +-
 inc/dg/enums.h                  |  4 +-
 inc/dg/multigrid.h              |  8 ++--
 inc/dg/topology/average.h       |  2 +-
 inc/dg/topology/average_mpi.h   |  2 +-
 inc/dg/topology/filter.h        |  2 +
 inc/dg/topology/interpolation.h |  8 ++--
 inc/geometries/average.h        | 10 ++---
 inc/geometries/geometries_doc.h |  1 +
 13 files changed, 88 insertions(+), 32 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d5e892fb8..76974dd18 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,28 +11,81 @@ doxygen documentation, READMEs or tex writeups.
  - json utility functions get, get_idx in json_utilities.h which adds a small abstraction layer that gives a user more control over what happens if a variable is not found
  - json utility functions file2Json, and string2Json in json_utilities.h which adds a small abstraction layer that gives a user more control over what happens if an error happens during the parsing of a file
  - "easy output" netcdf utility functions that are particularly useful for MPI output: either write data in parallel or funnel through the master thread
- - new include files dg/file/json_utilities.h and dg/exblas/exblas.h
+ - new include files dg/file/file.h, dg/file/json_utilities.h and dg/exblas/exblas.h
  - new class dg::Gradient for gradient and variation
- - new tensor functions tensor::
- - new class dg::Advection for the upwind scheme
- - dg::blas1::dot and dg::blas2::dot and corresponding exblas functions now detect NaN and Inf errors
+ - new class dg::Advection for the upwind advection scheme
+ - new blas1::reduce function for custom reductions
+ - new "exchangeable" dg::x::DVec, dg::x::HVec, ..., dg::x::CartesianGrid2d, ..., dg::x::IHMatrix, ... typedefs. The idea is that these resolve to either shared memory or mpi distributed memory versions depending on the MPI_VERSION macro. This helps merging shared and mpi programs into single ones.
+ - added "simple" mode to Average computation, which is beneficial for GPU computing
+ - add dg::integrate that computes an indefinite integral of a function (essentially the opposite of the derivative)
+ - add dg::ModalFilter inclusive tests
+ - new compose function that concatenates two or more functors to one
+ - add dg::cooRZP2X coordinate transformation functions to easily transform between Cylindrical and Cartesian coordinates
+ - interpolate function has an additional dg::space parameter to indicate nodal or modal values
+ - Grid classes now have host_vector and host_grid member typedefs
+ - new tensor functions dg::tensor::scalar_product2d and dg::scalar_product3d that can compute uE2 in one go
+ - new extended tensor functions dg::tensor::multiply2d and dg::multiply3d that can compute uE2 in one go
+ - new single step timestepper ShuOsher including new ShuOsherTableau and ConversToShuOsherTableau classes to hold corresponding coefficients
+ - new ShuOsher tableaus SSPRK
+ - new Runge Kutta embedded tableaus tsitouras, the default timesteppers in Julia
+ - new implicit RK tableau trapezoidal, implicit midpoint and sdirk-2-1-2
+ - new class Simpsons that implements Simpsons rule for (time) integration
+ - new implicit timesteppers DIRKStep and ImplicitRungeKutta
+ - implicit time steppers give access to solver
+ - Redesign of multistep time steppers consistent with Runge-Kutta ones in terms of MultistepTableau and ConvertsToMultistepTableau
+ - a host of new explicit, implicit and semi-implicit multistep tableaus
+ - experimental "filtered" multistep time-steppers that allow modal filtering (first tests are not promising though)
+ - new experimental multigrid solvers involving Chebyshev iterations as smoother (but none are better than nested iterations so they remain experimental)
+ - new class EVE (courtesy of Eduard Reiter)
+ - new class ChebyshevIterations and ChebyshevPreconditioner (for chebyshev iterations)
+ - new solvers LGMRES, BICGSTABL, and AndersonAcceleration (courtesy of Aslak Poulsen)
+ - new FixedPointSolver and AndersonSolver for nonlinear problems in time
+ - new class Gradient that computes gradients and variations
+ - a host of new functors for the evaluate and pullback functions
+ - FluxSurfaceIntegral, FluxVolumeIntegral and SafetyFactorAverage classes
 ### Changed
  - namespace file changed to **dg::file** and exblas changed to **dg::exblas** (for consistency reasons, everything should go into the dg namespace, which in particular reduces the chance for name-clashes to just one, namely 'dg')
  - changed file paths **dg/file/file.h**, **dg/geometries/geometries.h** , **dg/file/nc_utilities.h**
+ - Moved variation member function into new class Gradient (previously in ArakawaX and Poisson)
  - std=c++14 We use the C++-14 standard now (previously 11)
  - vectorclass dependency changed to vectorclass/version1 (previously we used a custom upload on feltor-dev repository)
  - default cuda compute capability bumped to sm-61 (previously sm-35)
- - Marconi config now uses jsoncpp module (previously manually installed)
- - Moved variation memeber function into new class Gradient (previously in ArakawaX and Poisson)
- - blas1::dot and blas2::dot now both do not accumulate rest of multiplication
+ - marconi config now uses jsoncpp module (previously manually installed)
+ - blas1::dot and blas2::dot now both do not accumulate rest of multiplication (inconsistent before)
+ - swapped input and output parameters in dg::blas1::evaluate first subroutine
+ - the fast_interpolation and fast_projection functions now can also double / divide the polynomial coefficient consistent with the grids
+ - change shift_topologic() shared RealTopology member functions to shift() and have an additional negative parameter that indicates sign swaps
+ - clarify and unify the behaviour of the interpolation functions when points lie outside the grid boundaries
+ - split and join functions have an additional real_type template parameter
+ - stopping criterion for bisection1d function
+ - multistep time-stepper now initialize with Runge-Kutta timesteppers of corresponding order
+ - Multigrid nested iteration algorithm now allows accuracies for each stage separately (which can give a significant speed-up)
+ - dg::inverse( bc) function is now a free-standing function to invert a boundary condition
+ - Elliptic classes now have jump_weighting and multiply_sigma functions
+ - CG operator now has a test-frequency parameter to control the number of times the error condition is evaluated
+ - Extrapolation class now has a derive member function to interpolate the derivative of the interpolating polynomial
+ - Adapt all src and diag project to changed file and json utilities and the moved variation member
+ - Rename all input files with correct json file-ending
+ - Complete redesign of src/feltor and src/lamb_dipole
+ - Merge toefl_hpc with old toefl_mpi program
+
 ### Deprecated
+ - Karniadakis time-stepper is now superceded by the ImExMultistep class
 ### Removed
- - removed diag/feltordiag.cu
+ - remove diag/feltordiag.cu
+ - remove dg::MemoryTraits and associated dimensionality and memory_category traits in favor of direct host_vector and host_grid typedefs in topology classes
+ - old txt input files
+ - DeltaFunction and Alpha for the computation of flux-surface averages no longer needed
 ### Fixed
  - Fix bug: race condition in dg::blas1::dot and dg::blas2::dot on GPUs that led to hard to reproduce and seemingly unreasonable crashes
- - Fix bugs: std namespace in diag/probes.h
+ - Fix bug: std namespace in diag/probes.h
  - Fix bug: const in exblas::cpu::get_element 
  - Fix bug: correct  indices in exblas::cpu::make_vcl_vec8d
+ - Fix bug: infinite creation of MPI communicators in exblas::mpi_reduce_communicator . Lead to MPI crashes due to memory overflow.
+ - dg::blas1::dot and dg::blas2::dot and corresponding exblas functions now detect NaN and Inf errors
+ - correct capture of cuda-aware mpi, create a fall-back for cuda-unaware mpi-installations
+ - Fix bug: test for no-communication in mpi_communicator (indicated false positives)
+ - Fix bug: coefficient and initialization in Extrpolate
 
 ## [v5.1]
 ### Added
diff --git a/README.adoc b/README.adoc
index 011d71fd1..82fbd04ec 100644
--- a/README.adoc
+++ b/README.adoc
@@ -71,7 +71,7 @@ ____
 | *CPU*     | Any         |support for AVX and FMA instruction set
 | *Compiler*| gcc-5.1 or msvc-15 or icc-17.0 (C{plus}{plus}-14 standard)| gcc-9.3, OpenMP-4 support, avx, fma instruction set flags
 | *GPU*     | - | NVidia GPU with compute-capability > 6 and nvcc-11.0
-| *MPI*     | - | mpi installation compatible with compiler (must be cuda-aware in case hybrid MPI+GPU is the target system)
+| *MPI*     | - | mpi installation compatible with compiler (ideally cuda-aware in case hybrid MPI+GPU is the target system)
 |=======================================================================
 ____
 Our GPU backend uses the
diff --git a/config/README.md b/config/README.md
index 955dccb9f..0a73ee3f1 100644
--- a/config/README.md
+++ b/config/README.md
@@ -57,7 +57,7 @@ make blas_mpib device=gpu NVCCARCH='-arch sm_60' OPT=-O2
 ```
 
 ### General Remarks
- - If MPI is used in connection with the gpu backend, the mpi installation needs to be **cuda-aware**
+ - If MPI is used in connection with the gpu backend, the mpi installation would ideally be **cuda-aware** but does not need to be
  - If `icc` is used as the C++ compiler the `-restrict` option has to be used to enable the recognition of the restrict keyword
  - Support for OpenMP-4 is recommended (at least gcc-4.9 or icc-15), but not mandatory
  - The library headers are compliant with the c++14 standard but we reserve the right to upgrade that in future updates
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 1190e3f42..ae8bb0914 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -59,8 +59,8 @@ double result = dg::blas1::dot( two, three); // result = 600 (100*(2*3))
  * product of the input numbers reaches \c Inf or \c Nan then the behaviour
  * is undefined and the function may throw. See @ref dg::ISNFINITE and @ref
  * dg::ISNSANE in that case
- * @note Our implementation guarantees binary reproducible results.
- * The sum is computed with infinite precision and the result is rounded
+ * @note Our implementation guarantees **binary reproducible** results.
+ * The sum is computed with **infinite precision** and the result is rounded
  * to the nearest double precision number.
  * This is possible with the help of an adapted version of the \c dg::exblas library and
 * works for single and double precision.
diff --git a/inc/dg/blas2.h b/inc/dg/blas2.h
index acf81883d..37c88b8c9 100644
--- a/inc/dg/blas2.h
+++ b/inc/dg/blas2.h
@@ -64,8 +64,8 @@ inline std::vector<int64_t> doDot_superacc( const ContainerType1& x, const Matri
  * product of the input numbers reaches \c Inf or \c Nan then the behaviour
  * is undefined and the function may throw. See @ref dg::ISNFINITE and @ref
  * dg::ISNSANE in that case
- * @note Our implementation guarantees binary reproducible results up to and excluding the last mantissa bit of the result.
- * Furthermore, the sum is computed with infinite precision and the result is then rounded
+ * @note Our implementation guarantees **binary reproducible** results up to and excluding the last mantissa bit of the result.
+ * Furthermore, the sum is computed with **infinite precision** and the result is then rounded
  * to the nearest double precision number. Although the products are not computed with
  * infinite precision, the order of multiplication is guaranteed.
  * This is possible with the help of an adapted version of the \c dg::exblas library and
diff --git a/inc/dg/enums.h b/inc/dg/enums.h
index 44ffa871f..cb8e7769e 100644
--- a/inc/dg/enums.h
+++ b/inc/dg/enums.h
@@ -168,8 +168,8 @@ static inline direction inverse( direction dir)
 
 ///@brief Space of DG coefficients
 enum space{
-    lspace, //!< DG Polynomial space
-    xspace //!< Configuration space
+    lspace, //!< DG Polynomial space "modal values"
+    xspace //!< Configuration space "nodal values"
 };
 
 ///@brief 2d coordinates
diff --git a/inc/dg/multigrid.h b/inc/dg/multigrid.h
index 28406912f..bbf0b2ddd 100644
--- a/inc/dg/multigrid.h
+++ b/inc/dg/multigrid.h
@@ -163,7 +163,7 @@ struct MultigridCG2d
     ///@return A copyable object; what it contains is undefined, its size is important
     const Container& copyable() const {return m_x[0];}
     /**
-     * @brief Nested iterations (USE THIS ONE!)
+     * @brief USE THIS ONE Nested iterations
      *
      * Equivalent to the following
      * -# Compute residual with given initial guess.
@@ -250,7 +250,7 @@ struct MultigridCG2d
     }
 
     /**
-     * @brief Nested iterations with Chebyshev as preconditioner for CG (experimental)
+     * @brief EXPERIMENTAL Nested iterations with Chebyshev as preconditioner for CG
      *
      * @note This function does the same as direct_solve but uses a
      * ChebyshevPreconditioner (with EVE to estimate the largest EV) at the coarse
@@ -367,7 +367,7 @@ struct MultigridCG2d
     }
 
     /**
-     * @brief Full multigrid cycles (experimental, use at own risk)
+     * @brief EXPERIMENTAL Full multigrid cycles (use at own risk)
      *
      * - Compute residual with given initial guess.
      * - If error larger than tolerance, do a full multigrid cycle with Chebeyshev iterations as smoother
@@ -431,7 +431,7 @@ struct MultigridCG2d
     }
 
     /**
-     * @brief A conjugate gradient with a full multigrid cycle as preconditioner (experimental, use at own risk)
+     * @brief EXPERIMENTAL A conjugate gradient with a full multigrid cycle as preconditioner (use at own risk)
      *
      * @copydoc hide_symmetric_op
      * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
diff --git a/inc/dg/topology/average.h b/inc/dg/topology/average.h
index a6d9743f1..0e28ec4fd 100644
--- a/inc/dg/topology/average.h
+++ b/inc/dg/topology/average.h
@@ -145,7 +145,7 @@ struct Average
     /**
      * @brief Compute the average as configured in the constructor
      *
-     * The compuatation is based on the exact, reproducible scalar product provided in the \c ::exblas library. It is divided in two steps
+     * The compuatation is based on the exact, reproducible scalar product provided in the \c dg::exblas library. It is divided in two steps
      *  - average the input field over the direction or plane given in the constructor
      *  - extend the lower dimensional result back to the original dimensionality
      *
diff --git a/inc/dg/topology/average_mpi.h b/inc/dg/topology/average_mpi.h
index fb581fa14..6486f7208 100644
--- a/inc/dg/topology/average_mpi.h
+++ b/inc/dg/topology/average_mpi.h
@@ -164,7 +164,7 @@ struct Average<MPI_Vector<container> >
     /**
      * @brief Compute the average as configured in the constructor
      *
-     * The compuatation is based on the exact, reproducible scalar product provided in the \c ::exblas library. It is divided in two steps
+     * The compuatation is based on the exact, reproducible scalar product provided in the \c dg::exblas library. It is divided in two steps
      *  - average the input field over the direction or plane given in the constructor
      *  - extend the lower dimensional result back to the original dimensionality
      *
diff --git a/inc/dg/topology/filter.h b/inc/dg/topology/filter.h
index 92c54a9eb..8f88477ae 100644
--- a/inc/dg/topology/filter.h
+++ b/inc/dg/topology/filter.h
@@ -123,6 +123,8 @@ modal_filter( UnaryOp op, const aRealMPITopology3d<real_type>& t)
  * \f[ y = V D V^{-1}\f]
  * where \f$ V\f$ is the Vandermonde matrix (the backward transformation matrix)
  * and \f$ D \f$ is a diagonal matrix with \f$ D_{ii} = \sigma(i)\f$
+ * @sa A discussion of the effects of the modal filter on advection schemes can be found here https://mwiesenberger.github.io/advection
+ * @note basically the result is that it is usually not advantageous to use a modal filter
  * @copydoc hide_matrix
  * @copydoc hide_ContainerType
  * @ingroup misc
diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index a81514e38..e419fc5ea 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -508,7 +508,7 @@ cusp::coo_matrix<int, real_type, cusp::host_memory> interpolation( const aRealTo
 }//namespace create
 
 /**
- * @brief Transform a vector from XSPACE to LSPACE
+ * @brief Transform a vector from dg::xspace (nodal values) to dg::lspace (modal values)
  *
  * @param in input
  * @param g grid
@@ -536,12 +536,12 @@ thrust::host_vector<real_type> forward_transform( const thrust::host_vector<real
  * @brief Interpolate a vector on a single point on a 1d Grid
  *
  * @param sp Indicate whether the elements of the vector
- * v are in xspace or lspace
+ * v are in xspace (nodal values) or lspace (modal values)
  *  (choose dg::xspace if you don't know what is going on here,
  *      It is faster to interpolate in dg::lspace so consider
  *      transforming v using dg::forward_transform( )
  *      if you do it very many times)
- * @param v The vector to interpolate in dg::xspace
+ * @param v The vector to interpolate
  * @param x X-coordinate of interpolation point
  * @param g The Grid on which to operate
  * @copydoc hide_bcx_doc
@@ -603,7 +603,7 @@ real_type interpolate(
  * @brief Interpolate a vector on a single point on a 2d Grid
  *
  * @param sp Indicate whether the elements of the vector
- * v are in xspace or lspace
+ * v are in xspace (nodal values) or lspace  (modal values)
  *  (choose dg::xspace if you don't know what is going on here,
  *      It is faster to interpolate in dg::lspace so consider
  *      transforming v using dg::forward_transform( )
diff --git a/inc/geometries/average.h b/inc/geometries/average.h
index cf4bd10c9..21237b31e 100644
--- a/inc/geometries/average.h
+++ b/inc/geometries/average.h
@@ -16,7 +16,7 @@ namespace geo
 
 /**
  * @brief Flux surface integral of the form
- \f[ \int dR dZ f(R,Z) \delta(\psi_p(R,Z)-\psi_0) g(R,Z) \f]
+ \f$ \int dR dZ f(R,Z) \delta(\psi_p(R,Z)-\psi_0) g(R,Z) \f$
 
  where for the width of the Gaussian shaped delta function we use the maximum of \c 0.5*h*GradPsip
      where \c h is the cell size in the grid
@@ -84,7 +84,7 @@ struct FluxSurfaceIntegral
 //This method for computing volumes is tested against flux-aligned grids in e.g. flux_t.cu
 /**
  * @brief Flux volume integral of the form
- \f[ \int dR dZ f(R,Z) \Theta(\psi_p(R,Z)-\psi_0) g(R,Z) \f]
+ \f$ \int dR dZ f(R,Z) \Theta(\psi_p(R,Z)-\psi_0) g(R,Z) \f$
 
  where \c Theta is the Heaviside function
  * @ingroup misc_geo
@@ -144,7 +144,7 @@ struct FluxVolumeIntegral
 
 /**
  * @brief Flux surface average (differential volume average) over quantity
- \f[ \langle f\rangle(\psi_0) = \frac{1}{A} \int dR dZ \delta(\psi_p(R,Z)-\psi_0) f(R,Z)H(R,Z) \f]
+ \f$ \langle f\rangle(\psi_0) = \frac{1}{A} \int dR dZ \delta(\psi_p(R,Z)-\psi_0) f(R,Z)H(R,Z) \f$
 
  with \f$ A = \int dRdZ \delta(\psi_p(R,Z)-\psi_0)H(R,Z)\f$
  where \c H is a weight function that can be used to e.g. cut away parts of the domain below the X-point or contain a volume form
@@ -201,7 +201,7 @@ struct FluxSurfaceAverage
 
 /**
  * @brief Class for the evaluation of the safety factor q based on a flux-surface integral
- * \f[ q(\psi_0) = \frac{1}{2\pi} \int dRdZ \frac{I(\psi_p)}{R} \delta(\psi_p - \psi_0)H(R,Z) \f]
+ * \f$ q(\psi_0) = \frac{1}{2\pi} \int dRdZ \frac{I(\psi_p)}{R} \delta(\psi_p - \psi_0)H(R,Z) \f$
 
 where \c H is a weights function that can optionally be used to cut away parts of the domain e.g. below the X-point.
  * @copydoc hide_container
@@ -245,7 +245,7 @@ struct SafetyFactorAverage
 
 /**
  * @brief Evaluation of the safety factor q based on direct integration of
- * \f[ q(\psi_0) = \frac{1}{2\pi} \int d\Theta \frac{B^\varphi}{B^\Theta} \f]
+ * \f$ q(\psi_0) = \frac{1}{2\pi} \int d\Theta \frac{B^\varphi}{B^\Theta} \f$
 
  * @attention Return value undefined if evaluated outside the closed fieldline region, but the function always returns, it won't throw an error or something
  * @copydoc hide_container
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index b3d51ff1f..cb9cf9d26 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -19,6 +19,7 @@
         @defgroup guenther The Guenther expansion
         @defgroup toroidal The Purely Toroidal expansion
         @defgroup circular The Circular expansion
+        @defgroup mod  Modification of any expansion
       @}
       @defgroup magnetic 3.2 Magnetic field, curvatures and associated functors
       @defgroup profiles 3.3 Penalization, weight and monitor metric functors
-- 
GitLab


From a078c48256ce7fc86d3b37d83ee0dcd551c447d4 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Sun, 14 Feb 2021 14:52:59 +0100
Subject: [PATCH 500/540] Introduce Variation

as an alternative name for Gradient
---
 diag/feltorSesoldiag.cpp    |  2 +-
 diag/feltorShwdiag.cpp      |  2 +-
 diag/impRdiag.cu            |  2 +-
 diag/normdiag.cu            |  2 +-
 diag/toeflEPdiag.cu         |  2 +-
 diag/toeflRdiag.cu          |  2 +-
 inc/dg/gradient.h           | 13 +++++++++++++
 src/ep/toeflR.cuh           |  2 +-
 src/feltorSH/feltor.cuh     |  2 +-
 src/feltorSHp/feltor.cuh    |  2 +-
 src/feltorSesol/feltor.cuh  |  2 +-
 src/feltorShw/feltor.cuh    |  2 +-
 src/reco2D/reconnection.cuh |  2 +-
 src/toefl/toeflR.cuh        |  2 +-
 14 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/diag/feltorSesoldiag.cpp b/diag/feltorSesoldiag.cpp
index 251ebd544..a5fb8093f 100644
--- a/diag/feltorSesoldiag.cpp
+++ b/diag/feltorSesoldiag.cpp
@@ -62,7 +62,7 @@ int main( int argc, char* argv[])
     dg::IHMatrix interp(dg::create::interpolation(xcoo,y0coo,g2d));
     dg::IHMatrix interp_in = dg::create::interpolation(g2d,g2d_in);
     dg::Poisson<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> poisson(g2d,  p.bc_x, p.bc_y,  p.bc_x, p.bc_y);
-    dg::Gradient<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> gradient(g2d, p.bc_x, p.bc_y);
+    dg::Variation<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> gradient(g2d, p.bc_x, p.bc_y);
 
 
     //2d field
diff --git a/diag/feltorShwdiag.cpp b/diag/feltorShwdiag.cpp
index eb603efdb..a3bde0ee4 100644
--- a/diag/feltorShwdiag.cpp
+++ b/diag/feltorShwdiag.cpp
@@ -69,7 +69,7 @@ int main( int argc, char* argv[])
     dg::IHMatrix interp(dg::create::interpolation(xcoo,y0coo,g2d));
     dg::IHMatrix interp_in = dg::create::interpolation(g2d,g2d_in);
     dg::Poisson<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> poisson(g2d,  p.bc_x, p.bc_y,  p.bc_x_phi, p.bc_y);
-    dg::Gradient<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> gradient(g2d, p.bc_x_phi, p.bc_y);
+    dg::Variation<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> gradient(g2d, p.bc_x_phi, p.bc_y);
     dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> pol(g2d,   p.bc_x_phi, p.bc_y, dg::normed, dg::centered);
     dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> lap(g2d,   p.bc_x, p.bc_y, dg::normed, dg::centered);
     
diff --git a/diag/impRdiag.cu b/diag/impRdiag.cu
index 611ef8040..cec06dc6e 100644
--- a/diag/impRdiag.cu
+++ b/diag/impRdiag.cu
@@ -122,7 +122,7 @@ int main( int argc, char* argv[])
   std::vector<dg::HVec> npe_h(3, dg::evaluate(dg::zero, g2d));
   //eval field
   dg::ArakawaX< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa(g2d);
-  dg::Gradient< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient(g2d);
+  dg::Variation< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient(g2d);
   //eval particle densities
   const dg::DVec binv( dg::evaluate(dg::LinearX(p.kappa, 1.), g2d));
   dg::DVec chi = dg::evaluate(dg::zero, g2d);
diff --git a/diag/normdiag.cu b/diag/normdiag.cu
index 5cccf3485..32b8bbdc2 100644
--- a/diag/normdiag.cu
+++ b/diag/normdiag.cu
@@ -61,7 +61,7 @@ int main( int argc, char* argv[])
     dg::DVec nG(dg::evaluate(prof,g2d));
     dg::DVec w2d = dg::create::weights( g2d);
     dg::Poisson<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> poisson(g2d,  p.bc_x, p.bc_y,  p.bc_x_phi, p.bc_y);
-    dg::Gradient<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient(g2d, p.bc_x_phi, p.bc_y);
+    dg::Variation<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient(g2d, p.bc_x_phi, p.bc_y);
     //open netcdf files
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     //set min and max timesteps
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index b19943f74..4b410c681 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -81,7 +81,7 @@ int main( int argc, char* argv[])
     dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
     dg::Grid1d g1d( 0., p.lx, p.n_out, p.Nx_out, p.bc_x);
     dg::ArakawaX< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa( g2d); 
-    dg::Gradient< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient( g2d); 
+    dg::Variation< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient( g2d); 
     double time = 0.;
     //2d field
     size_t count2d[3]  = {1, g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index 787d87131..332192ece 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -76,7 +76,7 @@ int main( int argc, char* argv[])
     dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
     dg::Grid1d g1d( 0., p.lx, p.n_out, p.Nx_out, p.bc_x);
     dg::ArakawaX< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa( g2d); 
-    dg::Gradient< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient( g2d); 
+    dg::Variation< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient( g2d); 
     double time = 0.;
     //2d field
     size_t count2d[3]  = {1, g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
diff --git a/inc/dg/gradient.h b/inc/dg/gradient.h
index 28bb0f7e6..ab1f06e72 100644
--- a/inc/dg/gradient.h
+++ b/inc/dg/gradient.h
@@ -187,6 +187,14 @@ class Gradient
 ///@ingroup matrixoperators
 template <class Geometry, class Matrix, class Container>
 using Gradient2d = Gradient<Geometry, Matrix, Container>;
+///@copydoc Gradient
+///@ingroup matrixoperators
+template <class Geometry, class Matrix, class Container>
+using Variation = Gradient<Geometry,Matrix,Container>;
+///@copydoc Gradient
+///@ingroup matrixoperators
+template <class Geometry, class Matrix, class Container>
+using Variation2d = Gradient<Geometry,Matrix,Container>;
 
 /**
  * @brief A 3d gradient \f$\chi\cdot\nabla\f$ and variation \f$ \nabla\phi \cdot \chi \nabla\phi\f$ operator
@@ -376,4 +384,9 @@ class Gradient3d
     bool m_multiplyZ = true;
 };
 
+///@copydoc Gradient3d
+///@ingroup matrixoperators
+template <class Geometry, class Matrix, class Container>
+using Variation3d = Gradient3d<Geometry,Matrix,Container>;
+
 } //namespace dg
diff --git a/src/ep/toeflR.cuh b/src/ep/toeflR.cuh
index 8eadab7e8..2da42732a 100644
--- a/src/ep/toeflR.cuh
+++ b/src/ep/toeflR.cuh
@@ -125,7 +125,7 @@ struct ToeflR
     dg::Elliptic<Geometry, Matrix, container> pol, laplaceM; //contains normalized laplacian
     dg::Helmholtz<Geometry,  Matrix, container> gamma1;
     dg::ArakawaX< Geometry, Matrix, container> arakawa; 
-    dg::Gradient< Geometry, Matrix, container> gradient; 
+    dg::Variation< Geometry, Matrix, container> gradient; 
 
     dg::Invert<container> invert_pol, invert_invgamma;
 
diff --git a/src/feltorSH/feltor.cuh b/src/feltorSH/feltor.cuh
index cb7c60afb..7efd543b8 100644
--- a/src/feltorSH/feltor.cuh
+++ b/src/feltorSH/feltor.cuh
@@ -103,7 +103,7 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
-    dg::Gradient< Geometry, Matrix, container> gradient; 
+    dg::Variation< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic<   Geometry, Matrix, container> lapperpM; 
     std::vector<container> multi_chi;
diff --git a/src/feltorSHp/feltor.cuh b/src/feltorSHp/feltor.cuh
index 42cb8490a..03f912d88 100644
--- a/src/feltorSHp/feltor.cuh
+++ b/src/feltorSHp/feltor.cuh
@@ -125,7 +125,7 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
-    dg::Gradient< Geometry, Matrix, container> gradient; 
+    dg::Variation< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic<   Geometry, Matrix, container> lapperpM; 
     std::vector<container> multi_chi;
diff --git a/src/feltorSesol/feltor.cuh b/src/feltorSesol/feltor.cuh
index 61141e521..38351c19c 100644
--- a/src/feltorSesol/feltor.cuh
+++ b/src/feltorSesol/feltor.cuh
@@ -101,7 +101,7 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
-    dg::Gradient< Geometry, Matrix, container> gradient; 
+    dg::Variation< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic< Geometry, Matrix, container > lapperpM; 
     std::vector<container> multi_chi;
diff --git a/src/feltorShw/feltor.cuh b/src/feltorShw/feltor.cuh
index 3cc809472..220cde979 100644
--- a/src/feltorShw/feltor.cuh
+++ b/src/feltorShw/feltor.cuh
@@ -106,7 +106,7 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
-    dg::Gradient< Geometry, Matrix, container> gradient; 
+    dg::Variation< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic< Geometry, Matrix, container > lapperp; 
     std::vector<container> multi_chi;
diff --git a/src/reco2D/reconnection.cuh b/src/reco2D/reconnection.cuh
index 37b9c9c01..b5a7e4068 100644
--- a/src/reco2D/reconnection.cuh
+++ b/src/reco2D/reconnection.cuh
@@ -202,7 +202,7 @@ struct Asela
 
     //matrices and solvers
     dg::ArakawaX< Geometry, Matrix, container > arakawa; 
-    dg::Gradient< Geometry, Matrix, container > gradient; 
+    dg::Variation< Geometry, Matrix, container > gradient; 
     dg::Elliptic<  Geometry, Matrix, container  > lapperp; //note the host vector    
     
     std::vector<container> multi_chi;
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index cff8e67a2..04d46649f 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -126,7 +126,7 @@ struct Explicit
     std::vector<dg::Elliptic<Geometry, Matrix, container> > multi_pol;
     std::vector<dg::Helmholtz<Geometry,  Matrix, container> > multi_gamma1;
     dg::ArakawaX< Geometry, Matrix, container> arakawa;
-    dg::Gradient< Geometry, Matrix, container> gradient;
+    dg::Variation< Geometry, Matrix, container> gradient;
 
     dg::MultigridCG2d<Geometry, Matrix, container> multigrid;
     dg::Extrapolation<container> old_phi, old_psi, old_gammaN;
-- 
GitLab


From 9951ff86056ad628445bf04ee7207447fcf747f9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 15 Feb 2021 13:05:39 +0100
Subject: [PATCH 501/540] Improve Documentation of geometries section

and add relevant changes to the Changelog
---
 CHANGELOG.md                           | 19 ++++-
 doc/header.html                        |  2 +-
 inc/geometries/ds.h                    | 41 ++++++-----
 inc/geometries/fluxfunctions.h         |  2 +
 inc/geometries/geometries.h            |  3 +
 inc/geometries/geometries_doc.h        |  9 ++-
 inc/geometries/guenther.h              |  4 +-
 inc/geometries/magnetic_field.h        | 54 +++++++++-----
 inc/geometries/make_field.h            | 97 ++++++++++++++++++++------
 inc/geometries/modified.h              | 21 ++++++
 inc/geometries/polynomial.h            |  2 +-
 inc/geometries/polynomial_parameters.h | 31 ++++++--
 inc/geometries/solovev.h               |  8 +--
 inc/geometries/solovev_parameters.h    | 32 +++++++--
 inc/geometries/taylor.h                |  2 +-
 inc/geometries/toroidal.h              |  4 +-
 16 files changed, 250 insertions(+), 81 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 76974dd18..b1f355aa4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,6 @@
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
-We do not (yet) follow semantic versioning.
 Only changes in code are reported here, we do not track changes in the
 doxygen documentation, READMEs or tex writeups.
 ## [v5.2]
@@ -43,6 +42,16 @@ doxygen documentation, READMEs or tex writeups.
  - new class Gradient that computes gradients and variations
  - a host of new functors for the evaluate and pullback functions
  - FluxSurfaceIntegral, FluxVolumeIntegral and SafetyFactorAverage classes
+ - new implementation: ds_centered_bc_along_field and dss_centered_bc_along_field that implement boundary condition "Stegmeir" style along the magnetic field lines
+ - new Fieldaligned member functions integrate_between_coarse_grid and interpolate_from_coarse_grid that allow field-aligned interpolations
+ - dg::geo::Periodify class and dg::geo::periodify function to extend flux-functions periodically beyond grid boundaries
+ - new dg::geometries::findCriticalPoint function that generalizes X-point and O-point identification
+ - new classes dg::geo::SquareNorm and dg::geo::ScalarProduct that work on cylindrical vector fields
+ - new set utility functors dg::geo::SetUnion, dg::geo::SetIntersection, and dg::geo::SetNot that help construct damping regions
+ - dg::geo::createMagneticField and dg::geo::createModifiedField with associated utility functions and classes that generalize the creation of magnetic flux functions and wall and sheath regions
+ - new polynomial expansion and associated dg::Horner2d functor for magnetic flux functions that can in particular approximate any experimental equilibrium
+ - new equilibrium, modifier and description fields for tokamak magnetic fields
+ - Sign reversal of magnetic field and associated flux functions is now possible
 ### Changed
  - namespace file changed to **dg::file** and exblas changed to **dg::exblas** (for consistency reasons, everything should go into the dg namespace, which in particular reduces the chance for name-clashes to just one, namely 'dg')
  - changed file paths **dg/file/file.h**, **dg/geometries/geometries.h** , **dg/file/nc_utilities.h**
@@ -68,6 +77,11 @@ doxygen documentation, READMEs or tex writeups.
  - Rename all input files with correct json file-ending
  - Complete redesign of src/feltor and src/lamb_dipole
  - Merge toefl_hpc with old toefl_mpi program
+ - bump Doxygen version to 1.8.17
+ - DS forward, backward, centered and dss functions are now free-standing, only requiring a fielaligned object, plus, and minus applications (this allows to reduce the number of times the plus and minus interpolation has to be applied)
+ - changed Fieldaligned members hp_inv to hbp to give more control
+ - dg::forward_transform function (previously dg::create::forward_transform)
+ - new dg::geo::MagneticFieldParameters struct to unify the representation of Meta-data in the TokamakMagneticField class (simplifies construction)
 
 ### Deprecated
  - Karniadakis time-stepper is now superceded by the ImExMultistep class
@@ -86,6 +100,8 @@ doxygen documentation, READMEs or tex writeups.
  - correct capture of cuda-aware mpi, create a fall-back for cuda-unaware mpi-installations
  - Fix bug: test for no-communication in mpi_communicator (indicated false positives)
  - Fix bug: coefficient and initialization in Extrpolate
+ - Fix bug: Fpsi safety-factor in case nan is encountered still works
+ - Fix bug: Fpsi safety-factor works up to the O-point
 
 ## [v5.1]
 ### Added
@@ -147,6 +163,7 @@ doxygen documentation, READMEs or tex writeups.
 - Optimization: implement fast EllSparseBlockMat kernel for z derivative
 - Optimization: change buffer layout in dg::NearestNeighborComm and CooSparseBlockMat kernels to avoid slow scatter/gather operations in mpi matrix-vector multiplication
 - Optimization: implement faster kernels for CooSparseBlockMat symv kernel to accelerate mpi symv with low computation to communication ratio
+- separate modification of fluxfunctions into mod namespace that works on flux functions in general (previously only solovev)
 
 ### Deprecated
 - dg::blas1::transfer (replaced by the more general dg::assign and dg::construct)
diff --git a/doc/header.html b/doc/header.html
index b0a68524e..46441565d 100644
--- a/doc/header.html
+++ b/doc/header.html
@@ -25,7 +25,7 @@ $extrastylesheet
   <ul class="menubar">
       <!-- <li><a href="../../../index.html">/</a><li>-->
     <li><a href="../../dg/html/modules.html">dg</a></li>
-    <li><a href="../../geometries/html/modules.html">dg::geometries</a></li>
+    <li><a href="../../geometries/html/modules.html">dg::geo</a></li>
     <li><a href="../../file/html/modules.html">dg::file</a></li>
     <li><a href="../../exblas/html/namespacedg_1_1exblas.html">dg::exblas</a></li>
   </ul>
diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 73799357b..936a84a24 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -219,6 +219,11 @@ struct ComputeDSSDIR{
 /*!@class hide_ds_dir
  * @param dir indicate the direction in the bracket operator and in symv
  */
+/*!@class hide_ds_freestanding
+ * @note This function computes the same as the corresponding member function of DS
+ * and you have to compute the einsPlus and einsMinus interpolations from dg::Fieldaligned yourself. The reasoning for this function is that you can re-use the latter interpolations if you compute for example both first and second derivative of a function.
+ */
+
 
 /*!@class hide_ds_attention
 @attention The \c div and \c symv member functions reliably converge only if fieldlines
@@ -674,6 +679,7 @@ void DS<G,I,M,container>::do_symv( double alpha, const container& f, double beta
 * @copydoc hide_ds_parameters4
 * @copydoc hide_ds_fp
 * @ingroup fieldaligned
+* @copydoc hide_ds_freestanding
 */
 template<class FieldAligned, class container>
 void ds_forward(const FieldAligned& fa, double alpha, const container& f, const container& fp, double beta, container& g)
@@ -681,9 +687,6 @@ void ds_forward(const FieldAligned& fa, double alpha, const container& f, const
     //direct
     dg::blas1::subroutine( detail::ComputeDSForward( alpha, beta),
             g, f, fp, fa.hp());
-    //m_fa(einsPlus, m_tempP, m_tempM);
-    //dg::blas1::subroutine( detail::ComputeDSForward( alpha, beta),
-    //        g, f, m_tempP, m_tempM, m_fa.hp());
 }
 /**
 * @brief backward derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
@@ -693,6 +696,7 @@ void ds_forward(const FieldAligned& fa, double alpha, const container& f, const
 * @copydoc hide_ds_parameters4
 * @copydoc hide_ds_fm
 * @ingroup fieldaligned
+* @copydoc hide_ds_freestanding
 */
 template<class FieldAligned, class container>
 void ds_backward( const FieldAligned& fa, double alpha, const container& fm, const container& f, double beta, container& g)
@@ -700,9 +704,6 @@ void ds_backward( const FieldAligned& fa, double alpha, const container& fm, con
     //direct
     dg::blas1::subroutine( detail::ComputeDSBackward( alpha, beta),
             g, f, fm, fa.hm());
-    //m_fa(einsMinus, m_tempM, m_tempP);
-    //dg::blas1::subroutine( detail::ComputeDSBackward( alpha, beta),
-    //        g, f, m_tempM, m_tempP, m_fa.hm());
 }
 /**
 * @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
@@ -713,6 +714,7 @@ void ds_backward( const FieldAligned& fa, double alpha, const container& fm, con
 * @copydoc hide_ds_fm
 * @copydoc hide_ds_fp
 * @ingroup fieldaligned
+* @copydoc hide_ds_freestanding
 */
 template<class FieldAligned, class container>
 void ds_centered( const FieldAligned& fa, double alpha, const container& fm,
@@ -732,9 +734,10 @@ void ds_centered( const FieldAligned& fa, double alpha, const container& fm,
  * to the numerical stability and toroidal resolution.
  * @param fa this object will be used to get grid distances
  * @copydoc hide_ds_parameters4
-* @copydoc hide_ds_fm
-* @copydoc hide_ds_fp
-* @ingroup fieldaligned
+ * @copydoc hide_ds_fm
+ * @copydoc hide_ds_fp
+ * @ingroup fieldaligned
+ * @copydoc hide_ds_freestanding
  */
 template<class FieldAligned, class container>
 void dss_centered( const FieldAligned& fa, double alpha, const container&
@@ -745,19 +748,19 @@ void dss_centered( const FieldAligned& fa, double alpha, const container&
 }
 
 /**
-* @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
-*
-* The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup.
+ * @brief centered derivative \f$ g = \alpha \vec v \cdot \nabla f + \beta g\f$
+ *
+ * The centered derivative is constructed by fitting a polynomial through the plus point the minus point and the center point and evaluating its derivative at the center point. For the exact resulting formula consult the <a href="./parallel.pdf" target="_blank">parallel derivative</a> writeup.
  * the boundary condition is implemented
  * along the field-line, that is the boundary condition is used as part of the polynomial interpolation.
  * @param fa this object will be used to get grid distances
-* @copydoc hide_ds_parameters4
-* @copydoc hide_ds_fm
-* @copydoc hide_ds_fp
-* @param bound either dg::NEU or dg::DIR (rest not implemented yet)
-* @param boundary_value first value is for incoming fieldlines, second one for outgoing
-* @ingroup fieldaligned
-*/
+ * @copydoc hide_ds_parameters4
+ * @copydoc hide_ds_fm
+ * @copydoc hide_ds_fp
+ * @param bound either dg::NEU or dg::DIR (rest not implemented yet)
+ * @param boundary_value first value is for incoming fieldlines, second one for outgoing
+ * @ingroup fieldaligned
+ */
 template<class FieldAligned, class container>
 void ds_centered_bc_along_field( const FieldAligned& fa, double alpha, const container& fm,
         const container& f, const container& fp, double beta, container& g,
diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index 26afd4154..e3289a4ff 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -141,6 +141,7 @@ struct ZCutter : public aCylindricalFunctor<ZCutter>
 
 /**
  * @brief This function uses the dg::Grid2d::shift member to extend another function beyond the grid boundaries
+ * @sa dg::geo::periodify
  */
 struct Periodify : public aCylindricalFunctor<Periodify>
 {
@@ -286,6 +287,7 @@ static inline int findCriticalPoint( const CylindricalFunctorsLvl2& psi, double&
     double Dinv = 1./(psipZZ*psipRR - psipRZ*psipRZ);
     while( (eps < eps_old || eps > 1e-7) && eps > 1e-10 && counter < 100)
     {
+        //newton iteration
         XN[0] = X[0] - Dinv*(psipZZ*psipR - psipRZ*psipZ);
         XN[1] = X[1] - Dinv*(-psipRZ*psipR + psipRR*psipZ);
         XN.swap(X);
diff --git a/inc/geometries/geometries.h b/inc/geometries/geometries.h
index e2c4e8d3e..0d6285054 100644
--- a/inc/geometries/geometries.h
+++ b/inc/geometries/geometries.h
@@ -24,6 +24,9 @@
 #include "guenther.h"
 #include "toroidal.h"
 #include "polynomial.h"
+#ifdef BOOST_VERSION
+#include "taylor.h"
+#endif
 #include "make_field.h"
 
 #include "fluxfunctions.h"
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index cb9cf9d26..ff4f74d8c 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -6,12 +6,14 @@
  *      All the grids introduced by this extension can be constructed with
  *      generator classes.
  * @defgroup grids 2. New geometric grids
- * @defgroup fluxfunctions 3. New functors surrounding the magnetic field geometry
+ * @defgroup geom_functors 3. New functors surrounding the magnetic field geometry
+ *  \f$ f(R,Z,\varphi)\f$
 
         All functors in this section model two or three-dimensional functions, i.e.
         they all overload the operator() like \c aCylindricalFunctor
  * @{
       @defgroup geom 3.1 Creating a flux function
+      \f$\psi_p(R,Z)\f$ dg::geo::createMagneticField
       @{
         @defgroup solovev The solovev expansion
         @defgroup polynomial The polynomial expansion
@@ -20,12 +22,17 @@
         @defgroup toroidal The Purely Toroidal expansion
         @defgroup circular The Circular expansion
         @defgroup mod  Modification of any expansion
+        \f$ H(\psi_p(R,Z))\f$, dg::geo::createModifiedField
+        @defgroup wall Wall and Sheath
       @}
       @defgroup magnetic 3.2 Magnetic field, curvatures and associated functors
       @defgroup profiles 3.3 Penalization, weight and monitor metric functors
+      @defgroup fluxfunctions 3.4. Utility functor functionality
  * @}
  * @defgroup fieldaligned 4. Fieldaligned derivatives
+ * \f$ \nabla_\parallel f\f$
  * @defgroup misc_geo 5. Miscellaneous additions
+ * \f$ q(\psi_p)\f$ and \f$ \nabla \psi_p ( R_X, Z_X) = 0\f$
  */
 /*! @mainpage
  * This extension adds new features to the FELTOR core dg library.
diff --git a/inc/geometries/guenther.h b/inc/geometries/guenther.h
index ba96b824f..739c21c96 100644
--- a/inc/geometries/guenther.h
+++ b/inc/geometries/guenther.h
@@ -8,8 +8,6 @@
 
 #include "magnetic_field.h"
 
-//TODO somebody document the functions as in solovev/geometry.h
-
 /*!@file
  *
  * MagneticField objects
@@ -155,7 +153,7 @@ static inline CylindricalFunctorsLvl1 createIpol( double I_0)
  * @param R_0 the major radius
  * @param I_0 the current
  * @return A magnetic field object
- * @ingroup geom
+ * @ingroup guenther
  */
 static inline dg::geo::TokamakMagneticField createGuentherField( double R_0, double I_0)
 {
diff --git a/inc/geometries/magnetic_field.h b/inc/geometries/magnetic_field.h
index 9e5aba919..cdc4373de 100644
--- a/inc/geometries/magnetic_field.h
+++ b/inc/geometries/magnetic_field.h
@@ -26,9 +26,9 @@ enum class equilibrium
 {
     solovev, //!< dg::geo::solovev::Psip
     taylor, //!< dg::geo::taylor::Psip
-    polynomial, ///!< dg::geo::polynomial::Psip
+    polynomial, //!< dg::geo::polynomial::Psip
     guenther, //!< dg::geo::guenther::Psip
-    toroidal, //!< dg::geo::toroidal::Psip
+    toroidal, //!< dg::geo::createToroidalField
     circular //!< dg::geo::circular::Psip
 };
 ///@brief How flux-function is modified
@@ -38,7 +38,12 @@ enum class modifier
     heaviside, //!< Psip is dampened to a constant outside a critical value
     sol_pfr //!< Psip is dampened in the SOL and PFR regions but not in the closed field line region
 };
-///@brief How flux function looks like. Decider on whether and what flux aligned grid to construct
+/**
+ * @brief How flux function looks like. Decider on whether and what flux aligned grid to construct
+ *
+ * The reason for this enum is that it is very hard to automatically detect if the construction
+ * of a flux aligned X-grid is possible, but for a human it is very easy to see.
+ */
 enum class description
 {
     standardO, //!< closed flux surfaces centered around an O-point located near (R_0, 0); flux-aligned grids can be constructed
@@ -77,12 +82,10 @@ static const std::map<std::string, description> str2description{
 /**
  * @brief Meta-data about the magnetic field in particular the flux function
  *
- * The purpose of this is to give a unified set of parameters for
- * all equilibria that can be used to stear program execution.
- *
- * For example it is very hard to automatically detect if the construction
- * of a flux aligned X-grid is possible, but for a human it is very easy.
- * Here we give the \c description specifier that can be used in an if-else statement.
+ * The purpose of this is to give a unified set of parameters for all
+ * equilibria that can be used to stear program execution based on
+ * characteristics of the magnetic flux functions (for example double X-point
+ * vs single X-point vs no X-point)
  */
 struct MagneticFieldParameters
 {
@@ -111,17 +114,29 @@ struct MagneticFieldParameters
         m_triangularity( triangularity),
         m_equilibrium( equ),
         m_modifier(mod), m_description( des){}
- //!< The minor radius; the purpose of this parameter is not to be exact but to serve as a refernce of how to setup the size of a simulation box
+    /**
+     * @brief The minor radius
+     *
+     * the purpose of this parameter is not to be exact but to serve as a refernce of how to setup the size of a simulation box
+     */
     double a() const{return m_a;}
- //!< (maximum Z - minimum Z of lcfs)/2a; 1 for a circle; the purpose of this parameter is not to be exact but more to be a reference of how to setup the aspect ratio of a simulation box
+    /**
+     * @brief \f$ e := \frac{\max Z_{\mathrm{lcfs}} - \min Z_{\mathrm{lcfs}}}{2a}\f$
+     *
+     * (1 for a circle); the purpose of this parameter is not to be exact but more to be a reference of how to setup the aspect ratio of a simulation box
+     */
     double elongation() const{return m_elongation;}
- //!< (R_0 - R_X) /a;  The purpose of this parameter is to find the approximate location of R_X (if an X-point is present, Z_X is given by elongation) the exact location can be computed by the \c findXpoint function
+    /**
+     * @brief \f$ \delta := \frac{R_0 - R_X}{a}\f$
+     *
+     * The purpose of this parameter is to find the approximate location of R_X (if an X-point is present, Z_X is given by elongation) the exact location can be computed by the \c findXpoint function
+     */
     double triangularity() const{return m_triangularity;}
- //!< the way the flux function is computed
+    /// the way the flux function is computed
     equilibrium getEquilibrium() const{return m_equilibrium;}
- //!<  the way the flux function is modified
+    ///  the way the flux function is modified
     modifier getModifier() const{return m_modifier;}
- //!< human readable descriptor of how the flux function looks
+    /// how the flux function looks
     description getDescription() const{return m_description;}
     private:
     double m_a,
@@ -181,6 +196,11 @@ struct TokamakMagneticField
 
     const CylindricalFunctorsLvl2& get_psip() const{return m_psip;}
     const CylindricalFunctorsLvl1& get_ipol() const{return m_ipol;}
+    /**
+     * @brief Access Meta-data of the field
+     *
+     * @return Meta-data
+     */
     const MagneticFieldParameters& params() const{return m_params;}
 
     private:
@@ -210,7 +230,7 @@ static inline CylindricalFunctorsLvl2 periodify( const CylindricalFunctorsLvl2&
 }
 ///@endcond
 /**
- * @brief Use dg::geo::Periodify to periodify every function the magnetic field
+ * @brief Use dg::geo::Periodify to periodify every function in the magnetic field
  *
  * Note that derivatives are periodified with dg::inverse boundary conditions
  * @param mag The magnetic field to periodify
@@ -220,7 +240,7 @@ static inline CylindricalFunctorsLvl2 periodify( const CylindricalFunctorsLvl2&
  * @param Z1 upper boundary in Z
  * @param bcx boundary condition in x (determines how function is periodified)
  * @param bcy boundary condition in y (determines how function is periodified)
- * @note So far this was only tested for Neumann boundary conditions. It is uncertain if Dirichlet boundary conditions work
+ * @attention So far this was only tested for Neumann boundary conditions. It is uncertain if Dirichlet boundary conditions work
  *
  * @return new periodified magnetic field
  */
diff --git a/inc/geometries/make_field.h b/inc/geometries/make_field.h
index 8f82d773c..aa9741e8e 100644
--- a/inc/geometries/make_field.h
+++ b/inc/geometries/make_field.h
@@ -12,18 +12,37 @@
  */
 namespace dg{
 namespace geo{
-///@addtogroup geom
-///@{
 
 /**
  * @brief Create a Magnetic field based on the given parameters
  *
  * This function abstracts the Magnetic field generation. It reads an
  * input Json file that tells this function via the "equilibrium" parameter which
- * field to generate and which parameters to expect in the file, for example if
- * "equilibrium" reads "toroidal", then only on additional parameter "R_0" is
- * read from the file and a field is constructed
- * @param gs Has to contain "equilibrium" which is converted dg::geo::equilibrium,
+ * field to generate and which parameters to expect in the file.
+ * See a list of possible combinations in the following
+ * @copydoc hide_solovev_json
+ * @copydoc hide_polynomial_json
+ *
+ * @code
+// Purely toroidal magnetic field
+{
+    "equilibrium" : "toroidal",
+    "R_0" : 10
+}
+// description "none" is chosen by default
+ * @endcode
+ * @code
+// Circular flux surfaces
+{
+    "equilibrium" : "circular",
+    "I_0" : 20
+    "R_0" : 10
+}
+// description "standardO" is chosen by default
+ * @endcode
+ * @sa \c dg::geo::description to see valid values for the %description field
+ *
+ * @param gs Has to contain "equilibrium" which is converted \c dg::geo::equilibrium,
  * i.e. "solovev", "polynomial", .... After that the respective parameters are created,
  * for example if "solovev", then the dg::geo::solovev::Parameters( gs, mode) is called and forwarded to dg::geo::createSolovevField(gp); similar for the rest
  * @param mode signifies what to do if an error occurs
@@ -57,6 +76,13 @@ static inline TokamakMagneticField createMagneticField( Json::Value gs, dg::file
             double R0 = dg::file::get( mode, gs, "R_0", 10).asDouble();
             return createCircularField( R0, I0);
         }
+#ifdef BOOST_VERSION
+        case equilibrium::taylor:
+        {
+            solovev::Parameters gp( gs, mode);
+            return createTaylorField( gp);
+        }
+#endif
         default:
         {
             solovev::Parameters gp( gs, mode);
@@ -82,6 +108,8 @@ void transform_psi( TokamakMagneticField mag, double& psi0, double& alpha0, doub
 }//namespace detail
 ///@endcond
 
+///@addtogroup wall
+///@{
 /**
  * @brief Modify Magnetic Field above or below certain Psi values according to given parameters
  *
@@ -90,16 +118,34 @@ void transform_psi( TokamakMagneticField mag, double& psi0, double& alpha0, doub
  * function with width alpha), i.e. we replace psi with IPolynomialHeaviside(psi).
  * This subsequently modifies all derivatives of psi and the poloidal
  * current in this region.
- * @param gs forwarded to dg::geo::createMagneticField
- * @param jsmod must contain the field "wall": "type" which has one of the values "none", then no other values are required; "heaviside" then requires
- *  "wall": "boundary" value where psi is modified to a constant psi0
- * "wall": "alpha" radius of the transition region where the modification acts (smaller is quicker);
- * or "sol_pfr", then requires
- *  "wall": "boundary" and
- * "wall": "alpha" must be arrays of size 2 to indicate values for the SOL and the PFR respectively
+ *
+@code
+{
+    "wall" :
+    {
+        "type": "none"
+    },
+    "wall":
+    {
+        "type": "heaviside",
+        "boundary": 1.1,
+        "alpha": 0.20
+    },
+    "wall":
+    {
+        "type": "sol_pfr",
+        "boundary": [1.1,0.998],
+        // First value indicates SOL, second the PFR
+        "alpha": [0.10,0.10]
+    },
+}
+@endcode
+@sa dg::geo::modification for possible values of "type" parameter
+ * @param gs forwarded to \c dg::geo::createMagneticField
+ * @param jsmod contains the fields described above to steer the creation of a modification region
  * @param mode Determines behaviour in case of an error
- * @param wall (out) On output contains the region where the wall is applied
- * @param transition (out) On output contains the region where the transition of Psip to a constant value occurs
+ * @param wall (out) On output contains the region where the wall is applied, the functor returns 1 where the wall is, 0 where there it is not and 0<f<1 in the transition region
+ * @param transition (out) On output contains the region where the transition of Psip to a constant value occurs, the functor returns 0<f<=1 for when there is a transition and 0 else
  * @note Per default the dampening happens nowhere
  * @return A magnetic field object
  * @attention This function is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
@@ -232,11 +278,9 @@ static inline TokamakMagneticField createModifiedField( Json::Value gs, Json::Va
         }
     }
 }
-///@}
 
-///@addtogroup profiles
-///@{
 
+/// A convenience function call for \c dg::geo::createModifiedField that ignores the transition parameter and returns the wall
 static inline CylindricalFunctor createWallRegion( Json::Value gs, Json::Value jsmod, dg::file::error mode)
 {
     CylindricalFunctor wall, transition;
@@ -249,14 +293,21 @@ static inline CylindricalFunctor createWallRegion( Json::Value gs, Json::Value j
  *
  * Check if any fieldlines that are not in the wall region intersect the boundary
  * and determine whether the poloidal field points towards or away from the wall
- * @param jsmod must contain the field
- * "sheath": "boundary" value where sheath region begins in units of minor radius a
- * "sheath": "alpha" radius of the transition region where the modification acts in units of minor radius a
+@code
+{
+    "sheath" :
+    {
+        "boundary": 0.1, //value where sheath region begins in uits of minor radius a
+        "alpha": 0.01 // radius of the transition region where the modification acts in units of minor radius a
+    }
+}
+@endcode
+ * @param jsmod most contain the field "sheath" as described above
  * @param mode Determines behaviour in case of an error
- * @param mag (in) the magnetic field to find the direction of the field
+ * @param mag (in) the magnetic field, used to find the direction of the field
  * towards or away from the sheath
  * @param wall (in) the penalization region that represents the actual
- * (perpendicular) wall without the divertor
+ * (perpendicular) wall without the divertor (if 0 on the boundary the boundary will be considered to be a sheath, else the boundary will be ignored)
  * @param R0 left boundary
  * @param R1 right boundary
  * @param Z0 bottom boundary
diff --git a/inc/geometries/modified.h b/inc/geometries/modified.h
index 55513d5ee..18d765f3b 100644
--- a/inc/geometries/modified.h
+++ b/inc/geometries/modified.h
@@ -28,6 +28,14 @@ namespace mod
 ///@addtogroup mod
 ///@{
 
+/**
+ * @brief \f$ \psi_{mod} := \begin{cases} H(\psi_p(R,Z))\text{ for } P(R,Z) \\
+ * \psi_p(R,Z) \text { else }
+ * \end{cases}
+ * \f$
+ *
+ * where H is the integrated dg::IPolynomialHeaviside function and P is a predicate that returns either true or false.
+ */
 struct Psip: public aCylindricalFunctor<Psip>
 {
     Psip( std::function<bool(double,double)> predicate, std::function<double(double,double)> psip, double psi0, double alpha, double sign = -1) :
@@ -151,6 +159,19 @@ struct PsipRZ: public aCylindricalFunctor<PsipRZ>
     std::function<bool(double,double)> m_pred;
 };
 
+
+/**
+ * @copydoc dg::geo::mod::Psip
+ *
+ * @param predicate P(R,Z) indicates the positions where Psi is to be modified (true) or not (false)
+ * @param psip the flux function to be modified
+ * @param psi0 parameter for dg::PolynomialHeaviside function
+ * @param alpha parameter for dg::PolynomialHeaviside function
+ * @param sign parameter for dg::PolynomialHeaviside function
+ * @sa nowhere, everywhere, HeavisideZ
+ *
+ * @return  the modified flux function
+ */
 static inline dg::geo::CylindricalFunctorsLvl2 createPsip(
         const std::function<bool(double,double)> predicate,
         const CylindricalFunctorsLvl2& psip,
diff --git a/inc/geometries/polynomial.h b/inc/geometries/polynomial.h
index 69d1f7149..774231646 100644
--- a/inc/geometries/polynomial.h
+++ b/inc/geometries/polynomial.h
@@ -165,7 +165,7 @@ static inline dg::geo::CylindricalFunctorsLvl1 createIpol( Parameters gp)
  * Based on \c dg::geo::polynomial::Psip(gp) and \c dg::geo::polynomial::Ipol(gp)
  * @param gp Polynomial parameters
  * @return A magnetic field object
- * @ingroup geom
+ * @ingroup polynomial
  */
 static inline dg::geo::TokamakMagneticField createPolynomialField(
     dg::geo::polynomial::Parameters gp)
diff --git a/inc/geometries/polynomial_parameters.h b/inc/geometries/polynomial_parameters.h
index b5eee42b3..9574c6a7a 100644
--- a/inc/geometries/polynomial_parameters.h
+++ b/inc/geometries/polynomial_parameters.h
@@ -14,9 +14,32 @@ namespace geo
 {
 namespace polynomial
 {
+/*! @class hide_polynomial_json
+ * @code
+// Polynomial geometry parameters
+{
+    "equilibrium" : "polynomial",
+    "M" : 8,
+    "N" : 8,
+    "PI" : -1.0,
+    "PP" : -1.0,
+    "R_0" : 906.38,
+    "c" :
+    [
+        -0.96689843290517163,
+        3.0863312163153722,
+        // ... M*N coefficients in total
+    ],
+    "description" : "standardX",
+    "elongation" : 1.5,
+    "inverseaspectratio" : 0.27593818984547458,
+    "triangularity" : 0.40000000000000002
+}
+@endcode
+*/
 /**
  * @brief Constructs and display geometric parameters for the polynomial fields
- * @ingroup geom
+ * @ingroup polynomial
  * @note include \c json/json.h before \c geometries.h in order to activate json functionality
  */
 struct Parameters
@@ -34,9 +57,9 @@ struct Parameters
 #ifdef JSONCPP_VERSION_STRING
     /**
      * @brief Construct from Json dataset
-     * @param js Can contain the variables "M" (1), "N" (1), "c" (0), "PP" (1.), "PI"
-     * (1.), "R_0" , "inverseaspectratio" , "elongation" (1), "triangularity"
-     * (0)
+     * @copydoc hide_polynomial_json
+     * @sa dg::geo::description to see valid values for the %description field
+     * @param js valid Json object (see code above to see the valid key : value pairs)
      * @param mode determine what happens when a key is missing
      * @note the default values in brackets are taken if the variables are not found in the input file
      * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
diff --git a/inc/geometries/solovev.h b/inc/geometries/solovev.h
index 815c56c36..f476d5dd7 100644
--- a/inc/geometries/solovev.h
+++ b/inc/geometries/solovev.h
@@ -371,7 +371,7 @@ static inline dg::geo::CylindricalFunctorsLvl1 createIpol( const Parameters& gp,
  * Based on \c dg::geo::solovev::Psip(gp) and \c dg::geo::solovev::Ipol(gp)
  * @param gp Solovev parameters
  * @return A magnetic field object
- * @ingroup geom
+ * @ingroup solovev
  */
 static inline dg::geo::TokamakMagneticField createSolovevField(
     dg::geo::solovev::Parameters gp)
@@ -384,8 +384,8 @@ static inline dg::geo::TokamakMagneticField createSolovevField(
 /**
  * @brief Create a modified Solovev Magnetic field
  *
- * Based on \c dg::geo::solovev::mod::Psip(gp) and
- * \c dg::geo::solovev::mod::Ipol(gp)
+ * Based on \c dg::geo::mod::Psip(gp) and
+ * \c dg::geo::solovev::Ipol(gp)
  * We modify psi above a certain value to a constant using the
  * \c dg::IPolynomialHeaviside function (an approximation to the integrated Heaviside
  * function with width alpha), i.e. we replace psi with IPolynomialHeaviside(psi).
@@ -396,7 +396,7 @@ static inline dg::geo::TokamakMagneticField createSolovevField(
  * @param alpha radius of the transition region where the modification acts (smaller is quicker)
  * @param sign determines which side of Psi to dampen (negative or positive, forwarded to \c dg::IPolynomialHeaviside)
  * @return A magnetic field object
- * @ingroup geom
+ * @ingroup solovev
  */
 static inline dg::geo::TokamakMagneticField createModifiedSolovevField(
     dg::geo::solovev::Parameters gp, double psi0, double alpha, double sign = -1)
diff --git a/inc/geometries/solovev_parameters.h b/inc/geometries/solovev_parameters.h
index f050b1ece..26af14742 100644
--- a/inc/geometries/solovev_parameters.h
+++ b/inc/geometries/solovev_parameters.h
@@ -14,9 +14,32 @@ namespace geo
 {
 namespace solovev
 {
+/*! @class hide_solovev_json
+ * @code
+// Solovev (and Taylor) geometry parameters
+{
+    "equilibrium": "solovev",
+    // Note that for the taylor field you need to include boost before the geometries header!
+    // "equilibrium" : "taylor",
+    "A": 0,
+    "R_0": 213.36,
+    "PP": 1,
+    "PI": 1,
+    "c":[
+        0.072597888572520090,
+        -0.14926096478076946,
+        // ... 12 coefficients in total
+    ],
+    "description" : "standardX",
+    "inverseaspectratio": 0.3211009174311926,
+    "triangularity": 0.3,
+    "elongation": 1.44
+}
+@endcode
+*/
 /**
  * @brief Constructs and display geometric parameters for the solovev and taylor fields
- * @ingroup geom
+ * @ingroup solovev
  * @note include \c json/json.h before \c geometries.h in order to activate json functionality
  */
 struct Parameters
@@ -33,9 +56,10 @@ struct Parameters
 #ifdef JSONCPP_VERSION_STRING
     /**
      * @brief Construct from Json dataset
-     * @param js Can contain the variables "A" (0), "c" (0), "PP" (1.), "PI"
-     * (1.), "R_0" , "inverseaspectratio" , "elongation" (1), "triangularity"
-     * (0)
+     * @copydoc hide_solovev_json
+     * @sa \c dg::geo::description to see valid values for the %description field
+     * @note the \c dg::geo::taylor field is chosen by setting "taylor" in the equilibrium field (but also note that you need to include boost for the taylor field)
+     * @param js valid Json object (see code above to see the valid key : value pairs)
      * @param mode determine what happens when a key is missing
      * @note the default values in brackets are taken if the variables are not found in the input file
      * @attention This Constructor is only defined if \c json/json.h is included before \c dg/geometries/geometries.h
diff --git a/inc/geometries/taylor.h b/inc/geometries/taylor.h
index 7b1f6adb0..d1a3d6bb9 100644
--- a/inc/geometries/taylor.h
+++ b/inc/geometries/taylor.h
@@ -311,7 +311,7 @@ static inline CylindricalFunctorsLvl1 createIpol( solovev::Parameters gp)
  * Based on \c dg::geo::taylor::Psip(gp) and \c dg::geo::taylor::Ipol(gp)
  * @param gp Solovev parameters
  * @return A magnetic field object
- * @ingroup geom
+ * @ingroup taylor
  * @attention The header \c taylor.h needs to be included seperately and depends on <a href="http://www.boost.org">boost</a>
  */
 static inline dg::geo::TokamakMagneticField createTaylorField( dg::geo::solovev::Parameters gp)
diff --git a/inc/geometries/toroidal.h b/inc/geometries/toroidal.h
index 2d3b0e53f..ef3439443 100644
--- a/inc/geometries/toroidal.h
+++ b/inc/geometries/toroidal.h
@@ -102,7 +102,7 @@ static inline CylindricalFunctorsLvl1 createIpol( double I0 )
  * \f[ \psi_p(R,Z) = 1, \quad I(\psi_p) = 1\f]
  * @param R0 the major radius
  * @return A magnetic field object
- * @ingroup geom
+ * @ingroup toroidal
  * @note The solovev field can also be made to model a todoidal slab field
  */
 static inline dg::geo::TokamakMagneticField createToroidalField( double R0)
@@ -118,7 +118,7 @@ static inline dg::geo::TokamakMagneticField createToroidalField( double R0)
  * @param R0 the major radius
  * @param I0 the current
  * @return A magnetic field object
- * @ingroup geom
+ * @ingroup toroidal
  */
 static inline dg::geo::TokamakMagneticField createCircularField( double R0, double I0)
 {
-- 
GitLab


From 5c77e2437b098f4f308b62095099d4667930c2ba Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 15 Feb 2021 13:30:51 +0100
Subject: [PATCH 502/540] Add formatting to Changelog

---
 CHANGELOG.md | 120 +++++++++++++++++++++++++--------------------------
 1 file changed, 60 insertions(+), 60 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b1f355aa4..2ec8b8577 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,99 +7,99 @@ doxygen documentation, READMEs or tex writeups.
 ## [v5.2]
 ### Added
  - M100 config file
- - json utility functions get, get_idx in json_utilities.h which adds a small abstraction layer that gives a user more control over what happens if a variable is not found
- - json utility functions file2Json, and string2Json in json_utilities.h which adds a small abstraction layer that gives a user more control over what happens if an error happens during the parsing of a file
+ - json utility functions `dg::file::get, dg::file::get_idx` in `dg/file/json_utilities.h` which adds a small abstraction layer that gives a user more control over what happens if a variable is not found
+ - json utility functions `dg::file::file2Json`, and `dg::file::string2Json` in json_utilities.h which adds a small abstraction layer that gives a user more control over what happens if an error happens during the parsing of a file
  - "easy output" netcdf utility functions that are particularly useful for MPI output: either write data in parallel or funnel through the master thread
- - new include files dg/file/file.h, dg/file/json_utilities.h and dg/exblas/exblas.h
- - new class dg::Gradient for gradient and variation
- - new class dg::Advection for the upwind advection scheme
- - new blas1::reduce function for custom reductions
- - new "exchangeable" dg::x::DVec, dg::x::HVec, ..., dg::x::CartesianGrid2d, ..., dg::x::IHMatrix, ... typedefs. The idea is that these resolve to either shared memory or mpi distributed memory versions depending on the MPI_VERSION macro. This helps merging shared and mpi programs into single ones.
- - added "simple" mode to Average computation, which is beneficial for GPU computing
- - add dg::integrate that computes an indefinite integral of a function (essentially the opposite of the derivative)
- - add dg::ModalFilter inclusive tests
- - new compose function that concatenates two or more functors to one
- - add dg::cooRZP2X coordinate transformation functions to easily transform between Cylindrical and Cartesian coordinates
- - interpolate function has an additional dg::space parameter to indicate nodal or modal values
- - Grid classes now have host_vector and host_grid member typedefs
- - new tensor functions dg::tensor::scalar_product2d and dg::scalar_product3d that can compute uE2 in one go
- - new extended tensor functions dg::tensor::multiply2d and dg::multiply3d that can compute uE2 in one go
- - new single step timestepper ShuOsher including new ShuOsherTableau and ConversToShuOsherTableau classes to hold corresponding coefficients
+ - new include files `dg/file/file.h`, `dg/file/json_utilities.h` and `dg/exblas/exblas.h`
+ - new class `dg::Gradient` (and a `dg::Variation` typedef) for gradient and variation
+ - new class `dg::Advection` for the upwind advection scheme
+ - new `dg::blas1::reduce` function for custom reductions
+ - new "exchangeable" `dg::x::DVec`, `dg::x::HVec`, ..., `dg::x::CartesianGrid2d`, ..., `dg::x::IHMatrix`, ... typedefs. The idea is that these resolve to either shared memory or mpi distributed memory versions depending on the MPI_VERSION macro. This helps merging shared and mpi programs into single ones.
+ - added "simple" mode to `dg::Average` computation, which is beneficial for GPU computing
+ - add `dg::integrate` that computes an indefinite integral of a function (essentially the opposite of the derivative)
+ - add` dg::ModalFilter` and tests thereof
+ - new `dg::compose` function that concatenates two or more functors to one
+ - add `dg::cooRZP2X` coordinate transformation functions to easily transform between Cylindrical and Cartesian coordinates
+ - interpolate function has an additional `dg::space` parameter to indicate nodal or modal values
+ - Grid classes now have `host_vector` and `host_grid` member typedefs
+ - new tensor functions `dg::tensor::scalar_product2d` and `dg::scalar_product3d` that can compute uE2 in one go
+ - new extended tensor functions `dg::tensor::multiply2d` and `dg::multiply3d` that can compute uE2 in one go
+ - new single step timestepper `dg::ShuOsher` including new `dg::ShuOsherTableau` and `dg::ConvertsToShuOsherTableau` classes to hold corresponding coefficients
  - new ShuOsher tableaus SSPRK
  - new Runge Kutta embedded tableaus tsitouras, the default timesteppers in Julia
  - new implicit RK tableau trapezoidal, implicit midpoint and sdirk-2-1-2
  - new class Simpsons that implements Simpsons rule for (time) integration
- - new implicit timesteppers DIRKStep and ImplicitRungeKutta
- - implicit time steppers give access to solver
- - Redesign of multistep time steppers consistent with Runge-Kutta ones in terms of MultistepTableau and ConvertsToMultistepTableau
+ - new implicit timesteppers `dg::DIRKStep` and `dg::ImplicitRungeKutta`
+ - Redesign of multistep time steppers consistent with Runge-Kutta ones in terms of `dg::MultistepTableau` and `dg::ConvertsToMultistepTableau`
  - a host of new explicit, implicit and semi-implicit multistep tableaus
  - experimental "filtered" multistep time-steppers that allow modal filtering (first tests are not promising though)
  - new experimental multigrid solvers involving Chebyshev iterations as smoother (but none are better than nested iterations so they remain experimental)
- - new class EVE (courtesy of Eduard Reiter)
- - new class ChebyshevIterations and ChebyshevPreconditioner (for chebyshev iterations)
- - new solvers LGMRES, BICGSTABL, and AndersonAcceleration (courtesy of Aslak Poulsen)
- - new FixedPointSolver and AndersonSolver for nonlinear problems in time
+ - new class `dg::EVE` that computes the largest Eigenvalue of a matrix (courtesy of Eduard Reiter)
+ - new class `dg::ChebyshevIterations` and `dg::ChebyshevPreconditioner` (for chebyshev iterations)
+ - new solvers `dg::LGMRES`, `dg::BICGSTABL`, and d`g::AndersonAcceleration` (courtesy of Aslak Poulsen)
+ - new `dg::FixedPointSolver` and `dg::AndersonSolver` for nonlinear problems in time
  - new class Gradient that computes gradients and variations
  - a host of new functors for the evaluate and pullback functions
- - FluxSurfaceIntegral, FluxVolumeIntegral and SafetyFactorAverage classes
- - new implementation: ds_centered_bc_along_field and dss_centered_bc_along_field that implement boundary condition "Stegmeir" style along the magnetic field lines
- - new Fieldaligned member functions integrate_between_coarse_grid and interpolate_from_coarse_grid that allow field-aligned interpolations
- - dg::geo::Periodify class and dg::geo::periodify function to extend flux-functions periodically beyond grid boundaries
- - new dg::geometries::findCriticalPoint function that generalizes X-point and O-point identification
- - new classes dg::geo::SquareNorm and dg::geo::ScalarProduct that work on cylindrical vector fields
- - new set utility functors dg::geo::SetUnion, dg::geo::SetIntersection, and dg::geo::SetNot that help construct damping regions
- - dg::geo::createMagneticField and dg::geo::createModifiedField with associated utility functions and classes that generalize the creation of magnetic flux functions and wall and sheath regions
- - new polynomial expansion and associated dg::Horner2d functor for magnetic flux functions that can in particular approximate any experimental equilibrium
+ - `dg::geo::FluxSurfaceIntegral`, `dg::geo::FluxVolumeIntegral` and `dg::geo::SafetyFactorAverage` classes
+ - new implementation: `dg::geo::ds_centered_bc_along_field` and `dg::geo::dss_centered_bc_along_field` that implement boundary condition "Stegmeir" style along the magnetic field lines
+ - new Fieldaligned member functions `integrate_between_coarse_grid` and `interpolate_from_coarse_grid` that allow field-aligned interpolations
+ - `dg::geo::Periodify` class and `dg::geo::periodify` function to extend flux-functions periodically beyond grid boundaries
+ - new `dg::geo::findCriticalPoint` function that generalizes X-point and O-point identification
+ - new classes `dg::geo::SquareNorm` and `dg::geo::ScalarProduct` that work on cylindrical vector fields
+ - new set utility functors `dg::geo::SetUnion`, `dg::geo::SetIntersection`, and `dg::geo::SetNot` that help construct damping regions
+ - `dg::geo::createMagneticField` and `dg::geo::createModifiedField` with associated utility functions and classes that generalize the creation of magnetic flux functions and wall and sheath regions
+ - new polynomial expansion and associated `dg::Horner2d` functor for magnetic flux functions that can in particular approximate any experimental equilibrium
  - new equilibrium, modifier and description fields for tokamak magnetic fields
  - Sign reversal of magnetic field and associated flux functions is now possible
 ### Changed
  - namespace file changed to **dg::file** and exblas changed to **dg::exblas** (for consistency reasons, everything should go into the dg namespace, which in particular reduces the chance for name-clashes to just one, namely 'dg')
- - changed file paths **dg/file/file.h**, **dg/geometries/geometries.h** , **dg/file/nc_utilities.h**
- - Moved variation member function into new class Gradient (previously in ArakawaX and Poisson)
- - std=c++14 We use the C++-14 standard now (previously 11)
+ - Moved **variation** member function into new class **dg::Variation** (previously in ArakawaX and Poisson)
+ - **std=c++14** We use the C++-14 standard now (previously 11)
  - vectorclass dependency changed to vectorclass/version1 (previously we used a custom upload on feltor-dev repository)
  - default cuda compute capability bumped to sm-61 (previously sm-35)
  - marconi config now uses jsoncpp module (previously manually installed)
- - blas1::dot and blas2::dot now both do not accumulate rest of multiplication (inconsistent before)
- - swapped input and output parameters in dg::blas1::evaluate first subroutine
+ - `dg::blas1::dot` and `dg::blas2::dot` and corresponding exblas functions now detect NaN and Inf errors
+ - `dg::blas1::dot` and `dg::blas2::dot` now both do not accumulate rest of multiplication (inconsistent before)
+ - all our mpi communications on GPUs now fall-back to host2host communication for cuda-unaware mpi-installations
+ - swapped input and output parameters in `dg::blas1::evaluate` first subroutine
  - the fast_interpolation and fast_projection functions now can also double / divide the polynomial coefficient consistent with the grids
- - change shift_topologic() shared RealTopology member functions to shift() and have an additional negative parameter that indicates sign swaps
+ - change `shift_topologic()` shared RealTopology member functions to `shift()` and have an additional `negative` parameter that indicates sign swaps
  - clarify and unify the behaviour of the interpolation functions when points lie outside the grid boundaries
  - split and join functions have an additional real_type template parameter
- - stopping criterion for bisection1d function
+ - improved stopping criterion for `dg::bisection1d` function
+ - implicit time steppers give access to solver
  - multistep time-stepper now initialize with Runge-Kutta timesteppers of corresponding order
  - Multigrid nested iteration algorithm now allows accuracies for each stage separately (which can give a significant speed-up)
- - dg::inverse( bc) function is now a free-standing function to invert a boundary condition
- - Elliptic classes now have jump_weighting and multiply_sigma functions
- - CG operator now has a test-frequency parameter to control the number of times the error condition is evaluated
- - Extrapolation class now has a derive member function to interpolate the derivative of the interpolating polynomial
- - Adapt all src and diag project to changed file and json utilities and the moved variation member
+ - `dg::inverse( bc)` function is now a free-standing function to invert a boundary condition
+ - `dg::Elliptic` classes now have `jump_weighting` and `multiply_sigma` member functions
+ - `dg::CG` operator now has a `test-frequency` parameter to control the number of times the error condition is evaluated
+ - `dg::Extrapolation` class now has a `derive` member function to interpolate the derivative of the interpolating polynomial
+ - Adapt all src and diag projects to changed file and json utilities and the moved variation member
  - Rename all input files with correct json file-ending
  - Complete redesign of src/feltor and src/lamb_dipole
  - Merge toefl_hpc with old toefl_mpi program
  - bump Doxygen version to 1.8.17
  - DS forward, backward, centered and dss functions are now free-standing, only requiring a fielaligned object, plus, and minus applications (this allows to reduce the number of times the plus and minus interpolation has to be applied)
- - changed Fieldaligned members hp_inv to hbp to give more control
- - dg::forward_transform function (previously dg::create::forward_transform)
- - new dg::geo::MagneticFieldParameters struct to unify the representation of Meta-data in the TokamakMagneticField class (simplifies construction)
+ - changed Fieldaligned members hp_inv to hbp
+ - changed name `dg::forward_transform` function (previously `dg::create::forward_transform`)
+ - new `dg::geo::MagneticFieldParameters` struct to unify the representation of Meta-data in the `dg::geo::TokamakMagneticField` class (simplifies construction)
 
 ### Deprecated
- - Karniadakis time-stepper is now superceded by the ImExMultistep class
+ - `dg::Karniadakis` time-stepper is now superceded by the `dg::ImExMultistep` class
 ### Removed
  - remove diag/feltordiag.cu
  - remove dg::MemoryTraits and associated dimensionality and memory_category traits in favor of direct host_vector and host_grid typedefs in topology classes
  - old txt input files
- - DeltaFunction and Alpha for the computation of flux-surface averages no longer needed
+ - `dg::geo::DeltaFunction` and `dg::geo::Alpha` for the computation of flux-surface averages no longer needed
 ### Fixed
- - Fix bug: race condition in dg::blas1::dot and dg::blas2::dot on GPUs that led to hard to reproduce and seemingly unreasonable crashes
+ - Fix bug: race condition in `dg::blas1::dot` and `dg::blas2::dot` on GPUs that led to hard to reproduce and seemingly unreasonable crashes
  - Fix bug: std namespace in diag/probes.h
- - Fix bug: const in exblas::cpu::get_element 
- - Fix bug: correct  indices in exblas::cpu::make_vcl_vec8d
- - Fix bug: infinite creation of MPI communicators in exblas::mpi_reduce_communicator . Lead to MPI crashes due to memory overflow.
- - dg::blas1::dot and dg::blas2::dot and corresponding exblas functions now detect NaN and Inf errors
- - correct capture of cuda-aware mpi, create a fall-back for cuda-unaware mpi-installations
- - Fix bug: test for no-communication in mpi_communicator (indicated false positives)
- - Fix bug: coefficient and initialization in Extrpolate
+ - Fix bug: const in `exblas::cpu::get_element` 
+ - Fix bug: correct  indices in `exblas::cpu::make_vcl_vec8d`
+ - Fix bug: infinite creation of MPI communicators in `exblas::mpi_reduce_communicator` . Lead to MPI crashes due to memory overflow.
+ - correct capture of cuda-aware mpi, 
+ - Fix bug: test for no-communication in mpi_communicator.h (indicated false positives)
+ - Fix bug: coefficient and initialization in `dg::Extrpolate`
  - Fix bug: Fpsi safety-factor in case nan is encountered still works
  - Fix bug: Fpsi safety-factor works up to the O-point
 
@@ -167,9 +167,9 @@ doxygen documentation, READMEs or tex writeups.
 
 ### Deprecated
 - dg::blas1::transfer (replaced by the more general dg::assign and dg::construct)
-- the header geometries/geometries.h (now dg/geometries/geometries.h is
+- the header geometries/geometries.h (now **dg/geometries/geometries.h** is
   preferred for unified access and easier recognition as a dg header file)
-- the header file/nc\_utilities.h ( now dg/file/nc\_utilities.h is preferred for
+- the header file/nc\_utilities.h ( now **dg/file/nc_utilities.h** is preferred for
   unified access and easier recognition as a dg header file)
 - the Helmholtz2 class (it's much faster to use Helmholtz twice)
 
-- 
GitLab


From 39724c07a0353d8020af98b02531a650d97e33f0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 16 Feb 2021 14:31:26 +0100
Subject: [PATCH 503/540] Fix documentation of volume element and metric

our metric function actually returns the inverse of the metric not the
metric itself
---
 inc/dg/topology/base_geometry.h | 18 ++++++++++--------
 inc/dg/topology/multiply.h      | 12 ++++++++++++
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/inc/dg/topology/base_geometry.h b/inc/dg/topology/base_geometry.h
index fa1d41a6f..fa50d7da4 100644
--- a/inc/dg/topology/base_geometry.h
+++ b/inc/dg/topology/base_geometry.h
@@ -28,14 +28,15 @@ struct aRealGeometry2d : public aRealTopology2d<real_type>
         return do_compute_jacobian();
     }
     /**
-    * @brief The Metric tensor of the coordinate system
+    * @brief The (inverse) metric tensor of the coordinate system
     *
-    *  The elements are the contravariant elements (if x,y are the coordinates)
+    *  The elements of the inverse metric tensor are the contravariant elements
+    *  of the metric \f$g\f$. If x,y are the coordinates, then
     \f[
-    g = \begin{pmatrix} g^{xx}(x,y) & g^{xy}(x,y) \\  & g^{yy}(x,y) \end{pmatrix}
+    g^{-1} = \begin{pmatrix} g^{xx}(x,y) & g^{xy}(x,y) \\  & g^{yy}(x,y) \end{pmatrix}
     \f]
     * @return symmetric tensor
-    * @note use the dg::tensor functions to compute the volume element from here
+    * @note use the \c dg::tensor::volume2d function to compute the volume element from here
     * @note per default this will be the identity tensor
     */
     SparseTensor<thrust::host_vector<real_type> > metric()const {
@@ -103,16 +104,17 @@ struct aRealGeometry3d : public aRealTopology3d<real_type>
         return do_compute_jacobian();
     }
     /**
-    * @brief The (contravariant) metric tensor of the coordinate system
+    * @brief The (inverse) metric tensor of the coordinate system
     *
-    *  The elements are the contravariant elements (if x,y,z are the coordinates)
+    *  The elements of the inverse metric tensor are the contravariant elements
+    *  of the metric \f$g\f$. If x,y,z are the coordinates, then
     \f[
-    g = \begin{pmatrix} g^{xx}(x,y,z) & g^{xy}(x,y,z) & g^{zz}(x,y,z)\\
+    g^{-1} = \begin{pmatrix} g^{xx}(x,y,z) & g^{xy}(x,y,z) & g^{zz}(x,y,z)\\
       & g^{yy}(x,y,z) & g^{yz}(x,y,z) \\
       & & g^{zz}(x,y,z)\end{pmatrix}
     \f]
     * @return symmetric tensor
-    * @note use the dg::tensor functions to compute the volume element from here
+    * @note use the \c dg::tensor::volume function to compute the volume element from here
     * @note per default this will be the identity tensor
     */
     SparseTensor<thrust::host_vector<real_type> > metric()const {
diff --git a/inc/dg/topology/multiply.h b/inc/dg/topology/multiply.h
index 0100aeeb2..00c60ae12 100644
--- a/inc/dg/topology/multiply.h
+++ b/inc/dg/topology/multiply.h
@@ -365,6 +365,12 @@ ContainerType determinant( const SparseTensor<ContainerType>& t)
     ContainerType vol=determinant2d(t);
     dg::blas1::transform(vol, vol, dg::InvSqrt<>());
     @endcode
+ *  @note The function is called volume because when you apply it to the inverse metric
+    tensor of our grids then you obtain the volume
+    \f[ \sqrt{g} = 1 / \sqrt{ \det( g^{-1})}\f]
+    @code
+    ContainerType vol = volume2d( g.metric());
+    @endcode
  * @param t the input tensor
  * @return the inverse square root of the determinant of \c t
  * @copydoc hide_ContainerType
@@ -386,6 +392,12 @@ ContainerType volume2d( const SparseTensor<ContainerType>& t)
     ContainerType vol=determinant(t);
     dg::blas1::transform(vol, vol, dg::InvSqrt<>());
     @endcode
+ *  @note The function is called volume because when you apply it to the inverse metric
+    tensor of our grids then you obtain the volume
+    \f[ \sqrt{g} = 1 / \sqrt{ \det( g^{-1})}\f]
+    @code
+    ContainerType vol = dg::tensor::volume( g.metric());
+    @endcode
  * @param t the input tensor
  * @return the inverse square root of the determinant of \c t
  * @copydoc hide_ContainerType
-- 
GitLab


From 0167741a6fa71251bded88706f1e93d5ebffbd0e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 16 Feb 2021 14:43:42 +0100
Subject: [PATCH 504/540] Fix typedef issue with tIHMatrix in x namespace

---
 inc/dg/topology/interpolation.h  | 13 +++----
 inc/dg/topology/mpi_projection.h | 59 +++++++++++++++-----------------
 2 files changed, 33 insertions(+), 39 deletions(-)

diff --git a/inc/dg/topology/interpolation.h b/inc/dg/topology/interpolation.h
index e419fc5ea..ec562b626 100644
--- a/inc/dg/topology/interpolation.h
+++ b/inc/dg/topology/interpolation.h
@@ -18,19 +18,16 @@ namespace dg{
 ///@addtogroup typedefs
 ///@{
 template<class real_type>
-using tIHMatrix = cusp::csr_matrix<int, real_type, cusp::host_memory>;
+using IHMatrix_t = cusp::csr_matrix<int, real_type, cusp::host_memory>;
 template<class real_type>
-using tIDMatrix = cusp::csr_matrix<int, real_type, cusp::device_memory>;
-using IHMatrix = tIHMatrix<double>;
-using IDMatrix = tIDMatrix<double>;
+using IDMatrix_t = cusp::csr_matrix<int, real_type, cusp::device_memory>;
+using IHMatrix = IHMatrix_t<double>;
+using IDMatrix = IDMatrix_t<double>;
 //typedef cusp::csr_matrix<int, double, cusp::host_memory> IHMatrix; //!< CSR host Matrix
 //typedef cusp::csr_matrix<int, double, cusp::device_memory> IDMatrix; //!< CSR device Matrix
 #ifndef MPI_VERSION
 namespace x{
-template<class real_type>
-using tIHMatrix = tIHMatrix<real_type>;
-template<class real_type>
-using tIDMatrix = tIDMatrix<real_type>;
+//introduce into namespace x
 using IHMatrix = IHMatrix;
 using IDMatrix = IDMatrix;
 } //namespace x
diff --git a/inc/dg/topology/mpi_projection.h b/inc/dg/topology/mpi_projection.h
index 70fcaa8ad..8dfe8b729 100644
--- a/inc/dg/topology/mpi_projection.h
+++ b/inc/dg/topology/mpi_projection.h
@@ -15,18 +15,15 @@ namespace dg
 ///@addtogroup typedefs
 ///@{
 template<class real_type>
-using tMIHMatrix = MPIDistMat< tIHMatrix<real_type>, GeneralComm< dg::iHVec, thrust::host_vector<real_type>> >;
+using MIHMatrix_t = MPIDistMat< IHMatrix_t<real_type>, GeneralComm< dg::iHVec, thrust::host_vector<real_type>> >;
 template<class real_type>
-using tMIDMatrix = MPIDistMat< tIDMatrix<real_type>, GeneralComm< dg::iDVec, thrust::device_vector<real_type>> >;
-using MIHMatrix = tMIHMatrix<double>;
-using MIDMatrix = tMIDMatrix<double>;
+using MIDMatrix_t = MPIDistMat< IDMatrix_t<real_type>, GeneralComm< dg::iDVec, thrust::device_vector<real_type>> >;
+using MIHMatrix = MIHMatrix_t<double>;
+using MIDMatrix = MIDMatrix_t<double>;
 //typedef MPIDistMat< dg::IHMatrix, GeneralComm< dg::iHVec, dg::HVec > > MIHMatrix; //!< MPI distributed CSR host Matrix
 //typedef MPIDistMat< dg::IDMatrix, GeneralComm< dg::iDVec, dg::DVec > > MIDMatrix; //!< MPI distributed CSR device Matrix
 namespace x{
-template<class real_type>
-using tIHMatrix = tMIHMatrix<real_type>;
-template<class real_type>
-using tIDMatrix = tMIDMatrix<real_type>;
+//introduce into namespace x
 using IHMatrix = MIHMatrix;
 using IDMatrix = MIDMatrix;
 } //namespace x
@@ -82,7 +79,7 @@ static void global2bufferIdx( const cusp::array1d<int, cusp::host_memory>& globa
  * @ingroup mpi_structures
  */
 template<class ConversionPolicy, class real_type>
-dg::tMIHMatrix<real_type> convert( const dg::tIHMatrix<real_type>& global, const ConversionPolicy& policy)
+dg::MIHMatrix_t<real_type> convert( const dg::IHMatrix_t<real_type>& global, const ConversionPolicy& policy)
 {
     dg::iHVec unique_global_idx;
     cusp::array1d<int, cusp::host_memory> buffer_idx;
@@ -95,18 +92,18 @@ dg::tMIHMatrix<real_type> convert( const dg::tIHMatrix<real_type>& global, const
         for(unsigned i=0; i<local_idx.size(); i++)
             success = policy.global2localIdx(global.column_indices[i], local_idx[i], pids[i]);
         assert( success);
-        dg::tIHMatrix<real_type> local( global.num_rows, policy.local_size(), global.values.size());
+        dg::IHMatrix_t<real_type> local( global.num_rows, policy.local_size(), global.values.size());
         comm = dg::GeneralComm< dg::iHVec, thrust::host_vector<real_type>>();
         local.row_offsets=global.row_offsets;
         local.column_indices=local_idx;
         local.values=global.values;
-        return dg::tMIHMatrix<real_type>( local, comm, dg::row_dist);
+        return dg::MIHMatrix_t<real_type>( local, comm, dg::row_dist);
     }
-    dg::tIHMatrix<real_type> local( global.num_rows, comm.buffer_size(), global.values.size());
+    dg::IHMatrix_t<real_type> local( global.num_rows, comm.buffer_size(), global.values.size());
     local.row_offsets=global.row_offsets;
     local.column_indices=buffer_idx;
     local.values=global.values;
-    dg::tMIHMatrix<real_type> matrix(   local, comm, dg::row_dist);
+    dg::MIHMatrix_t<real_type> matrix(   local, comm, dg::row_dist);
     return matrix;
 }
 
@@ -118,47 +115,47 @@ namespace create
 
 ///@copydoc dg::create::interpolation(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
-dg::tMIHMatrix<real_type> interpolation( const aRealMPITopology2d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
+dg::MIHMatrix_t<real_type> interpolation( const aRealMPITopology2d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
 {
-    return tMIHMatrix<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
+    return MIHMatrix_t<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
 ///@copydoc dg::create::interpolation(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
-dg::tMIHMatrix<real_type> interpolation( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology3d<real_type>& g_old)
+dg::MIHMatrix_t<real_type> interpolation( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology3d<real_type>& g_old)
 {
-    return tMIHMatrix<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
+    return MIHMatrix_t<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
 ///@copydoc dg::create::interpolation(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
-dg::tMIHMatrix<real_type> interpolation( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
+dg::MIHMatrix_t<real_type> interpolation( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
 {
-    return tMIHMatrix<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
+    return MIHMatrix_t<real_type>( interpolation( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
 
 ///@copydoc dg::create::interpolationT(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
-dg::tMIHMatrix<real_type> interpolationT( const aRealMPITopology2d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
+dg::MIHMatrix_t<real_type> interpolationT( const aRealMPITopology2d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
 {
-    return tMIHMatrix<real_type>( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
+    return MIHMatrix_t<real_type>( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
 ///@copydoc dg::create::interpolationT(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
-dg::tMIHMatrix<real_type> interpolationT( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology3d<real_type>& g_old)
+dg::MIHMatrix_t<real_type> interpolationT( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology3d<real_type>& g_old)
 {
-    return tMIHMatrix<real_type>( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
+    return MIHMatrix_t<real_type>( interpolationT( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
 
 ///@copydoc dg::create::projection(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
-dg::tMIHMatrix<real_type> projection( const aRealMPITopology2d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
+dg::MIHMatrix_t<real_type> projection( const aRealMPITopology2d<real_type>& g_new, const aRealMPITopology2d<real_type>& g_old)
 {
-    return tMIHMatrix<real_type>( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
+    return MIHMatrix_t<real_type>( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
 ///@copydoc dg::create::projection(const RealGrid1d&,const RealGrid1d&)
 template<class real_type>
-dg::tMIHMatrix<real_type> projection( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology3d<real_type>& g_old)
+dg::MIHMatrix_t<real_type> projection( const aRealMPITopology3d<real_type>& g_new, const aRealMPITopology3d<real_type>& g_old)
 {
-    return tMIHMatrix<real_type>( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
+    return MIHMatrix_t<real_type>( projection( g_new.local(), g_old.local()), GeneralComm<iHVec, thrust::host_vector<real_type>>());
 }
 
 /**
@@ -167,9 +164,9 @@ dg::tMIHMatrix<real_type> projection( const aRealMPITopology3d<real_type>& g_new
  * @copydetails interpolation(const thrust::host_vector<real_type>&,const thrust::host_vector<real_type>&,const aRealTopology2d<real_type>&,dg::bc,dg::bc)
  */
 template<class real_type>
-dg::tMIHMatrix<real_type> interpolation( const thrust::host_vector<real_type>& x, const thrust::host_vector<real_type>& y, const aRealMPITopology2d<real_type>& g, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
+dg::MIHMatrix_t<real_type> interpolation( const thrust::host_vector<real_type>& x, const thrust::host_vector<real_type>& y, const aRealMPITopology2d<real_type>& g, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU)
 {
-    dg::tIHMatrix<real_type> mat = dg::create::interpolation( x,y, g.global(), bcx, bcy);
+    dg::IHMatrix_t<real_type> mat = dg::create::interpolation( x,y, g.global(), bcx, bcy);
     return convert(  mat, g);
 }
 
@@ -179,9 +176,9 @@ dg::tMIHMatrix<real_type> interpolation( const thrust::host_vector<real_type>& x
  * @copydetails interpolation(const thrust::host_vector<real_type>&,const thrust::host_vector<real_type>&,const thrust::host_vector<real_type>&,const aRealTopology3d<real_type>&,dg::bc,dg::bc,dg::bc)
  */
 template<class real_type>
-dg::tMIHMatrix<real_type> interpolation( const thrust::host_vector<real_type>& x, const thrust::host_vector<real_type>& y, const thrust::host_vector<real_type>& z, const aRealMPITopology2d<real_type>& g, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU, dg::bc bcz = dg::PER)
+dg::MIHMatrix_t<real_type> interpolation( const thrust::host_vector<real_type>& x, const thrust::host_vector<real_type>& y, const thrust::host_vector<real_type>& z, const aRealMPITopology2d<real_type>& g, dg::bc bcx = dg::NEU, dg::bc bcy = dg::NEU, dg::bc bcz = dg::PER)
 {
-    dg::tIHMatrix<real_type> mat = dg::create::interpolation( x,y,z, g.global(), bcx, bcy, bcz);
+    dg::IHMatrix_t<real_type> mat = dg::create::interpolation( x,y,z, g.global(), bcx, bcy, bcz);
     return convert(  mat, g);
 }
 
-- 
GitLab


From 1804ca3e23e5f9368ee199f3af62ae2b422cc782 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 16 Feb 2021 21:06:09 +0100
Subject: [PATCH 505/540] Fix timestepper in feltor/manufactured.cu

---
 src/feltor/manufactured.cu | 2 +-
 src/lamb_dipole/shu_b.cu   | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/feltor/manufactured.cu b/src/feltor/manufactured.cu
index 3b5a580cf..a28ddf629 100644
--- a/src/feltor/manufactured.cu
+++ b/src/feltor/manufactured.cu
@@ -83,7 +83,7 @@ int main( int argc, char* argv[])
     //    feltor::FeltorSpecialSolver<
     //        dg::CylindricalGrid3d, dg::IDMatrix, dg::DMatrix, dg::DVec>
     //    > karniadakis( grid, p, mag);
-    dg::ExplicitMultistep< std::array<std::array<dg::DVec,2>,2 > > mp("TVB", 3, y0);
+    dg::ExplicitMultistep< std::array<std::array<dg::DVec,2>,2 > > mp("TVB-3-3", y0);
     double time = 0, TMAX = 0.1;
     mp.init( feltor, time, y0, p.dt);
     while( time < TMAX)
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 4894e5c14..36c823eb6 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -4,7 +4,6 @@
 #include <thrust/host_vector.h>
 
 #include "dg/algorithm.h"
-#include "dg/file/json_utilities.h"
 
 #ifndef WITHOUT_GLFW
 #include "draw/host_window.h"
-- 
GitLab


From 3a329c50859ee73773b6b6957f63a1e6028dc1c2 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Tue, 16 Feb 2021 21:08:37 +0100
Subject: [PATCH 506/540] Make blas1 functions be aware of NaN and Inf

---
 inc/dg/blas1.h             | 79 +++++++++++++++++++++++++-------------
 inc/dg/blas1_t.cu          | 38 +++++++++++++++++-
 inc/dg/dg_doc.h            |  2 +-
 inc/dg/subroutines.h       | 12 ++++++
 inc/dg/topology/multiply.h |  8 ++--
 5 files changed, 106 insertions(+), 33 deletions(-)

diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index ae8bb0914..25b8d345b 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -34,6 +34,10 @@ namespace dg{
  */
 namespace blas1
 {
+///@cond
+template< class ContainerType, class BinarySubroutine, class Functor, class ContainerType0, class ...ContainerTypes>
+inline void evaluate( ContainerType& y, BinarySubroutine f, Functor g, const ContainerType0& x0, const ContainerTypes& ...xs);
+///@endcond
 
 ///@addtogroup blas1
 ///@{
@@ -44,6 +48,13 @@ namespace blas1
  * If the vector sizes do not match, the result is undefined.
  * The compiler chooses the implementation and parallelization of this function based on given template parameters. For a full set of rules please refer to \ref dispatch.
  */
+/**
+ * @class hide_naninf
+ * @attention only result vectors that are **write-only** and do not alias
+ * input vectors contain correct results when the result vector contains NaN
+ * or Inf on input. In particular, \c dg::blas1::scal( y, 0 ) does not remove
+ * NaN or Inf from y while \c dg::blas1::copy( 0, y ) does.
+ */
 
 /*! @brief \f$ x^T y\f$ Binary reproducible Euclidean dot product between two vectors
  *
@@ -120,11 +131,12 @@ inline get_value_type<ContainerType> reduce( const ContainerType& x, get_value_t
  * explicit pointwise assignment \f$ y_i = x_i\f$
  * @copydoc hide_iterations
  * @param source vector to copy
- * @param target destination
+ * @param target (write-only) destination
  * @note in contrast to the (deprecated) \c blas1::transfer functions the \c copy function uses
  * the execution policy to determine the implementation and thus works
  * only on types with same execution policy
  * @note catches self-assignment
+ * @copydoc hide_naninf
  * @copydoc hide_ContainerType
  */
 template<class ContainerTypeIn, class ContainerTypeOut>
@@ -144,7 +156,8 @@ dg::DVec two( 100,2);
 dg::blas1::scal( two,  0.5 )); // result[i] = 1.
 @endcode
  * @param alpha Scalar
- * @param x ContainerType x
+ * @param x (read/write) x
+ * @copydoc hide_naninf
  * @copydoc hide_ContainerType
  */
 template< class ContainerType>
@@ -165,7 +178,8 @@ dg::DVec two( 100,2);
 dg::blas1::plus( two,  2. )); // result[i] = 4.
 @endcode
  * @param alpha Scalar
- * @param x ContainerType x
+ * @param x (read/write) x
+ * @copydoc hide_naninf
  * @copydoc hide_ContainerType
  */
 template< class ContainerType>
@@ -188,7 +202,8 @@ dg::blas1::axpby( 2, two, 3., three); // three[i] = 13 (2*2+3*3)
  * @param alpha Scalar
  * @param x ContainerType x may alias y
  * @param beta Scalar
- * @param y ContainerType y contains solution on output
+ * @param y (read/write) ContainerType y contains solution on output
+ * @copydoc hide_naninf
  * @copydoc hide_ContainerType
  */
 template< class ContainerType, class ContainerType1>
@@ -221,7 +236,8 @@ dg::blas1::axpbypgz( 2.5, two, 2., five, -3.,result);
  * @param beta Scalar
  * @param y ContainerType y may alias result
  * @param gamma Scalar
- * @param z ContainerType contains solution on output
+ * @param z (read/write) ContainerType contains solution on output
+ * @copydoc hide_naninf
  * @copydoc hide_ContainerType
  */
 template< class ContainerType, class ContainerType1, class ContainerType2>
@@ -266,33 +282,35 @@ dg::blas1::axpby( 2, two, 3., three, result); // result[i] = 13 (2*2+3*3)
  * @param x ContainerType x may alias z
  * @param beta Scalar
  * @param y ContainerType y may alias z
- * @param z ContainerType z contains solution on output
+ * @param z (write-only) ContainerType z contains solution on output
+ * @copydoc hide_naninf
  * @copydoc hide_ContainerType
  */
 template< class ContainerType, class ContainerType1, class ContainerType2>
 inline void axpby( get_value_type<ContainerType> alpha, const ContainerType1& x, get_value_type<ContainerType> beta, const ContainerType2& y, ContainerType& z)
 {
-    dg::blas1::axpbypgz( alpha, x,  beta, y, 0., z);
+    dg::blas1::evaluate( z , dg::equals(), dg::PairSum(), alpha, x, beta, y);
 }
 
 /**
-* @brief \f$ y = \alpha x_1 x_2 + \beta y\f$
-*
-* Multiplies two vectors element by element: \f[ y_i = \alpha x_{1i}x_{2i} + \beta y_i\f]
-* @copydoc hide_iterations
+ * @brief \f$ y = \alpha x_1 x_2 + \beta y\f$
+ *
+ * Multiplies two vectors element by element: \f[ y_i = \alpha x_{1i}x_{2i} + \beta y_i\f]
+ * @copydoc hide_iterations
 
 @code
 dg::DVec two( 100,2), three( 100,3), result(100,6);
 dg::blas1::pointwiseDot(2., two,  three, -4., result );
 // result[i] = -12. (2*2*3-4*6)
 @endcode
-* @param alpha scalar
-* @param x1 ContainerType x1
-* @param x2 ContainerType x2 may alias x1
-* @param beta scalar
-* @param y  ContainerType y contains result on output ( may alias x1 or x2)
-* @copydoc hide_ContainerType
-*/
+ * @param alpha scalar
+ * @param x1 ContainerType x1
+ * @param x2 ContainerType x2 may alias x1
+ * @param beta scalar
+ * @param y (read/write)  ContainerType y contains result on output ( may alias x1 or x2)
+ * @copydoc hide_naninf
+ * @copydoc hide_ContainerType
+ */
 template< class ContainerType, class ContainerType1, class ContainerType2>
 inline void pointwiseDot( get_value_type<ContainerType> alpha, const ContainerType1& x1, const ContainerType2& x2, get_value_type<ContainerType> beta, ContainerType& y)
 {
@@ -300,6 +318,7 @@ inline void pointwiseDot( get_value_type<ContainerType> alpha, const ContainerTy
         dg::blas1::scal(y, beta);
         return;
     }
+    //not sure this is necessary performance-wise, subroutine does allow aliases
     if( std::is_same<ContainerType, ContainerType1>::value && &x1==(const ContainerType1*)&y){
         dg::blas1::subroutine( dg::AxyPby<get_value_type<ContainerType>>(alpha,beta), x2, y );
 
@@ -324,13 +343,14 @@ dg::blas1::pointwiseDot( two,  three, result ); // result[i] = 6.
 @endcode
 * @param x1 ContainerType x1
 * @param x2 ContainerType x2 may alias x1
-* @param y  ContainerType y contains result on output ( may alias x1 or x2)
+* @param y (write-only) ContainerType y contains result on output ( may alias x1 or x2)
+* @copydoc hide_naninf
 * @copydoc hide_ContainerType
 */
 template< class ContainerType, class ContainerType1, class ContainerType2>
 inline void pointwiseDot( const ContainerType1& x1, const ContainerType2& x2, ContainerType& y)
 {
-    dg::blas1::pointwiseDot( 1., x1, x2, 0., y );
+    dg::blas1::evaluate( y, dg::equals(), dg::PairSum(), x1,x2);
 }
 
 /**
@@ -349,7 +369,8 @@ dg::blas1::pointwiseDot(2., two,  three, four, -4., result );
 * @param x2 ContainerType x2 may alias x1
 * @param x3 ContainerType x3 may alias x1 and/or x2
 * @param beta scalar
-* @param y  ContainerType y contains result on output ( may alias x1,x2 or x3)
+* @param y  (read/write) ContainerType y contains result on output ( may alias x1,x2 or x3)
+* @copydoc hide_naninf
 * @copydoc hide_ContainerType
 */
 template< class ContainerType, class ContainerType1, class ContainerType2, class ContainerType3>
@@ -378,7 +399,8 @@ dg::blas1::pointwiseDivide( 3, two,  three, 5, result );
 * @param x1 ContainerType x1
 * @param x2 ContainerType x2 may alias x1
 * @param beta scalar
-* @param y  ContainerType y contains result on output ( may alias x1 and/or x2)
+* @param y  (read/write) ContainerType y contains result on output ( may alias x1 and/or x2)
+* @copydoc hide_naninf
 * @copydoc hide_ContainerType
 */
 template< class ContainerType, class ContainerType1, class ContainerType2>
@@ -409,13 +431,14 @@ dg::blas1::pointwiseDivide( two,  three, result );
 @endcode
 * @param x1 ContainerType x1
 * @param x2 ContainerType x2 may alias x1
-* @param y  ContainerType y contains result on output ( may alias x1 and/or x2)
+* @param y  (write-only) ContainerType y contains result on output ( may alias x1 and/or x2)
+* @copydoc hide_naninf
 * @copydoc hide_ContainerType
 */
 template< class ContainerType, class ContainerType1, class ContainerType2>
 inline void pointwiseDivide( const ContainerType1& x1, const ContainerType2& x2, ContainerType& y)
 {
-    dg::blas1::pointwiseDivide(1., x1, x2, 0.,y );
+    dg::blas1::evaluate( y, dg::equals(), dg::divides(), x1, x2);
 }
 
 /**
@@ -436,8 +459,9 @@ dg::blas1::pointwiseDot(2., two,  three, -4., four, five, 2., result );
 * @param x2 ContainerType x2
 * @param y2 ContainerType y2
 * @param gamma scalar
-* @param z  ContainerType z contains result on output
+* @param z  (read/write) ContainerType z contains result on output
 * @note all aliases are allowed
+* @copydoc hide_naninf
 * @copydoc hide_ContainerType
 */
 template<class ContainerType, class ContainerType1, class ContainerType2, class ContainerType3, class ContainerType4>
@@ -468,10 +492,11 @@ dg::blas1::transform( two, result, dg::EXP<double>());
 // result[i] = 7.389056... (e^2)
 @endcode
  * @param x ContainerType x may alias y
- * @param y ContainerType y contains result, may alias x
+ * @param y (write-only) ContainerType y contains result, may alias x
  * @param op unary Operator to use on every element
  * @tparam UnaryOp Functor with signature: \c value_type \c operator()( value_type)
  * @note \c UnaryOp must be callable on the device in use. In particular, with CUDA it must be of functor tpye (@b not a function) and its signatures must contain the \__device__ specifier. (s.a. \ref DG_DEVICE)
+ * @copydoc hide_naninf
  * @copydoc hide_ContainerType
  */
 template< class ContainerType, class ContainerType1, class UnaryOp>
@@ -502,6 +527,7 @@ dg::blas1::evaluate( result, dg::equals(), function, pi2, pi3);
  * @param x0 first input
  * @param xs more input
  * @note all aliases allowed
+ * @copydoc hide_naninf
  * @copydoc hide_ContainerType
  *
  */
@@ -564,6 +590,7 @@ except the scalar product, which is not trivially parallel.
  * @tparam Subroutine a function or functor with an arbitrary number of arguments and no return type; taking a \c value_type argument for each input argument in the call
  * and a <tt> value_type&  </tt> argument for each output argument.
  * \c Subroutine must be callable on the device in use. In particular, with CUDA it must be a functor (@b not a function) and its signature must contain the \__device__ specifier. (s.a. \ref DG_DEVICE)
+ * @copydoc hide_naninf
  * @copydoc hide_ContainerType
  */
 template< class Subroutine, class ContainerType, class ...ContainerTypes>
diff --git a/inc/dg/blas1_t.cu b/inc/dg/blas1_t.cu
index 80cb6b4a0..49f5407e0 100644
--- a/inc/dg/blas1_t.cu
+++ b/inc/dg/blas1_t.cu
@@ -23,6 +23,13 @@ int main()
     thrust::device_vector<double> v1p( 500, 2.0002), v2p( 500, 3.00003), v3p(500,5.0005), v4p(500,4.00004);
     Vector v1(v1p), v2(v2p), v3(v3p), v4(v4p), v5(v4p);
     dg::exblas::udouble ud;
+    v3[0] = 1./0.; //we test here if nan breaks code
+    dg::blas1::copy( v3p, v3); ud.d = v3[0];
+    std::cout << "copy (x=x)            ";
+    if ( !std::isfinite( v3[0]))
+        std::cerr << "Error: Result has NaN!\n";
+    else
+        std::cout <<ud.i-4617316080911554445<<std::endl;
     dg::blas1::scal( v3, 3e-10); ud.d = v3[0];
     std::cout << "scal (x=ax)           "<<ud.i-4474825110624711575<<std::endl;
     dg::blas1::plus( v3, 3e-10); ud.d = v3[0];
@@ -31,20 +38,45 @@ int main()
     std::cout << "fma (y=ax+y)          "<<ud.i-4633360230582305548<<std::endl;
     dg::blas1::axpby( 3e-10, v1, -2e-10 , v2); ud.d = v2[0];
     std::cout << "axpby (y=ax+by)       "<<ud.i-4408573477492505937<<std::endl;
+    v5[0] = 1./0.; //we test here if nan breaks code
+    dg::blas1::axpby( 3e-10, v1, -2. , v2, v5); ud.d = v5[0];
+    std::cout << "axpby (z=ax+by)       ";
+    if ( !std::isfinite( v5[0]))
+        std::cerr << "Error: Result has NaN!\n";
+    else
+        std::cout <<ud.i-4468869610430797025<<std::endl;
     dg::blas1::axpbypgz( 2.5, v1, 7.e+10, v2, -0.125, v3); ud.d = v3[0];
     std::cout << "axpbypgz (y=ax+by+gz) "<<ud.i-4617320336812948958<<std::endl;
+    v3[0] = 1./0.; //we test here if nan breaks code
     dg::blas1::pointwiseDot( v1, v2, v3); ud.d = v3[0];
-    std::cout << "pDot (z=xy)           "<<ud.i-4413077932784031586<<std::endl;
+    std::cout << "pDot (z=xy)           ";
+    if ( !std::isfinite( v3[0]))
+        std::cerr << "Error: Result has NaN!\n";
+    else
+        std::cout <<ud.i-4413077932784031586<<std::endl;
     dg::blas1::pointwiseDot( 0.2, v1, v2, +0.4e10, v3); ud.d = v3[0];
     std::cout << "pDot ( z=axy+bz)      "<<ud.i-4556605413983777388<<std::endl;
+    v5 = v4p;
     dg::blas1::pointwiseDot( -0.2, v1, v2, 0.4, v3, v4, 0.1, v5); ud.d = v5[0];
     std::cout << "pDot (z=axy+bfh+gz)   "<<ud.i-4601058031075598447<<std::endl;
     dg::blas1::pointwiseDot( 0.2, v1, v2,v4, 0.4, v3); ud.d = v3[0];
     std::cout << "pDot (z=awxy + bz)    "<<ud.i-4550507856334720009<<std::endl;
+    v5[0] = 1./0.; //we test here if nan breaks code
+    dg::blas1::pointwiseDivide( v1,v2,v5); ud.d = v5[0];
+    std::cout << "pDivide (z=x/y)       ";
+    if ( !std::isfinite( v5[0]))
+        std::cerr << "Error: result has NaN!\n";
+    else
+        std::cout <<ud.i-4810082017219139146<<std::endl;
     dg::blas1::pointwiseDivide( 5.,v1,v2,-1,v3); ud.d = v3[0];
     std::cout << "pDivide (z=ax/y+bz)   "<<ud.i-4820274520177585116<<std::endl;
+    v3[0] = 1./0.; //we test here if nan breaks code
     dg::blas1::transform( v1, v3, dg::EXP<>()); ud.d = v3[0];
-    std::cout << "transform y=exp(x)    "<<ud.i-4620007020034741378<<std::endl;
+    std::cout << "transform y=exp(x)    ";
+    if ( !std::isfinite( v3[0]))
+        std::cerr << "Error: result has NaN!\n";
+    else
+        std::cout <<ud.i-4620007020034741378<<std::endl;;
     }
 
     //v1 = 2, v2 = 3
@@ -74,6 +106,8 @@ int main()
     std::cout << "2.5*2+ 2.*5-0.125*96 = " << w3[0][0] <<" (3)\n";
     dg::blas1::pointwiseDivide( 5.,w1,5.,-1,w3);
     std::cout << "5*2/5-1*3 = " << w3[0][0] <<" (-1)\n";
+    dg::blas1::pointwiseDivide( w1,5.,w3);
+    std::cout << "2/5 = " << w3[0][0] <<" (0.4)\n";
     dg::blas1::copy( w2, w1);
     std::cout << "5 = " << w1[0][0] <<" (5)"<< std::endl;
     dg::blas1::scal( w1, 0.4);
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index ab2227f26..276f51c90 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -135,7 +135,7 @@
  *          @defgroup binary_operators blas1::evaluate binary operators
  *              Binary subroutines for the dg::blas1::evaluate function
  *
- *          @defgroup variadic_evaluates blas1::evaluate variadic subroutines
+ *          @defgroup variadic_evaluates blas1::evaluate variadic functors
  *              Functors to use in the dg::blas1::evaluate function
  *
  *          @defgroup variadic_subroutines blas1::subroutine subroutines
diff --git a/inc/dg/subroutines.h b/inc/dg/subroutines.h
index 9e5fd983a..defb62138 100644
--- a/inc/dg/subroutines.h
+++ b/inc/dg/subroutines.h
@@ -56,6 +56,15 @@ DG_DEVICE void operator()( T1 x, T2& y) const
 
 ///@addtogroup variadic_evaluates
 ///@{
+///\f$ y = x_1/x_2
+struct divides
+{
+    template< class T1, class T2>
+DG_DEVICE T1 operator()( T1 x1, T2 x2) const
+    {
+        return x1/x2;
+    }
+};
 
 ///@brief \f$ y = \sum_i x_i \f$
 struct Sum
@@ -264,11 +273,13 @@ DG_DEVICE void operator()( T x, T y, T& z)const{
         T temp = z*m_b;
         z = DG_FMA( m_a*x, y, temp);
     }
+    ///\f$ y = ax_1x_2x_3 +by \f$
 DG_DEVICE
     void operator()( T x1, T x2, T x3, T& y)const{
         T temp = y*m_b;
         y = DG_FMA( m_a*x1, x2*x3, temp);
     }
+    /// \f$ z = ax_1y_1+bx_2y_2+gz \f$
 DG_DEVICE
     void operator()( T x1, T y1, T x2, T y2, T& z)const{
         T temp = z*m_g;
@@ -285,6 +296,7 @@ template<class T>
 struct PointwiseDivide
 {
     PointwiseDivide( T a, T b): m_a(a), m_b(b){}
+    ///\f$ z = az/y +bz \f$
 DG_DEVICE
     void operator()( T y, T& z)const{
         T temp = z*m_b;
diff --git a/inc/dg/topology/multiply.h b/inc/dg/topology/multiply.h
index 00c60ae12..2e9fb954b 100644
--- a/inc/dg/topology/multiply.h
+++ b/inc/dg/topology/multiply.h
@@ -112,6 +112,10 @@ struct InverseTensorMultiply3d{
               +t02*DG_FMA(t10, t21, (-t20*t11));
     }
 };
+///@}
+
+///@addtogroup variadic_evaluates
+///@{
 
 /// \f$ y = \lambda\mu v_i T_{ij} w_j \f$
 template<class value_type>
@@ -150,10 +154,6 @@ struct TensorDot3d{
         return lambda*mu*DG_FMA(v0,tmp0 , DG_FMA(v1,tmp1 , v2*tmp2));
     }
 };
-///@}
-
-///@addtogroup variadic_evaluates
-///@{
 
 ///\f$ y = t_{00} t_{11} - t_{10}t_{01} \f$
 template<class value_type>
-- 
GitLab


From cfc7c9f547f3a4faf0a94b24dea916ef284ba677 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 12:44:35 +0100
Subject: [PATCH 507/540] Fix bug in pushForwardPerp in transform.h

the error likely only affected hector.h
---
 inc/dg/topology/base_geometryX.h |  2 +-
 inc/dg/topology/transform.h      | 61 ++++++++++++--------
 inc/dg/topology/transform_t.cu   | 98 ++++++++++++++++++++++++++++++++
 3 files changed, 135 insertions(+), 26 deletions(-)
 create mode 100644 inc/dg/topology/transform_t.cu

diff --git a/inc/dg/topology/base_geometryX.h b/inc/dg/topology/base_geometryX.h
index 05190d2b0..955570c4d 100644
--- a/inc/dg/topology/base_geometryX.h
+++ b/inc/dg/topology/base_geometryX.h
@@ -153,7 +153,7 @@ thrust::host_vector<real_type> pullback( const Functor& f, const aRealGeometryX2
     return vec;
 }
 
-///@copydoc pullback(const Functor&,const aRealGeometry2d&)
+///@copydoc pullback(const Functor&,const aRealGeometry3d&)
 ///@ingroup pullback
 template< class Functor, class real_type>
 thrust::host_vector<real_type> pullback( const Functor& f, const aRealGeometryX3d<real_type>& g)
diff --git a/inc/dg/topology/transform.h b/inc/dg/topology/transform.h
index 508271754..1d4a13a38 100644
--- a/inc/dg/topology/transform.h
+++ b/inc/dg/topology/transform.h
@@ -8,9 +8,10 @@
 namespace dg
 {
 /**
- * @brief Pull back a function defined in physical coordinates to the curvilinear (computational) coordinate system
+ * @brief \f$ f_i = f( x(\zeta_i, \eta_i), y(\zeta_i, \eta_i))\f$
  *
- * Pullback is equivalent to the following:
+ * Pull back a function defined in physical coordinates to the curvilinear (computational) coordinate system.
+ * The pullback is equivalent to the following:
  *
  * -# generate the list of physical space coordinates (e.g. in 2d \f$ x_i = x(\zeta_i, \eta_i),\ y_i = y(\zeta_i, \eta_i)\f$ for all \c i) using the map member of the grid e.g. aRealGeometry2d::map()
  * -#  evaluate the given function or functor at these coordinates and store the result in the output vector (e.g. in 2d  \f$ v_i = f(x_i,y_i)\f$ for all \c i)
@@ -38,8 +39,11 @@ thrust::host_vector<real_type> pullback( const Functor& f, const aRealGeometry2d
     return vec;
 }
 
-///@copydoc pullback(const Functor&,const aRealGeometry2d&)
-///@ingroup pullback
+/**
+ * @brief \f$ f_i = f( x(\zeta_i, \eta_i, \nu_i), y(\zeta_i, \eta_i, \nu_i), z(\zeta_i,\eta_i,\nu_i))\f$
+ * @copydetails pullback(const Functor&,const aRealGeometry2d&)
+ * @ingroup pullback
+ */
 template< class Functor, class real_type>
 thrust::host_vector<real_type> pullback( const Functor& f, const aRealGeometry3d<real_type>& g)
 {
@@ -64,8 +68,11 @@ MPI_Vector<thrust::host_vector<real_type> > pullback( const Functor& f, const aR
     return MPI_Vector<thrust::host_vector<real_type> >( vec, g.communicator());
 }
 
-///@copydoc pullback(const Functor&,const aRealGeometry2d&)
-///@ingroup pullback
+/**
+ * @brief \f$ f_i = f( x(\zeta_i, \eta_i, \nu_i), y(\zeta_i, \eta_i, \nu_i), z(\zeta_i,\eta_i,\nu_i))\f$
+ * @copydetails pullback(const Functor&,const aRealGeometry2d&)
+ * @ingroup pullback
+ */
 template< class Functor, class real_type>
 MPI_Vector<thrust::host_vector<real_type> > pullback( const Functor& f, const aRealMPIGeometry3d<real_type>& g)
 {
@@ -79,10 +86,12 @@ MPI_Vector<thrust::host_vector<real_type> > pullback( const Functor& f, const aR
 #endif //MPI_VERSION
 
 /**
- * @brief Push forward a vector from cylindrical or Cartesian to a new coordinate system
+ * @brief \f$ \bar v = J v\f$
  *
- * Computes \f[ v^x(x,y) = x_R (x,y) v^R(R(x,y), Z(x,y)) + x_Z v^Z(R(x,y), Z(x,y)) \\
-                v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
+ * Push forward a vector from cylindrical or Cartesian to a new coordinate system.
+ * Applies the Jacobian matrix \f$ {\bar v} = J  v\f$:
+ * \f[ v^x(x,y) = x_R (x,y) v^R(R(x,y), Z(x,y)) + x_Z v^Z(R(x,y), Z(x,y)) \\
+       v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ...
  * @tparam Functor1 Binary or Ternary functor
  * @tparam Functor2 Binary or Ternary functor
@@ -108,10 +117,12 @@ void pushForwardPerp( const Functor1& vR, const Functor2& vZ,
 }
 
 /**
- * @brief Push forward a vector from cylindrical or Cartesian to a new coordinate system
+ * @brief \f$ {\bar v} = J  v\f$
  *
- * Computes \f[ v^x(x,y) = x_R (x,y) v^R(R(x,y), Z(x,y)) + x_Z v^Z(R(x,y), Z(x,y)) \\
-                v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
+ * Push forward a vector from cylindrical or Cartesian to a new coordinate system.
+ * Applies the Jacobian matrix \f$ {\bar v} = J  v\f$:
+ * \f[ v^x(x,y) = x_R (x,y) v^R(R(x,y), Z(x,y)) + x_Z v^Z(R(x,y), Z(x,y)) \\
+       v^y(x,y) = y_R (x,y) v^R(R(x,y), Z(x,y)) + y_Z v^Z(R(x,y), Z(x,y)) \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ...
  * @tparam Functor1 Binary or Ternary functor
  * @tparam Functor2 Binary or Ternary functor
@@ -142,12 +153,14 @@ void pushForward( const Functor1& vR, const Functor2& vZ, const Functor3& vPhi,
 }
 
 /**
- * @brief Push forward a symmetric 2d tensor from cylindrical or Cartesian to a new coordinate system
+ * @brief \f$ \bar \chi = J \chi J^T\f$
  *
- * Computes \f[
- \chi^{xx}(x,y) = x_R x_R \chi^{RR} + 2x_Rx_Z \chi^{RZ} + x_Zx_Z\chi^{ZZ} \\
- \chi^{xy}(x,y) = x_R x_R \chi^{RR} + (x_Ry_Z+y_Rx_Z) \chi^{RZ} + x_Zx_Z\chi^{ZZ} \\
- \chi^{yy}(x,y) = y_R y_R \chi^{RR} + 2y_Ry_Z \chi^{RZ} + y_Zy_Z\chi^{ZZ} \\
+ * Push forward a symmetric 2d tensor from cylindrical or Cartesian to a new coordinate system.
+ * Applies the Jacobian matrix \f$ \bar \chi = J \chi J^T\f$:
+ *\f[
+ \chi^{xx}(x,y) = x_R^2 \chi^{RR} + 2x_Rx_Z \chi^{RZ} + x_Z^2\chi^{ZZ} \\
+ \chi^{xy}(x,y) = x_Ry_R \chi^{RR} + (x_Ry_Z+y_Rx_Z) \chi^{RZ} + x_Zy_Z\chi^{ZZ} \\
+ \chi^{yy}(x,y) = y_R^2 \chi^{RR} + 2y_Ry_Z \chi^{RZ} + y_Z^2\chi^{ZZ} \\
                \f]
    where \f$ x_R = \frac{\partial x}{\partial R}\f$, ...
  * @tparam FunctorRR Binary or Ternary functor
@@ -183,15 +196,13 @@ void pushForwardPerp( const FunctorRR& chiRR, const FunctorRZ& chiRZ, const Func
     chi.idx(2,2)=1;
     chi.values() = values;
     //we do not need 3rd dimension here
-
     container tmp00(jac.value(0,0)), tmp01(tmp00), tmp10(tmp00), tmp11(tmp00);
-    // multiply Chi*t -> tmp
-    dg::tensor::multiply2d( chi, jac.value(0,0), jac.value(1,0), tmp00, tmp10);
-    dg::tensor::multiply2d( chi, jac.value(0,1), jac.value(1,1), tmp01, tmp11);
-    // multiply tT * tmp -> Chi
-    SparseTensor<container> transpose = jac.transpose();
-    dg::tensor::multiply2d( transpose, tmp00, tmp01, chi.values()[2], chi.values()[3]);
-    dg::tensor::multiply2d( transpose, tmp10, tmp11, chi.values()[3], chi.values()[4]);
+    // multiply Chi*J^T -> tmp ( Matrix-Matrix multiplication: "line x column")
+    dg::tensor::multiply2d( chi, jac.value(0,0), jac.value(0,1), tmp00, tmp10);
+    dg::tensor::multiply2d( chi, jac.value(1,0), jac.value(1,1), tmp01, tmp11);
+    // multiply J * tmp -> Chi
+    dg::tensor::multiply2d( jac, tmp00, tmp10, chi.values()[2], chi.values()[3]);
+    dg::tensor::multiply2d( jac, tmp01, tmp11, chi.values()[3], chi.values()[4]);
 }
 
 namespace create{
diff --git a/inc/dg/topology/transform_t.cu b/inc/dg/topology/transform_t.cu
new file mode 100644
index 000000000..d4ded6771
--- /dev/null
+++ b/inc/dg/topology/transform_t.cu
@@ -0,0 +1,98 @@
+#include <iostream>
+
+
+#include "dg/backend/typedefs.h"
+#include "base_geometry.h"
+#include "transform.h"
+#include "tensor.h"
+
+
+
+struct TestGrid : public dg::aRealGeometry2d<double>
+{
+    using real_type = double;
+
+    TestGrid( ): dg::aRealGeometry2d<real_type>(0.,1.,0.,1.,1,2,2,dg::PER,dg::PER){}
+    virtual TestGrid* clone()const override final{
+        return new TestGrid(*this);
+    }
+    private:
+    virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny) override final{
+        aRealTopology2d<real_type>::do_set(new_n,new_Nx,new_Ny);
+    }
+    virtual dg::SparseTensor<thrust::host_vector<real_type> > do_compute_metric()const override final {
+        dg::SparseTensor<thrust::host_vector<real_type> > metric(*this);
+        metric.values().resize(5);
+        metric.values()[2] = dg::evaluate( dg::CONSTANT(2), *this);
+        metric.values()[3] = dg::evaluate( dg::CONSTANT(3), *this);
+        metric.values()[4] = dg::evaluate( dg::CONSTANT(4), *this);
+        metric.idx( 0,0) = 2;
+        metric.idx( 1,0) = metric.idx(0,1) = 3;
+        metric.idx( 1,1) = 4;
+        return metric;
+        //( 2, 3 )
+        //( 3, 4 )
+    }
+    virtual dg::SparseTensor<thrust::host_vector<real_type> > do_compute_jacobian()const  override final{
+        dg::SparseTensor<thrust::host_vector<real_type> > jac(*this);
+        jac.values().resize(6);
+        jac.values()[2] = dg::evaluate( dg::CONSTANT(0.1), *this);
+        jac.values()[3] = dg::evaluate( dg::CONSTANT(0.2), *this);
+        jac.values()[4] = dg::evaluate( dg::CONSTANT(0.3), *this);
+        jac.values()[5] = dg::evaluate( dg::CONSTANT(0.4), *this);
+        jac.idx(0,0) = 2; jac.idx( 0,1) = 3;
+        jac.idx(1,0) = 4; jac.idx( 1,1) = 5;
+        return jac;
+        //( 0.1, 0.2 )
+        //( 0.3, 0.4 )
+    }
+    virtual std::vector<thrust::host_vector<real_type> > do_compute_map()const override final{
+        std::vector<thrust::host_vector<real_type> > map(2);
+        map[0] = std::vector<real_type>{{0,1,1,0}};
+        map[1] = std::vector<real_type>{{0,0,1,1}};
+        return map;
+    }
+};
+
+double test_function( double x, double y){
+    return ( x*x + y*y);
+}
+
+int main()
+{
+    TestGrid  test;
+    dg::HVec points = dg::pullback( test_function, test);
+    std::cout << "Test transform functionality!\n";
+    std::cout << "Test pullback\n";
+    std::cout << points[0]<< " (0) "<<points[1]<<" (1) "<<points[2]<<" (2) "<<points[3]<< " (1)\n";
+    if( points[0] != 0 || points[1] != 1 || points[2] != 2 || points[3] != 1)
+        std::cerr << "Pullback FAILED\n";
+    else
+        std::cerr << "Pullback PASSED\n";
+    std::cout << "Test pushForwardPerp\n";
+    dg::HVec vx(points), vy(points), vz(points);
+    dg::pushForwardPerp( dg::CONSTANT(4), dg::CONSTANT(5), vx, vy, test);
+    std::cout << vx[0]<< " (1.4) "<<vy[0]<< " (3.2)\n";
+    if( vx[0] - 1.4 > 1e-15 || vy[0] - 3.2 > 1e-15 )
+        std::cerr << "PushForwardPerp FAILED\n";
+    else
+        std::cerr << "PushForwardPerp PASSED\n";
+    std::cout << "Test pushForward\n";
+    dg::pushForward( dg::CONSTANT(4), dg::CONSTANT(5), dg::CONSTANT(6), vx, vy, vz, test);
+    std::cout << vx[0]<< " (1.4) "<<vy[0]<<" (3.2) "<<vz[0]<< " (6)\n";
+    if( vx[0] - 1.4 > 1e-15 || vy[0] - 3.2 > 1e-15 || vz[0] - 6 > 1e-15 )
+        std::cerr << "PushForward FAILED\n";
+    else
+        std::cerr << "PushForward PASSED\n";
+    std::cout << "Test pushForwardPerp Tensor\n";
+    dg::SparseTensor<dg::HVec> tensor;
+    dg::pushForwardPerp( dg::CONSTANT(2), dg::CONSTANT(3), dg::CONSTANT(4), tensor, test);
+    std::cout << tensor.value(0,0)[0]<<" (0.30) "<< tensor.value(0,1)[0]<<" (0.68) "<<std::endl;
+    std::cout << tensor.value(1,0)[0]<<" (0.68) "<< tensor.value(1,1)[0]<<" (1.54)"<<std::endl;
+    if( tensor.value(0,0)[0] - 0.30 > 1e-15 || tensor.value(0,1)[0] - 0.68 > 1e-15 ||
+        tensor.value(1,0)[0] - 0.68 > 1e-15 || tensor.value(1,1)[0] - 1.54 > 1e-15)
+        std::cerr << "PushForwardPerp Tensor FAILED\n";
+    else
+        std::cerr << "PushForwardPerp Tensor PASSED\n";
+    return 0;
+}
-- 
GitLab


From e95f33a05188b8a94829b5bd6ab09dd5f78d03cd Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 14:42:18 +0100
Subject: [PATCH 508/540] Fix old js name in geometries programs to json

---
 inc/geometries/average_t.cu                    | 2 +-
 inc/geometries/conformalX_elliptic_b.cu        | 2 +-
 inc/geometries/conformal_elliptic_b.cu         | 2 +-
 inc/geometries/ds_curv_mpit.cu                 | 2 +-
 inc/geometries/flux_t.cu                       | 2 +-
 inc/geometries/geometryX_elliptic_b.cu         | 4 ++--
 inc/geometries/geometryX_refined_elliptic_b.cu | 4 ++--
 inc/geometries/geometry_elliptic_b.cu          | 2 +-
 inc/geometries/geometry_elliptic_mpib.cu       | 2 +-
 inc/geometries/magnetic_field_t.cu             | 2 +-
 inc/geometries/ribeiroX_t.cu                   | 4 ++--
 inc/geometries/ribeiro_mpit.cu                 | 2 +-
 inc/geometries/ribeiro_t.cu                    | 2 +-
 inc/geometries/separatrix_orthogonal_t.cu      | 2 +-
 inc/geometries/simple_orthogonal_t.cu          | 2 +-
 15 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/inc/geometries/average_t.cu b/inc/geometries/average_t.cu
index 6d4d77e1b..86e32a271 100644
--- a/inc/geometries/average_t.cu
+++ b/inc/geometries/average_t.cu
@@ -15,7 +15,7 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/conformalX_elliptic_b.cu b/inc/geometries/conformalX_elliptic_b.cu
index 0b0183228..cca39771f 100644
--- a/inc/geometries/conformalX_elliptic_b.cu
+++ b/inc/geometries/conformalX_elliptic_b.cu
@@ -88,7 +88,7 @@ int main(int argc, char**argv)
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is( "geometry_params_Xpoint.js");
+        std::ifstream is( "geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/conformal_elliptic_b.cu b/inc/geometries/conformal_elliptic_b.cu
index 50bf2dfdd..a98c90b06 100644
--- a/inc/geometries/conformal_elliptic_b.cu
+++ b/inc/geometries/conformal_elliptic_b.cu
@@ -87,7 +87,7 @@ int main(int argc, char**argv)
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/ds_curv_mpit.cu b/inc/geometries/ds_curv_mpit.cu
index 9979409ac..4e989aa7a 100644
--- a/inc/geometries/ds_curv_mpit.cu
+++ b/inc/geometries/ds_curv_mpit.cu
@@ -40,7 +40,7 @@ int main(int argc, char * argv[])
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/flux_t.cu b/inc/geometries/flux_t.cu
index 36e6d9550..c91f410d7 100644
--- a/inc/geometries/flux_t.cu
+++ b/inc/geometries/flux_t.cu
@@ -46,7 +46,7 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index aa53883a7..d0d338a6f 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -12,8 +12,8 @@
 #include "separatrix_orthogonal.h"
 #include "testfunctors.h"
 
-//const char* parameters = "geometry_params_Xpoint_taylor.js";
-const char* parameters = "geometry_params_Xpoint.js";
+//const char* parameters = "geometry_params_Xpoint_taylor.json";
+const char* parameters = "geometry_params_Xpoint.json";
 
 int main(int argc, char**argv)
 {
diff --git a/inc/geometries/geometryX_refined_elliptic_b.cu b/inc/geometries/geometryX_refined_elliptic_b.cu
index ef231e0a1..0905a6590 100644
--- a/inc/geometries/geometryX_refined_elliptic_b.cu
+++ b/inc/geometries/geometryX_refined_elliptic_b.cu
@@ -8,8 +8,8 @@
 //#include "taylor.h"
 #include "testfunctors.h"
 
-//const char* parameters = "geometry_params_Xpoint_taylor.js";
-const char* parameters = "geometry_params_Xpoint.js";
+//const char* parameters = "geometry_params_Xpoint_taylor.json";
+const char* parameters = "geometry_params_Xpoint.json";
 
 int main(int argc, char**argv)
 {
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index 885a0f527..40977947d 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -23,7 +23,7 @@ int main(int argc, char**argv)
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index ab2bfeb77..5a4cadb1c 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -26,7 +26,7 @@ int main(int argc, char**argv)
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/magnetic_field_t.cu b/inc/geometries/magnetic_field_t.cu
index 2b229c663..45fa27d9c 100644
--- a/inc/geometries/magnetic_field_t.cu
+++ b/inc/geometries/magnetic_field_t.cu
@@ -26,7 +26,7 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index c4cf681a1..6da0e4c95 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -73,8 +73,8 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        //std::ifstream is("geometry_params_Xpoint_taylor.js");
-        std::ifstream is("geometry_params_Xpoint.js");
+        //std::ifstream is("geometry_params_Xpoint_taylor.json");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index a1b50732e..cb4a2bf6e 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -35,7 +35,7 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index d76c4d916..9839250ac 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -47,7 +47,7 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 483e4bcef..8cc3b79f3 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -81,7 +81,7 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index e84621399..946ffd126 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -47,7 +47,7 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
-- 
GitLab


From c975c867d88234a0ffd4857f11350cb4d5408648 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 14:42:54 +0100
Subject: [PATCH 509/540] Fix Hector

some bugs were still from when we modernized the grids ...
Now it does the same as in feltor v3.1 when the paper was published
---
 inc/geometries/geometries_doc.h |   1 +
 inc/geometries/hector.h         | 207 +++++++++++++++++---------------
 inc/geometries/hector_t.cu      |  26 ++--
 3 files changed, 127 insertions(+), 107 deletions(-)

diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index ff4f74d8c..7c69e0d62 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -2,6 +2,7 @@
 /*!
  *
  * @defgroup generators_geo 1. Grid generators
+ * \f$( x,\ y,\ \zeta_x,\ \zeta_y,\ \eta_x,\ \eta_y)\f$ \c dg::geo::Hector
  *
  *      All the grids introduced by this extension can be constructed with
  *      generator classes.
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 643af9c1c..8d8f33b16 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -116,7 +116,7 @@ void compute_zev(
         for( unsigned i=1; i<v_vec.size(); i++)
         {
             temp = end;
-            dg::stepperRK( "Feagin-17-8-10",  iter, v_vec[i-1], begin, v_vec[i], end, steps);
+            dg::stepperRK( "Feagin-17-8-10",  iter, v_vec[i-1], temp, v_vec[i], end, steps);
             eta[i] = end[1];
         }
         temp = end;
@@ -193,10 +193,7 @@ void transform(
     dg::blas1::transfer( u_zeta, uh_zeta);
     dg::blas1::transfer( u_eta, uh_eta);
     dg::SparseTensor<thrust::host_vector<double> > jac = g2d.jacobian();
-    dg::blas1::pointwiseDot( uh_zeta, jac.value(0,0), u_x);
-    dg::blas1::pointwiseDot( 1., uh_eta, jac.value(1,0) , 1., u_x);
-    dg::blas1::pointwiseDot( uh_zeta, jac.value(0,1), u_y);
-    dg::blas1::pointwiseDot( 1., uh_eta, jac.value(1,1), 1., u_y);
+    dg::tensor::multiply2d( jac.transpose(), uh_zeta, uh_eta, u_x, u_y);
 }
 
 }//namespace detail
@@ -205,6 +202,11 @@ void transform(
 /**
  * @brief The High PrEcision Conformal grid generaTOR
  *
+ * @note implements the algorithm described in <a href =
+ * "https://doi.org/10.1016/j.jcp.2017.03.056"> M. Wiesenberger, M. Held, L.
+ * Einkemmer Streamline integration as a method for two-dimensional elliptic
+ * grid generation Journal of Computational Physics 340, 435-450 (2017) </a>
+ *
  * @snippet hector_t.cu doxygen
  * @ingroup generators_geo
  * @tparam IMatrix The interpolation matrix type
@@ -229,18 +231,18 @@ struct Hector : public aGenerator2d
      * @param verbose If true convergence details are printed to std::cout
      */
     Hector( const CylindricalFunctorsLvl2& psi, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) :
-        g2d_(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
+        m_g2d(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
         //first construct u_
-        container u = construct_grid_and_u( dg::geo::Constant(1), dg::geo::detail::LaplacePsi(psi), psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
-        construct( u, psi0, psi1, dg::geo::Constant(1.), dg::geo::Constant(0.), dg::geo::Constant(1.) );
-        conformal_=orthogonal_=true;
+        container u = construct_grid_and_u( dg::geo::Constant(1), dg::geo::detail::LaplacePsi(psi), psi0, psi1, X0, Y0, eps_u , verbose);
+        construct( u, psi0, psi1, dg::geo::Constant(1.), dg::geo::Constant(0.), dg::geo::Constant(1.), verbose);
+        m_conformal=m_orthogonal=true;
         ////we actually don't need u_ but it makes a good testcase
         //container psi__;
-        //dg::blas1::transfer(dg::pullback( psi, g2d_), psi__);
+        //dg::blas1::transfer(dg::pullback( psi, m_g2d), psi__);
         //dg::blas1::axpby( +1., psi__, 1.,  u); //u = c0(\tilde u + \psi-\psi_0)
         //dg::blas1::plus( u,-psi0);
-        //dg::blas1::scal( u, c0_);
+        //dg::blas1::scal( u, m_c0);
         //dg::blas1::transfer( u, u_);
     }
 
@@ -260,20 +262,20 @@ struct Hector : public aGenerator2d
      * @param verbose If true convergence details are printed to std::cout
      */
     Hector( const CylindricalFunctorsLvl2& psi, const CylindricalFunctorsLvl1& chi, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) :
-        g2d_(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
+        m_g2d(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
         dg::geo::detail::LaplaceAdaptPsi lapAdaPsi( psi, chi);
         //first construct u_
-        container u = construct_grid_and_u( chi.f(), lapAdaPsi, psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
-        construct( u, psi0, psi1, chi.f(),dg::geo::Constant(0), chi.f() );
-        orthogonal_=true;
-        conformal_=false;
+        container u = construct_grid_and_u( chi.f(), lapAdaPsi, psi0, psi1, X0, Y0, eps_u , verbose);
+        construct( u, psi0, psi1, chi.f(),dg::geo::Constant(0), chi.f(), verbose );
+        m_orthogonal=true;
+        m_conformal=false;
         ////we actually don't need u_ but it makes a good testcase
         //container psi__;
-        //dg::blas1::transfer(dg::pullback( psi, g2d_), psi__);
+        //dg::blas1::transfer(dg::pullback( psi, m_g2d), psi__);
         //dg::blas1::axpby( +1., psi__, 1.,  u); //u = c0(\tilde u + \psi-\psi_0)
         //dg::blas1::plus( u,-psi0);
-        //dg::blas1::scal( u, c0_);
+        //dg::blas1::scal( u, m_c0);
         //dg::blas1::transfer( u, u_);
     }
 
@@ -294,19 +296,19 @@ struct Hector : public aGenerator2d
      */
     Hector( const CylindricalFunctorsLvl2& psi,const CylindricalSymmTensorLvl1& chi,
             double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) :
-        g2d_(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
+        m_g2d(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
         //first construct u_
         container u = construct_grid_and_u( psi, chi,
-                psi0, psi1, X0, Y0, n, Nx, Ny, eps_u , verbose);
-        construct( u, psi0, psi1, chi.xx(), chi.xy(), chi.yy());
-        orthogonal_=conformal_=false;
+                psi0, psi1, X0, Y0, eps_u , verbose);
+        construct( u, psi0, psi1, chi.xx(), chi.xy(), chi.yy(), verbose);
+        m_orthogonal=m_conformal=false;
         ////we actually don't need u_ but it makes a good testcase
         //container psi__;
-        //dg::blas1::transfer(dg::pullback( psi, g2d_), psi__);
+        //dg::blas1::transfer(dg::pullback( psi, m_g2d), psi__);
         //dg::blas1::axpby( +1., psi__, 1.,  u); //u = c0(\tilde u + \psi-\psi_0)
         //dg::blas1::plus( u,-psi0);
-        //dg::blas1::scal( u, c0_);
+        //dg::blas1::scal( u, m_c0);
         //dg::blas1::transfer( u, u_);
     }
 
@@ -316,18 +318,18 @@ struct Hector : public aGenerator2d
      *
      * @return  orthogonal zeta, eta grid
      */
-    const dg::geo::CurvilinearGrid2d& internal_grid() const {return g2d_;}
+    const dg::geo::CurvilinearGrid2d& internal_grid() const {return m_g2d;}
     virtual Hector* clone() const{return new Hector(*this);}
-    bool isConformal() const {return conformal_;}
+    bool isConformal() const {return m_conformal;}
     private:
-    virtual double do_width() const {return lu_;}
+    virtual double do_width() const {return m_lu;}
     virtual double do_height() const {return 2.*M_PI;}
     /**
      * @brief True if orthogonal constructor was used
      *
      * @return true if orthogonal constructor was used
      */
-    virtual bool do_isOrthogonal() const {return orthogonal_;}
+    virtual bool do_isOrthogonal() const {return m_orthogonal;}
     virtual void do_generate( const thrust::host_vector<double>& u1d,
                      const thrust::host_vector<double>& v1d,
                      thrust::host_vector<double>& x,
@@ -338,20 +340,20 @@ struct Hector : public aGenerator2d
                      thrust::host_vector<double>& vy) const
     {
         thrust::host_vector<double> eta_init, zeta, eta;
-        detail::compute_zev( etaV_, v1d, eta_init, g2d_);
+        detail::compute_zev( m_etaV, v1d, eta_init, m_g2d);
         thrust::host_vector<double> zeta_init( eta_init.size(), 0.);
-        detail::construct_grid( zetaU_, etaU_, u1d, zeta_init, eta_init, zeta, eta, g2d_);
+        detail::construct_grid( m_zetaU, m_etaU, u1d, zeta_init, eta_init, zeta, eta, m_g2d);
         //the box is periodic in eta and the y=0 line needs not to coincide with the eta=0 line
         for( unsigned i=0; i<eta.size(); i++)
             eta[i] = fmod(eta[i]+2.*M_PI, 2.*M_PI);
-        dg::IHMatrix Q = dg::create::interpolation( zeta, eta, g2d_);
-
-        dg::blas2::symv( Q, g2d_.map()[0], x);
-        dg::blas2::symv( Q, g2d_.map()[1], y);
-        dg::blas2::symv( Q, ux_, ux);
-        dg::blas2::symv( Q, uy_, uy);
-        dg::blas2::symv( Q, vx_, vx);
-        dg::blas2::symv( Q, vy_, vy);
+        dg::IHMatrix Q = dg::create::interpolation( zeta, eta, m_g2d);
+
+        dg::blas2::symv( Q, m_g2d.map()[0], x);
+        dg::blas2::symv( Q, m_g2d.map()[1], y);
+        dg::blas2::symv( Q, m_ux, ux);
+        dg::blas2::symv( Q, m_uy, uy);
+        dg::blas2::symv( Q, m_vx, vx);
+        dg::blas2::symv( Q, m_vy, vy);
         ////Test if u1d is u
         //thrust::host_vector<double> u(u1d.size()*v1d.size());
         //dg::blas2::symv( Q, u_, u);
@@ -364,40 +366,44 @@ struct Hector : public aGenerator2d
         //std::cout << "Error in u is "<<eps<<std::endl;
     }
 
-    container construct_grid_and_u( const CylindricalFunctor& chi, const CylindricalFunctor& lapChiPsi, double psi0, double psi1, double X0, double Y0, unsigned n, unsigned Nx, unsigned Ny, double eps_u , bool verbose)
+    container construct_grid_and_u( const CylindricalFunctor& chi, const CylindricalFunctor& lapChiPsi, double psi0, double psi1, double X0, double Y0, double eps_u , bool verbose)
     {
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
-        dg::geo::CurvilinearGrid2d g2d_old = g2d_;
+        dg::geo::CurvilinearGrid2d g2d_old = m_g2d;
         container adapt = dg::pullback(chi, g2d_old);
         dg::Elliptic2d<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
         ellipticD_old.set_chi( adapt);
 
         container u_old = dg::evaluate( dg::zero, g2d_old), u(u_old);
+        dg::CG<container > invert_old( u_old, g2d_old.size());
         container lapu = dg::pullback( lapChiPsi, g2d_old);
-        dg::Invert<container > invert_old( u_old, n*n*Nx*Ny, eps_u);
-        unsigned number = invert_old( ellipticD_old, u_old, lapu);
+        dg::blas2::symv( ellipticD_old.weights(), lapu, lapu);
+        unsigned number = invert_old( ellipticD_old, u_old, lapu, ellipticD_old.precond(), ellipticD_old.inv_weights(), eps_u);
+        if(verbose) std::cout << "Nx "<<m_g2d.Nx()<<" Ny "<<m_g2d.Ny()<<std::flush;
+        if(verbose) std::cout <<" iter "<<number<<" error "<<eps<<"\n";
         while( (eps < eps_old||eps > 1e-7) && eps > eps_u)
         {
             eps = eps_old;
-            g2d_.multiplyCellNumbers(2,2);
-            if(verbose) std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
-            dg::Elliptic2d<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
-            adapt = dg::pullback(chi, g2d_);
+            m_g2d.multiplyCellNumbers(2,2);
+            if(verbose) std::cout << "Nx "<<m_g2d.Nx()<<" Ny "<<m_g2d.Ny()<<std::flush;
+            dg::Elliptic2d<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD( m_g2d, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+            adapt = dg::pullback(chi, m_g2d);
             ellipticD.set_chi( adapt);
-            lapu = dg::pullback( lapChiPsi, g2d_);
-            const container vol2d = dg::create::weights( g2d_);
-            const IMatrix Q = dg::create::interpolation( g2d_, g2d_old);
-            u = dg::evaluate( dg::zero, g2d_);
-            container u_diff( u);
+            const container vol2d = dg::create::weights( m_g2d);
+            const IMatrix Q = dg::create::interpolation( m_g2d, g2d_old);
+            container u_diff = dg::evaluate( dg::zero, m_g2d);
             dg::blas2::gemv( Q, u_old, u_diff);
+            u = u_diff;
 
-            dg::Invert<container > invert( u_diff, n*n*Nx*Ny, 0.1*eps_u);
-            number = invert( ellipticD, u, lapu);
+            dg::CG<container > invert( u_diff, m_g2d.size());
+            lapu = dg::pullback( lapChiPsi, m_g2d);
+            dg::blas2::symv( ellipticD.weights(), lapu, lapu);
+            number = invert( ellipticD, u, lapu, ellipticD.precond(), ellipticD.inv_weights(), 0.1*eps_u);
             dg::blas1::axpby( 1. ,u, -1., u_diff);
             eps = sqrt( dg::blas2::dot( u_diff, vol2d, u_diff) / dg::blas2::dot( u, vol2d, u) );
             if(verbose) std::cout <<" iter "<<number<<" error "<<eps<<"\n";
-            g2d_old = g2d_;
+            g2d_old = m_g2d;
             u_old = u;
             number++;//get rid of warning
         }
@@ -405,12 +411,12 @@ struct Hector : public aGenerator2d
     }
 
     container construct_grid_and_u( const CylindricalFunctorsLvl2& psi,
-            const CylindricalSymmTensorLvl1& chi, double psi0, double psi1, double X0, double Y0, unsigned n, unsigned Nx, unsigned Ny, double eps_u, bool verbose )
+            const CylindricalSymmTensorLvl1& chi, double psi0, double psi1, double X0, double Y0, double eps_u, bool verbose )
     {
         dg::geo::detail::LaplaceChiPsi lapChiPsi( psi, chi);
         //first find u( \zeta, \eta)
         double eps = 1e10, eps_old = 2e10;
-        dg::geo::CurvilinearGrid2d g2d_old = g2d_;
+        dg::geo::CurvilinearGrid2d g2d_old = m_g2d;
         dg::Elliptic2d<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD_old( g2d_old, dg::DIR, dg::PER, dg::not_normed, dg::centered);
         dg::SparseTensor<container> chi_t;
         dg::pushForwardPerp( chi.xx(), chi.xy(), chi.yy(), chi_t, g2d_old);
@@ -418,50 +424,53 @@ struct Hector : public aGenerator2d
         ellipticD_old.set_chi( chi_t);
 
         container u_old = dg::evaluate( dg::zero, g2d_old), u(u_old);
+        dg::CG<container > invert_old( u_old, g2d_old.size());
         container lapu = dg::pullback( lapChiPsi, g2d_old);
-        dg::Invert<container > invert_old( u_old, n*n*Nx*Ny, eps_u);
-        unsigned number = invert_old( ellipticD_old, u_old, lapu);
+        dg::blas2::symv( ellipticD_old.weights(), lapu, lapu);
+        unsigned number = invert_old( ellipticD_old, u_old, lapu, ellipticD_old.precond(), ellipticD_old.inv_weights(), eps_u);
         while( (eps < eps_old||eps > 1e-7) && eps > eps_u)
         {
             eps = eps_old;
-            g2d_.multiplyCellNumbers(2,2);
-            if(verbose)std::cout << "Nx "<<Nx<<" Ny "<<Ny<<std::flush;
-            dg::Elliptic2d<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD( g2d_, dg::DIR, dg::PER, dg::not_normed, dg::centered);
-            dg::pushForwardPerp( chi.xx(), chi.xy(), chi.yy(), chi_t, g2d_);
+            m_g2d.multiplyCellNumbers(2,2);
+            if(verbose) std::cout << "Nx "<<m_g2d.Nx()<<" Ny "<<m_g2d.Ny()<<std::flush;
+            dg::Elliptic2d<dg::geo::CurvilinearGrid2d, Matrix, container> ellipticD( m_g2d, dg::DIR, dg::PER, dg::not_normed, dg::centered);
+            dg::pushForwardPerp( chi.xx(), chi.xy(), chi.yy(), chi_t, m_g2d);
 
             ellipticD.set_chi( chi_t );
-            lapu = dg::pullback( lapChiPsi, g2d_);
-            const container vol2d = dg::create::weights( g2d_);
-            const IMatrix Q = dg::create::interpolation( g2d_, g2d_old);
-            u = dg::evaluate( dg::zero, g2d_);
-            container u_diff( u);
+            const container vol2d = dg::create::weights( m_g2d);
+            const IMatrix Q = dg::create::interpolation( m_g2d, g2d_old);
+            container u_diff = dg::evaluate( dg::zero, m_g2d);
             dg::blas2::gemv( Q, u_old, u_diff);
+            u = u_diff;
 
-            dg::Invert<container > invert( u_diff, n*n*Nx*Ny, 0.1*eps_u);
-            number = invert( ellipticD, u, lapu);
+            dg::CG<container > invert( u_diff, m_g2d.size());
+            lapu = dg::pullback( lapChiPsi, m_g2d);
+            dg::blas2::symv( ellipticD.weights(), lapu, lapu);
+            number = invert( ellipticD, u, lapu, ellipticD.precond(), ellipticD.inv_weights(), 0.1*eps_u);
             dg::blas1::axpby( 1. ,u, -1., u_diff);
             eps = sqrt( dg::blas2::dot( u_diff, vol2d, u_diff) / dg::blas2::dot( u, vol2d, u) );
             if(verbose) std::cout <<" iter "<<number<<" error "<<eps<<"\n";
-            g2d_old = g2d_;
+            g2d_old = m_g2d;
             u_old = u;
             number++;//get rid of warning
         }
         return u;
     }
 
-    void construct(const container& u, double psi0, double psi1, const CylindricalFunctor& chi_XX, const CylindricalFunctor& chi_XY, const CylindricalFunctor& chi_YY)
+    void construct(const container& u, double psi0, double psi1, const CylindricalFunctor& chi_XX, const CylindricalFunctor& chi_XY, const CylindricalFunctor& chi_YY, bool verbose)
     {
         //now compute u_zeta and u_eta
-        Matrix dzeta = dg::create::dx( g2d_, dg::DIR);
-        Matrix deta = dg::create::dy( g2d_, dg::PER);
+        Matrix dzeta = dg::create::dx( m_g2d, dg::DIR);
+        Matrix deta = dg::create::dy( m_g2d, dg::PER);
         container u_zeta(u), u_eta(u);
         dg::blas2::symv( dzeta, u, u_zeta);
-        dg::blas1::plus( u_zeta, (psi1-psi0)/g2d_.lx());
+        dg::blas1::plus( u_zeta, (psi1-psi0)/m_g2d.lx());
         dg::blas2::symv( deta, u, u_eta);
 
 
+
         dg::SparseTensor<container> chi;
-        dg::pushForwardPerp( chi_XX, chi_XY, chi_YY, chi, g2d_);
+        dg::pushForwardPerp( chi_XX, chi_XY, chi_YY, chi, m_g2d);
 
         //now compute ZetaU and EtaU
         container temp_zeta(u), temp_eta(u);
@@ -471,11 +480,11 @@ struct Hector : public aGenerator2d
         container zetaU=temp_zeta, etaU=temp_eta;
         dg::blas1::pointwiseDivide( zetaU, temp_scalar, zetaU);
         dg::blas1::pointwiseDivide(  etaU, temp_scalar,  etaU);
+
         //now compute etaV and its inverse
         container etaVinv(u_zeta), etaV(etaVinv);
-        dg::blas1::pointwiseDot( etaVinv, chi.value(0,0), etaVinv);
-        container perp_vol = dg::tensor::volume(g2d_.metric());
-        dg::blas1::pointwiseDot( etaVinv, perp_vol, etaVinv);
+        container perp_vol = dg::tensor::volume(m_g2d.metric());
+        dg::blas1::pointwiseDot( 1., u_zeta, perp_vol, chi.value(0,0), 0., etaVinv);
         dg::blas1::transform( etaVinv, etaV, dg::INVERT<double>());
         thrust::host_vector<double> etaVinv_h;
         dg::blas1::transfer( etaVinv, etaVinv_h);
@@ -487,30 +496,30 @@ struct Hector : public aGenerator2d
         dg::blas1::pointwiseDot( v_eta, perp_vol, v_eta);
 
         //construct c0 and scale all vector components with it
-        c0_ = fabs( detail::construct_c0( etaVinv_h, g2d_));
-        if( psi1 < psi0) c0_*=-1;
-        lu_ = c0_*(psi1-psi0);
-        //std::cout << "c0 is "<<c0_<<"\n";
-        dg::blas1::scal(  etaV, 1./c0_);
-        dg::blas1::scal( zetaU, 1./c0_);
-        dg::blas1::scal(  etaU, 1./c0_);
-        dg::blas1::scal( u_zeta, c0_);
-        dg::blas1::scal( v_zeta, c0_);
-        dg::blas1::scal(  u_eta, c0_);
-        dg::blas1::scal(  v_eta, c0_);
+        m_c0 = fabs( detail::construct_c0( etaVinv_h, m_g2d));
+        if( psi1 < psi0) m_c0*=-1;
+        m_lu = m_c0*(psi1-psi0);
+        if(verbose) std::cout << "c0 is "<<m_c0<<"\n";
+        dg::blas1::scal(  etaV, 1./m_c0);
+        dg::blas1::scal( zetaU, 1./m_c0);
+        dg::blas1::scal(  etaU, 1./m_c0);
+        dg::blas1::scal( u_zeta, m_c0);
+        dg::blas1::scal( v_zeta, m_c0);
+        dg::blas1::scal(  u_eta, m_c0);
+        dg::blas1::scal(  v_eta, m_c0);
         //transfer to host
-        detail::transform( u_zeta, u_eta, ux_, uy_, g2d_);
-        detail::transform( v_zeta, v_eta, vx_, vy_, g2d_);
-        dg::assign( etaV, etaV_);
-        dg::assign( etaU, etaU_);
-        dg::assign( zetaU, zetaU_);
+        detail::transform( u_zeta, u_eta, m_ux, m_uy, m_g2d);
+        detail::transform( v_zeta, v_eta, m_vx, m_vy, m_g2d);
+        dg::assign( etaV, m_etaV);
+        dg::assign( etaU, m_etaU);
+        dg::assign( zetaU, m_zetaU);
     }
     private:
-    bool conformal_, orthogonal_;
-    double c0_, lu_;
-    thrust::host_vector<double> u_, ux_, uy_, vx_, vy_;
-    thrust::host_vector<double> etaV_, zetaU_, etaU_;
-    dg::geo::CurvilinearGrid2d g2d_;
+    bool m_conformal, m_orthogonal;
+    double m_c0, m_lu;
+    thrust::host_vector<double> m_ux, m_uy, m_vx, m_vy;
+    thrust::host_vector<double> m_etaV, m_zetaU, m_etaU;
+    dg::geo::CurvilinearGrid2d m_g2d;
 
 };
 
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 89e0f0d90..702d13a4b 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -51,7 +51,7 @@ int main( int argc, char* argv[])
     Json::Value js;
     if( argc==1)
     {
-        std::ifstream is("geometry_params_Xpoint.js");
+        std::ifstream is("geometry_params_Xpoint.json");
         is >> js;
     }
     else
@@ -63,14 +63,14 @@ int main( int argc, char* argv[])
     dg::geo::solovev::Parameters gp(js);
     {const dg::geo::CylindricalFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp);
     std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";}
-    std::cout << "Type psi_0 and psi_1\n";
+    std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
     gp.display( std::cout);
     dg::Timer t;
-    //solovev::detail::Fpsi fpsi( gp, -10);
-    std::cout << "Constructing conformal grid ... \n";
-    int construction = 0;
+    std::cout << "Type construction method: 0 (conformal), 1 (adaption), 2 (monitor metric)\n";
+    int construction = 2;
+    std::cin >> construction;
     t.tic();
     //![doxygen]
     std::unique_ptr< dg::geo::aGenerator2d > hector;
@@ -78,17 +78,20 @@ int main( int argc, char* argv[])
     //%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
     if( construction == 0)
     {
+        std::cout << "Constructing conformal grid ... \n";
         hector.reset( new dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>(
                 psip, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true));
     }
     else if( construction == 1)
     {
+        std::cout << "Constructing weighted grid ... \n";
         dg::geo::CylindricalFunctorsLvl1 nc = dg::geo::make_NablaPsiInvCollective( psip);
         hector.reset( new dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>(
                 psip, nc, psi_0, psi_1, gp.R_0, 0., nGrid, NxGrid, NyGrid, epsHector, true));
     }
     else
     {
+        std::cout << "Constructing Monitor metric grid ... \n";
         dg::geo::CylindricalSymmTensorLvl1 lc = dg::geo::make_LiseikinCollective(
                 psip, 0.1, 0.001);
         hector.reset( new dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>(
@@ -165,13 +168,20 @@ int main( int argc, char* argv[])
     std::cout << "TEST VOLUME IS:\n";
     dg::HVec volume = dg::create::volume( g2d);
     dg::HVec ones2d = dg::evaluate( dg::one, g2d);
-    double volumeUV = dg::blas1::dot( vol, ones2d);
+    double volumeUV = dg::blas1::dot( volume, ones2d);
+    std::cout << "volumeUV is "<< volumeUV<<std::endl;
 
     volume = dg::create::volume( dynamic_cast<dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>*>( hector.get())->internal_grid());
     ones2d = dg::evaluate( dg::one, dynamic_cast<dg::geo::Hector<dg::IDMatrix, dg::DMatrix, dg::DVec>*>( hector.get())->internal_grid());
-    double volumeZE = dg::blas1::dot( vol, ones2d);
-    std::cout << "volumeUV is "<< volumeUV<<std::endl;
+    double volumeZE = dg::blas1::dot( volume, ones2d);
     std::cout << "volumeZE is "<< volumeZE<<std::endl;
+    auto iris = dg::compose( dg::Iris(psi_0, psi_1), psip.f());
+    dg::CartesianGrid2d g2dC( gp.R_0 -2.0*gp.a, gp.R_0 + 2.0*gp.a, -2.0*gp.a,2.0*gp.a,3, 2e2, 2e2, dg::PER, dg::PER);
+    dg::HVec vec  = dg::evaluate( iris, g2dC);
+    dg::HVec onesC = dg::evaluate( dg::one, g2dC);
+    dg::HVec g2d_weights = dg::create::volume( g2dC);
+    double volumeRZ = dg::blas2::dot( vec, g2d_weights, onesC);
+    std::cout << "volumeRZ is "<< volumeRZ<<std::endl;
     std::cout << "relative difference in volume is "<<fabs(volumeUV - volumeZE)/volumeZE<<std::endl;
     err = nc_close( ncid);
     return 0;
-- 
GitLab


From 6efdabfe76418e3e5f31d79f8e5460d172547c5c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 14:45:04 +0100
Subject: [PATCH 510/540] Fix missing end equation in subroutines docu

---
 inc/dg/subroutines.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inc/dg/subroutines.h b/inc/dg/subroutines.h
index defb62138..baeecf0c9 100644
--- a/inc/dg/subroutines.h
+++ b/inc/dg/subroutines.h
@@ -56,7 +56,7 @@ DG_DEVICE void operator()( T1 x, T2& y) const
 
 ///@addtogroup variadic_evaluates
 ///@{
-///\f$ y = x_1/x_2
+///\f$ y = x_1/x_2 \f$
 struct divides
 {
     template< class T1, class T2>
-- 
GitLab


From 4d6b1f936a734f2871f178dcb0fd853e5c1cf7fa Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 15:29:32 +0100
Subject: [PATCH 511/540] Change u_ to m_u in hector

---
 inc/geometries/hector.h | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index 8d8f33b16..b16ffd3e2 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -233,17 +233,17 @@ struct Hector : public aGenerator2d
     Hector( const CylindricalFunctorsLvl2& psi, double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) :
         m_g2d(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
-        //first construct u_
+        //first construct m_u
         container u = construct_grid_and_u( dg::geo::Constant(1), dg::geo::detail::LaplacePsi(psi), psi0, psi1, X0, Y0, eps_u , verbose);
         construct( u, psi0, psi1, dg::geo::Constant(1.), dg::geo::Constant(0.), dg::geo::Constant(1.), verbose);
         m_conformal=m_orthogonal=true;
-        ////we actually don't need u_ but it makes a good testcase
+        ////we actually don't need m_u but it makes a good testcase
         //container psi__;
         //dg::blas1::transfer(dg::pullback( psi, m_g2d), psi__);
         //dg::blas1::axpby( +1., psi__, 1.,  u); //u = c0(\tilde u + \psi-\psi_0)
         //dg::blas1::plus( u,-psi0);
         //dg::blas1::scal( u, m_c0);
-        //dg::blas1::transfer( u, u_);
+        //dg::blas1::transfer( u, m_u);
     }
 
     /**
@@ -265,18 +265,18 @@ struct Hector : public aGenerator2d
         m_g2d(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
         dg::geo::detail::LaplaceAdaptPsi lapAdaPsi( psi, chi);
-        //first construct u_
+        //first construct m_u
         container u = construct_grid_and_u( chi.f(), lapAdaPsi, psi0, psi1, X0, Y0, eps_u , verbose);
         construct( u, psi0, psi1, chi.f(),dg::geo::Constant(0), chi.f(), verbose );
         m_orthogonal=true;
         m_conformal=false;
-        ////we actually don't need u_ but it makes a good testcase
+        ////we actually don't need m_u but it makes a good testcase
         //container psi__;
         //dg::blas1::transfer(dg::pullback( psi, m_g2d), psi__);
         //dg::blas1::axpby( +1., psi__, 1.,  u); //u = c0(\tilde u + \psi-\psi_0)
         //dg::blas1::plus( u,-psi0);
         //dg::blas1::scal( u, m_c0);
-        //dg::blas1::transfer( u, u_);
+        //dg::blas1::transfer( u, m_u);
     }
 
     /**
@@ -298,18 +298,18 @@ struct Hector : public aGenerator2d
             double psi0, double psi1, double X0, double Y0, unsigned n = 13, unsigned Nx = 2, unsigned Ny = 10, double eps_u = 1e-10, bool verbose=false) :
         m_g2d(dg::geo::RibeiroFluxGenerator(psi, psi0, psi1, X0, Y0,1), n, Nx, Ny, dg::DIR)
     {
-        //first construct u_
+        //first construct m_u
         container u = construct_grid_and_u( psi, chi,
                 psi0, psi1, X0, Y0, eps_u , verbose);
         construct( u, psi0, psi1, chi.xx(), chi.xy(), chi.yy(), verbose);
         m_orthogonal=m_conformal=false;
-        ////we actually don't need u_ but it makes a good testcase
+        ////we actually don't need m_u but it makes a good testcase
         //container psi__;
         //dg::blas1::transfer(dg::pullback( psi, m_g2d), psi__);
         //dg::blas1::axpby( +1., psi__, 1.,  u); //u = c0(\tilde u + \psi-\psi_0)
         //dg::blas1::plus( u,-psi0);
         //dg::blas1::scal( u, m_c0);
-        //dg::blas1::transfer( u, u_);
+        //dg::blas1::transfer( u, m_u);
     }
 
 
@@ -356,7 +356,7 @@ struct Hector : public aGenerator2d
         dg::blas2::symv( Q, m_vy, vy);
         ////Test if u1d is u
         //thrust::host_vector<double> u(u1d.size()*v1d.size());
-        //dg::blas2::symv( Q, u_, u);
+        //dg::blas2::symv( Q, m_u, u);
         //dg::HVec u2d(u1d.size()*v1d.size());
         //for( unsigned i=0; i<v1d.size(); i++)
         //    for( unsigned j=0; j<u1d.size(); j++)
-- 
GitLab


From 1e9b9fdfa8b5918a710c571e1b6670d7f2d2d750 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 17:06:41 +0100
Subject: [PATCH 512/540] Move variation function to Elliptic

- as suggested by Markus
- this makes the symv function slower by 1 memop per element
---
 diag/compare.cpp            |  10 +-
 diag/feltorSesoldiag.cpp    |   6 +-
 diag/feltorShwdiag.cpp      |   9 +-
 diag/fftwdiag.cpp           |   4 +-
 diag/growthrate.cpp         |   2 +-
 diag/impRdiag.cu            |  25 ++--
 diag/normdiag.cu            |   4 +-
 diag/reco2Ddiag.cu          |   3 +-
 diag/toeflEPdiag.cu         |   4 +-
 diag/toeflRdiag.cu          |   4 +-
 inc/dg/elliptic.h           | 255 +++++++++++++++++++++---------------
 inc/dg/elliptic2d_b.cu      |  34 +++--
 inc/dg/elliptic2d_mpib.cu   |  27 ++--
 inc/dg/elliptic_b.cu        |  23 +++-
 inc/dg/elliptic_mpib.cu     |  18 ++-
 inc/dg/gradient.h           | 135 +++----------------
 inc/dg/gradient_t.cu        |  29 +---
 src/ep/toeflR.cuh           |   6 +-
 src/feltorSH/feltor.cuh     |   4 +-
 src/feltorSHp/feltor.cuh    |   5 +-
 src/feltorSesol/feltor.cuh  |   5 +-
 src/feltorShw/feltor.cuh    |   6 +-
 src/impurities/toeflI.cuh   |   4 +-
 src/reco2D/reconnection.cuh |   6 +-
 src/toefl/toeflR.cuh        |   4 +-
 25 files changed, 296 insertions(+), 336 deletions(-)

diff --git a/diag/compare.cpp b/diag/compare.cpp
index f1b2334ee..dd254de54 100644
--- a/diag/compare.cpp
+++ b/diag/compare.cpp
@@ -2,7 +2,7 @@
 #include <iomanip>
 
 #include "dg/blas1.h"
-#include "file/nc_utilities.h"
+#include "dg/file/nc_utilities.h"
 
 
 //This program reads in two netcdf files and tries to compare the fields 
@@ -37,9 +37,9 @@ int main( int argc, char** argv)
     err = nc_inq_dimid( ncid1, "y", &dimIDs1[1]);
     err = nc_inq_dimid( ncid2, "y", &dimIDs2[1]);
     try{ err = nc_inq_dimid( ncid1, "z", &dimIDs1[2]);}
-    catch( dg::file::NC_Error) { numDims1=2; }
+    catch( dg::file::NC_Error&) { numDims1=2; }
     try{ err = nc_inq_dimid( ncid2, "z", &dimIDs2[2]);}
-    catch( dg::file::NC_Error) { numDims2=2; }
+    catch( dg::file::NC_Error&) { numDims2=2; }
     if( numDims1 != numDims2)
     {
         std::cerr << "Files not of same dimensionality!!\n";
@@ -67,13 +67,13 @@ int main( int argc, char** argv)
         err = nc_inq_varid(ncid1, "electrons", &dataID1);
         err = nc_inq_varid(ncid2, "electrons", &dataID2);
     }
-    catch( dg::file::NC_Error)
+    catch( dg::file::NC_Error&)
     {
         try{
             err = nc_inq_varid(ncid1, "T", &dataID1);
             err = nc_inq_varid(ncid2, "T", &dataID2);
         }
-        catch( dg::file::NC_Error)
+        catch( dg::file::NC_Error&)
         {
             std::cerr <<"Neither electrons nor T found!\n";
             return -1;
diff --git a/diag/feltorSesoldiag.cpp b/diag/feltorSesoldiag.cpp
index a5fb8093f..80ba71a88 100644
--- a/diag/feltorSesoldiag.cpp
+++ b/diag/feltorSesoldiag.cpp
@@ -62,7 +62,7 @@ int main( int argc, char* argv[])
     dg::IHMatrix interp(dg::create::interpolation(xcoo,y0coo,g2d));
     dg::IHMatrix interp_in = dg::create::interpolation(g2d,g2d_in);
     dg::Poisson<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> poisson(g2d,  p.bc_x, p.bc_y,  p.bc_x, p.bc_y);
-    dg::Variation<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> gradient(g2d, p.bc_x, p.bc_y);
+    dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> pol(g2d, p.bc_x, p.bc_y);
 
 
     //2d field
@@ -186,9 +186,9 @@ int main( int argc, char* argv[])
             polavg(logn[1],temp1d,false);
             err_out = nc_put_vara_double( ncid_out, dataIDs1d[3],   start1d, count1d, temp1d.data()); 
             polavg(phi,temp1d,false);
-            gradient.variation(phi,temp2);
+            pol.variation(phi,temp2);
             double T_perp = 0.5*dg::blas2::dot( npe[1], w2d, temp2);
-            gradient.variation(temp,temp2);
+            pol.variation(temp,temp2);
             double T_perp_zonal = 0.5*dg::blas2::dot( npe[1], w2d, temp2);   
             double T_perpratio = T_perp_zonal/T_perp;
             dg::blas2::gemv( poisson.dyrhs(), phi, temp2); 
diff --git a/diag/feltorShwdiag.cpp b/diag/feltorShwdiag.cpp
index a3bde0ee4..e67af5790 100644
--- a/diag/feltorShwdiag.cpp
+++ b/diag/feltorShwdiag.cpp
@@ -69,7 +69,6 @@ int main( int argc, char* argv[])
     dg::IHMatrix interp(dg::create::interpolation(xcoo,y0coo,g2d));
     dg::IHMatrix interp_in = dg::create::interpolation(g2d,g2d_in);
     dg::Poisson<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> poisson(g2d,  p.bc_x, p.bc_y,  p.bc_x_phi, p.bc_y);
-    dg::Variation<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> gradient(g2d, p.bc_x_phi, p.bc_y);
     dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> pol(g2d,   p.bc_x_phi, p.bc_y, dg::normed, dg::centered);
     dg::Elliptic<dg::CartesianGrid2d, dg::HMatrix, dg::HVec> lap(g2d,   p.bc_x, p.bc_y, dg::normed, dg::centered);
     
@@ -252,10 +251,10 @@ int main( int argc, char* argv[])
 		    
                 }
 
-                gradient.variation(phi,temp2);
+                pol.variation(phi,temp2);
                 Tperp = 0.5*dg::blas2::dot( one, w2d, temp2);   // 0.5   u_E^2            
                 polavg(phi,temp);      // <N u_E^2 > 
-                gradient.variation(temp,temp2);
+                pol.variation(temp,temp2);
                 Tperpz = 0.5*dg::blas2::dot( one, w2d, temp2);   //0.5 ( D_x <phi> )^2 
                 Tperpratio = Tperpz/Tperp;
                 dg::blas2::gemv( poisson.dyrhs(), phi, temp2); 
@@ -321,10 +320,10 @@ int main( int argc, char* argv[])
 		    dg::blas1::transform(navgtilde[i], navgtilde[i], dg::PLUS<>(-1.0));
                 }
                                     
-                gradient.variation(phi,temp2);
+                pol.variation(phi,temp2);
                 Tperp = 0.5*dg::blas2::dot( one, w2d, temp2);   // 0.5   u_E^2            
                 polavg(phi,temp);      // <N u_E^2 > 
-                gradient.variation(temp,temp2);
+                pol.variation(temp,temp2);
                 Tperpz = 0.5*dg::blas2::dot( one, w2d, temp2);   //0.5 ( D_x <phi> )^2 
                 Tperpratio = Tperpz/Tperp;
                 dg::blas2::gemv( poisson.dyrhs(), phi, temp2); 
diff --git a/diag/fftwdiag.cpp b/diag/fftwdiag.cpp
index e1bb2c3ff..472fb8664 100644
--- a/diag/fftwdiag.cpp
+++ b/diag/fftwdiag.cpp
@@ -59,8 +59,8 @@ int main( int argc, char* argv[])
     dg::Grid1d g1dx_f( kxmin, kxmax,1., Nkx,  p.bc_x);
     dg::Grid1d g1dy_f( kymin, kymax,1., Nky,  p.bc_y);
     
-    unsigned i_mode = 0;
-    unsigned j_mode = 1*p.sigma;
+    //unsigned i_mode = 0;
+    //unsigned j_mode = 1*p.sigma;
 
     //2d field netcdf vars of input.nc
     size_t count2d[3]  = {1, g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
diff --git a/diag/growthrate.cpp b/diag/growthrate.cpp
index 6131a653c..d39be8af2 100644
--- a/diag/growthrate.cpp
+++ b/diag/growthrate.cpp
@@ -65,7 +65,7 @@ int main( int argc, char* argv[])
     double x2sum=0.;
     double xysum=0.;
     double n=0.;
-    double a=0.;
+    //double a=0.;
     double gamma=0.;
     double b=0.;
     double c=0.;
diff --git a/diag/impRdiag.cu b/diag/impRdiag.cu
index cec06dc6e..fdd40911b 100644
--- a/diag/impRdiag.cu
+++ b/diag/impRdiag.cu
@@ -122,7 +122,6 @@ int main( int argc, char* argv[])
   std::vector<dg::HVec> npe_h(3, dg::evaluate(dg::zero, g2d));
   //eval field
   dg::ArakawaX< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa(g2d);
-  dg::Variation< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient(g2d);
   //eval particle densities
   const dg::DVec binv( dg::evaluate(dg::LinearX(p.kappa, 1.), g2d));
   dg::DVec chi = dg::evaluate(dg::zero, g2d);
@@ -133,18 +132,18 @@ int main( int argc, char* argv[])
   //calculation variables per species
   double mass_[num_species] = {}, cn[num_species] = {};
   double posX = 0, posY =0 ;
-  double NposX = 0, NposY =0 ; //
+  double NposX = 0;//, NposY =0 ; //
   double posX_init[num_species] = {}, posY_init[num_species] = {};
-  double NposX_init[num_species] = {}, NposY_init[num_species] = {}; //
+  double NposX_init[num_species] = {};//, NposY_init[num_species] = {}; //
   double posX_old[num_species] ={}, posY_old[num_species] = {};
-  double NposX_old[num_species] ={}, NposY_old[num_species] = {};
+  double NposX_old[num_species] ={};//, NposY_old[num_species] = {};
   double posX_max = 0, posY_max = 0;
   double posX_max_old[num_species] = {}, posY_max_old[num_species] = {};
   double posX_max_hs = 0, posY_max_hs = 0;
   double velX[num_species] = {}, velY[num_species] = {};
-  double NvelX[num_species] = {}, NvelY[num_species] = {};
+  double NvelX[num_species] = {};//, NvelY[num_species] = {};
   double velX_old[num_species] = {}, velY_old[num_species] = {};
-  double NvelX_old[num_species] = {}, NvelY_old[num_species] = {};
+  //double NvelX_old[num_species] = {}, NvelY_old[num_species] = {};
   double velX_max = 0, velY_max = 0;
   double velCOM = 0;
   double accX = 0, accY = 0;
@@ -279,20 +278,20 @@ int main( int argc, char* argv[])
       // N*posX/Y, N*velX/Y
       if (i==0)
       { NposX_init[j] = dg::blas2::dot( xvec, w2d, ntilde[j]);
-        NposY_init[j] = dg::blas2::dot( yvec, w2d, ntilde[j]);
+        //NposY_init[j] = dg::blas2::dot( yvec, w2d, ntilde[j]);
       }
       if (i>0)
       { NposX = dg::blas2::dot( xvec, w2d, ntilde[j])-NposX_init[j];
-        NposY = dg::blas2::dot( yvec, w2d, ntilde[j])-NposY_init[j];
+        //NposY = dg::blas2::dot( yvec, w2d, ntilde[j])-NposY_init[j];
       }
       if (i==0)
-      { NvelX_old[j] = -NposX/deltaT;
-        NvelY_old[j] = -NposY/deltaT;
+      { //NvelX_old[j] = -NposX/deltaT;
+        //NvelY_old[j] = -NposY/deltaT;
         NposX_old[j] = NposX;
-        NposY_old[j] = NposY;
+        //NposY_old[j] = NposY;
       }
       NvelX[j] = (NposX - NposX_old[j])/deltaT;
-      NvelY[j] = (NposY - NposY_old[j])/deltaT;
+      //NvelY[j] = (NposY - NposY_old[j])/deltaT;
 
       // init pos/vel calculations as soon as mass > 1.E-15, otherwise NAN
       if ( mass_[j]>1.E-15 && (reset_flag[j]))
@@ -374,7 +373,7 @@ int main( int argc, char* argv[])
       m[j]++;
     }
     //field
-    gradient.variation(field[0], helper);
+    pol.variation(field[0], helper);
     double energy[5] = {};
     energy[0] = dg::blas2::dot(lnn[0], w2d, npe[0]);
     energy[1] = p.a[1]*p.tau[1]*dg::blas2::dot(npe[1], w2d, lnn[1]);
diff --git a/diag/normdiag.cu b/diag/normdiag.cu
index 32b8bbdc2..6d4604f5d 100644
--- a/diag/normdiag.cu
+++ b/diag/normdiag.cu
@@ -61,7 +61,7 @@ int main( int argc, char* argv[])
     dg::DVec nG(dg::evaluate(prof,g2d));
     dg::DVec w2d = dg::create::weights( g2d);
     dg::Poisson<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> poisson(g2d,  p.bc_x, p.bc_y,  p.bc_x_phi, p.bc_y);
-    dg::Variation<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient(g2d, p.bc_x_phi, p.bc_y);
+    dg::Elliptic<dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol(g2d, p.bc_x_phi, p.bc_y);
     //open netcdf files
     err = nc_open( argv[1], NC_NOWRITE, &ncid);
     //set min and max timesteps
@@ -163,7 +163,7 @@ int main( int argc, char* argv[])
         }
         
 
-        gradient.variation(phi,uE2);
+        pol.variation(phi,uE2);
         uE2norm= 0.5*dg::blas2::dot( one, w2d,uE2);   // 0.5   u_E^2    
         nlnnnorm = dg::blas2::dot(ne,w2d,logne);
         NiuE2norm = 0.5*dg::blas2::dot(Ni, w2d,uE2);
diff --git a/diag/reco2Ddiag.cu b/diag/reco2Ddiag.cu
index 69bfc8b5c..d83137610 100644
--- a/diag/reco2Ddiag.cu
+++ b/diag/reco2Ddiag.cu
@@ -65,7 +65,7 @@ int main( int argc, char* argv[])
     steps-=1;
     imax = steps/p.itstp;
     double deltaT = p.dt*p.itstp;     //define timestep
-    double apar0=0.,gamma=0.,psiX=0.,logpsiX=0.,logpsiXold=0.;
+    double gamma=0.,psiX=0.,logpsiX=0.,logpsiXold=0.;
 
     for( unsigned i=imin; i<=imax; i++)//timestepping
     {
@@ -84,7 +84,6 @@ int main( int argc, char* argv[])
         dg::blas2::gemv(interp, apar, aparx0y0);
             
         
-//         apar0=aparx0y0[0];
         dg::blas2::gemv(interp, apareq, apareqx0y0);
             
 //            std::cout << time << " " <<  aparx0y0[0]<< " " << apareqx0y0[0] <<"\n";
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index 4b410c681..b03a6e9e1 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -81,7 +81,7 @@ int main( int argc, char* argv[])
     dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
     dg::Grid1d g1d( 0., p.lx, p.n_out, p.Nx_out, p.bc_x);
     dg::ArakawaX< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa( g2d); 
-    dg::Variation< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient( g2d); 
+    dg::Elliptic< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol( g2d); 
     double time = 0.;
     //2d field
     size_t count2d[3]  = {1, g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
@@ -279,7 +279,7 @@ int main( int argc, char* argv[])
         double Ue, Ui, Uphi;
         for( unsigned j=0; j<2; j++)
             dg::blas1::transform( npe[j], lnn[j], dg::LN<double>()); 
-        gradient.variation(phi, helper); 
+        pol.variation(phi, helper); 
         Ue = p.z[0]*p.tau[0]*dg::blas2::dot( lnn[0], w2d, npe[0]);
         Ui = p.z[1]*p.tau[1]*dg::blas2::dot( lnn[1], w2d, npe[1]);
         Uphi = 0.5*dg::blas2::dot( npe[1], w2d, helper); 
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index 332192ece..0dca27fff 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -76,7 +76,7 @@ int main( int argc, char* argv[])
     dg::Grid2d g2d( 0., p.lx, 0.,p.ly, p.n_out, p.Nx_out, p.Ny_out, p.bc_x, p.bc_y);
     dg::Grid1d g1d( 0., p.lx, p.n_out, p.Nx_out, p.bc_x);
     dg::ArakawaX< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> arakawa( g2d); 
-    dg::Variation< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> gradient( g2d); 
+    dg::Elliptic< dg::CartesianGrid2d, dg::DMatrix, dg::DVec> pol( g2d); 
     double time = 0.;
     //2d field
     size_t count2d[3]  = {1, g2d.n()*g2d.Ny(), g2d.n()*g2d.Nx()};
@@ -274,7 +274,7 @@ int main( int argc, char* argv[])
         double Ue, Ui, Uphi;
         for( unsigned j=0; j<2; j++)
             dg::blas1::transform( npe[j], lnn[j], dg::LN<double>()); 
-        gradient.variation(phi, helper);
+        pol.variation(phi, helper);
         if(p.equations == "global" || p.equations == "ralf_global")
         {
             Ue = dg::blas2::dot( lnn[0], w2d, npe[0]);
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index e59327ae6..1d00274ef 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -19,44 +19,47 @@ namespace dg
 {
 
 /**
- * @brief A 2d negative elliptic differential operator \f$ -\nabla \cdot ( \chi \nabla ) \f$
+ * @brief A 2d negative elliptic differential operator \f$ -\nabla \cdot ( \mathbf{\chi}\cdot \nabla ) \f$
  *
  * @ingroup matrixoperators
  *
- * The term discretized is \f[ -\nabla \cdot ( \chi \nabla ) \f]
- * where \f$ \nabla \f$ is the two-dimensional nabla and \f$\chi\f$ is a
- * (possibly spatially dependent) tensor.
- * In general coordinates that means
+ * The term discretized is \f[ -\nabla \cdot ( \mathbf{\chi} \cdot \nabla ) \f] where \f$
+ * \nabla \f$ is the two-dimensional nabla and \f$\chi = \sigma
+ * \mathbf{\tau}\f$ is a (possibly spatially dependent) tensor with scalar part
+ * \f$ \sigma\f$ (usually the volume form) and tensor part \f$ \tau\f$ (usually
+ * the inverse metric). In general coordinates that means
  * \f[ -\frac{1}{\sqrt{g}}\left(
  * \partial_x\left(\sqrt{g}\left(\chi^{xx}\partial_x + \chi^{xy}\partial_y \right)\right)
  + \partial_y\left(\sqrt{g} \left(\chi^{yx}\partial_x + \chi^{yy}\partial_y \right)\right) \right)\f]
  is discretized.
- Per default, \f$ \chi\f$ is the metric tensor but you can set it to any tensor
+ Per default, \f$ \chi = \sqrt{g} g^{-1}\f$ but you can set it to any tensor
  you like (in order for the operator to be invertible \f$\chi\f$ should be
  symmetric and positive definite though).
+
  Note that the local discontinuous Galerkin discretization adds so-called jump terms
  \f[ D^\dagger \chi D + \alpha \chi_{on/off} J \f]
  where \f$\alpha\f$  is a scale factor ( = jfactor), \f$ D \f$ contains the discretizations of the above derivatives, and \f$ J\f$ is a self-adjoint matrix.
  (The symmetric part of \f$J\f$ is added @b before the volume element is divided). The adjoint of a matrix is defined with respect to the volume element including dG weights.
  Usually, the default \f$ \alpha=1 \f$ is a good choice.
  However, in some cases, e.g. when \f$ \chi \f$ exhibits very large variations
- \f$ \alpha=0.1\f$ or \f$ \alpha=0.01\f$ might be better values. 
+ \f$ \alpha=0.1\f$ or \f$ \alpha=0.01\f$ might be better values.
  In a time dependent problem the value of \f$\alpha\f$ determines the
- numerical diffusion, i.e. for too low values numerical oscillations may appear. 
+ numerical diffusion, i.e. for too low values numerical oscillations may appear.
  The \f$ \chi_{on/off} \f$ in the jump term serves to weight the jump term with \f$ \chi \f$. This can be switched either on or off with off being the default.
  Also note that a forward discretization has more diffusion than a centered discretization.
 
  The following code snippet demonstrates the use of \c Elliptic in an inversion problem
  * @snippet elliptic2d_b.cu invert
  * @copydoc hide_geometry_matrix_container
- * This class has the \c SelfMadeMatrixTag so it can be used in blas2::symv functions
+ * This class has the \c SelfMadeMatrixTag so it can be used in \c blas2::symv functions
  * and thus in a conjugate gradient solver.
- * @note The constructors initialize \f$ \chi=g\f$ so that a negative laplacian operator
- * results
- * @note The inverse of \f$ \chi\f$ makes a good general purpose preconditioner
+ * @note The constructors initialize \f$ \chi=\sqrt{g}g^{-1}\f$ so that a
+ * negative laplacian operator results
+ * @note The inverse of \f$ \sigma\f$ makes a good general purpose preconditioner
  * @note the jump term \f$ \alpha J\f$  adds artificial numerical diffusion as discussed above
+ * @note Since the pattern arises quite often (because of the ExB velocity \f$ u_E^2\f$ in the ion gyro-centre potential)
+ * this class also can compute the variation integrand \f$ \lambda^2\nabla \phi\cdot \chi\cdot\nabla\phi\f$
  * @attention Pay attention to the negative sign which is necessary to make the matrix @b positive @b definite
- *
  */
 template <class Geometry, class Matrix, class Container>
 class Elliptic
@@ -117,10 +120,8 @@ class Elliptic
         dg::assign( dg::create::inv_weights(g),   m_precond);
         m_temp = m_tempx = m_tempy = m_inv_weights;
         m_chi=g.metric();
-        m_vol=dg::tensor::volume(m_chi);
-        dg::tensor::scal( m_chi, m_vol);
+        m_sigma = m_vol = dg::tensor::volume(m_chi);
         dg::assign( dg::create::weights(g), m_weights_wo_vol);
-        dg::assign( dg::evaluate(dg::one, g), m_sigma);
     }
 
     /**
@@ -136,26 +137,47 @@ class Elliptic
         *this = Elliptic( std::forward<Params>( ps)...);
     }
 
-    ///@copydoc Elliptic3d::set_chi(const ContainerType0&)
+    /**
+     * @brief Change scalar part in Chi tensor
+     *
+     * Internally, we split the tensor \f$\chi = \sigma\mathbf{\tau}\f$ into
+     * a scalar part \f$ \sigma\f$ and a tensor part \f$ \tau\f$ and you can
+     * set each part seperately. This functions sets the scalar part.
+     *
+     * @param sigma The new scalar part in \f$\chi\f$
+     * @note The class will take care of the volume element in the divergence so do not multiply it to \c sigma yourself
+     * @attention If some or all elements of sigma are zero the preconditioner
+     * is invalidated and the operator can no longer be inverted. The symv
+     * function can still be called however.
+     * @tparam ContainerType0 must be usable with \c Container in \ref dispatch
+     */
     template<class ContainerType0>
     void set_chi( const ContainerType0& sigma)
     {
-        dg::blas1::pointwiseDivide( sigma, m_sigma, m_tempx);
+        dg::blas1::pointwiseDot( sigma, m_vol, m_sigma);
         //update preconditioner
-        dg::blas1::pointwiseDivide( m_precond, m_tempx, m_precond);
-        dg::tensor::scal( m_chi, m_tempx);
-        dg::blas1::copy( sigma, m_sigma);
+        dg::blas1::pointwiseDivide( m_inv_weights, sigma, m_precond);
+        // sigma is possibly zero, which will invalidate the preconditioner
+        // it is important to call this blas1 function because it can
+        // overwrite NaN in m_precond in the next update
     }
     /**
-     * @copydoc Elliptic3d::set_chi(const SparseTensor<ContainerType0>&)
-     * @note the 3d parts in \c tau will be ignored
+     * @brief Change tensor part in Chi tensor
+     *
+     * We split the tensor \f$\chi = \sigma\mathbf{\tau}\f$ into
+     * a scalar part \f$ \sigma\f$ and a tensor part \f$ \tau\f$ and you can
+     * set each part seperately. This functions sets the tensor part.
+     *
+     * @note The class will take care of the volume element in the divergence so do not multiply it to \c tau yourself
+     *
+     * @param tau The new tensor part in \f$\chi\f$ (must be positive definite)
+     * @note the 3d parts in \c tau will be ignored for 2d computations
+     * @tparam ContainerType0 must be usable in \c dg::assign to \c Container
      */
     template<class ContainerType0>
     void set_chi( const SparseTensor<ContainerType0>& tau)
     {
         m_chi = SparseTensor<Container>(tau);
-        dg::tensor::scal( m_chi, m_sigma);
-        dg::tensor::scal( m_chi, m_vol);
     }
 
     /**
@@ -179,9 +201,9 @@ class Elliptic
     /**
      * @brief Return the default preconditioner to use in conjugate gradient
      *
-     * Currently returns the inverse weights without volume elment divided by the scalar part of \f$ \chi\f$.
-     * This is especially good when \f$ \chi\f$ exhibits large amplitudes or variations
-     * @return the inverse of \f$\chi\f$.
+     * Currently returns the inverse weights without volume elment divided by the scalar part of \f$ \sigma\f$.
+     * This is especially good when \f$ \sigma\f$ exhibits large amplitudes or variations
+     * @return the inverse of \f$\sigma\f$.
      */
     const Container& precond()const {
         return m_precond;
@@ -243,34 +265,13 @@ class Elliptic
      */
     template<class ContainerType0, class ContainerType1>
     void symv( value_type alpha, const ContainerType0& x, value_type beta, ContainerType1& y)
-    {
-        multiply_sigma( alpha, 1., x, beta, y);
-    }
-
-    /**
-     * @brief Compute elliptic term with a possibly zero prefactor and add to output
-     *
-     * i.e this function computes \f[ y = -\alpha\nabla \cdot ( \sigma\chi \nabla x )  + \beta y\f]
-     * This is in principle possible also with the \c set_chi() and \c symv() functions
-     * however sometimes you have a \c sigma with explicit zeros or negative values.
-     * Then you need to use this function because \c set_chi() won't allow a \c sigma with zeros
-     * @note This function does not change the internal \c chi tensor
-     * @param alpha a scalar
-     * @param sigma The prefactor for the \c chi tensor
-     * @param x left-hand-side
-     * @param beta a scalar
-     * @param y result
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerType0, class ContainerType1, class ContainerType2>
-    void multiply_sigma( value_type alpha, const ContainerType2& sigma, const ContainerType0& x, value_type beta, ContainerType1& y)
     {
         //compute gradient
         dg::blas2::gemv( m_rightx, x, m_tempx); //R_x*f
         dg::blas2::gemv( m_righty, x, m_tempy); //R_y*f
 
         //multiply with tensor (note the alias)
-        dg::tensor::multiply2d(sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
+        dg::tensor::multiply2d(m_sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
 
         //now take divergence
         dg::blas2::symv( m_lefty, m_tempy, m_temp);
@@ -283,7 +284,7 @@ class Elliptic
             {
                 dg::blas2::symv( m_jfactor, m_jumpX, x, 0., m_tempx);
                 dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
-                dg::tensor::multiply2d(sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
+                dg::tensor::multiply2d(m_sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
                 dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
             }
             else
@@ -297,6 +298,49 @@ class Elliptic
         if( m_no == not_normed)//multiply weights without volume
             dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
     }
+
+    /**
+     * @brief \f$ \sigma = (\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
+     *
+     * @param phi the vector to take the variation of
+     * @param sigma (inout) the variation
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerType0, class ContainerType1>
+    void variation(const ContainerType0& phi, ContainerType1& sigma){
+        variation(1., 1., phi, 0., sigma);
+    }
+    /**
+     * @brief \f$ \sigma = \lambda^2(\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
+     *
+     * @param lambda input prefactor
+     * @param phi the vector to take the variation of
+     * @param sigma (out) the variation
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
+    void variation(const ContainerTypeL& lambda, const ContainerType0& phi, ContainerType1& sigma){
+        variation(1.,lambda, phi, 0., sigma);
+    }
+    /**
+     * @brief \f$ \sigma = \alpha \lambda^2 (\nabla\phi\cdot\chi\cdot\nabla \phi) + \beta \sigma\f$
+     *
+     * @param alpha scalar input prefactor
+     * @param lambda input prefactor
+     * @param phi the vector to take the variation of
+     * @param beta the output prefactor
+     * @param sigma (inout) the variation
+     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
+     */
+    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
+    void variation(value_type alpha, const ContainerTypeL& lambda, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
+    {
+        dg::blas2::gemv( m_rightx, phi, m_tempx); //R_x*f
+        dg::blas2::gemv( m_righty, phi, m_tempy); //R_y*f
+        dg::tensor::scalar_product2d(alpha, lambda, m_tempx, m_tempy, m_chi, lambda, m_tempx, m_tempy, beta, sigma);
+    }
+
+
     /**
      * @brief Determine if weights are multiplied to make operator symmetric or not
      *
@@ -323,20 +367,26 @@ using Elliptic2d = Elliptic<Geometry, Matrix, Container>;
 
 //Elliptic3d is tested in inc/geometries/elliptic3d_t.cu
 /**
- * @brief A 3d negative elliptic differential operator \f$ -\nabla \cdot ( \chi \nabla ) \f$
+ * @brief A 3d negative elliptic differential operator \f$ -\nabla \cdot ( \mathbf{\chi}\cdot \nabla ) \f$
  *
  * @ingroup matrixoperators
  *
- * The term discretized is \f[ -\nabla \cdot ( \mathbf \chi\cdot \nabla ) \f]
- * where \f$ \mathbf \chi \f$ is a positive semi-definit tensor.
- * In general coordinates that means
+ * The term discretized is \f[ -\nabla \cdot ( \mathbf{\chi} \cdot \nabla ) \f] where \f$
+ * \nabla \f$ is the two-dimensional nabla and \f$\chi = \sigma
+ * \mathbf{\tau}\f$ is a (possibly spatially dependent) tensor with scalar part
+ * \f$ \sigma\f$ (usually the volume form) and tensor part \f$ \tau\f$ (usually
+ * the inverse metric). In general coordinates that means
  * \f[ -\frac{1}{\sqrt{g}}\left(
  * \partial_x\left(\sqrt{g}\left(\chi^{xx}\partial_x + \chi^{xy}\partial_y + \chi^{xz}\partial_z \right)\right)
  + \partial_y\left(\sqrt{g}\left(\chi^{yx}\partial_x + \chi^{yy}\partial_y + \chi^{yz}\partial_z \right)\right)
  + \partial_z\left(\sqrt{g}\left(\chi^{zx}\partial_x + \chi^{zy}\partial_y + \chi^{zz}\partial_z \right)\right)
  \right)\f]
- is discretized. Note that the local discontinuous Galerkin discretization adds so-called
- jump terms
+ is discretized.
+ Per default, \f$ \chi = \sqrt{g} g^{-1}\f$ but you can set it to any tensor
+ you like (in order for the operator to be invertible \f$\chi\f$ should be
+ symmetric and positive definite though).
+
+ Note that the local discontinuous Galerkin discretization adds so-called jump terms
  \f[ D^\dagger \chi D + \alpha\chi_{on/off} J \f]
  where \f$\alpha\f$  is a scale factor ( = jfactor), \f$ D \f$ contains the discretizations of the above derivatives, and \f$ J\f$ is a self-adjoint matrix.
  (The symmetric part of \f$J\f$ is added @b before the volume element is divided). The adjoint of a matrix is defined with respect to the volume element including dG weights.
@@ -348,16 +398,18 @@ using Elliptic2d = Elliptic<Geometry, Matrix, Container>;
  The \f$ \chi_{on/off} \f$ in the jump term serves to weight the jump term with \f$ \chi \f$. This can be switched either on or off with off being the default.
  Also note that a forward discretization has more diffusion than a centered discretization.
 
- The following code snippet demonstrates the use of \c Elliptic in an inversion problem
+ The following code snippet demonstrates the use of \c Elliptic3d in an inversion problem
  * @snippet elliptic_b.cu invert
  * @copydoc hide_geometry_matrix_container
  * This class has the \c SelfMadeMatrixTag so it can be used in \c blas2::symv functions
  * and thus in a conjugate gradient solver.
- * @note The constructors initialize \f$ \chi=g\f$ so that a negative laplacian operator
- * results
+ * @note The constructors initialize \f$ \chi=\sqrt{g}g^{-1}\f$ so that a
+ * negative laplacian operator results
+ * @note The inverse of \f$ \sigma\f$ makes a good general purpose preconditioner
  * @note the jump term \f$ \alpha J\f$  adds artificial numerical diffusion as discussed above
+ * @note Since the pattern arises quite often (because of the ExB velocity \f$ u_E^2\f$ in the ion gyro-centre potential)
+ * this class also can compute the variation integrand \f$ \lambda^2\nabla \phi\cdot \chi\cdot\nabla\phi\f$
  * @attention Pay attention to the negative sign which is necessary to make the matrix @b positive @b definite
- *
  */
 template <class Geometry, class Matrix, class Container>
 class Elliptic3d
@@ -420,10 +472,8 @@ class Elliptic3d
         dg::assign( dg::create::inv_weights(g),   m_precond);
         m_temp = m_tempx = m_tempy = m_tempz = m_inv_weights;
         m_chi=g.metric();
-        m_vol=dg::tensor::volume(m_chi);
-        dg::tensor::scal( m_chi, m_vol);
+        m_sigma = m_vol = dg::tensor::volume(m_chi);
         dg::assign( dg::create::weights(g), m_weights_wo_vol);
-        dg::assign( dg::evaluate(dg::one, g), m_sigma);
     }
     ///@copydoc Elliptic::construct()
     template<class ...Params>
@@ -433,43 +483,23 @@ class Elliptic3d
         *this = Elliptic3d( std::forward<Params>( ps)...);
     }
 
-    /**
-     * @brief Change scalar part in Chi tensor
-     *
-     * Internally, we split the tensor \f$\chi = \sigma\mathbf{\tau}\f$ into
-     * a scalar part \f$ \sigma\f$ and a tensor part \f$ \tau\f$ and you can
-     * set each part seperately. This functions sets the scalar part.
-     *
-     * @param sigma The new scalar part in \f$\chi\f$ (all elements must be >0)
-     * @tparam ContainerType0 must be usable with \c Container in \ref dispatch
-     */
+    ///@copydoc Elliptic2d::set_chi(const ContainerType0&)
     template<class ContainerType0>
     void set_chi( const ContainerType0& sigma)
     {
-        dg::blas1::pointwiseDivide( sigma, m_sigma, m_tempx);
+        dg::blas1::pointwiseDot( sigma, m_vol, m_sigma);
         //update preconditioner
-        dg::blas1::pointwiseDivide( m_precond, m_tempx, m_precond);
-        dg::tensor::scal( m_chi, m_tempx);
-        dg::blas1::copy( sigma, m_sigma);
+        dg::blas1::pointwiseDivide( m_inv_weights, sigma, m_precond);
+        // sigma is possibly zero, which will invalidate the preconditioner
+        // it is important to call this blas1 function because it can
+        // overwrite NaN in m_precond in the next update
     }
-    /**
-     * @brief Change tensor part in Chi tensor
-     *
-     * Internally, we split the tensor \f$\chi = \sigma\mathbf{\tau}\f$ into
-     * a scalar part \f$ \sigma\f$ and a tensor part \f$ \tau\f$ and you can
-     * set each part seperately. This functions sets the tensor part.
-     *
-     * @note The class will take care of the volume element in the divergence so do not multiply it to \c tau yourself
-     *
-     * @param tau The new tensor part in \f$\chi\f$ (must be positive definite)
-     * @tparam ContainerType0 must be usable in \c dg::assign to \c Container
-     */
+
+    ///@copydoc Elliptic2d::set_chi(const SparseTensor<ContainerType0>&)
     template<class ContainerType0>
     void set_chi( const SparseTensor<ContainerType0>& tau)
     {
         m_chi = SparseTensor<Container>(tau);
-        dg::tensor::scal( m_chi, m_sigma);
-        dg::tensor::scal( m_chi, m_vol);
     }
 
     ///@copydoc Elliptic::inv_weights()
@@ -512,12 +542,6 @@ class Elliptic3d
     ///@copydoc Elliptic::symv(value_type,const ContainerType0&,value_type,ContainerType1&)
     template<class ContainerType0, class ContainerType1>
     void symv( value_type alpha, const ContainerType0& x, value_type beta, ContainerType1& y)
-    {
-        multiply_sigma( alpha, 1., x, beta, y);
-    }
-    ///@copydoc Elliptic::multiply_sigma(value_type,const ContainerType2&,const ContainerType0&,value_type,ContainerType1&)
-    template<class ContainerType0, class ContainerType1, class ContainerType2>
-    void multiply_sigma( value_type alpha, const ContainerType2& sigma, const ContainerType0& x, value_type beta, ContainerType1& y)
     {
         //compute gradient
         dg::blas2::gemv( m_rightx, x, m_tempx); //R_x*f
@@ -527,14 +551,14 @@ class Elliptic3d
             dg::blas2::gemv( m_rightz, x, m_tempz); //R_z*f
 
             //multiply with tensor (note the alias)
-            dg::tensor::multiply3d(sigma, m_chi, m_tempx, m_tempy, m_tempz, 0., m_tempx, m_tempy, m_tempz);
+            dg::tensor::multiply3d(m_sigma, m_chi, m_tempx, m_tempy, m_tempz, 0., m_tempx, m_tempy, m_tempz);
             //now take divergence
             dg::blas2::symv( -1., m_leftz, m_tempz, 0., m_temp);
             dg::blas2::symv( -1., m_lefty, m_tempy, 1., m_temp);
         }
         else
         {
-            dg::tensor::multiply2d(sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
+            dg::tensor::multiply2d(m_sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
             dg::blas2::symv( -1.,m_lefty, m_tempy, 0., m_temp);
         }
         dg::blas2::symv( -1., m_leftx, m_tempx, 1., m_temp);
@@ -546,7 +570,7 @@ class Elliptic3d
             {
                 dg::blas2::symv( m_jfactor, m_jumpX, x, 0., m_tempx);
                 dg::blas2::symv( m_jfactor, m_jumpY, x, 0., m_tempy);
-                dg::tensor::multiply2d(m_chi, m_tempx, m_tempy, m_tempx, m_tempy);
+                dg::tensor::multiply2d(m_sigma, m_chi, m_tempx, m_tempy, 0., m_tempx, m_tempy);
                 dg::blas1::axpbypgz(1.0,m_tempx,1.0,m_tempy,1.0,m_temp);
             }
             else
@@ -561,11 +585,30 @@ class Elliptic3d
             dg::blas1::pointwiseDot( alpha, m_weights_wo_vol, m_temp, beta, y);
     }
 
-    /**
-     * @brief Determine if weights are multiplied to make operator symmetric or not
-     *
-     * @param new_norm new setting
-     */
+    ///@copydoc Elliptic::variation(const ContainerType0&,ContainerType1&)
+    template<class ContainerType0, class ContainerType1>
+    void variation(const ContainerType0& phi, ContainerType1& sigma){
+        variation(1.,1., phi, 0., sigma);
+    }
+    ///@copydoc Elliptic::variation(const ContainerTypeL&,const ContainerType0&,ContainerType1&){
+    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
+    void variation(const ContainerTypeL& lambda, const ContainerType0& phi, ContainerType1& sigma){
+        variation(1.,lambda, phi, 0., sigma);
+    }
+    ///@copydoc Elliptic::variation(value_type,const ContainerTypeL&,const ContainerType0&,value_type,ContainerType1&)
+    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
+    void variation(value_type alpha, const ContainerTypeL& lambda, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
+    {
+        dg::blas2::gemv( m_rightx, phi, m_tempx); //R_x*f
+        dg::blas2::gemv( m_righty, phi, m_tempy); //R_y*f
+        if( m_multiplyZ)
+            dg::blas2::gemv( m_rightz, phi, m_tempz); //R_y*f
+        else
+            dg::blas1::scal( m_tempz, 0.);
+        dg::tensor::scalar_product3d(alpha, lambda,  m_tempx, m_tempy, m_tempz, m_chi, lambda, m_tempx, m_tempy, m_tempz, beta, sigma);
+    }
+
+    ///@copydoc Elliptic::set_norm(dg::norm)
     void set_norm( dg::norm new_norm) {
         m_no = new_norm;
     }
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index f8a0c24a7..cc1511f35 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -27,7 +27,9 @@ double rhs( double x, double y) { return 2.*sin(x)*sin(y)*(amp*sin(x)*sin(y)+1)-
 //double rhs( double x, double y) { return 2.*sin( x)*sin(y);}
 //double rhs( double x, double y) { return 2.*sin(x)*sin(y)*(sin(x)*sin(y)+1)-sin(x)*sin(x)*cos(y)*cos(y)-cos(x)*cos(x)*sin(y)*sin(y)+(x*sin(x)-cos(x))*sin(y) + x*sin(x)*sin(y);}
 double sol(double x, double y)  { return sin( x)*sin(y);}
-double der(double x, double y)  { return cos( x)*sin(y);}
+double derX(double x, double y)  { return cos( x)*sin(y);}
+double derY(double x, double y)  { return sin( x)*cos(y);}
+double vari(double x, double y)  { return pol(x,y)*pol(x,y)*(derX(x,y)*derX(x,y) + derY(x,y)*derY(x,y));}
 
 
 int main()
@@ -44,9 +46,9 @@ int main()
 	/*std::cout << "Type n, Nx and Ny and epsilon and jfactor (1)! \n";
     std::cin >> n >> Nx >> Ny; //more N means less iterations for same error
     std::cin >> eps >> jfactor;*/
-    bool jump_weight;
-    std::cout << "Jump weighting on or off? Type 1 for true or 0 for false: \n";
-    std::cin >> jump_weight;
+    bool jump_weight = false;
+    //std::cout << "Jump weighting on or off? Type 1 for true or 0 for false (default): \n";
+    //std::cin >> jump_weight;
     std::cout << "Computation on: "<< n <<" x "<< Nx <<" x "<< Ny << std::endl;
     //std::cout << "# of 2d cells                 "<< Nx*Ny <<std::endl;
 
@@ -63,7 +65,8 @@ int main()
     dg::DVec temp = x;
     //compute error
     const dg::DVec solution = dg::evaluate( sol, grid);
-    const dg::DVec derivati = dg::evaluate( der, grid);
+    const dg::DVec derivati = dg::evaluate( derX, grid);
+    const dg::DVec variatio = dg::evaluate( vari, grid);
     const double norm = dg::blas2::dot( w2d, solution);
     dg::DVec error( solution);
     dg::exblas::udouble res;
@@ -90,6 +93,11 @@ int main()
     for(unsigned u=0; u<stages; u++)
     {
         multi_pol[u].construct( multigrid.grid(u), dg::not_normed, dg::centered, jfactor);
+        //this tests if elliptic can recover from NaN in the preconditioner
+        multi_pol[u].set_chi(0.);
+        // here we test if we can set the tensor part in elliptic
+        multi_pol[u].set_chi( multigrid.grid(u).metric());
+        // now set the actual scalar part in chi
         multi_pol[u].set_chi( multi_chi[u]);
         multi_pol[u].set_jump_weighting(jump_weight);
     }
@@ -110,18 +118,18 @@ int main()
     double err = dg::blas2::dot( w2d, error);
     err = sqrt( err/norm); res.d = err;
     std::cout << " "<<err << "\t"<<res.i<<"\n";
-    }
-
     dg::DMatrix DX = dg::create::dx( grid);
     dg::blas2::gemv( DX, x, error);
     dg::blas1::axpby( 1.,derivati,-1., error);
-    double err = dg::blas2::dot( w2d, error);
+    err = dg::blas2::dot( w2d, error);
     const double norm_der = dg::blas2::dot( w2d, derivati);
-    std::cout << "L2 Norm of relative error in derivative is "<<std::setprecision(16)<< sqrt( err/norm_der)<<std::endl;
+    std::cout << "L2 Norm of relative error in derivative is\n "<<std::setprecision(16)<< sqrt( err/norm_der)<<std::endl;
     //derivative converges with p-1, for p = 1 with 1/2
 
+    }
+
     {
-        std::cout << "Forward Elliptic\n";
+    std::cout << "Forward Elliptic\n";
     x = temp;
     //![invert]
     //create an Elliptic object without volume form (not normed)
@@ -145,6 +153,12 @@ int main()
     //output the relative error
     std::cout << " "<<sqrt( err/norm) << "\n";
     //![invert]
+    std::cout << "Compute variation in forward Elliptic\n";
+    pol_forward.variation( 1., chi, x, 0., error);
+    dg::blas1::axpby( 1., variatio, -1., error);
+    err = dg::blas2::dot( w2d, error);
+    norm = dg::blas2::dot( w2d, variatio);
+    std::cout << " "<<sqrt( err/norm) << "\n";
     }
 
     {
diff --git a/inc/dg/elliptic2d_mpib.cu b/inc/dg/elliptic2d_mpib.cu
index 871213512..290cf35d5 100644
--- a/inc/dg/elliptic2d_mpib.cu
+++ b/inc/dg/elliptic2d_mpib.cu
@@ -13,12 +13,12 @@
 //
 //global relative error in L2 norm is O(h^P)
 //as a rule of thumb with n=4 the true error is err = 1e-3 * eps as long as eps > 1e3*err
-using value_type = float;
-using Vector = dg::fMDVec;
-using Matrix = dg::fMDMatrix;
-//using value_type = double;
-//using Vector = dg::MDVec;
-//using Matrix = dg::MDMatrix;
+//using value_type = float;
+//using Vector = dg::fMDVec;
+//using Matrix = dg::fMDMatrix;
+using value_type = double;
+using Vector = dg::MDVec;
+using Matrix = dg::MDMatrix;
 
 const value_type lx = M_PI;
 const value_type ly = 2.*M_PI;
@@ -35,7 +35,9 @@ value_type rhs( value_type x, value_type y) { return 2.*sin(x)*sin(y)*(amp*sin(x
 //value_type rhs( value_type x, value_type y) { return 2.*sin( x)*sin(y);}
 //value_type rhs( value_type x, value_type y) { return 2.*sin(x)*sin(y)*(sin(x)*sin(y)+1)-sin(x)*sin(x)*cos(y)*cos(y)-cos(x)*cos(x)*sin(y)*sin(y)+(x*sin(x)-cos(x))*sin(y) + x*sin(x)*sin(y);}
 value_type sol(value_type x, value_type y)  { return sin( x)*sin(y);}
-value_type der(value_type x, value_type y)  { return cos( x)*sin(y);}
+value_type derX(value_type x, value_type y)  { return cos( x)*sin(y);}
+value_type derY(value_type x, value_type y)  { return sin( x)*cos(y);}
+value_type vari(value_type x, value_type y)  { return pol(x,y)*pol(x,y)*(derX(x,y)*derX(x,y) + derY(x,y)*derY(x,y));}
 
 
 int main(int argc, char* argv[] )
@@ -48,7 +50,7 @@ int main(int argc, char* argv[] )
     int rank;
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
     dg::Timer t;
-    value_type eps = 1e-4;
+    value_type eps = 1e-6;
 
     //if(rank==0)std::cout << "Type epsilon! \n";
     //if(rank==0)std::cin >> eps;
@@ -95,7 +97,8 @@ int main(int argc, char* argv[] )
 
     //compute error
     const Vector solution = dg::evaluate( sol, grid);
-    const Vector derivati = dg::evaluate( der, grid);
+    const Vector derivati = dg::evaluate( derX, grid);
+    const Vector variatio = dg::evaluate( vari, grid);
     Vector error( solution);
     dg::blas1::axpby( 1.,x,-1., error);
 
@@ -109,6 +112,12 @@ int main(int argc, char* argv[] )
     norm = dg::blas2::dot( w2d, derivati);
     if(rank==0)std::cout << "L2 Norm of relative error in derivative is "<<sqrt( err/norm)<<std::endl;
     //derivative converges with p-1, for p = 1 with 1/2
+    if(rank==0)std::cout << "Compute variation in forward Elliptic      ";
+    multi_pol[0].variation( 1., chi, x, 0., error);
+    dg::blas1::axpby( 1., variatio, -1., error);
+    err = dg::blas2::dot( w2d, error);
+    norm = dg::blas2::dot( w2d, variatio);
+    if(rank==0)std::cout <<sqrt( err/norm) << "\n";
 
     MPI_Finalize();
     return 0;
diff --git a/inc/dg/elliptic_b.cu b/inc/dg/elliptic_b.cu
index fdb302614..dcbdf868e 100644
--- a/inc/dg/elliptic_b.cu
+++ b/inc/dg/elliptic_b.cu
@@ -19,13 +19,20 @@ const double lx = 2.*M_PI;
 const double ly = 2.*M_PI;
 const double lz = 2.*M_PI;
 double fct(double x, double y, double z){ return sin(x-R_0)*sin(y)*sin(z);}
-double derivative( double x, double y, double z){return cos(x-R_0)*sin(y)*sin(z);}
+double fctX( double x, double y, double z){return cos(x-R_0)*sin(y)*sin(z);}
+double fctY(double x, double y, double z){ return sin(x-R_0)*cos(y)*sin(z);}
+double fctZ(double x, double y, double z){ return sin(x-R_0)*sin(y)*cos(z);}
 double laplace2d_fct( double x, double y, double z) { return -1./x*cos(x-R_0)*sin(y)*sin(z) + 2.*fct(x,y,z);}
 double laplace3d_fct( double x, double y, double z) { return -1./x*cos(x-R_0)*sin(y)*sin(z) + 2.*fct(x,y,z) + 1./x/x*fct(x,y,z);}
 dg::bc bcx = dg::DIR;
 dg::bc bcy = dg::DIR;
 dg::bc bcz = dg::PER;
 double initial( double x, double y, double z) {return sin(0);}
+double variation3d( double x, double y, double z) {
+    return (fctX(x,y,z)*fctX(x,y,z)
+        + fctY(x,y,z)*fctY(x,y,z)
+        + fctZ(x,y,z)*fctZ(x,y,z)/x/x)*fct(x,y,z)*fct(x,y,z);
+}
 
 
 int main()
@@ -38,7 +45,7 @@ int main()
     std::cout << "Type epsilon! \n";
     std::cin >> eps;
     bool jump_weight;
-    std::cout << "Jump weighting on or off? Type 1 for true or 0 for false: \n";
+    std::cout << "Jump weighting on or off? Type 1 for true or 0 for false (default): \n";
     std::cin >> jump_weight;
     std::cout << "TEST CYLINDRICAL LAPLACIAN\n";
     //std::cout << "Create Laplacian\n";
@@ -51,7 +58,7 @@ int main()
     dg::Elliptic3d<dg::aGeometry3d, dg::DMatrix, dg::DVec> laplace(grid, dg::not_normed, dg::centered);
 
     laplace.set_jump_weighting(jump_weight);
-    
+
     dg::CG< dg::DVec > pcg( x, n*n*Nx*Ny*Nz);
 
     const dg::DVec solution = dg::evaluate ( fct, grid);
@@ -75,13 +82,21 @@ int main()
     dg::exblas::udouble res;
     norm = sqrt(normerr/norm); res.d = norm;
     std::cout << "L2 Norm of relative error is:               " <<res.d<<"\t"<<res.i<<std::endl;
-    const dg::DVec deriv = dg::evaluate( derivative, grid);
+    const dg::DVec deriv = dg::evaluate( fctX, grid);
     dg::DMatrix DX = dg::create::dx( grid);
     dg::blas2::gemv( DX, x, error);
     dg::blas1::axpby( 1., deriv, -1., error);
     normerr = dg::blas2::dot( w3d, error);
     norm = dg::blas2::dot( w3d, deriv);
     std::cout << "L2 Norm of relative error in derivative is: " <<sqrt( normerr/norm)<<std::endl;
+    std::cout << "Compute variation in Elliptic               ";
+    const dg::DVec variatio = dg::evaluate ( variation3d, grid);
+    laplace.variation( solution, x, error);
+    dg::blas1::axpby( 1., variatio, -1., error);
+    norm = dg::blas2::dot( w3d, variatio);
+    normerr = dg::blas2::dot( w3d, error);
+    std::cout <<sqrt( normerr/norm) << "\n";
+
 
     std::cout << "TEST SPLIT SOLUTION\n";
     x = dg::evaluate( initial, grid);
diff --git a/inc/dg/elliptic_mpib.cu b/inc/dg/elliptic_mpib.cu
index 967886b43..bacd664e9 100644
--- a/inc/dg/elliptic_mpib.cu
+++ b/inc/dg/elliptic_mpib.cu
@@ -24,13 +24,20 @@ const value_type lx = 2.*M_PI;
 const value_type ly = 2.*M_PI;
 const value_type lz = 2.*M_PI;
 value_type fct(value_type x, value_type y, value_type z){ return sin(x-R_0)*sin(y)*sin(z);}
-value_type derivative( value_type x, value_type y, value_type z){return cos(x-R_0)*sin(y)*sin(z);}
+value_type fctX( value_type x, value_type y, value_type z){return cos(x-R_0)*sin(y)*sin(z);}
+value_type fctY(value_type x, value_type y, value_type z){ return sin(x-R_0)*cos(y)*sin(z);}
+value_type fctZ(value_type x, value_type y, value_type z){ return sin(x-R_0)*sin(y)*cos(z);}
 value_type laplace2d_fct( value_type x, value_type y, value_type z) { return -1./x*cos(x-R_0)*sin(y)*sin(z) + 2.*fct(x,y,z);}
 value_type laplace3d_fct( value_type x, value_type y, value_type z) { return -1./x*cos(x-R_0)*sin(y)*sin(z) + 2.*fct(x,y,z) + 1./x/x*fct(x,y,z);}
 dg::bc bcx = dg::DIR;
 dg::bc bcy = dg::DIR;
 dg::bc bcz = dg::PER;
 value_type initial( value_type x, value_type y, value_type z) {return sin(0);}
+value_type variation3d( value_type x, value_type y, value_type z) {
+    return (fctX(x,y,z)*fctX(x,y,z)
+        + fctY(x,y,z)*fctY(x,y,z)
+        + fctZ(x,y,z)*fctZ(x,y,z)/x/x)*fct(x,y,z)*fct(x,y,z);
+}
 
 
 int main( int argc, char* argv[])
@@ -65,7 +72,7 @@ int main( int argc, char* argv[])
 
     if(rank==0)std::cout<<"Expand right hand side\n";
     const Vector solution = dg::evaluate ( fct, grid);
-    const Vector deriv = dg::evaluate( derivative, grid);
+    const Vector deriv = dg::evaluate( fctX, grid);
     Vector b = dg::evaluate ( laplace3d_fct, grid);
     //compute W b
     dg::blas2::symv( w3d, b, b);
@@ -89,6 +96,13 @@ int main( int argc, char* argv[])
     normerr = dg::blas2::dot( w3d, error);
     norm = dg::blas2::dot( w3d, deriv);
     if(rank==0)std::cout << "L2 Norm of relative error in derivative is: " <<sqrt( normerr/norm)<<std::endl;
+    if(rank==0)std::cout << "Compute variation in Elliptic               ";
+    const Vector variatio = dg::evaluate ( variation3d, grid);
+    laplace.variation( solution, x, error);
+    dg::blas1::axpby( 1., variatio, -1., error);
+    norm = dg::blas2::dot( w3d, variatio);
+    normerr = dg::blas2::dot( w3d, error);
+    if(rank==0)std::cout <<sqrt( normerr/norm) << "\n";
 
     if(rank==0)std::cout << "TEST SPLIT SOLUTION\n";
     x = dg::evaluate( initial, grid);
diff --git a/inc/dg/gradient.h b/inc/dg/gradient.h
index ab1f06e72..6b9c94136 100644
--- a/inc/dg/gradient.h
+++ b/inc/dg/gradient.h
@@ -19,11 +19,11 @@ namespace dg
 {
 
 /**
- * @brief A 2d gradient \f$\chi\cdot\nabla\f$ and variation \f$ \nabla\phi \cdot \chi \nabla\phi\f$ operator
+ * @brief A 2d gradient \f$\chi\cdot\nabla\f$ operator
  *
  * @ingroup matrixoperators
  *
- * The terms discretized are the gradient \f[\chi\cdot\nabla\f] and the variation \f[ \nabla\phi \cdot \chi \nabla\phi\f]
+ * The term discretized is the gradient \f[\chi\cdot\nabla\f]
  * where \f$ \nabla \f$ is the two-dimensional nabla and \f$\chi\f$ is a
  * tensor (usually the metric).
  *
@@ -35,14 +35,15 @@ namespace dg
  you like.
 
  * @copydoc hide_geometry_matrix_container
- * @note The constructors initialize \f$ \chi=g\f$ so that a traditional
+ * @note The constructors initialize \f$ \chi=g^{-1}\f$ so that a traditional
  * gradient results
- * @attention This a convenience class. It is often more
- * efficient to compute the simple derivatives of a vector yourself, because
- * you can re-use them in other places; the same goes for the storage of the
- * metric tensor, it often can be re-used at other places.  To compute the above
- * expressions you then simply use the relevant tensor functions
- * \c dg::tensor::multiply2d and \c dg::tensor::scalar_product2d
+ * @attention This a convenience class that saves you the construction of
+ * derivatives, the metric and temporary storage and 3 lines of application
+ * code. It is often more efficient to compute the simple derivatives of a
+ * vector yourself, because you can re-use them in other places; the same goes
+ * for the storage of the metric tensor, it often can be re-used at other
+ * places.  To compute the above expressions you then simply use the relevant
+ * tensor functions \c dg::tensor::multiply2d and \c dg::tensor::scalar_product2d
  */
 template <class Geometry, class Matrix, class Container>
 class Gradient
@@ -137,46 +138,6 @@ class Gradient
         dg::blas2::gemv( m_righty, f, m_tempy); //R_y*f
         dg::tensor::multiply2d(lambda, m_chi, m_tempx, m_tempy, mu, vx, vy);
     }
-    /**
-     * @brief \f$ \sigma = (\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
-     *
-     * @param phi the vector to take the variation of
-     * @param sigma (inout) the variation
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerType0, class ContainerType1>
-    void variation(const ContainerType0& phi, ContainerType1& sigma){
-        variation(1., 1., phi, 0., sigma);
-    }
-    /**
-     * @brief \f$ \sigma = \lambda^2(\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
-     *
-     * @param lambda input prefactor
-     * @param phi the vector to take the variation of
-     * @param sigma (out) the variation
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
-    void variation(const ContainerTypeL& lambda, const ContainerType0& phi, ContainerType1& sigma){
-        variation(1.,lambda, phi, 0., sigma);
-    }
-    /**
-     * @brief \f$ \sigma = \alpha \lambda^2 (\nabla\phi\cdot\chi\cdot\nabla \phi) + \beta \sigma\f$
-     *
-     * @param alpha scalar input prefactor
-     * @param lambda input prefactor
-     * @param phi the vector to take the variation of
-     * @param beta the output prefactor
-     * @param sigma (inout) the variation
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
-    void variation(value_type alpha, const ContainerTypeL& lambda, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
-    {
-        dg::blas2::gemv( m_rightx, phi, m_tempx); //R_x*f
-        dg::blas2::gemv( m_righty, phi, m_tempy); //R_y*f
-        dg::tensor::scalar_product2d(alpha, lambda, m_tempx, m_tempy, m_chi, lambda, m_tempx, m_tempy, beta, sigma);
-    }
     private:
     Matrix m_rightx, m_righty;
     Container m_tempx, m_tempy;
@@ -187,21 +148,13 @@ class Gradient
 ///@ingroup matrixoperators
 template <class Geometry, class Matrix, class Container>
 using Gradient2d = Gradient<Geometry, Matrix, Container>;
-///@copydoc Gradient
-///@ingroup matrixoperators
-template <class Geometry, class Matrix, class Container>
-using Variation = Gradient<Geometry,Matrix,Container>;
-///@copydoc Gradient
-///@ingroup matrixoperators
-template <class Geometry, class Matrix, class Container>
-using Variation2d = Gradient<Geometry,Matrix,Container>;
 
 /**
- * @brief A 3d gradient \f$\chi\cdot\nabla\f$ and variation \f$ \nabla\phi \cdot \chi \nabla\phi\f$ operator
+ * @brief A 3d gradient \f$\chi\cdot\nabla\f$ operator
  *
  * @ingroup matrixoperators
  *
- * The terms discretized are the gradient \f$\chi\cdot\nabla\f$ and the variation \f[ \nabla\phi \cdot \chi \nabla\phi\f]
+ * The term discretized is the gradient \f$\chi\cdot\nabla\f$
  * where \f$ \mathbf \chi \f$ is a tensor (usually the metric).
  * In general coordinates that means
  * \f[ \chi\cdot\nabla =
@@ -212,14 +165,15 @@ using Variation2d = Gradient<Geometry,Matrix,Container>;
  Per default, \f$ \chi\f$ is the metric tensor but you can set it to any tensor
  you like.
  * @copydoc hide_geometry_matrix_container
- * @note The constructors initialize \f$ \chi=g\f$ so that a traditional gradient
+ * @note The constructors initialize \f$ \chi=g^{-1}\f$ so that a traditional gradient
  * results
- * @attention This a convenience class. It is often more
- * efficient to compute the simple derivatives of a vector yourself, because
- * you can re-use them in other places; the same goes for the storage of the
- * metric tensor, it often can be re-used at other places.  To compute the above
- * expressions you then simply use the relevant tensor functions
- * \c dg::tensor::multiply3d and \c dg::tensor::scalar_product3d
+ * @attention This a convenience class that saves you the construction of
+ * derivatives, the metric and temporary storage and 4 lines of application
+ * code. It is often more efficient to compute the simple derivatives of a
+ * vector yourself, because you can re-use them in other places; the same goes
+ * for the storage of the metric tensor, it often can be re-used at other
+ * places.  To compute the above expressions you then simply use the relevant
+ * tensor functions \c dg::tensor::multiply3d and \c dg::tensor::scalar_product3d
  */
 template <class Geometry, class Matrix, class Container>
 class Gradient3d
@@ -333,50 +287,6 @@ class Gradient3d
             dg::blas1::scal( m_tempz, 0.);
         dg::tensor::multiply3d(lambda, m_chi, m_tempx, m_tempy, m_tempz, mu, vx, vy, vz);
     }
-    /**
-     * @brief \f$ \sigma = (\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
-     *
-     * @param phi the vector to take the variation of
-     * @param sigma (out) the variation
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerType0, class ContainerType1>
-    void variation(const ContainerType0& phi, ContainerType1& sigma){
-        variation(1.,1., phi, 0., sigma);
-    }
-    /**
-     * @brief \f$ \sigma = \lambda^2(\nabla\phi\cdot\chi\cdot\nabla \phi) \f$
-     *
-     * @param lambda input prefactor
-     * @param phi the vector to take the variation of
-     * @param sigma (out) the variation
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
-    void variation(const ContainerTypeL& lambda, const ContainerType0& phi, ContainerType1& sigma){
-        variation(1.,lambda, phi, 0., sigma);
-    }
-    /**
-     * @brief \f$ \sigma = \alpha\lambda^2 (\nabla\phi\cdot\chi\cdot\nabla \phi) + \beta \sigma\f$
-     *
-     * @param alpha scalar input prefactor
-     * @param lambda input prefactor
-     * @param phi the vector to take the variation of
-     * @param beta the output prefactor
-     * @param sigma (inout) the variation
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerTypeL, class ContainerType0, class ContainerType1>
-    void variation(value_type alpha, const ContainerTypeL& lambda, const ContainerType0& phi, value_type beta, ContainerType1& sigma)
-    {
-        dg::blas2::gemv( m_rightx, phi, m_tempx); //R_x*f
-        dg::blas2::gemv( m_righty, phi, m_tempy); //R_y*f
-        if( m_multiplyZ)
-            dg::blas2::gemv( m_rightz, phi, m_tempz); //R_y*f
-        else
-            dg::blas1::scal( m_tempz, 0.);
-        dg::tensor::scalar_product3d(alpha, lambda,  m_tempx, m_tempy, m_tempz, m_chi, lambda, m_tempx, m_tempy, m_tempz, beta, sigma);
-    }
     private:
     Matrix m_rightx, m_righty, m_rightz;
     Container m_tempx, m_tempy, m_tempz;
@@ -384,9 +294,4 @@ class Gradient3d
     bool m_multiplyZ = true;
 };
 
-///@copydoc Gradient3d
-///@ingroup matrixoperators
-template <class Geometry, class Matrix, class Container>
-using Variation3d = Gradient3d<Geometry,Matrix,Container>;
-
 } //namespace dg
diff --git a/inc/dg/gradient_t.cu b/inc/dg/gradient_t.cu
index 9d2c37b93..4cc2da2cb 100644
--- a/inc/dg/gradient_t.cu
+++ b/inc/dg/gradient_t.cu
@@ -3,7 +3,6 @@
 
 #include "gradient.h"
 
-//![function]
 const double lx = 2*M_PI;
 const double ly = 2*M_PI;
 dg::bc bcx = dg::PER;
@@ -18,9 +17,6 @@ double dxphi( double x, double y) {
 double dyphi( double x, double y) {
     return -sin(x)*sin(y);
 }
-double variation( double x, double y) {
-    return dxphi(x,y)*dxphi(x,y) + dyphi(x,y)*dyphi(x,y);
-}
 double phi3d( double x, double y,double z) {
     return sin(x)*cos(y)*cos(z);
 }
@@ -33,16 +29,11 @@ double dyphi3d( double x, double y, double z) {
 double dzphi3d( double x, double y, double z) {
     return -sin(x)*cos(y)*sin(z)/x/x;
 }
-double variation3d( double x, double y, double z) {
-    return dxphi3d(x,y,z)*dxphi3d(x,y,z)
-        + dyphi3d(x,y,z)*dyphi3d(x,y,z)
-        + dzphi3d(x,y,z)*dzphi3d(x,y,z)*x*x;
-}
 
 // There are more tests in geometries/geometry_advection_(mpi)b.cu
 int main()
 {
-    std::cout<<"This program tests the execution of the gradient and variation scheme! A test is passed if the number in the second column shows exactly zero!\n";
+    std::cout<<"This program tests the execution of the gradient! A test is passed if the number in the second column shows exactly zero!\n";
     unsigned n = 5, Nx = 32, Ny = 48;
     std::cout << "TEST 2D\n";
     std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<std::endl;
@@ -54,17 +45,15 @@ int main()
     const dg::DVec ph = dg::construct<dg::DVec>( dg::evaluate( phi, grid));
     const dg::DVec phx = dg::construct<dg::DVec>( dg::evaluate( dxphi, grid));
     const dg::DVec phy = dg::construct<dg::DVec>( dg::evaluate( dyphi, grid));
-    const dg::DVec var = dg::construct<dg::DVec>( dg::evaluate( variation, grid));
-    dg::DVec dxph(ph), dyph(ph), va(ph);
+    dg::DVec dxph(ph), dyph(ph);
 
     // create a Gradient object
     dg::Gradient<dg::aGeometry2d, dg::DMatrix, dg::DVec> gradient( grid);
 
     //apply arakawa scheme
     gradient.gradient( ph, dxph, dyph);
-    gradient.variation( ph, va);
 
-    int64_t binary[] = {4500635718861276907,4487444521638156650,4499885861996435701};
+    int64_t binary[] = {4500635718861276907,4487444521638156650};
     dg::exblas::udouble res;
     dg::DVec w2d = dg::create::weights( grid);
 
@@ -74,9 +63,6 @@ int main()
     dg::blas1::axpby( 1., phy, -1., dyph);
     res.d = sqrt(dg::blas2::dot( w2d, dyph)); //don't forget sqrt when computing errors
     std::cout << "Gy Distance to solution "<<res.d<<"\t\t"<<res.i-binary[1]<<std::endl;
-    dg::blas1::axpby( 1., var, -1., va);
-    res.d = sqrt(dg::blas2::dot( w2d, va)); //don't forget sqrt when computing errors
-    std::cout << "V  Distance to solution "<<res.d<<"\t\t"<<res.i-binary[2]<<std::endl;
     //periocid bc       |  dirichlet bc
     //n = 1 -> p = 2    |
     //n = 2 -> p = 1    |
@@ -95,17 +81,15 @@ int main()
     const dg::DVec phx3d = dg::construct<dg::DVec>( dg::evaluate( dxphi3d, grid3d));
     const dg::DVec phy3d = dg::construct<dg::DVec>( dg::evaluate( dyphi3d, grid3d));
     const dg::DVec phz3d = dg::construct<dg::DVec>( dg::evaluate( dzphi3d, grid3d));
-    const dg::DVec var3d = dg::construct<dg::DVec>( dg::evaluate( variation3d, grid3d));
-    dg::DVec dxph3d(ph3d), dyph3d(ph3d), dzph3d(ph3d), va3d(ph3d);
+    dg::DVec dxph3d(ph3d), dyph3d(ph3d), dzph3d(ph3d);
 
     // create a Gradient object
     dg::Gradient3d<dg::aGeometry3d, dg::DMatrix, dg::DVec> gradient3d( grid3d);
 
     //apply arakawa scheme
     gradient3d.gradient( ph3d, dxph3d, dyph3d, dzph3d);
-    gradient3d.variation( ph3d, va3d);
 
-    int64_t binary3d[] = {4504451755369532568,4491224193368827475,4549042274897523598,4550331496568322612};
+    int64_t binary3d[] = {4504451755369532568,4491224193368827475,4549042274897523598};
     dg::exblas::udouble res3d;
     dg::DVec w3d = dg::create::weights( grid3d);
 
@@ -118,9 +102,6 @@ int main()
     dg::blas1::axpby( 1., phz3d, -1., dzph3d);
     res3d.d = sqrt(dg::blas2::dot( w3d, dzph3d)); //don't forget sqrt when computing errors
     std::cout << "Gz Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[2]<<std::endl;
-    dg::blas1::axpby( 1., var3d, -1., va3d);
-    res3d.d = sqrt(dg::blas2::dot( w3d, va3d)); //don't forget sqrt when computing errors
-    std::cout << "V  Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[3]<<std::endl;
     //periocid bc       |  dirichlet bc
     //n = 1 -> p = 2    |
     //n = 2 -> p = 1    |
diff --git a/src/ep/toeflR.cuh b/src/ep/toeflR.cuh
index 2da42732a..81e40bad4 100644
--- a/src/ep/toeflR.cuh
+++ b/src/ep/toeflR.cuh
@@ -125,7 +125,6 @@ struct ToeflR
     dg::Elliptic<Geometry, Matrix, container> pol, laplaceM; //contains normalized laplacian
     dg::Helmholtz<Geometry,  Matrix, container> gamma1;
     dg::ArakawaX< Geometry, Matrix, container> arakawa; 
-    dg::Variation< Geometry, Matrix, container> gradient; 
 
     dg::Invert<container> invert_pol, invert_invgamma;
 
@@ -150,7 +149,6 @@ ToeflR< Geometry, M, container>::ToeflR( const Geometry& grid, const Parameters&
     laplaceM( grid, dg::normed, dg::centered),
     gamma1(  grid, 0., dg::centered),
     arakawa( grid),
-    gradient( grid, dg::centered),
     invert_pol(      omega, omega.size(), p.eps_pol),
     invert_invgamma( omega, omega.size(), p.eps_gamma),
     w2d( dg::create::volume(grid)), v2d( dg::create::inv_volume(grid)), one( dg::evaluate(dg::one, grid)),
@@ -171,7 +169,7 @@ void ToeflR<G, M, container>::compute_psi( const container& phi)
         if(  number == invert_invgamma.get_max())
             throw dg::Fail( eps_gamma);
 
-        gradient.variation(binv, phi, omega); //needed also in local energy theorem
+        pol.variation(binv, phi, omega); //needed also in local energy theorem
         dg::blas1::axpby( 1., psi[i], -0.5*mu[i], omega, psi[i]);   //psi  Gamma phi - 0.5 u_E^2
     }
 }
@@ -233,7 +231,7 @@ void ToeflR<G, M, container>::operator()(double t, const std::vector<container>&
         double Ue = z[0]*tau[0]*dg::blas2::dot( lny[0], w2d, ype[0]);
         double Up = z[1]*tau[1]*dg::blas2::dot( lny[1], w2d, ype[1]);
         double Uphi = 0.5*dg::blas2::dot( ype[0], w2d, omega) + 0.5*dg::blas2::dot( ype[1], w2d, omega); 
-        gradient.variation(potential_, omega); 
+        pol.variation(potential_, omega); 
         double UE = debye_*dg::blas2::dot( one, w2d, omega);
         energy_ = Ue + Up + Uphi + UE;
         //std::cout << "Ue "<<Ue<< "Up "<<Up<< "Uphi "<<Uphi<< "UE "<<UE<<"\n";
diff --git a/src/feltorSH/feltor.cuh b/src/feltorSH/feltor.cuh
index 7efd543b8..ea64c8dc8 100644
--- a/src/feltorSH/feltor.cuh
+++ b/src/feltorSH/feltor.cuh
@@ -103,7 +103,6 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
-    dg::Variation< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic<   Geometry, Matrix, container> lapperpM; 
     std::vector<container> multi_chi;
@@ -131,7 +130,6 @@ Explicit<Grid, Matrix, container>::Explicit( const Grid& g, eule::Parameters p):
     phi( 2, chi),chii(chi),uE2(chi),// (phi,psi), (chi_i), u_ExB
     ype(4,chi), logype(ype), // y+(bgamp+profamp) , log(ype)
     poisson(g, g.bcx(), g.bcy(), g.bcx(), g.bcy()), //first N/U then phi BCC
-    gradient(g, g.bcx(), g.bcy(), dg::centered),
     lapperpM ( g,g.bcx(), g.bcy(),     dg::normed,         dg::centered),
     invert_pol(         omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma(    omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
@@ -245,7 +243,7 @@ container& Explicit<G, Matrix,container>::compute_psi(const container& ti,contai
         }
     }
 
-    gradient.variation(binv, potential, uE2); // (nabla_perp phi)^2
+    multi_pol[0].variation(binv, potential, uE2); // (nabla_perp phi)^2
     dg::blas1::axpby( 1., phi[1], -0.5, uE2,phi[1]);             //psi  Gamma phi - 0.5 u_E^2
     return phi[1];    
 }
diff --git a/src/feltorSHp/feltor.cuh b/src/feltorSHp/feltor.cuh
index 03f912d88..642b82a42 100644
--- a/src/feltorSHp/feltor.cuh
+++ b/src/feltorSHp/feltor.cuh
@@ -125,8 +125,6 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
-    dg::Variation< Geometry, Matrix, container> gradient; 
-
     dg::Elliptic<   Geometry, Matrix, container> lapperpM; 
     std::vector<container> multi_chi;
     std::vector<dg::Elliptic<   Geometry, Matrix, container> > multi_pol;     
@@ -153,7 +151,6 @@ Explicit<Grid, Matrix, container>::Explicit( const Grid& g, eule::Parameters p):
     phi( 2, chi),chii(chi),uE2(chi),// (phi,psi), (chi_i), u_ExB
     n(2,chi), logn(n), pr(n), logpr(n), te(n), logte(n), tetilde(n),
     poisson(g, g.bcx(), g.bcy(), g.bcx(), g.bcy()), //first  N,P then phi BC
-    gradient(g, g.bcx(), g.bcy(), dg::centered), //first  N,P then phi BC
     lapperpM ( g,g.bcx(), g.bcy(),     dg::normed,         dg::centered),
     invert_pol(         omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma(    omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
@@ -266,7 +263,7 @@ container& Explicit<G, Matrix,container>::compute_psi(const container& ti,contai
             old_psi.update( phi[1]);
         }
     }
-    gradient.variation(binv,potential, uE2); // (nabla_perp phi)^2
+    multi_pol[0].variation(binv,potential, uE2); // (nabla_perp phi)^2
     dg::blas1::axpby( 1., phi[1], -0.5, uE2,phi[1]);             //psi  Gamma phi - 0.5 u_E^2        
     return phi[1];    
 }
diff --git a/src/feltorSesol/feltor.cuh b/src/feltorSesol/feltor.cuh
index 38351c19c..f9a14d25d 100644
--- a/src/feltorSesol/feltor.cuh
+++ b/src/feltorSesol/feltor.cuh
@@ -101,8 +101,6 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
-    dg::Variation< Geometry, Matrix, container> gradient; 
-
     dg::Elliptic< Geometry, Matrix, container > lapperpM; 
     std::vector<container> multi_chi;
     std::vector<dg::Elliptic<Geometry, Matrix, container> > multi_pol;
@@ -132,7 +130,6 @@ Explicit<Grid, Matrix, container>::Explicit( const Grid& g, eule::Parameters p):
     w2d( dg::create::weights(g)), v2d( dg::create::inv_weights(g)), 
     phi( 2, chi), npe(phi), logn(phi),
     poisson(g, g.bcx(), g.bcy(), p.bc_x_phi, g.bcy()), //first N/U then phi BCC
-    gradient(g, p.bc_x_phi, g.bcy(), dg::centered), //first N/U then phi BCC
     lapperpM ( g,g.bcx(), g.bcy(),       dg::normed,         dg::centered),
     invert_pol(         omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma(   omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
@@ -212,7 +209,7 @@ container& Explicit<G, Matrix,container>::compute_psi( container& potential)
         if(  number[0] == invert_invgamma.get_max())
             throw dg::Fail( p.eps_gamma);
     }
-    gradient.variation(binv, potential, omega);        // omega = u_E^2
+    multi_pol[0].variation(binv, potential, omega);        // omega = u_E^2
     dg::blas1::axpby( 1., phi[1], -0.5, omega,phi[1]);             //psi  Gamma phi - 0.5 u_E^2
     return phi[1];    
 }
diff --git a/src/feltorShw/feltor.cuh b/src/feltorShw/feltor.cuh
index 220cde979..436d9a0b5 100644
--- a/src/feltorShw/feltor.cuh
+++ b/src/feltorShw/feltor.cuh
@@ -106,7 +106,6 @@ struct Explicit
 
     //matrices and solvers
     dg::Poisson< Geometry, Matrix, container> poisson; 
-    dg::Variation< Geometry, Matrix, container> gradient; 
 
     dg::Elliptic< Geometry, Matrix, container > lapperp; 
     std::vector<container> multi_chi;
@@ -140,7 +139,6 @@ Explicit<Grid, Matrix, container>::Explicit( const Grid& g, eule::Parameters p):
     profne(dg::evaluate(dg::ExpProfX(p.nprofileamp, p.bgprofamp,p.invkappa),g)),
     profNi(profne),
     poisson(g, g.bcx(), g.bcy(), p.bc_x_phi, g.bcy()), //first N then phi BCC
-    gradient(g, p.bc_x_phi, g.bcy(), dg::centered),
     lapperp ( g,g.bcx(), g.bcy(),       dg::normed,          dg::centered),
     invert_pol(         omega, p.Nx*p.Ny*p.n*p.n, p.eps_pol),
     invert_invgamma(   omega, p.Nx*p.Ny*p.n*p.n, p.eps_gamma),
@@ -179,7 +177,7 @@ container& Explicit<Grid, Matrix, container>::compute_psi( container& potential)
             std::vector<unsigned> number = multigrid.direct_solve( multi_gammaPhi, phi[1], potential, p.eps_gamma);
             old_psi.update( phi[1]);
         }
-        gradient.variation(binv,potential, omega);         // omega = u_E^2
+        multi_pol[0].variation(binv,potential, omega);         // omega = u_E^2
         dg::blas1::axpby( 1., phi[1], -0.5, omega, phi[1]);   //psi =  Gamma phi - 0.5 u_E^2
     }
     if (p.modelmode==2)
@@ -498,7 +496,7 @@ void Explicit<Grid, Matrix, container>::operator()(double ttt, const std::vector
             S[i]    = 0.5*z[i]*p.tau[i]*dg::blas2::dot( y[i], w2d, y[i]); // N_tilde^2
         }
         mass_ = dg::blas2::dot( one, w2d, y[0] ); //take real ion density which is electron density!!
-        gradient.variation( phi[0], omega);
+        multi_pol[0].variation( phi[0], omega);
         double Tperp = 0.5*p.mu[1]*dg::blas2::dot( one, w2d, omega);   //= 0.5 mu_i u_E^2
         energy_ = S[0] + S[1]  + Tperp; 
         evec[0] = S[0], evec[1] = S[1], evec[2] = Tperp;
diff --git a/src/impurities/toeflI.cuh b/src/impurities/toeflI.cuh
index 304809edb..a03ba96f0 100644
--- a/src/impurities/toeflI.cuh
+++ b/src/impurities/toeflI.cuh
@@ -120,7 +120,6 @@ private:
     //matrices and solvers
     Helmholtz< Geometry, Matrix, container > gamma1;
     ArakawaX< Geometry, Matrix, container> arakawa; 
-    Gradient< Geometry, Matrix, container> gradient; 
     dg::Elliptic< Geometry, Matrix, container > pol, laplaceM; 
     dg::Invert<container> invert_pol, invert_invgamma;
 
@@ -140,7 +139,6 @@ ToeflI< Geometry, Matrix, container>::ToeflI( const Geometry& grid, imp::Paramet
     gamma_n( 2, chi),
     gamma1(  grid, -0.5*p.tau[1]),
     arakawa( grid),
-    gradient( grid),
     pol(     grid, not_normed, centered),
     laplaceM( grid, normed, centered),
     invert_pol(      omega, omega.size(), p.eps_pol),
@@ -157,7 +155,7 @@ const container& ToeflI<G, M, container>::compute_psi( const container& potentia
     gamma1.alpha() = -0.5*p.tau[idx]*p.mu[idx];
     invert_invgamma( gamma1, phi[idx], potential);
 
-    gradient.variation(binv,potential, omega); // u_E^2
+    pol.variation(binv,potential, omega); // u_E^2
 
     dg::blas1::axpby( 1., phi[idx], -0.5*p.mu[idx], omega, phi[idx]);   //psi  Gamma phi - 0.5 u_E^2
     return phi[idx];
diff --git a/src/reco2D/reconnection.cuh b/src/reco2D/reconnection.cuh
index b5a7e4068..95019bac9 100644
--- a/src/reco2D/reconnection.cuh
+++ b/src/reco2D/reconnection.cuh
@@ -202,7 +202,6 @@ struct Asela
 
     //matrices and solvers
     dg::ArakawaX< Geometry, Matrix, container > arakawa; 
-    dg::Variation< Geometry, Matrix, container > gradient; 
     dg::Elliptic<  Geometry, Matrix, container  > lapperp; //note the host vector    
     
     std::vector<container> multi_chi;
@@ -225,7 +224,6 @@ template<class Grid, class IMatrix, class Matrix, class container>
 Asela<Grid, IMatrix, Matrix, container>::Asela( const Grid& g, Parameters p): 
     //////////the arakawa operators ////////////////////////////////////////
     arakawa(g, g.bcx(), g.bcy()),
-    gradient(g, g.bcx(), g.bcy(), dg::centered ),
     //////////the elliptic and Helmholtz operators//////////////////////////
     lapperp (     g, g.bcx(), g.bcy(),   dg::normed,        dg::centered),
     multigrid( g, 3),
@@ -348,7 +346,7 @@ container& Asela<Geometry, IMatrix, Matrix,container>::compute_psi( container& p
         if(  number[0] == invert_invgamma.get_max())
         throw dg::Fail( p.eps_gamma); 
     }
-    gradient.variation(potential, omega); 
+    multi_pol[0].variation(potential, omega); 
     dg::blas1::axpby( 1., phi[1], -0.5, omega,phi[1]);        
     return phi[1];  
 }
@@ -454,7 +452,7 @@ void Asela<Geometry, IMatrix, Matrix, container>::operator()( double time,  cons
     }
     mass_ = dg::blas2::dot( one, w2d, y[0] ); //take real ion density which is electron density!!
     double Tperp = 0.5*p.mu[1]*dg::blas2::dot( npe[1], w2d, omega);   // Tperp = 0.5 mu_i N_i u_E^2
-    gradient.variation( apar[0], omega); // |nabla_\perp Aparallel|^2 
+    multi_pol[0].variation( apar[0], omega); // |nabla_\perp Aparallel|^2 
     double Uapar = 0.5*p.beta*dg::blas2::dot( one, w2d, omega); // Uapar = 0.5 beta |nabla_\perp Aparallel|^2
     energy_ = S[0] + S[1]  + Tperp + Tpar[0] + Tpar[1]; 
     evec[0] = S[0], evec[1] = S[1], evec[2] = Tperp, evec[3] = Tpar[0], evec[4] = Tpar[1]; evec[5] = Uapar;
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index 04d46649f..33369d18f 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -126,7 +126,6 @@ struct Explicit
     std::vector<dg::Elliptic<Geometry, Matrix, container> > multi_pol;
     std::vector<dg::Helmholtz<Geometry,  Matrix, container> > multi_gamma1;
     dg::ArakawaX< Geometry, Matrix, container> arakawa;
-    dg::Variation< Geometry, Matrix, container> gradient;
 
     dg::MultigridCG2d<Geometry, Matrix, container> multigrid;
     dg::Extrapolation<container> old_phi, old_psi, old_gammaN;
@@ -152,7 +151,6 @@ Explicit< Geometry, M, container>::Explicit( const Geometry& grid, const Paramet
     pol(     grid, dg::not_normed, dg::centered, p.jfactor),
     laplaceM( grid, dg::normed, dg::centered),
     arakawa( grid),
-    gradient( grid, dg::centered),
     multigrid( grid, 3),
     old_phi( 2, chi), old_psi( 2, chi), old_gammaN( 2, chi),
     w2d( dg::create::volume(grid)), one( dg::evaluate(dg::one, grid)),
@@ -187,7 +185,7 @@ const container& Explicit<G, M, container>::compute_psi( double t, const contain
         }
     }
     //compute (nabla phi)^2
-    gradient.variation(potential, omega);
+    pol.variation(potential, omega);
     //compute psi
     if(equations == "global")
     {
-- 
GitLab


From bcab553f194384800201b4ba174583eb68b5fcee Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 17:16:09 +0100
Subject: [PATCH 513/540] Adapt CHANGELOG with recent fixes

---
 CHANGELOG.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2ec8b8577..7e205905c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,7 +11,7 @@ doxygen documentation, READMEs or tex writeups.
  - json utility functions `dg::file::file2Json`, and `dg::file::string2Json` in json_utilities.h which adds a small abstraction layer that gives a user more control over what happens if an error happens during the parsing of a file
  - "easy output" netcdf utility functions that are particularly useful for MPI output: either write data in parallel or funnel through the master thread
  - new include files `dg/file/file.h`, `dg/file/json_utilities.h` and `dg/exblas/exblas.h`
- - new class `dg::Gradient` (and a `dg::Variation` typedef) for gradient and variation
+ - new class `dg::Gradient` for gradient
  - new class `dg::Advection` for the upwind advection scheme
  - new `dg::blas1::reduce` function for custom reductions
  - new "exchangeable" `dg::x::DVec`, `dg::x::HVec`, ..., `dg::x::CartesianGrid2d`, ..., `dg::x::IHMatrix`, ... typedefs. The idea is that these resolve to either shared memory or mpi distributed memory versions depending on the MPI_VERSION macro. This helps merging shared and mpi programs into single ones.
@@ -38,7 +38,6 @@ doxygen documentation, READMEs or tex writeups.
  - new class `dg::ChebyshevIterations` and `dg::ChebyshevPreconditioner` (for chebyshev iterations)
  - new solvers `dg::LGMRES`, `dg::BICGSTABL`, and d`g::AndersonAcceleration` (courtesy of Aslak Poulsen)
  - new `dg::FixedPointSolver` and `dg::AndersonSolver` for nonlinear problems in time
- - new class Gradient that computes gradients and variations
  - a host of new functors for the evaluate and pullback functions
  - `dg::geo::FluxSurfaceIntegral`, `dg::geo::FluxVolumeIntegral` and `dg::geo::SafetyFactorAverage` classes
  - new implementation: `dg::geo::ds_centered_bc_along_field` and `dg::geo::dss_centered_bc_along_field` that implement boundary condition "Stegmeir" style along the magnetic field lines
@@ -53,13 +52,14 @@ doxygen documentation, READMEs or tex writeups.
  - Sign reversal of magnetic field and associated flux functions is now possible
 ### Changed
  - namespace file changed to **dg::file** and exblas changed to **dg::exblas** (for consistency reasons, everything should go into the dg namespace, which in particular reduces the chance for name-clashes to just one, namely 'dg')
- - Moved **variation** member function into new class **dg::Variation** (previously in ArakawaX and Poisson)
+ - Moved **variation** member function into **dg::Elliptic** (previously in ArakawaX and Poisson)
  - **std=c++14** We use the C++-14 standard now (previously 11)
  - vectorclass dependency changed to vectorclass/version1 (previously we used a custom upload on feltor-dev repository)
  - default cuda compute capability bumped to sm-61 (previously sm-35)
  - marconi config now uses jsoncpp module (previously manually installed)
  - `dg::blas1::dot` and `dg::blas2::dot` and corresponding exblas functions now detect NaN and Inf errors
  - `dg::blas1::dot` and `dg::blas2::dot` now both do not accumulate rest of multiplication (inconsistent before)
+ - All blas1 functions that do not read or alias their result vector now overwrite NaN and Inf
  - all our mpi communications on GPUs now fall-back to host2host communication for cuda-unaware mpi-installations
  - swapped input and output parameters in `dg::blas1::evaluate` first subroutine
  - the fast_interpolation and fast_projection functions now can also double / divide the polynomial coefficient consistent with the grids
@@ -102,6 +102,8 @@ doxygen documentation, READMEs or tex writeups.
  - Fix bug: coefficient and initialization in `dg::Extrpolate`
  - Fix bug: Fpsi safety-factor in case nan is encountered still works
  - Fix bug: Fpsi safety-factor works up to the O-point
+ - Fix bug: `dg::pushForwardPerp` on functors computed wrong result (only affects `dg::geo::Hector`)
+ - Fix bug(s): several bugs in `dg::geo::Hector` which computed wrong grid (happened probably when we changed the grid design to polymorphic)
 
 ## [v5.1]
 ### Added
-- 
GitLab


From bc209b72c4f907768ac8d2be8d97f7708f778f99 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 20:15:17 +0100
Subject: [PATCH 514/540] Remove Gradient class again

 - we don't use it anywhere
 - and it is just three lines of code really
---
 inc/dg/algorithm.h   |   1 -
 inc/dg/gradient.h    | 297 -------------------------------------------
 inc/dg/gradient_t.cu | 113 ----------------
 3 files changed, 411 deletions(-)
 delete mode 100644 inc/dg/gradient.h
 delete mode 100644 inc/dg/gradient_t.cu

diff --git a/inc/dg/algorithm.h b/inc/dg/algorithm.h
index 4d776f32d..5ad99a7a7 100644
--- a/inc/dg/algorithm.h
+++ b/inc/dg/algorithm.h
@@ -29,7 +29,6 @@
 #include "refined_elliptic.h"
 #include "arakawa.h"
 #include "advection.h"
-#include "gradient.h"
 #include "poisson.h"
 #include "simpsons.h"
 #include "topology/average.h"
diff --git a/inc/dg/gradient.h b/inc/dg/gradient.h
deleted file mode 100644
index 6b9c94136..000000000
--- a/inc/dg/gradient.h
+++ /dev/null
@@ -1,297 +0,0 @@
-#pragma once
-
-#include "blas.h"
-#include "enums.h"
-#include "backend/memory.h"
-#include "topology/evaluation.h"
-#include "topology/derivatives.h"
-#ifdef MPI_VERSION
-#include "topology/mpi_derivatives.h"
-#include "topology/mpi_evaluation.h"
-#endif
-#include "topology/geometry.h"
-
-/*! @file
-
-  @brief General gradient operator
-  */
-namespace dg
-{
-
-/**
- * @brief A 2d gradient \f$\chi\cdot\nabla\f$ operator
- *
- * @ingroup matrixoperators
- *
- * The term discretized is the gradient \f[\chi\cdot\nabla\f]
- * where \f$ \nabla \f$ is the two-dimensional nabla and \f$\chi\f$ is a
- * tensor (usually the metric).
- *
- * In general coordinates that means
- * \f[\chi\cdot\nabla = \left(\chi^{xx}\partial_x + \chi^{xy}\partial_y \right)
- + \left(\chi^{yx}\partial_x + \chi^{yy}\partial_y \right) \f]
- is discretized.
- Per default, \f$ \chi\f$ is the metric tensor but you can set it to any tensor
- you like.
-
- * @copydoc hide_geometry_matrix_container
- * @note The constructors initialize \f$ \chi=g^{-1}\f$ so that a traditional
- * gradient results
- * @attention This a convenience class that saves you the construction of
- * derivatives, the metric and temporary storage and 3 lines of application
- * code. It is often more efficient to compute the simple derivatives of a
- * vector yourself, because you can re-use them in other places; the same goes
- * for the storage of the metric tensor, it often can be re-used at other
- * places.  To compute the above expressions you then simply use the relevant
- * tensor functions \c dg::tensor::multiply2d and \c dg::tensor::scalar_product2d
- */
-template <class Geometry, class Matrix, class Container>
-class Gradient
-{
-    public:
-    using geometry_type = Geometry;
-    using matrix_type = Matrix;
-    using container_type = Container;
-    using value_type = get_value_type<Container>;
-    ///@brief empty object ( no memory allocation)
-    Gradient(){}
-    /**
-     * @brief Construct from Grid
-     *
-     * @param g The Grid, boundary conditions are taken from here
-     * @param dir Direction of the right first derivative in x and y
-     *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
-     * @note chi is assumed the metric per default
-     */
-    Gradient( const Geometry& g, direction dir = centered):
-        Gradient( g, g.bcx(), g.bcy(), dir)
-    {
-    }
-
-    /**
-     * @brief Construct from grid and boundary conditions
-     * @param g The Grid
-     * @param bcx boundary condition in x
-     * @param bcy boundary contition in y
-     * @param dir Direction of the right first derivative in x and y
-     *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
-     * @note chi is assumed the metric per default
-     */
-    Gradient( const Geometry& g, bc bcx, bc bcy, direction dir = centered)
-    {
-        dg::blas2::transfer( dg::create::dx( g, bcx, dir), m_rightx);
-        dg::blas2::transfer( dg::create::dy( g, bcy, dir), m_righty);
-        m_chi=g.metric();
-        m_tempx = m_tempy = dg::construct<Container>( dg::evaluate( dg::zero, g));
-    }
-
-    /**
-    * @brief Perfect forward parameters to one of the constructors
-    *
-    * @tparam Params deduced by the compiler
-    * @param ps parameters forwarded to constructors
-    */
-    template<class ...Params>
-    void construct( Params&& ...ps)
-    {
-        //construct and swap
-        *this = Gradient( std::forward<Params>( ps)...);
-    }
-
-    ///@copydoc Gradient3d::chi()
-    SparseTensor<Container>& chi( ){return m_chi;}
-    ///@brief Access the Chi tensor
-    const SparseTensor<Container>& chi( ) const{return m_chi;}
-
-    /**
-     * @brief \f$ \vec v=\chi \cdot\nabla f \f$
-     *
-     * @param f the vector to take the gradient of
-     * @param vx (output) x-component
-     * @param vy (output) y-component
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerType0, class ContainerType1, class ContainerType2>
-    void gradient( const ContainerType0& f, ContainerType1& vx, ContainerType2& vy){
-        dg::blas2::gemv( m_rightx, f, vx); //R_x*f
-        dg::blas2::gemv( m_righty, f, vy); //R_y*f
-        dg::tensor::multiply2d(1., m_chi, vx, vy, 0., vx, vy);
-    }
-
-    /**
-     * @brief \f$ \vec v = \lambda \chi\cdot\nabla f + \mu \vec v \f$
-     *
-     * @param lambda a prefactor
-     * @param f the vector to take the gradient of
-     * @param mu the output prefactor
-     * @param vx (inout) x-component
-     * @param vy (inout) y-component
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerType0, class ContainerType1, class ContainerType2,
-        class ContainerType3, class ContainerType4>
-    void gradient(const ContainerType0& lambda, const ContainerType1& f, const
-            ContainerType2& mu, ContainerType3& vx, ContainerType4& vy)
-    {
-        //compute gradient
-        dg::blas2::gemv( m_rightx, f, m_tempx); //R_x*f
-        dg::blas2::gemv( m_righty, f, m_tempy); //R_y*f
-        dg::tensor::multiply2d(lambda, m_chi, m_tempx, m_tempy, mu, vx, vy);
-    }
-    private:
-    Matrix m_rightx, m_righty;
-    Container m_tempx, m_tempy;
-    SparseTensor<Container> m_chi;
-};
-
-///@copydoc Gradient
-///@ingroup matrixoperators
-template <class Geometry, class Matrix, class Container>
-using Gradient2d = Gradient<Geometry, Matrix, Container>;
-
-/**
- * @brief A 3d gradient \f$\chi\cdot\nabla\f$ operator
- *
- * @ingroup matrixoperators
- *
- * The term discretized is the gradient \f$\chi\cdot\nabla\f$
- * where \f$ \mathbf \chi \f$ is a tensor (usually the metric).
- * In general coordinates that means
- * \f[ \chi\cdot\nabla =
- * \left(\chi^{xx}\partial_x + \chi^{xy}\partial_y + \chi^{xz}\partial_z \right)
- + \left(\chi^{yx}\partial_x + \chi^{yy}\partial_y + \chi^{yz}\partial_z \right)
- + \left(\chi^{zx}\partial_x + \chi^{zy}\partial_y + \chi^{zz}\partial_z \right)
- \f]
- Per default, \f$ \chi\f$ is the metric tensor but you can set it to any tensor
- you like.
- * @copydoc hide_geometry_matrix_container
- * @note The constructors initialize \f$ \chi=g^{-1}\f$ so that a traditional gradient
- * results
- * @attention This a convenience class that saves you the construction of
- * derivatives, the metric and temporary storage and 4 lines of application
- * code. It is often more efficient to compute the simple derivatives of a
- * vector yourself, because you can re-use them in other places; the same goes
- * for the storage of the metric tensor, it often can be re-used at other
- * places.  To compute the above expressions you then simply use the relevant
- * tensor functions \c dg::tensor::multiply3d and \c dg::tensor::scalar_product3d
- */
-template <class Geometry, class Matrix, class Container>
-class Gradient3d
-{
-    public:
-    using geometry_type = Geometry;
-    using matrix_type = Matrix;
-    using container_type = Container;
-    using value_type = get_value_type<Container>;
-    ///@brief empty object ( no memory allocation)
-    Gradient3d(){}
-    /**
-     * @brief Construct from Grid
-     *
-     * @param g The Grid; boundary conditions are taken from here
-     * @param dir Direction of the right first derivative in x and y
-     *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
-     * the direction of the z derivative is always \c dg::centered
-     * @note chi is assumed the metric per default
-     */
-    Gradient3d( const Geometry& g, direction dir = centered):
-        Gradient3d( g, g.bcx(), g.bcy(), g.bcz(), dir)
-    {
-    }
-
-    /**
-     * @brief Construct from grid and boundary conditions
-     * @param g The Grid
-     * @param bcx boundary condition in x
-     * @param bcy boundary contition in y
-     * @param bcz boundary contition in z
-     * @param dir Direction of the right first derivative in x and y
-     *  (i.e. \c dg::forward, \c dg::backward or \c dg::centered),
-     * the direction of the z derivative is always \c dg::centered
-     * @note chi is assumed the metric per default
-     */
-    Gradient3d( const Geometry& g, bc bcx, bc bcy, bc bcz, direction dir = centered)
-    {
-        dg::blas2::transfer( dg::create::dx( g, bcx, dir), m_rightx);
-        dg::blas2::transfer( dg::create::dy( g, bcy, dir), m_righty);
-        dg::blas2::transfer( dg::create::dz( g, bcz, dg::centered), m_rightz);
-        m_chi=g.metric();
-        m_tempx = m_tempy = m_tempz = dg::construct<Container>( dg::evaluate( dg::zero, g));
-    }
-    ///@copydoc Gradient::construct()
-    template<class ...Params>
-    void construct( Params&& ...ps)
-    {
-        //construct and swap
-        *this = Gradient3d( std::forward<Params>( ps)...);
-    }
-
-    ///@brief Access the Chi tensor
-    SparseTensor<Container>& chi( ){return m_chi;}
-    ///@brief Access the Chi tensor
-    const SparseTensor<Container>& chi( ) const{return m_chi;}
-
-    /**
-     * @brief Restrict the problem to the first 2 dimensions
-     *
-     * This effectively makes the behaviour of dg::Gradient3d
-     * identical to the dg::Gradient class.
-     * @param compute_in_2d if true, the gradient and variaton functions replace all derivatives in z with 0, false reverts to the original behaviour.
-     */
-    void set_compute_in_2d( bool compute_in_2d ) {
-        m_multiplyZ = !compute_in_2d;
-    }
-    /**
-     * @brief \f$ \vec v=\chi \cdot\nabla f \f$
-     *
-     * @param f the vector to take the gradient of
-     * @param vx (output) x-component
-     * @param vy (output) y-component
-     * @param vz (output) z-component (0, if set_compute_in_2d(true) was set)
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerType0, class ContainerType1, class ContainerType2, class ContainerType3>
-    void gradient( const ContainerType0& f, ContainerType1& vx, ContainerType2& vy, ContainerType3& vz){
-        //compute gradient
-        dg::blas2::gemv( m_rightx, f, vx); //R_x*f
-        dg::blas2::gemv( m_righty, f, vy); //R_y*f
-        if( m_multiplyZ)
-            dg::blas2::gemv( m_rightz, f, vz); //R_y*f
-        else
-            dg::blas1::scal( vz, 0.);
-        dg::tensor::multiply3d(1., m_chi, vx, vy, vz, 0., vx, vy, vz);
-    }
-
-    /**
-     * @brief \f$ \vec v = \lambda \chi\cdot\nabla f + \mu \vec v \f$
-     *
-     * @param lambda a prefactor
-     * @param f the vector to take the gradient of
-     * @param mu the output prefactor
-     * @param vx (inout) x-component
-     * @param vy (inout) y-component
-     * @param vz (inout) z-component
-     * @tparam ContainerTypes must be usable with \c Container in \ref dispatch
-     */
-    template<class ContainerType0, class ContainerType1, class ContainerType2,
-        class ContainerType3, class ContainerType4, class ContainerType5>
-    void gradient(const ContainerType0& lambda, const ContainerType1& f, const
-            ContainerType2& mu, ContainerType3& vx, ContainerType4& vy, ContainerType5& vz)
-    {
-        //compute gradient
-        dg::blas2::gemv( m_rightx, f, m_tempx); //R_x*f
-        dg::blas2::gemv( m_righty, f, m_tempy); //R_y*f
-        if( m_multiplyZ)
-            dg::blas2::gemv( m_rightz, f, m_tempz); //R_y*f
-        else
-            dg::blas1::scal( m_tempz, 0.);
-        dg::tensor::multiply3d(lambda, m_chi, m_tempx, m_tempy, m_tempz, mu, vx, vy, vz);
-    }
-    private:
-    Matrix m_rightx, m_righty, m_rightz;
-    Container m_tempx, m_tempy, m_tempz;
-    SparseTensor<Container> m_chi;
-    bool m_multiplyZ = true;
-};
-
-} //namespace dg
diff --git a/inc/dg/gradient_t.cu b/inc/dg/gradient_t.cu
deleted file mode 100644
index 4cc2da2cb..000000000
--- a/inc/dg/gradient_t.cu
+++ /dev/null
@@ -1,113 +0,0 @@
-#include <iostream>
-#include <iomanip>
-
-#include "gradient.h"
-
-const double lx = 2*M_PI;
-const double ly = 2*M_PI;
-dg::bc bcx = dg::PER;
-dg::bc bcy = dg::PER;
-
-double phi( double x, double y) {
-    return sin(x)*cos(y);
-}
-double dxphi( double x, double y) {
-    return cos(x)*cos(y);
-}
-double dyphi( double x, double y) {
-    return -sin(x)*sin(y);
-}
-double phi3d( double x, double y,double z) {
-    return sin(x)*cos(y)*cos(z);
-}
-double dxphi3d( double x, double y, double z) {
-    return cos(x)*cos(y)*cos(z);
-}
-double dyphi3d( double x, double y, double z) {
-    return -sin(x)*sin(y)*cos(z);
-}
-double dzphi3d( double x, double y, double z) {
-    return -sin(x)*cos(y)*sin(z)/x/x;
-}
-
-// There are more tests in geometries/geometry_advection_(mpi)b.cu
-int main()
-{
-    std::cout<<"This program tests the execution of the gradient! A test is passed if the number in the second column shows exactly zero!\n";
-    unsigned n = 5, Nx = 32, Ny = 48;
-    std::cout << "TEST 2D\n";
-    std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<std::endl;
-
-    // create a Cartesian grid on the domain [0,lx]x[0,ly]
-    const dg::CartesianGrid2d grid( 0, lx, 0, ly, n, Nx, Ny, bcx, bcy);
-
-    // evaluate left and right hand side on the grid
-    const dg::DVec ph = dg::construct<dg::DVec>( dg::evaluate( phi, grid));
-    const dg::DVec phx = dg::construct<dg::DVec>( dg::evaluate( dxphi, grid));
-    const dg::DVec phy = dg::construct<dg::DVec>( dg::evaluate( dyphi, grid));
-    dg::DVec dxph(ph), dyph(ph);
-
-    // create a Gradient object
-    dg::Gradient<dg::aGeometry2d, dg::DMatrix, dg::DVec> gradient( grid);
-
-    //apply arakawa scheme
-    gradient.gradient( ph, dxph, dyph);
-
-    int64_t binary[] = {4500635718861276907,4487444521638156650};
-    dg::exblas::udouble res;
-    dg::DVec w2d = dg::create::weights( grid);
-
-    dg::blas1::axpby( 1., phx, -1., dxph);
-    res.d = sqrt(dg::blas2::dot( w2d, dxph)); //don't forget sqrt when computing errors
-    std::cout << "Gx Distance to solution "<<res.d<<"\t\t"<<res.i-binary[0]<<std::endl;
-    dg::blas1::axpby( 1., phy, -1., dyph);
-    res.d = sqrt(dg::blas2::dot( w2d, dyph)); //don't forget sqrt when computing errors
-    std::cout << "Gy Distance to solution "<<res.d<<"\t\t"<<res.i-binary[1]<<std::endl;
-    //periocid bc       |  dirichlet bc
-    //n = 1 -> p = 2    |
-    //n = 2 -> p = 1    |
-    //n = 3 -> p = 3    |        3
-    //n = 4 -> p = 3    |
-    //n = 5 -> p = 5    |
-    std::cout << "TEST 3D\n";
-    unsigned Nz = 100;
-    std::cout << "Computing on the Grid " <<n<<" x "<<Nx<<" x "<<Ny <<" x "<<Nz<<std::endl;
-
-    // create a Cylindrical grid on the domain [0,lx]x[0,ly]
-    const dg::CylindricalGrid3d grid3d( M_PI, 3*M_PI, -M_PI, M_PI, 0., 2*M_PI, n, Nx, Ny, Nz, bcx, bcy, dg::PER);
-
-    // evaluate left and right hand side on the grid
-    const dg::DVec ph3d = dg::construct<dg::DVec>( dg::evaluate( phi3d, grid3d));
-    const dg::DVec phx3d = dg::construct<dg::DVec>( dg::evaluate( dxphi3d, grid3d));
-    const dg::DVec phy3d = dg::construct<dg::DVec>( dg::evaluate( dyphi3d, grid3d));
-    const dg::DVec phz3d = dg::construct<dg::DVec>( dg::evaluate( dzphi3d, grid3d));
-    dg::DVec dxph3d(ph3d), dyph3d(ph3d), dzph3d(ph3d);
-
-    // create a Gradient object
-    dg::Gradient3d<dg::aGeometry3d, dg::DMatrix, dg::DVec> gradient3d( grid3d);
-
-    //apply arakawa scheme
-    gradient3d.gradient( ph3d, dxph3d, dyph3d, dzph3d);
-
-    int64_t binary3d[] = {4504451755369532568,4491224193368827475,4549042274897523598};
-    dg::exblas::udouble res3d;
-    dg::DVec w3d = dg::create::weights( grid3d);
-
-    dg::blas1::axpby( 1., phx3d, -1., dxph3d);
-    res3d.d = sqrt(dg::blas2::dot( w3d, dxph3d)); //don't forget sqrt when computing errors
-    std::cout << "Gx Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[0]<<std::endl;
-    dg::blas1::axpby( 1., phy3d, -1., dyph3d);
-    res3d.d = sqrt(dg::blas2::dot( w3d, dyph3d)); //don't forget sqrt when computing errors
-    std::cout << "Gy Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[1]<<std::endl;
-    dg::blas1::axpby( 1., phz3d, -1., dzph3d);
-    res3d.d = sqrt(dg::blas2::dot( w3d, dzph3d)); //don't forget sqrt when computing errors
-    std::cout << "Gz Distance to solution "<<res3d.d<<"\t\t"<<res3d.i-binary3d[2]<<std::endl;
-    //periocid bc       |  dirichlet bc
-    //n = 1 -> p = 2    |
-    //n = 2 -> p = 1    |
-    //n = 3 -> p = 3    |        3
-    //n = 4 -> p = 3    |
-    //n = 5 -> p = 5    |
-    std::cout << "\nContinue with topology/average_t.cu !\n\n";
-    return 0;
-}
-- 
GitLab


From 936bff839406801d3f1b5dee8480ab556c235bf5 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 20:17:09 +0100
Subject: [PATCH 515/540] Fix bug in mpi curvilinear product grid -> perp

---
 inc/geometries/mpi_curvilinear.h | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index 93e1f2d14..c9f70f7ae 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -199,6 +199,9 @@ RealCurvilinearMPIGrid2d<real_type>::RealCurvilinearMPIGrid2d( const RealCurvili
         metric_.values()[i].data().resize(s);
     for( unsigned i=0; i<map_.size(); i++)
         map_[i].data().resize(s);
+    jac_.set_communicator( this->communicator());
+    metric_.set_communicator( this->communicator());
+    map_.set_communicator( this->communicator());
 }
 ///@endcond
 //
-- 
GitLab


From 5ad30475d2b2e2054827b2e28be3a0f5d7f87eb6 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 20:18:11 +0100
Subject: [PATCH 516/540] Remove get_poloidal_comm from MPI Topology

Seems to serve no purpose and isn't used anywhere
---
 inc/dg/topology/mpi_grid.h | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/inc/dg/topology/mpi_grid.h b/inc/dg/topology/mpi_grid.h
index 04017f180..40ebac2d0 100644
--- a/inc/dg/topology/mpi_grid.h
+++ b/inc/dg/topology/mpi_grid.h
@@ -133,12 +133,6 @@ struct aRealMPITopology2d
      * @return Communicator
      */
     MPI_Comm communicator() const{return comm;}
-    MPI_Comm get_poloidal_comm() const{
-        int remain[] = {false, true};
-        MPI_Comm comm1d;
-        MPI_Cart_sub( comm, remain, &comm1d);
-        return comm1d;
-    }
     /**
      * @brief The Discrete Legendre Transformation
      *
-- 
GitLab


From c950df3e8fd2cdec3e36d30a2cc2044210f35000 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 20:19:00 +0100
Subject: [PATCH 517/540] Move variation in geometries advection -> elliptic

---
 inc/geometries/geometry_advection_b.cu    | 42 ++++---------------
 inc/geometries/geometry_advection_mpib.cu | 41 +++++--------------
 inc/geometries/geometry_elliptic_b.cu     | 49 +++++++++++------------
 inc/geometries/geometry_elliptic_mpib.cu  | 28 ++++++++-----
 inc/geometries/testfunctors.h             | 14 +++++++
 5 files changed, 74 insertions(+), 100 deletions(-)

diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index 074b11f9a..e56d1ea4c 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -72,19 +72,6 @@ struct ArakawaDirPer
     FuncDirPer2 g_;
 };
 
-struct VariationDirPer
-{
-    VariationDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4. ){}
-    double operator()(double R, double Z, double phi) const {
-        return this->operator()(R,Z);}
-
-    double operator()(double R, double Z) const {
-        return f_.dR( R,Z)*f_.dR(R,Z) + f_.dZ(R,Z)*f_.dZ(R,Z);
-    }
-    private:
-    dg::geo::FuncDirPer f_;
-};
-
 struct CurvatureDirPer
 {
     CurvatureDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4.), curvR(c,+1), curvZ(c,+1){}
@@ -103,7 +90,7 @@ struct CurvatureDirPer
 
 int main(int argc, char** argv)
 {
-    std::cout << "Type n (5), Nx (10), Ny (80)\n";
+    std::cout << "Type n (5), Nx (8), Ny (80)\n";
     unsigned n, Nx, Ny;
     std::cin >> n>> Nx>>Ny;
     Json::Value js;
@@ -127,13 +114,12 @@ int main(int argc, char** argv)
     std::cout << "Psi_0 = "<<psi_0<<" psi_1 = "<<psi_1<<std::endl;
     //gp.display( std::cout);
     dg::Timer t;
-    //solovev::detail::Fpsi fpsi( gp, -10);
     std::cout << "Constructing grid ... \n";
     t.tic();
-    //dg::geo::RibeiroFluxGenerator ribeiro( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::geo::FluxGenerator ribeiro( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
-    //dg::geo::SimpleOrthogonal ribeiro( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::geo::CurvilinearGrid2d grid(ribeiro, n, Nx, Ny, dg::DIR); //2d
+    //dg::geo::RibeiroFluxGenerator generator( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::FluxGenerator generator( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::SimpleOrthogonal generator( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::CurvilinearGrid2d grid(generator, n, Nx, Ny, dg::DIR); //2d
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s"<<std::endl;
     grid.display();
@@ -142,16 +128,11 @@ int main(int argc, char** argv)
     std::cout <<std::fixed<< std::setprecision(6)<<std::endl;
 
 
-    dg::geo::FuncDirPer left(mag, psi_0, psi_1, 4);
-    FuncDirPer2 right( mag, psi_0, psi_1);
-    ArakawaDirPer jacobian( mag, psi_0, psi_1);
-    VariationDirPer variationLHS(mag, psi_0, psi_1);
 
-    const dg::DVec lhs = dg::pullback( left, grid);
+    const dg::DVec lhs = dg::pullback( dg::geo::FuncDirPer(mag, psi_0, psi_1, 4), grid);
     dg::DVec jac(lhs);
-    const dg::DVec rhs = dg::pullback( right, grid);
-    const dg::DVec sol = dg::pullback ( jacobian, grid);
-    const dg::DVec variation = dg::pullback ( variationLHS, grid);
+    const dg::DVec rhs = dg::pullback( FuncDirPer2( mag, psi_0, psi_1), grid);
+    const dg::DVec sol = dg::pullback ( ArakawaDirPer( mag, psi_0, psi_1), grid);
     dg::DVec eins = dg::evaluate( dg::one, grid);
 
     ///////////////////////////////////////////////////////////////////////
@@ -189,13 +170,6 @@ int main(int argc, char** argv)
     dg::blas1::axpby( 1., sol, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     std::cout << "          Rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
-    ///////////////////////////////////////////////////////////////////////
-    std::cout << "TESTING VARIATION\n";
-    dg::Gradient<dg::aGeometry2d, dg::DMatrix, dg::DVec> gradient( grid);
-    gradient.variation( lhs, jac);
-    dg::blas1::axpby( 1., variation, -1., jac);
-    result = dg::blas2::dot( jac, vol, jac);
-    std::cout << "               distance to solution "<<sqrt( result)<<std::endl; //don't forget sqrt when comuting errors
 
     ////////////////////////////transform curvature components////////
     std::cout << "TESTING CURVATURE 3D\n";
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index b7df42838..ced13fae0 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -71,19 +71,6 @@ struct ArakawaDirPer
     FuncDirPer2 g_;
 };
 
-struct VariationDirPer
-{
-    VariationDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4. ){}
-    double operator()(double R, double Z, double phi) const {
-        return this->operator()(R,Z);}
-
-    double operator()(double R, double Z) const {
-        return f_.dR( R,Z)*f_.dR(R,Z) + f_.dZ(R,Z)*f_.dZ(R,Z);
-    }
-    private:
-    dg::geo::FuncDirPer f_;
-};
-
 struct CurvatureDirPer
 {
     CurvatureDirPer( dg::geo::TokamakMagneticField c, double psi_0, double psi_1): f_(c, psi_0, psi_1,4.), curvR(c,+1), curvZ(c,+1){}
@@ -121,8 +108,8 @@ int main(int argc, char** argv)
         is >> js;
     }
     dg::geo::solovev::Parameters gp(js);
-    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField( gp);
-    if(rank==0)std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField( gp);
+    if(rank==0)std::cout << "Psi min "<<mag.psip()(gp.R_0, 0)<<"\n";
     if(rank==0)std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     if(rank==0)std::cin >> psi_0>> psi_1;
@@ -136,8 +123,9 @@ int main(int argc, char** argv)
         MPI_Comm planeComm;
         int remain_dims[] = {true,true,false}; //true true false
         MPI_Cart_sub( comm, remain_dims, &planeComm);
-    //dg::geo::RibeiroFluxGenerator generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
-    dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::RibeiroFluxGenerator generator( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::FluxGenerator generator( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::SimpleOrthogonal generator( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
     Geometry grid(generator, n, Nx, Ny, dg::DIR, dg::PER, planeComm); //2d
     t.toc();
     if(rank==0)std::cout << "Construction took "<<t.diff()<<"s\n";
@@ -145,16 +133,14 @@ int main(int argc, char** argv)
     dg::MDVec vol = dg::create::volume( grid);
     if(rank==0)std::cout <<std::fixed<< std::setprecision(2)<<std::endl;
 
-    dg::geo::FuncDirPer left(c, psi_0, psi_1, 4);
-    FuncDirPer2 right( c, psi_0, psi_1);
-    ArakawaDirPer jacobian( c, psi_0, psi_1);
-    VariationDirPer variationLHS(c, psi_0, psi_1);
+    dg::geo::FuncDirPer left(mag, psi_0, psi_1, 4);
+    FuncDirPer2 right( mag, psi_0, psi_1);
+    ArakawaDirPer jacobian( mag, psi_0, psi_1);
 
     const dg::MDVec lhs = dg::pullback( left, grid);
     dg::MDVec jac(lhs);
     const dg::MDVec rhs = dg::pullback( right, grid);
     const dg::MDVec sol = dg::pullback ( jacobian, grid);
-    const dg::MDVec variation = dg::pullback ( variationLHS, grid);
     dg::MDVec eins = dg::evaluate( dg::one, grid);
 
     ///////////////////////////////////////////////////////////////////////
@@ -186,19 +172,12 @@ int main(int argc, char** argv)
     dg::blas1::axpby( 1., sol, -1., jac);
     result = dg::blas2::dot( jac, vol, jac);
     if(rank==0)std::cout << "          Rel. distance to solution "<<sqrt( result/norm)<<std::endl; //don't forget sqrt when comuting errors
-    ///////////////////////////////////////////////////////////////////////
-    if(rank==0)std::cout << "TESTING VARIATION 3D\n";
-    dg::Gradient<Geometry, dg::MDMatrix, dg::MDVec> gradient( grid);
-    gradient.variation( lhs, jac);
-    dg::blas1::axpby( 1., variation, -1., jac);
-    result = dg::blas2::dot( jac, vol, jac);
-    if(rank==0)std::cout << "               distance to solution "<<sqrt( result)<<std::endl; //don't forget sqrt when comuting errors
 
     ////////////////////////////transform curvature components////////
     if(rank==0)std::cout << "TESTING CURVATURE 3D\n";
     dg::MDVec curvX, curvY;
     dg::MHVec tempX, tempY;
-    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(c,+1), dg::geo::CurvatureNablaBZ(c,+1), tempX, tempY, grid);
+    dg::pushForwardPerp(dg::geo::CurvatureNablaBR(mag,+1), dg::geo::CurvatureNablaBZ(mag,+1), tempX, tempY, grid);
     dg::blas1::transfer(  tempX, curvX);
     dg::blas1::transfer(  tempY, curvY);
     dg::MDMatrix dx, dy;
@@ -211,7 +190,7 @@ int main(int argc, char** argv)
     dg::blas1::pointwiseDot( 1., tempy, curvY, 1.,  tempx);
     norm = dg::blas2::dot( tempx, vol, tempx);
 
-    CurvatureDirPer curv(c, psi_0, psi_1);
+    CurvatureDirPer curv(mag, psi_0, psi_1);
     dg::MDVec curvature;
     dg::blas1::transfer( dg::pullback(curv, grid), curvature);
 
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index 40977947d..4f9b4951b 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -5,19 +5,19 @@
 #include "dg/algorithm.h"
 #include "dg/file/nc_utilities.h"
 
+#include "flux.h"
 #include "solovev.h"
 #include "guenther.h"
 #include "simple_orthogonal.h"
 #include "curvilinear.h"
 #include "testfunctors.h"
 
-
 int main(int argc, char**argv)
 {
-    std::cout << "Type n, Nx, Ny, Nz\n";
+    std::cout << "Type n (3), Nx (8), Ny (80), Nz (1)\n";
     unsigned n, Nx, Ny, Nz;
     std::cin >> n>> Nx>>Ny>>Nz;
-    std::cout << "Type psi_0 and psi_1\n";
+    std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
     double psi_0, psi_1;
     std::cin >> psi_0>> psi_1;
     Json::Value js;
@@ -33,15 +33,17 @@ int main(int argc, char**argv)
     }
     //write parameters from file into variables
     dg::geo::solovev::Parameters gp(js);
-    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     gp.display( std::cout);
     dg::Timer t;
-    std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
+    std::cout << "Psi min "<<mag.psip()(gp.R_0, 0)<<"\n";
     std::cout << "Constructing grid ... \n";
     t.tic();
-    dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::SimpleOrthogonal generator( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::FluxGenerator generator( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR);
     std::unique_ptr<dg::aGeometry2d> g2d( g3d.perp_grid() );
+
     dg::Elliptic<dg::aGeometry2d, dg::DMatrix, dg::DVec> pol( *g2d, dg::not_normed, dg::forward);
     t.toc();
     std::cout << "Construction took "<<t.diff()<<"s\n";
@@ -58,35 +60,23 @@ int main(int argc, char**argv)
     ncerr = nc_def_var( ncid, "num_solution", NC_DOUBLE, 2, dim2d, &functionID);
     ncerr = nc_def_var( ncid, "ana_solution", NC_DOUBLE, 2, dim2d, &function2ID);
 
-    dg::HVec X( g2d->size()), Y(X); //P = dg::pullback( dg::coo3, g);
-    for( unsigned i=0; i<g2d->size(); i++)
-    {
-        X[i] = g2d->map()[0][i];
-        Y[i] = g2d->map()[1][i];
-    }
+    dg::HVec X( g2d->map()[0]), Y(g2d->map()[1]);
     ncerr = nc_put_var_double( ncid, coordsID[0], X.data());
     ncerr = nc_put_var_double( ncid, coordsID[1], Y.data());
     ///////////////////////////////////////////////////////////////////////////
     dg::DVec x = dg::evaluate( dg::zero, *g2d);
-    //const dg::DVec b =    dg::pullback( dg::geo::EllipticDirNeuM(c, psi_0, psi_1, 440, -220, 40., 1), *g2d);
-    //const dg::DVec chi =  dg::pullback( dg::geo::BmodTheta(c), *g2d);
-    //const dg::DVec solution = dg::pullback( dg::geo::FuncDirNeu(c,psi_0, psi_1, 440, -220, 40.,1 ), *g2d);
-    const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), *g2d);
-    const dg::DVec chi =  dg::pullback( dg::geo::Bmodule(c), *g2d);
-    const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, psi_1, 4), *g2d);
-    //const dg::DVec b =        dg::pullback( dg::geo::LaplacePsi(gp), *g2d);
-    //const dg::DVec chi =      dg::pullback( dg::one, *g2d);
-    //const dg::DVec solution =     dg::pullback( psip, *g2d);
-
+    const dg::DVec b =    dg::pullback( dg::geo::EllipticDirPerM(mag, psi_0, psi_1, 4), *g2d);
+    const dg::DVec chi =  dg::pullback( dg::geo::Bmodule(mag), *g2d);
+    const dg::DVec solution = dg::pullback( dg::geo::FuncDirPer(mag, psi_0, psi_1, 4), *g2d);
     const dg::DVec vol3d = dg::create::volume( *g2d);
     pol.set_chi( chi);
     //compute error
     dg::DVec error( solution);
     const double eps = 1e-10;
+    dg::Invert<dg::DVec > invert( x, n*n*Nx*Ny*Nz, eps);
     std::cout << "eps \t # iterations \t error \t hx_max\t hy_max \t time/iteration \n";
     std::cout << eps<<"\t";
     t.tic();
-    dg::Invert<dg::DVec > invert( x, n*n*Nx*Ny*Nz, eps);
     unsigned number = invert(pol, x,b);// vol3d, v3d );
     std::cout <<number<<"\t";
     t.toc();
@@ -96,16 +86,23 @@ int main(int argc, char**argv)
     std::cout << sqrt( err/norm) << "\t";
 
     dg::SparseTensor<dg::DVec> metric = g2d->metric();
-    dg::DVec gyy = metric.value(1,1), gxx=metric.value(0,0), vol = dg::tensor::volume(metric);
+    dg::DVec gyy = metric.value(1,1), gxx=metric.value(0,0), volume = dg::tensor::volume(metric);
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
-    dg::blas1::pointwiseDot( gxx, vol, gxx);
-    dg::blas1::pointwiseDot( gyy, vol, gyy);
+    dg::blas1::pointwiseDot( gxx, volume, gxx);
+    dg::blas1::pointwiseDot( gyy, volume, gyy);
     dg::blas1::scal( gxx, g2d->hx());
     dg::blas1::scal( gyy, g2d->hy());
     std::cout << *thrust::max_element( gxx.begin(), gxx.end()) << "\t";
     std::cout << *thrust::max_element( gyy.begin(), gyy.end()) << "\t";
     std::cout<<t.diff()/(double)number<<"s"<<std::endl;
+    ///////////////////////////////////////////////////////////////////////
+    std::cout << "TESTING VARIATION\n";
+    pol.variation( x, x);
+    const dg::DVec variation = dg::pullback( dg::geo::VariationDirPer( mag, psi_0, psi_1), *g2d);
+    dg::blas1::axpby( 1., variation, -1., x);
+    double result = dg::blas2::dot( x, vol3d, x);
+    std::cout << "               distance to solution "<<sqrt( result)<<std::endl; //don't forget sqrt when comuting errors
 
     dg::blas1::transfer( error, X );
     ncerr = nc_put_var_double( ncid, psiID, X.data());
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 5a4cadb1c..09b648e0d 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -12,8 +12,10 @@
 //#include "guenther.h"
 #include "mpi_curvilinear.h"
 #include "simple_orthogonal.h"
+#include "flux.h"
 #include "testfunctors.h"
 
+typedef  dg::geo::CurvilinearMPIGrid2d Geometry;
 
 int main(int argc, char**argv)
 {
@@ -35,8 +37,8 @@ int main(int argc, char**argv)
         is >> js;
     }
     dg::geo::solovev::Parameters gp(js);
-    dg::geo::TokamakMagneticField c = dg::geo::createSolovevField(gp);
-    if(rank==0)std::cout << "Psi min "<<c.psip()(gp.R_0, 0)<<"\n";
+    dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
+    if(rank==0)std::cout << "Psi min "<<mag.psip()(gp.R_0, 0)<<"\n";
     if(rank==0)std::cout << "Type psi_0 and psi_1\n";
     double psi_0, psi_1;
     if(rank==0)std::cin >> psi_0>> psi_1;
@@ -46,7 +48,8 @@ int main(int argc, char**argv)
     if(rank==0)std::cout << "Constructing grid ... \n";
     dg::Timer t;
     t.tic();
-    dg::geo::SimpleOrthogonal generator( c.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    //dg::geo::SimpleOrthogonal generator( mag.get_psip(), psi_0, psi_1, gp.R_0, 0., 1);
+    dg::geo::FluxGenerator generator( mag.get_psip(), mag.get_ipol(), psi_0, psi_1, gp.R_0, 0., 1);
     dg::geo::CurvilinearProductMPIGrid3d g3d( generator, n, Nx, Ny,Nz, dg::DIR, dg::PER, dg::PER, comm);
     std::unique_ptr<dg::aMPIGeometry2d> g2d(g3d.perp_grid());
     dg::Elliptic<dg::aMPIGeometry2d, dg::MDMatrix, dg::MDVec> pol( *g2d, dg::not_normed, dg::forward);
@@ -70,9 +73,9 @@ int main(int argc, char**argv)
     dg::file::put_var_double( ncid, coordsID[1], *g2d, Y);
     ///////////////////////////////////////////////////////////////////////////
     dg::MDVec x =    dg::evaluate( dg::zero, *g2d);
-    const dg::MDVec b =    dg::pullback( dg::geo::EllipticDirPerM(c, psi_0, psi_1, 4), *g2d);
-    const dg::MDVec chi =  dg::pullback( dg::geo::Bmodule(c), *g2d);
-    const dg::MDVec solution = dg::pullback( dg::geo::FuncDirPer(c, psi_0, psi_1, 4), *g2d);
+    const dg::MDVec b =    dg::pullback( dg::geo::EllipticDirPerM(mag, psi_0, psi_1, 4), *g2d);
+    const dg::MDVec chi =  dg::pullback( dg::geo::Bmodule(mag), *g2d);
+    const dg::MDVec solution = dg::pullback( dg::geo::FuncDirPer(mag, psi_0, psi_1, 4), *g2d);
     const dg::MDVec vol3d = dg::create::volume( *g2d);
     pol.set_chi( chi);
     //compute error
@@ -91,17 +94,24 @@ int main(int argc, char**argv)
     if(rank==0)std::cout << sqrt( err/norm) << "\t";
 
     dg::SparseTensor<dg::MDVec> metric = g2d->metric();
-    dg::MDVec gyy = metric.value(1,1), gxx=metric.value(0,0), vol = dg::tensor::volume(metric);
+    dg::MDVec gyy = metric.value(1,1), gxx=metric.value(0,0), volume = dg::tensor::volume(metric);
     dg::blas1::transform( gxx, gxx, dg::SQRT<double>());
     dg::blas1::transform( gyy, gyy, dg::SQRT<double>());
-    dg::blas1::pointwiseDot( gxx, vol, gxx);
-    dg::blas1::pointwiseDot( gyy, vol, gyy);
+    dg::blas1::pointwiseDot( gxx, volume, gxx);
+    dg::blas1::pointwiseDot( gyy, volume, gyy);
     dg::blas1::scal( gxx, g2d->hx());
     dg::blas1::scal( gyy, g2d->hy());
     if(rank==0)std::cout << "(Max elements on first process)\t";
     if(rank==0)std::cout << *thrust::max_element( gxx.data().begin(), gxx.data().end()) << "\t";
     if(rank==0)std::cout << *thrust::max_element( gyy.data().begin(), gyy.data().end()) << "\t";
     if(rank==0)std::cout<<t.diff()/(double)number<<"s"<<std::endl;
+    ///////////////////////////////////////////////////////////////////////
+    if(rank==0)std::cout << "TESTING VARIATION\n";
+    pol.variation( x, x);
+    const dg::MDVec variation = dg::pullback( dg::geo::VariationDirPer( mag, psi_0, psi_1), *g2d);
+    dg::blas1::axpby( 1., variation, -1., x);
+    double result = dg::blas2::dot( x, vol3d, x);
+    if(rank==0)std::cout << "               distance to solution "<<sqrt( result)<<std::endl; //don't forget sqrt when comuting errors
 
     dg::MHVec transfer;
     dg::assign( error, transfer);
diff --git a/inc/geometries/testfunctors.h b/inc/geometries/testfunctors.h
index 6a5bb0cdc..9d159b439 100644
--- a/inc/geometries/testfunctors.h
+++ b/inc/geometries/testfunctors.h
@@ -395,6 +395,20 @@ struct FuncDirPer
     double psi0_, psi1_, k_;
     const TokamakMagneticField c_;
 };
+// Variation of FuncDirPer
+struct VariationDirPer
+{
+    VariationDirPer( dg::geo::TokamakMagneticField mag, double psi_0, double psi_1): m_f(mag, psi_0, psi_1,4. ){}
+    double operator()(double R, double Z, double phi) const {
+        return this->operator()(R,Z);}
+
+    double operator()(double R, double Z) const {
+        return m_f.dR( R,Z)*m_f.dR(R,Z) + m_f.dZ(R,Z)*m_f.dZ(R,Z);
+    }
+    private:
+    dg::geo::FuncDirPer m_f;
+};
+
 
 //takes the magnetic field as chi
 struct EllipticDirPerM
-- 
GitLab


From cb4bd582e1a6f21d3cf0113b7de1b8d13baae69e Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Wed, 17 Feb 2021 23:55:10 +0100
Subject: [PATCH 518/540] easy_output functions no longer destroy input

- this is just unexpected for an "easy" output function
- now it respects const-ness
---
 inc/file/Makefile        |  4 +--
 inc/file/easy_output.h   | 55 +++++++++++++++++++++-------------------
 inc/file/netcdf_mpit.cpp | 45 +++++++++++++-------------------
 inc/file/netcdf_t.cpp    |  8 +++---
 4 files changed, 53 insertions(+), 59 deletions(-)

diff --git a/inc/file/Makefile b/inc/file/Makefile
index 4cade7eca..bb4b7daf9 100644
--- a/inc/file/Makefile
+++ b/inc/file/Makefile
@@ -9,10 +9,10 @@ INCLUDE+= -I../    # other project libraries
 
 all: netcdf_t netcdf_mpit
 
-netcdf_t: netcdf_t.cpp nc_utilities.h
+netcdf_t: netcdf_t.cpp nc_utilities.h easy_output.h
 	$(CC) $< -o $@ $(CFLAGS) -g $(INCLUDE) $(LIBS)
 
-netcdf_mpit: netcdf_mpit.cpp nc_utilities.h
+netcdf_mpit: netcdf_mpit.cpp nc_utilities.h easy_output.h
 	$(MPICC) $< -o $@ $(MPICFLAGS) $(INCLUDE) $(LIBS)
 
 .PHONY: doc clean
diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
index d9cd82ba4..16cbb7b08 100644
--- a/inc/file/easy_output.h
+++ b/inc/file/easy_output.h
@@ -108,11 +108,12 @@ struct NC_Error_Handle
 * This version is for a time-independent variable,
 * i.e. writes a single variable in one go and is actually equivalent
 * to \c nc_put_var_double. The dimensionality is given by the grid.
+* @note This function throws a \c dg::file::NC_Error if an error occurs
 * @tparam host_vector Type with \c data() member that returns pointer to first element in CPU (host) adress space, meaning it cannot be a GPU vector
 * @param ncid Forwarded to \c nc_put_vara_double
 * @param varid  Forwarded to \c nc_put_vara_double
 * @param grid The grid from which to construct \c start and \c count variables to forward to \c nc_put_vara_double
-* @param data data is forwarded to \c nc_put_vara_double, may be destroyed on output in the mpi version.
+* @param data data is forwarded to \c nc_put_vara_double
 * @param parallel This parameter is ignored in the serial version.
 * In the MPI version this parameter indicates whether each process
 * writes to the file independently in parallel (\c true)
@@ -123,8 +124,8 @@ struct NC_Error_Handle
 * Note that serious performance penalties have been observed on some platforms for parallel netcdf.
 */
 template<class host_vector>
-void put_var_double(int ncid, int varid, dg::aTopology2d& grid,
-    host_vector& data, bool parallel = false)
+void put_var_double(int ncid, int varid, const dg::aTopology2d& grid,
+    const host_vector& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[2] = {0,0}, count[2];
@@ -141,12 +142,13 @@ void put_var_double(int ncid, int varid, dg::aTopology2d& grid,
 * This version is for a time-dependent variable,
 * i.e. writes a single time-slice into the file.
 * The dimensionality is given by the grid.
+* @note This function throws a \c dg::file::NC_Error if an error occurs
 * @tparam host_vector Type with \c data() member that returns pointer to first element in CPU (host) adress space, meaning it cannot be a GPU vector
 * @param ncid Forwarded to \c nc_put_vara_double
 * @param varid  Forwarded to \c nc_put_vara_double
 * @param slice The number of the time-slice to write (first element of the \c startp array in \c nc_put_vara_double)
 * @param grid The grid from which to construct \c start and \c count variables to forward to \c nc_put_vara_double
-* @param data data is forwarded to \c nc_put_vara_double, may be destroyed on output in the mpi version.
+* @param data data is forwarded to \c nc_put_vara_double, 
 * @param parallel This parameter is ignored in the serial version.
 * In the MPI version this parameter indicates whether each process
 * writes to the file independently in parallel (\c true)
@@ -158,35 +160,33 @@ void put_var_double(int ncid, int varid, dg::aTopology2d& grid,
 */
 template<class host_vector>
 void put_vara_double(int ncid, int varid, unsigned slice,
-    dg::aTopology2d& grid, host_vector& data, bool parallel = false)
+    const dg::aTopology2d& grid, const host_vector& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[3] = {slice,0,0}, count[3];
     count[0] = 1;
     count[1] = grid.n()*grid.Ny();
     count[2] = grid.n()*grid.Nx();
-    err = nc_put_vara_double( ncid, varid, start, count,
-        data.data());
+    err = nc_put_vara_double( ncid, varid, start, count, data.data());
 }
 
 ///@copydoc put_var_double()
 template<class host_vector>
-void put_var_double(int ncid, int varid, dg::aTopology3d& grid,
-    host_vector& data, bool parallel = false)
+void put_var_double(int ncid, int varid, const dg::aTopology3d& grid,
+    const host_vector& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[3] = {0,0,0}, count[3];
     count[0] = grid.Nz();
     count[1] = grid.n()*grid.Ny();
     count[2] = grid.n()*grid.Nx();
-    err = nc_put_vara_double( ncid, varid, start, count,
-        data.data());
+    err = nc_put_vara_double( ncid, varid, start, count, data.data());
 }
 
 ///@copydoc put_vara_double()
 template<class host_vector>
 void put_vara_double(int ncid, int varid, unsigned slice,
-    dg::aTopology3d& grid, host_vector& data, bool parallel = false)
+    const dg::aTopology3d& grid, const host_vector& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[4] = {slice, 0,0,0}, count[4];
@@ -194,15 +194,14 @@ void put_vara_double(int ncid, int varid, unsigned slice,
     count[1] = grid.Nz();
     count[2] = grid.n()*grid.Ny();
     count[3] = grid.n()*grid.Nx();
-    err = nc_put_vara_double( ncid, varid, start, count,
-        data.data());
+    err = nc_put_vara_double( ncid, varid, start, count, data.data());
 }
 
 #ifdef MPI_VERSION
 ///@copydoc put_var_double()
 template<class host_vector>
-void put_var_double(int ncid, int varid, dg::aMPITopology2d& grid,
-    dg::MPI_Vector<host_vector>& data, bool parallel = false)
+void put_var_double(int ncid, int varid, const dg::aMPITopology2d& grid,
+    const dg::MPI_Vector<host_vector>& data, bool parallel = false)
 {
     file::NC_Error_Handle err;
     size_t start[3] = {0,0}, count[2];
@@ -221,15 +220,16 @@ void put_var_double(int ncid, int varid, dg::aMPITopology2d& grid,
             MPI_Cart_coords( comm, rrank, 2, &coords[2*rrank]);
         if(rank==0)
         {
+            host_vector receive( data.data());
             for( int rrank=0; rrank<size; rrank++)
             {
                 if(rrank!=0)
-                    MPI_Recv( data.data().data(), local_size, MPI_DOUBLE,
+                    MPI_Recv( receive.data(), local_size, MPI_DOUBLE,
                           rrank, rrank, comm, &status);
                 start[0] = coords[2*rrank+1]*count[0],
                 start[1] = coords[2*rrank+0]*count[1],
                 err = nc_put_vara_double( ncid, varid, start, count,
-                    data.data().data());
+                    receive.data());
             }
         }
         else
@@ -251,7 +251,7 @@ void put_var_double(int ncid, int varid, dg::aMPITopology2d& grid,
 ///@copydoc put_vara_double()
 template<class host_vector>
 void put_vara_double(int ncid, int varid, unsigned slice,
-    dg::aMPITopology2d& grid, dg::MPI_Vector<host_vector>& data,
+    const dg::aMPITopology2d& grid, const dg::MPI_Vector<host_vector>& data,
     bool parallel = false)
 {
     file::NC_Error_Handle err;
@@ -281,15 +281,16 @@ void put_vara_double(int ncid, int varid, unsigned slice,
             MPI_Cart_coords( comm, rrank, 2, &coords[2*rrank]);
         if(rank==0)
         {
+            host_vector receive( data.data());
             for( int rrank=0; rrank<size; rrank++)
             {
                 if(rrank!=0)
-                    MPI_Recv( data.data().data(), local_size, MPI_DOUBLE,
+                    MPI_Recv( receive.data(), local_size, MPI_DOUBLE,
                           rrank, rrank, comm, &status);
                 start[1] = coords[2*rrank+1]*count[1],
                 start[2] = coords[2*rrank+0]*count[2],
                 err = nc_put_vara_double( ncid, varid, start, count,
-                    data.data().data());
+                    receive.data());
             }
         }
         else
@@ -302,7 +303,7 @@ void put_vara_double(int ncid, int varid, unsigned slice,
 ///@copydoc put_var_double()
 template<class host_vector>
 void put_var_double(int ncid, int varid,
-    dg::aMPITopology3d& grid, dg::MPI_Vector<host_vector>& data,
+    const dg::aMPITopology3d& grid, const dg::MPI_Vector<host_vector>& data,
     bool parallel = false)
 {
     file::NC_Error_Handle err;
@@ -323,16 +324,17 @@ void put_var_double(int ncid, int varid,
             MPI_Cart_coords( comm, rrank, 3, &coords[3*rrank]);
         if(rank==0)
         {
+            host_vector receive( data.data());
             for( int rrank=0; rrank<size; rrank++)
             {
                 if(rrank!=0)
-                    MPI_Recv( data.data().data(), local_size, MPI_DOUBLE,
+                    MPI_Recv( receive.data(), local_size, MPI_DOUBLE,
                           rrank, rrank, comm, &status);
                 start[0] = coords[3*rrank+2]*count[0],
                 start[1] = coords[3*rrank+1]*count[1],
                 start[2] = coords[3*rrank+0]*count[2];
                 err = nc_put_vara_double( ncid, varid, start, count,
-                    data.data().data());
+                    receive.data());
             }
         }
         else
@@ -387,16 +389,17 @@ void put_vara_double(int ncid, int varid, unsigned slice,
             MPI_Cart_coords( comm, rrank, 3, &coords[3*rrank]);
         if(rank==0)
         {
+            host_vector receive( data.data());
             for( int rrank=0; rrank<size; rrank++)
             {
                 if(rrank!=0)
-                    MPI_Recv( data.data().data(), local_size, MPI_DOUBLE,
+                    MPI_Recv( receive.data(), local_size, MPI_DOUBLE,
                           rrank, rrank, comm, &status);
                 start[1] = coords[3*rrank+2]*count[1],
                 start[2] = coords[3*rrank+1]*count[2],
                 start[3] = coords[3*rrank+0]*count[3];
                 err = nc_put_vara_double( ncid, varid, start, count,
-                    data.data().data());
+                    receive.data());
             }
         }
         else
diff --git a/inc/file/netcdf_mpit.cpp b/inc/file/netcdf_mpit.cpp
index 620675a15..717eb6288 100644
--- a/inc/file/netcdf_mpit.cpp
+++ b/inc/file/netcdf_mpit.cpp
@@ -1,10 +1,10 @@
 #include <iostream>
 #include <string>
 #include <mpi.h>
-#include <netcdf_par.h>
 #include <cmath>
 
 #include "dg/algorithm.h"
+#define _FILE_INCLUDED_BY_DG_
 #include "nc_utilities.h"
 
 double function( double x, double y, double z){return sin(x)*sin(y)*cos(z);}
@@ -21,53 +21,44 @@ int main(int argc, char* argv[])
     double Tmax=2.*M_PI;
     double NT = 10;
     double dt = Tmax/NT;
-    dg::Grid1d gx( 0, 2.*M_PI, 3, 10);
-    dg::Grid1d gy( 0, 2.*M_PI, 3, 10);
-    dg::Grid1d gz( 0, 2.*M_PI, 1, 20);
-    dg::Grid3d g( gx, gy, gz);
+    double x0 = 0., x1 = 2.*M_PI;
+    MPI_Comm comm;
+    std::stringstream ss;
+    ss<< "2 1 2";
+    dg::mpi_init3d( dg::PER, dg::PER, dg::PER, comm, ss);
+    dg::MPIGrid3d grid( x0,x1,x0,x1,x0,x1,3,10,10,20, comm);
     std::string hello = "Hello world\n";
-    thrust::host_vector<double> data = dg::evaluate( function, g);
+    dg::MPI_Vector<thrust::host_vector<double>> data = dg::evaluate( function, grid);
 
     //create NetCDF File
-    int ncid;
+    int ncid=0;
     dg::file::NC_Error_Handle err;
-    MPI_Info info = MPI_INFO_NULL;
-    err = nc_create_par( "testmpi.nc", NC_NETCDF4|NC_MPIIO|NC_CLOBBER, MPI_COMM_WORLD, info, &ncid);
-    err = nc_put_att_text( ncid, NC_GLOBAL, "input", hello.size(), hello.data());
+    if(rank==0)err = nc_create( "testmpi.nc", NC_NETCDF4|NC_CLOBBER, &ncid);
+    if(rank==0)err = nc_put_att_text( ncid, NC_GLOBAL, "input", hello.size(), hello.data());
 
     int dimids[4], tvarID;
-    err = dg::file::define_dimensions( ncid, dimids, &tvarID, g);
+    if(rank==0)err = dg::file::define_dimensions( ncid, dimids, &tvarID, grid);
     int dataID;
-    err = nc_def_var( ncid, "data", NC_DOUBLE, 4, dimids, &dataID);
+    if(rank==0)err = nc_def_var( ncid, "data", NC_DOUBLE, 4, dimids, &dataID);
 
     /* Write metadata to file. */
-    err = nc_enddef(ncid);
+    if(rank==0)err = nc_enddef(ncid);
 
-    //err = nc_enddef( ncid);
-    size_t start[4] = {0, rank*g.Nz()/size, 0, 0};
-    size_t count[4] = {1, g.Nz()/size, g.Ny()*g.n(), g.Nx()*g.n()};
-    if( rank==0) std::cout<< "Write from "<< start[0]<< " "<<start[1]<<" "<<start[2]<<" "<<start[3]<<std::endl;
-    if( rank==0) std::cout<< "Number of elements "<<count[0]<< " "<<count[1]<<" "<<count[2]<<" "<<count[3]<<std::endl;
-    err = nc_var_par_access(ncid, dataID , NC_COLLECTIVE);
-    err = nc_var_par_access(ncid, tvarID , NC_COLLECTIVE);
     size_t Tcount=1, Tstart=0;
     double time = 0;
     //err = nc_close(ncid);
     for(unsigned i=0; i<=NT; i++)
     {
         if(rank==0)std::cout<<"Write timestep "<<i<<"\n";
-        //err = nc_open_par( "testmpi.nc", NC_WRITE|NC_MPIIO, MPI_COMM_WORLD, info, &ncid); //doesn't work I don't know why
         time = i*dt;
         Tstart = i;
-        data = dg::evaluate( function, g);
+        data = dg::evaluate( function, grid);
         dg::blas1::scal( data, cos( time));
-        start[0] = i;
         //write dataset (one timeslice)
-        err = nc_put_vara_double( ncid, dataID, start, count, data.data() + start[1]*count[2]*count[3]);
-        err = nc_put_vara_double( ncid, tvarID, &Tstart, &Tcount, &time);
-        //err = nc_close(ncid);
+        dg::file::put_vara_double( ncid, dataID, i, grid, data, false);
+        if(rank==0)err = nc_put_vara_double( ncid, tvarID, &Tstart, &Tcount, &time);
     }
-    err = nc_close(ncid);
+    if(rank==0)err = nc_close(ncid);
     MPI_Finalize();
     return 0;
 }
diff --git a/inc/file/netcdf_t.cpp b/inc/file/netcdf_t.cpp
index 4327351b8..1f36a0dae 100644
--- a/inc/file/netcdf_t.cpp
+++ b/inc/file/netcdf_t.cpp
@@ -4,6 +4,7 @@
 #include <cmath>
 
 #include "dg/algorithm.h"
+#define _FILE_INCLUDED_BY_DG_
 #include "nc_utilities.h"
 
 double function( double x, double y, double z){return sin(x)*sin(y)*cos(z);}
@@ -28,7 +29,6 @@ int main()
     int ncid;
     dg::file::NC_Error_Handle err;
     err = nc_create( "test.nc", NC_NETCDF4|NC_CLOBBER, &ncid); //for netcdf4
-    //err = nc_create( "test.nc", NC_CLOBBER, &ncid);
     err = nc_put_att_text( ncid, NC_GLOBAL, "input", hello.size(), hello.data());
 
     int dim_ids[4], tvarID;
@@ -63,9 +63,9 @@ int main()
         dg::blas1::scal( dataX, cos( time));
         dg::blas1::scal( dataY, cos( time));
         dg::blas1::scal( dataZ, cos( time));
-        err = nc_put_vara_double( ncid, vectorID[0], start, count, dataX.data());
-        err = nc_put_vara_double( ncid, vectorID[1], start, count, dataY.data());
-        err = nc_put_vara_double( ncid, vectorID[2], start, count, dataZ.data());
+        dg::file::put_vara_double( ncid, vectorID[0], i, g, dataX);
+        dg::file::put_vara_double( ncid, vectorID[1], i, g, dataY);
+        dg::file::put_vara_double( ncid, vectorID[2], i, g, dataZ);
         //write time
         err = nc_put_vara_double( ncid, tvarID, &Tstart, &Tcount, &time);
     }
-- 
GitLab


From 23a005879fb92d529329057f75be3d157b24e978 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Feb 2021 00:06:45 +0100
Subject: [PATCH 519/540] Second Fix of MPI Curvilinear perp grid

- the error was that the perp metric contained the 3rd dimension
(is this an indication of a mis-design? Or a mis-use of the
tensor::volume function?)
---
 inc/geometries/curvilinear.h     |  72 +++++++++---------
 inc/geometries/mpi_curvilinear.h | 124 +++++++++++++++++--------------
 2 files changed, 105 insertions(+), 91 deletions(-)

diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index 79b55d503..b27ac0d39 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -89,7 +89,7 @@ struct RealCurvilinearGrid2d : public dg::aRealGeometry2d<real_type>
 {
     ///@copydoc hide_grid_parameters2d
     RealCurvilinearGrid2d( const aRealGenerator2d<real_type>& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx=dg::DIR, bc bcy=dg::PER):
-        dg::aRealGeometry2d<real_type>( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), handle_(generator)
+        dg::aRealGeometry2d<real_type>( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, dg::PER), m_handle(generator)
     {
         construct( n,Nx,Ny);
     }
@@ -101,7 +101,7 @@ struct RealCurvilinearGrid2d : public dg::aRealGeometry2d<real_type>
     explicit RealCurvilinearGrid2d( RealCurvilinearProductGrid3d<real_type> g);
 
     ///read access to the generator
-    const aRealGenerator2d<real_type>& generator() const{return *handle_;}
+    const aRealGenerator2d<real_type>& generator() const{return *m_handle;}
     virtual RealCurvilinearGrid2d* clone()const override final{return new RealCurvilinearGrid2d(*this);}
     private:
     virtual void do_set(unsigned new_n, unsigned new_Nx, unsigned new_Ny) override final
@@ -111,15 +111,15 @@ struct RealCurvilinearGrid2d : public dg::aRealGeometry2d<real_type>
     }
     void construct( unsigned n, unsigned Nx, unsigned Ny);
     virtual SparseTensor<thrust::host_vector<real_type> > do_compute_jacobian( ) const override final{
-        return jac_;
+        return m_jac;
     }
     virtual SparseTensor<thrust::host_vector<real_type>> do_compute_metric( ) const override final{
-        return metric_;
+        return m_metric;
     }
-    virtual std::vector<thrust::host_vector<real_type>> do_compute_map()const override final{return map_;}
-    dg::SparseTensor<thrust::host_vector<real_type>> jac_, metric_;
-    std::vector<thrust::host_vector<real_type>> map_;
-    dg::ClonePtr<aRealGenerator2d<real_type>> handle_;
+    virtual std::vector<thrust::host_vector<real_type>> do_compute_map()const override final{return m_map;}
+    dg::SparseTensor<thrust::host_vector<real_type>> m_jac, m_metric;
+    std::vector<thrust::host_vector<real_type>> m_map;
+    dg::ClonePtr<aRealGenerator2d<real_type>> m_handle;
 };
 
 
@@ -138,15 +138,15 @@ struct RealCurvilinearProductGrid3d : public dg::aRealProductGeometry3d<real_typ
     RealCurvilinearProductGrid3d( const aRealGenerator2d<real_type>& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx=dg::DIR, bc bcy=dg::PER, bc bcz=dg::PER):
         dg::aRealProductGeometry3d<real_type>( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz)
     {
-        map_.resize(3);
-        handle_ = generator;
+        m_map.resize(3);
+        m_handle = generator;
         constructPerp( n, Nx, Ny);
         constructParallel(Nz);
     }
 
 
     ///@copydoc RealCurvilinearGrid2d::generator()const
-    const aRealGenerator2d<real_type> & generator() const{return *handle_;}
+    const aRealGenerator2d<real_type> & generator() const{return *m_handle;}
     virtual RealCurvilinearProductGrid3d* clone()const override final{return new RealCurvilinearProductGrid3d(*this);}
     private:
     virtual RealCurvilinearGrid2d<real_type>* do_perp_grid() const override final;
@@ -159,22 +159,22 @@ struct RealCurvilinearProductGrid3d : public dg::aRealProductGeometry3d<real_typ
     //construct phi and lift rest to 3d
     void constructParallel(unsigned Nz)
     {
-        map_[2]=dg::evaluate(dg::cooZ3d, *this);
+        m_map[2]=dg::evaluate(dg::cooZ3d, *this);
         unsigned size = this->size();
         unsigned size2d = this->n()*this->n()*this->Nx()*this->Ny();
         //resize for 3d values
         for( unsigned r=0; r<6;r++)
-            jac_.values()[r].resize(size);
-        map_[0].resize(size);
-        map_[1].resize(size);
+            m_jac.values()[r].resize(size);
+        m_map[0].resize(size);
+        m_map[1].resize(size);
         //lift to 3D grid
         for( unsigned k=1; k<Nz; k++)
             for( unsigned i=0; i<size2d; i++)
             {
                 for(unsigned r=0; r<6; r++)
-                    jac_.values()[r][k*size2d+i] = jac_.values()[r][(k-1)*size2d+i];
-                map_[0][k*size2d+i] = map_[0][(k-1)*size2d+i];
-                map_[1][k*size2d+i] = map_[1][(k-1)*size2d+i];
+                    m_jac.values()[r][k*size2d+i] = m_jac.values()[r][(k-1)*size2d+i];
+                m_map[0][k*size2d+i] = m_map[0][(k-1)*size2d+i];
+                m_map[1][k*size2d+i] = m_map[1][(k-1)*size2d+i];
             }
     }
     //construct 2d plane
@@ -184,22 +184,22 @@ struct RealCurvilinearProductGrid3d : public dg::aRealProductGeometry3d<real_typ
         dg::Grid1d gY1d( this->y0(), this->y1(), n, Ny);
         thrust::host_vector<real_type> x_vec = dg::evaluate( dg::cooX1d, gX1d);
         thrust::host_vector<real_type> y_vec = dg::evaluate( dg::cooX1d, gY1d);
-        jac_ = SparseTensor< thrust::host_vector<real_type>>( x_vec);//unit tensor
-        jac_.values().resize( 6);
-        handle_->generate( x_vec, y_vec, map_[0], map_[1], jac_.values()[2], jac_.values()[3], jac_.values()[4], jac_.values()[5]);
-        jac_.idx(0,0) = 2, jac_.idx(0,1) = 3, jac_.idx(1,0)=4, jac_.idx(1,1) = 5;
+        m_jac = SparseTensor< thrust::host_vector<real_type>>( x_vec);//unit tensor
+        m_jac.values().resize( 6);
+        m_handle->generate( x_vec, y_vec, m_map[0], m_map[1], m_jac.values()[2], m_jac.values()[3], m_jac.values()[4], m_jac.values()[5]);
+        m_jac.idx(0,0) = 2, m_jac.idx(0,1) = 3, m_jac.idx(1,0)=4, m_jac.idx(1,1) = 5;
     }
     virtual SparseTensor<thrust::host_vector<real_type> > do_compute_jacobian( ) const override final{
-        return jac_;
+        return m_jac;
     }
     virtual SparseTensor<thrust::host_vector<real_type> > do_compute_metric( ) const override final
     {
-        return detail::square( jac_, map_[0], handle_->isOrthogonal());
+        return detail::square( m_jac, m_map[0], m_handle->isOrthogonal());
     }
-    virtual std::vector<thrust::host_vector<real_type> > do_compute_map()const override final{return map_;}
-    std::vector<thrust::host_vector<real_type> > map_;
-    SparseTensor<thrust::host_vector<real_type> > jac_;
-    dg::ClonePtr<aRealGenerator2d<real_type>> handle_;
+    virtual std::vector<thrust::host_vector<real_type> > do_compute_map()const override final{return m_map;}
+    std::vector<thrust::host_vector<real_type> > m_map;
+    SparseTensor<thrust::host_vector<real_type> > m_jac;
+    dg::ClonePtr<aRealGenerator2d<real_type>> m_handle;
 };
 
 using CurvilinearGrid2d         = dg::geo::RealCurvilinearGrid2d<double>;
@@ -215,19 +215,21 @@ using CurvilinearProductGrid3d  = CurvilinearProductGrid3d ;
 ///@cond
 template<class real_type>
 RealCurvilinearGrid2d<real_type>::RealCurvilinearGrid2d( RealCurvilinearProductGrid3d<real_type> g):
-    dg::aRealGeometry2d<real_type>( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy() ), handle_(g.generator())
+    dg::aRealGeometry2d<real_type>( g.x0(), g.x1(), g.y0(), g.y1(), g.n(), g.Nx(), g.Ny(), g.bcx(), g.bcy() ), m_handle(g.generator())
 {
     g.set( this->n(), this->Nx(), this->Ny(), 1); //shouldn't trigger 2d grid generator
-    map_=g.map();
-    jac_=g.jacobian();
-    metric_=g.metric();
-    dg::blas1::copy( 1., metric_.values()[3]); //set pp to 1
-    map_.pop_back();
+    m_map=g.map();
+    m_jac=g.jacobian();
+    m_metric=g.metric();
+    // we rely on the fact that the 3d grid uses square to compute its metric
+    // so the (2,2) entry is value 3 that we need to set to 1
+    dg::blas1::copy( 1., m_metric.values()[3]);
+    m_map.pop_back();
 }
 template<class real_type>
 void RealCurvilinearGrid2d<real_type>::construct( unsigned n, unsigned Nx, unsigned Ny)
 {
-    RealCurvilinearProductGrid3d<real_type> g( *handle_, n,Nx,Ny,1,this->bcx());
+    RealCurvilinearProductGrid3d<real_type> g( *m_handle, n,Nx,Ny,1,this->bcx());
     *this = RealCurvilinearGrid2d<real_type>(g);
 }
 template<class real_type>
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index c9f70f7ae..ab4945e89 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -30,7 +30,7 @@ struct RealCurvilinearMPIGrid2d : public dg::aRealMPIGeometry2d<real_type>
     /// @param comm a two-dimensional Cartesian communicator
     /// @note the paramateres given in the constructor are global parameters
     RealCurvilinearMPIGrid2d( const aRealGenerator2d<real_type>& generator, unsigned n, unsigned Nx, unsigned Ny, dg::bc bcx, dg::bc bcy, MPI_Comm comm):
-        dg::aRealMPIGeometry2d<real_type>( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, bcy, comm), handle_(generator)
+        dg::aRealMPIGeometry2d<real_type>( 0, generator.width(), 0., generator.height(), n, Nx, Ny, bcx, bcy, comm), m_handle(generator)
     {
         //generate global 2d grid and then reduce to local
         RealCurvilinearGrid2d<real_type> g(generator, n, Nx, Ny);
@@ -40,11 +40,11 @@ struct RealCurvilinearMPIGrid2d : public dg::aRealMPIGeometry2d<real_type>
     explicit RealCurvilinearMPIGrid2d( const RealCurvilinearProductMPIGrid3d<real_type>& g);
 
     ///read access to the generator
-    const aRealGenerator2d<real_type>& generator() const{return *handle_;}
+    const aRealGenerator2d<real_type>& generator() const{return *m_handle;}
     virtual RealCurvilinearMPIGrid2d* clone()const override final{return new RealCurvilinearMPIGrid2d(*this);}
     virtual RealCurvilinearGrid2d<real_type>* global_geometry()const override final{
         return new RealCurvilinearGrid2d<real_type>(
-                *handle_,
+                *m_handle,
                 global().n(), global().Nx(), global().Ny(),
                 global().bcx(), global().bcy());
     }
@@ -54,7 +54,7 @@ struct RealCurvilinearMPIGrid2d : public dg::aRealMPIGeometry2d<real_type>
     virtual void do_set( unsigned new_n, unsigned new_Nx, unsigned new_Ny) override final
     {
         dg::aRealMPITopology2d<real_type>::do_set(new_n, new_Nx, new_Ny);
-        RealCurvilinearGrid2d<real_type> g( *handle_, new_n, new_Nx, new_Ny);
+        RealCurvilinearGrid2d<real_type> g( *m_handle, new_n, new_Nx, new_Ny);
         divide_and_conquer(g);//distribute to processes
     }
     void divide_and_conquer(const RealCurvilinearGrid2d<real_type>& g_)
@@ -65,30 +65,31 @@ struct RealCurvilinearMPIGrid2d : public dg::aRealMPIGeometry2d<real_type>
         for( unsigned i=0; i<3; i++)
             for( unsigned j=0; j<3; j++)
             {
-                metric_.idx(i,j) = metric.idx(i,j);
-                jac_.idx(i,j) = jacobian.idx(i,j);
+                m_metric.idx(i,j) = metric.idx(i,j);
+                m_jac.idx(i,j) = jacobian.idx(i,j);
             }
-        jac_.values().resize( jacobian.values().size());
+        // Here we set the communicator implicitly
+        m_jac.values().resize( jacobian.values().size());
         for( unsigned i=0; i<jacobian.values().size(); i++)
-            jac_.values()[i] = global2local( jacobian.values()[i], *this);
-        metric_.values().resize( metric.values().size());
+            m_jac.values()[i] = global2local( jacobian.values()[i], *this);
+        m_metric.values().resize( metric.values().size());
         for( unsigned i=0; i<metric.values().size(); i++)
-            metric_.values()[i] = global2local( metric.values()[i], *this);
-        map_.resize(map.size());
+            m_metric.values()[i] = global2local( metric.values()[i], *this);
+        m_map.resize(map.size());
         for( unsigned i=0; i<map.size(); i++)
-            map_[i] = global2local( map[i], *this);
+            m_map[i] = global2local( map[i], *this);
     }
 
     virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> do_compute_jacobian( ) const override final{
-        return jac_;
+        return m_jac;
     }
     virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> do_compute_metric( ) const override final{
-        return metric_;
+        return m_metric;
     }
-    virtual std::vector<MPI_Vector<thrust::host_vector<real_type>>> do_compute_map()const override final{return map_;}
-    dg::SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> jac_, metric_;
-    std::vector<MPI_Vector<thrust::host_vector<real_type>>> map_;
-    dg::ClonePtr<aRealGenerator2d<real_type>> handle_;
+    virtual std::vector<MPI_Vector<thrust::host_vector<real_type>>> do_compute_map()const override final{return m_map;}
+    dg::SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> m_jac, m_metric;
+    std::vector<MPI_Vector<thrust::host_vector<real_type>>> m_map;
+    dg::ClonePtr<aRealGenerator2d<real_type>> m_handle;
 };
 
 /**
@@ -105,9 +106,9 @@ struct RealCurvilinearProductMPIGrid3d : public dg::aRealProductMPIGeometry3d<re
     /// @note the paramateres given in the constructor are global parameters
     RealCurvilinearProductMPIGrid3d( const aRealGenerator2d<real_type>& generator, unsigned n, unsigned Nx, unsigned Ny, unsigned Nz, bc bcx, bc bcy, bc bcz, MPI_Comm comm):
         dg::aRealProductMPIGeometry3d<real_type>( 0, generator.width(), 0., generator.height(), 0., 2.*M_PI, n, Nx, Ny, Nz, bcx, bcy, bcz, comm),
-        handle_( generator)
+        m_handle( generator)
     {
-        map_.resize(3);
+        m_map.resize(3);
         RealCurvilinearMPIGrid2d<real_type> g(generator,n,Nx,Ny, bcx, bcy, this->get_perp_comm());
         constructPerp( g);
         constructParallel(this->local().Nz());
@@ -115,11 +116,11 @@ struct RealCurvilinearProductMPIGrid3d : public dg::aRealProductMPIGeometry3d<re
 
 
     ///read access to the generator
-    const aRealGenerator2d<real_type>& generator() const{return *handle_;}
+    const aRealGenerator2d<real_type>& generator() const{return *m_handle;}
     virtual RealCurvilinearProductMPIGrid3d* clone()const{return new RealCurvilinearProductMPIGrid3d(*this);}
     virtual RealCurvilinearProductGrid3d<real_type>* global_geometry()const{
         return new RealCurvilinearProductGrid3d<real_type>(
-                *handle_,
+                *m_handle,
                 global().n(), global().Nx(), global().Ny(), global().Nz(),
                 global().bcx(), global().bcy(), global().bcz());
     }
@@ -132,20 +133,20 @@ struct RealCurvilinearProductMPIGrid3d : public dg::aRealProductMPIGeometry3d<re
         dg::aRealMPITopology3d<real_type>::do_set(new_n, new_Nx, new_Ny, new_Nz);
         if( !( new_n == this->n() && new_Nx == global().Nx() && new_Ny == global().Ny() ) )
         {
-            RealCurvilinearMPIGrid2d<real_type> g( *handle_,new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), this->get_perp_comm());
+            RealCurvilinearMPIGrid2d<real_type> g( *m_handle,new_n,new_Nx,new_Ny, this->bcx(), this->bcy(), this->get_perp_comm());
             constructPerp( g);
         }
         constructParallel(this->local().Nz());
     }
     void constructPerp( RealCurvilinearMPIGrid2d<real_type>& g2d)
     {
-        jac_=g2d.jacobian();
-        map_=g2d.map();
+        m_jac=g2d.jacobian();
+        m_map=g2d.map();
     }
     void constructParallel( unsigned localNz )
     {
-        map_.resize(3);
-        map_[2]=dg::evaluate(dg::cooZ3d, *this);
+        m_map.resize(3);
+        m_map[2]=dg::evaluate(dg::cooZ3d, *this);
         unsigned size = this->local().size();
         unsigned size2d = this->n()*this->n()*this->local().Nx()*this->local().Ny();
         //resize for 3d values
@@ -153,55 +154,66 @@ struct RealCurvilinearProductMPIGrid3d : public dg::aRealProductMPIGeometry3d<re
         exblas::mpi_reduce_communicator( comm, &comm_mod, &comm_mod_reduce);
         for( unsigned r=0; r<6;r++)
         {
-            jac_.values()[r].data().resize(size);
-            jac_.values()[r].set_communicator( comm, comm_mod, comm_mod_reduce);
+            m_jac.values()[r].data().resize(size);
+            m_jac.values()[r].set_communicator( comm, comm_mod, comm_mod_reduce);
         }
-        map_[0].data().resize(size);
-        map_[0].set_communicator( comm, comm_mod, comm_mod_reduce);
-        map_[1].data().resize(size);
-        map_[1].set_communicator( comm, comm_mod, comm_mod_reduce);
+        m_map[0].data().resize(size);
+        m_map[0].set_communicator( comm, comm_mod, comm_mod_reduce);
+        m_map[1].data().resize(size);
+        m_map[1].set_communicator( comm, comm_mod, comm_mod_reduce);
         //lift to 3D grid
         for( unsigned k=1; k<localNz; k++)
             for( unsigned i=0; i<size2d; i++)
             {
                 for(unsigned r=0; r<6; r++)
-                    jac_.values()[r].data()[k*size2d+i] = jac_.values()[r].data()[(k-1)*size2d+i];
-                map_[0].data()[k*size2d+i] = map_[0].data()[(k-1)*size2d+i];
-                map_[1].data()[k*size2d+i] = map_[1].data()[(k-1)*size2d+i];
+                    m_jac.values()[r].data()[k*size2d+i] = m_jac.values()[r].data()[(k-1)*size2d+i];
+                m_map[0].data()[k*size2d+i] = m_map[0].data()[(k-1)*size2d+i];
+                m_map[1].data()[k*size2d+i] = m_map[1].data()[(k-1)*size2d+i];
             }
     }
     virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> do_compute_jacobian( ) const override final{
-        return jac_;
+        return m_jac;
     }
     virtual SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> do_compute_metric( ) const override final{
-        return detail::square( jac_, map_[0], handle_->isOrthogonal());
+        return detail::square( m_jac, m_map[0], m_handle->isOrthogonal());
     }
-    virtual std::vector<MPI_Vector<thrust::host_vector<real_type>>> do_compute_map()const override final{return map_;}
-    dg::SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> jac_;
-    std::vector<MPI_Vector<thrust::host_vector<real_type>>> map_;
-    ClonePtr<dg::geo::aRealGenerator2d<real_type>> handle_;
+    virtual std::vector<MPI_Vector<thrust::host_vector<real_type>>> do_compute_map()const override final{return m_map;}
+    dg::SparseTensor<MPI_Vector<thrust::host_vector<real_type>>> m_jac;
+    std::vector<MPI_Vector<thrust::host_vector<real_type>>> m_map;
+    ClonePtr<dg::geo::aRealGenerator2d<real_type>> m_handle;
 };
 ///@cond
 template<class real_type>
 RealCurvilinearMPIGrid2d<real_type>::RealCurvilinearMPIGrid2d( const RealCurvilinearProductMPIGrid3d<real_type>& g):
     dg::aRealMPIGeometry2d<real_type>( g.global().x0(), g.global().x1(), g.global().y0(), g.global().y1(), g.global().n(), g.global().Nx(), g.global().Ny(), g.global().bcx(), g.global().bcy(), g.get_perp_comm() ),
-    handle_(g.generator())
+    m_handle(g.generator())
 {
-    map_=g.map();
-    jac_=g.jacobian();
-    metric_=g.metric();
+    m_map=g.map();
+    m_jac=g.jacobian();
+    m_metric=g.metric();
     //now resize to 2d
-    map_.pop_back();
+    m_map.pop_back();
     unsigned s = this->local().size();
-    for( unsigned i=0; i<jac_.values().size(); i++)
-        jac_.values()[i].data().resize(s);
-    for( unsigned i=0; i<metric_.values().size(); i++)
-        metric_.values()[i].data().resize(s);
-    for( unsigned i=0; i<map_.size(); i++)
-        map_[i].data().resize(s);
-    jac_.set_communicator( this->communicator());
-    metric_.set_communicator( this->communicator());
-    map_.set_communicator( this->communicator());
+    MPI_Comm comm = g.get_perp_comm(), comm_mod, comm_mod_reduce;
+    exblas::mpi_reduce_communicator( comm, &comm_mod, &comm_mod_reduce);
+    for( unsigned i=0; i<m_jac.values().size(); i++)
+    {
+        m_jac.values()[i].data().resize(s);
+        m_jac.values()[i].set_communicator( comm, comm_mod, comm_mod_reduce);
+    }
+    for( unsigned i=0; i<m_metric.values().size(); i++)
+    {
+        m_metric.values()[i].data().resize(s);
+        m_metric.values()[i].set_communicator( comm, comm_mod, comm_mod_reduce);
+    }
+    // we rely on the fact that the 3d grid uses square to compute its metric
+    // so the (2,2) entry is value 3 that we need to set to 1
+    dg::blas1::copy( 1., m_metric.values()[3]);
+    for( unsigned i=0; i<m_map.size(); i++)
+    {
+        m_map[i].data().resize(s);
+        m_map[i].set_communicator( comm, comm_mod, comm_mod_reduce);
+    }
 }
 ///@endcond
 //
-- 
GitLab


From 3597e0486e64919a9a2996484f12fbc60e86da2d Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Feb 2021 00:13:14 +0100
Subject: [PATCH 520/540] Fix const in file::put_var function

---
 inc/dg/backend/mpi_vector.h     | 14 ++++++++++++--
 inc/dg/dg_doc.h                 | 20 ++++++++++----------
 inc/dg/topology/base_geometry.h | 15 ++++++++++++++-
 inc/dg/topology/mpi_base.h      |  2 +-
 inc/file/easy_output.h          |  2 +-
 5 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index e3ea6f93e..730ab58c9 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -38,7 +38,7 @@ struct MPI_Vector
     /**
      * @brief construct a vector
      *
-     * calls \c exblas::mpi_reduce_communicator() (collective call)
+     * calls \c dg::exblas::mpi_reduce_communicator() (collective call)
      * @param data internal data copy
      * @param comm MPI communicator (may not be \c MPI_COMM_NULL)
      */
@@ -80,8 +80,18 @@ struct MPI_Vector
      * @return returns MPI_COMM_NULL to processes not part of that group
      */
     MPI_Comm communicator_mod_reduce() const{return m_comm128Reduce;}
+
     /**
-    * @brief Set the communicators with \c exblas::mpi_reduce_communicator
+    * @brief Set the communicators with \c dg::exblas::mpi_reduce_communicator
+    *
+    * The reason why you can't just set the comm and need three parameters is
+    * that generating communicators involves communication, which you might want to
+    * avoid when you do it many times. So you have to call the function as
+    * @code
+    * MPI_Comm comm = MPI_COMM_WORLD, comm_mod, comm_mod_reduce;
+    * dg::exblas::mpi_reduce_communicator( comm, &comm_mod, &comm_mod_reduce);
+    * mpi_vector.set_communicator( comm, comm_mod, comm_mod_reduce);
+    * @endcode
     */
     void set_communicator(MPI_Comm comm, MPI_Comm comm_mod, MPI_Comm comm_mod_reduce){
         m_comm = comm;
diff --git a/inc/dg/dg_doc.h b/inc/dg/dg_doc.h
index 276f51c90..4af882803 100644
--- a/inc/dg/dg_doc.h
+++ b/inc/dg/dg_doc.h
@@ -186,7 +186,7 @@
   */
   /** @class hide_geometry
   * @tparam Geometry
-  A type that is or derives from one of the abstract geometry base classes ( \c aGeometry2d, \c aGeometry3d, \c aMPIGeometry2d, ...).
+  * A type that is or derives from one of the abstract geometry base classes ( \c aGeometry2d, \c aGeometry3d, \c aMPIGeometry2d, ...).
   */
 
   /** @class hide_container_geometry
@@ -202,7 +202,7 @@
 
   /** @class hide_geometry_matrix_container
   * @tparam Geometry
-  A type that is or derives from one of the abstract geometry base classes ( \c aGeometry2d, \c aGeometry3d, \c aMPIGeometry2d, ...). \c Geometry determines which \c Matrix and \c Container types can be used:
+  * A type that is or derives from one of the abstract geometry base classes ( \c aGeometry2d, \c aGeometry3d, \c aMPIGeometry2d, ...). \c Geometry determines which \c Matrix and \c Container types can be used:
   * @tparam Matrix
   * A class for which the blas2 functions are callable in connection with the \c Container class and to which the return type of \c create::dx() can be converted using \c dg::blas2::transfer.
   * The \c Matrix type can be one of:
@@ -218,15 +218,15 @@
   *  - \c dg::MHVec or \c dg::MDVec when \c Geometry is one of the MPI geometries
   */
 
- /** @class hide_symmetric_op
+/** @class hide_symmetric_op
  * @tparam SymmetricOp
- A class for which the \c blas2::symv(Matrix&, Vector1&, Vector2&) function is callable
- with the \c Container type as argument. Also, The functions \c %inv_weights() and \c %precond()
- need to be callable and return inverse weights and the preconditioner for the conjugate
- gradient method. \c SymmetricOp is assumed to be linear, symmetric and positive definite!
- @note you can make your own \c SymmetricOp by providing the member function \c void \c symv(const Container&, Container&);
-  and specializing \c TensorTraits with the \c SelfMadeMatrixTag as the \c tensor_category
-  */
+ * A class for which the \c blas2::symv(Matrix&, Vector1&, Vector2&) function is callable
+ * with the \c Container type as argument. Also, The functions \c %inv_weights() and \c %precond()
+ * need to be callable and return inverse weights and the preconditioner for the conjugate
+ * gradient method. \c SymmetricOp is assumed to be linear, symmetric and positive definite!
+ * @note you can make your own \c SymmetricOp by providing the member function \c void \c symv(const Container&, Container&);
+ * and specializing \c TensorTraits with the \c SelfMadeMatrixTag as the \c tensor_category
+ */
 
 /*! @mainpage Introduction
  *
diff --git a/inc/dg/topology/base_geometry.h b/inc/dg/topology/base_geometry.h
index fa50d7da4..0143b43fb 100644
--- a/inc/dg/topology/base_geometry.h
+++ b/inc/dg/topology/base_geometry.h
@@ -161,7 +161,20 @@ struct aRealGeometry3d : public aRealTopology3d<real_type>
     }
 };
 
-///@brief a 3d product space Geometry
+/**
+ * @brief A 3d product space Geometry \f$ g_{2d} \otimes g_{1d}\f$
+ *
+ * This class represents a product space of a 2d grid (the "perp_grid") and a 1d
+ * grid (the "parallel_grid").
+ * The special feature of the product space is that the metric is simply
+ * \f[ g = \begin{pmatrix}
+ *  (g_{2d}(x,y)) & 0 \\
+ *  0 & g_{1d}(x,y)
+ * \end{pmatrix}
+ * \f]
+ * That is the metric elements do not depend on the third coordinate.
+ * @tparam real_type The value type of the grid
+ */
 template<class real_type>
 struct aRealProductGeometry3d : public aRealGeometry3d<real_type>
 {
diff --git a/inc/dg/topology/mpi_base.h b/inc/dg/topology/mpi_base.h
index 1c1ab91d7..b1d9a3793 100644
--- a/inc/dg/topology/mpi_base.h
+++ b/inc/dg/topology/mpi_base.h
@@ -101,7 +101,7 @@ struct aRealMPIGeometry3d : public aRealMPITopology3d<real_type>
     }
 };
 
-///@brief a 3d product space MPI Geometry
+///@copydoc aRealProductGeometry3d
 template<class real_type>
 struct aRealProductMPIGeometry3d : public aRealMPIGeometry3d<real_type>
 {
diff --git a/inc/file/easy_output.h b/inc/file/easy_output.h
index 16cbb7b08..532c51e50 100644
--- a/inc/file/easy_output.h
+++ b/inc/file/easy_output.h
@@ -357,7 +357,7 @@ void put_var_double(int ncid, int varid,
 ///@copydoc put_vara_double()
 template<class host_vector>
 void put_vara_double(int ncid, int varid, unsigned slice,
-    dg::aMPITopology3d& grid, dg::MPI_Vector<host_vector>& data,
+    const dg::aMPITopology3d& grid, const dg::MPI_Vector<host_vector>& data,
     bool parallel = false)
 {
     file::NC_Error_Handle err;
-- 
GitLab


From df2c555fdc6f0c581b78e247830971060b8b2daa Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Feb 2021 00:15:20 +0100
Subject: [PATCH 521/540] Fix output in ribeiro_t and ribeiro_mpit

---
 inc/geometries/ribeiro_mpit.cu | 2 +-
 inc/geometries/ribeiro_t.cu    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index cb4a2bf6e..24f82242d 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -121,7 +121,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(vol, w2d, vol));
-    if(rank==0)std::cout << "Rel Error of volume form is "<<error<<"\n";
+    if(rank==0)std::cout << "Difference vol - 1/g_xx "<<error<<"\n";
 
     vol = dg::create::volume( g3d);
     dg::MHVec ones3d = dg::evaluate( dg::one, g3d);
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index 9839250ac..8a16f4646 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -141,7 +141,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(vol, w2d, vol));
-    std::cout << "Rel Error of volume form is "<<error<<"\n";
+    std::cout << "Difference vol - 1/g_xx "<<error<<"\n";
 
     vol = dg::create::volume( g3d);
     dg::HVec ones3d = dg::evaluate( dg::one, g3d);
-- 
GitLab


From 2289124d3e7f2d2f566cb7f25d86fca111dad5d3 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Feb 2021 00:31:04 +0100
Subject: [PATCH 522/540] Some docu on Alignment tensor

---
 inc/geometries/fluxfunctions.h  | 6 +++---
 inc/geometries/geometries_doc.h | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/inc/geometries/fluxfunctions.h b/inc/geometries/fluxfunctions.h
index e3289a4ff..777ee631c 100644
--- a/inc/geometries/fluxfunctions.h
+++ b/inc/geometries/fluxfunctions.h
@@ -458,7 +458,7 @@ struct SquareNorm : public aCylindricalFunctor<SquareNorm>
 };
 
 
-/*!@brief \f[ \chi^{ij} = b^ib^j\f]
+/*!@brief \f$ \chi^{ij} = b^ib^j\f$
  *
  * Creates the two times contravariant tensor that,
  * when applied to a covariant vector, creates a vector
@@ -491,7 +491,7 @@ dg::SparseTensor<dg::get_host_vector<Geometry3d>> createAlignmentTensor(
     t.values() = chi;
     return t;
 }
-/*!@brief \f[ \chi^{ij} = g^{ij} - b^ib^j\f]
+/*!@brief \f$ h^{ij} = g^{ij} - b^ib^j\f$
  *
  * Creates the two times contravariant tensor that,
  * when applied to a covariant vector, creates a vector
@@ -499,7 +499,7 @@ dg::SparseTensor<dg::get_host_vector<Geometry3d>> createAlignmentTensor(
  *
  * @param bhat The (unit) vector field \c b
  * @param g The vector field is pushed unto this grid
- * @return The tensor \c chi living on the coordinate system given by \c g
+ * @return The tensor \c h living on the coordinate system given by \c g
  * @tparam Geometry3d A three-dimensional geometry
  */
 template<class Geometry3d>
diff --git a/inc/geometries/geometries_doc.h b/inc/geometries/geometries_doc.h
index 7c69e0d62..35707beb8 100644
--- a/inc/geometries/geometries_doc.h
+++ b/inc/geometries/geometries_doc.h
@@ -29,6 +29,7 @@
       @defgroup magnetic 3.2 Magnetic field, curvatures and associated functors
       @defgroup profiles 3.3 Penalization, weight and monitor metric functors
       @defgroup fluxfunctions 3.4. Utility functor functionality
+      \f$ h^{ij}\f$
  * @}
  * @defgroup fieldaligned 4. Fieldaligned derivatives
  * \f$ \nabla_\parallel f\f$
-- 
GitLab


From b3075cf9233f762439fbffa94be26d2220ace578 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Feb 2021 00:37:54 +0100
Subject: [PATCH 523/540] Replace multiply_sigma function in feltor

---
 src/feltor/feltor.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/feltor/feltor.h b/src/feltor/feltor.h
index 783796cf1..63534da79 100644
--- a/src/feltor/feltor.h
+++ b/src/feltor/feltor.h
@@ -355,6 +355,7 @@ struct Explicit
     }
     const Container& lapMperpP (int i)
     {
+        m_lapperpP.set_chi( 1.);
         dg::blas2::gemv( m_lapperpP, m_phi[i], m_temp1);
         return m_temp1;
     }
@@ -1072,7 +1073,8 @@ void Explicit<Geometry, IMatrix, Matrix, Container>::operator()(
         dg::blas1::axpby( 1., m_s[0][0], 0.5*m_p.tau[1]*m_p.mu[1], m_temp0, m_s[0][1]);
         // potential part of FLR correction S_N += -div*(mu S_n grad*Phi/B^2)
         dg::blas1::pointwiseDot( m_p.mu[1], m_s[0][0], m_binv, m_binv, 0., m_temp0);
-        m_lapperpP.multiply_sigma( 1., m_temp0, m_phi[0], 1., m_s[0][1]);
+        m_lapperpP.set_chi( m_temp0);
+        m_lapperpP.symv( 1., m_phi[0], 1., m_s[0][1]);
 
         // S_U += - U S_N/N
         dg::blas1::pointwiseDot( -1.,  m_fields[1][0],  m_s[0][0], 0., m_temp0);
-- 
GitLab


From efa85e1e9566f9f51cf486b9c98945ee3941d9ef Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Feb 2021 00:50:03 +0100
Subject: [PATCH 524/540] Remove dg::blas1::transfer

- from blas1.h and all programs that use it
---
 CHANGELOG.md                                  |  5 +++-
 diag/feltorSHdiag.cu                          |  8 +++---
 diag/impRdiag.cu                              |  2 +-
 diag/probes.h                                 |  4 +--
 diag/toeflEPdiag.cu                           |  2 +-
 diag/toeflRdiag.cu                            |  2 +-
 inc/dg/blas1.h                                | 17 +----------
 inc/geometries/geometryX_elliptic_b.cu        |  8 +++---
 .../geometryX_refined_elliptic_b.cu           |  6 ++--
 inc/geometries/geometry_advection_b.cu        |  6 ++--
 inc/geometries/geometry_advection_mpib.cu     |  6 ++--
 inc/geometries/geometry_elliptic_b.cu         |  6 ++--
 inc/geometries/hector.h                       | 18 ++++++------
 inc/geometries/hector_t.cu                    |  2 +-
 inc/geometries/ribeiroX_t.cu                  | 12 ++++----
 inc/geometries/ribeiro_mpit.cu                |  4 +--
 inc/geometries/ribeiro_t.cu                   |  6 ++--
 inc/geometries/simple_orthogonal_t.cu         |  2 +-
 src/ep/toeflR.cu                              |  4 +--
 src/ep/toeflR.cuh                             |  2 +-
 src/ep/toefl_mpi.cu                           | 12 ++++----
 src/feltorSH/feltor.cu                        | 12 ++++----
 src/feltorSH/feltor_hpc.cu                    | 12 ++++----
 src/feltorSH/feltor_mpi.cu                    | 12 ++++----
 src/feltorSHp/Makefile                        |  5 +---
 src/feltorSHp/feltor.cu                       | 12 ++++----
 src/feltorSHp/feltor_hpc.cu                   | 20 ++++++-------
 src/feltorSHp/feltor_mpi.cu                   | 20 ++++++-------
 src/feltorSesol/feltor.cu                     |  8 +++---
 src/feltorSesol/feltor_hpc.cu                 | 16 +++++------
 src/feltorSesol/feltor_mpi.cu                 | 16 +++++------
 src/feltorShw/feltor.cu                       |  8 +++---
 src/feltorShw/feltor_hpc.cu                   | 16 +++++------
 src/feltorShw/feltor_mpi.cu                   | 16 +++++------
 src/hasegawa/hw.cu                            |  4 +--
 src/hasegawa/mima.cu                          |  4 +--
 src/impurities/toeflI.cu                      |  4 +--
 src/impurities/toefl_hpc.cu                   | 12 ++++----
 src/impurities/toefl_mpi.cu                   | 12 ++++----
 src/lamb_dipole/shu_b.cu                      |  2 +-
 src/polar/polar.cu                            |  2 +-
 src/reco2D/reconnection.cu                    | 16 +++++------
 src/reco2D/reconnection.cuh                   | 14 +++++-----
 src/reco2D/reconnection_hpc.cu                | 28 +++++++++----------
 src/reco2D/reconnection_mpi.cu                | 28 +++++++++----------
 src/toefl/toeflR.cu                           |  4 +--
 src/toefl/toeflR.cuh                          |  6 ++--
 src/toefl/toefl_hpc.cu                        |  4 +--
 48 files changed, 216 insertions(+), 231 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7e205905c..237171146 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -50,6 +50,7 @@ doxygen documentation, READMEs or tex writeups.
  - new polynomial expansion and associated `dg::Horner2d` functor for magnetic flux functions that can in particular approximate any experimental equilibrium
  - new equilibrium, modifier and description fields for tokamak magnetic fields
  - Sign reversal of magnetic field and associated flux functions is now possible
+ - new utility `dg::geo::createAlignmentTensor` and `dg::geo::createProjectionTensor` with respect to the magnetic unit vector
 ### Changed
  - namespace file changed to **dg::file** and exblas changed to **dg::exblas** (for consistency reasons, everything should go into the dg namespace, which in particular reduces the chance for name-clashes to just one, namely 'dg')
  - Moved **variation** member function into **dg::Elliptic** (previously in ArakawaX and Poisson)
@@ -71,7 +72,7 @@ doxygen documentation, READMEs or tex writeups.
  - multistep time-stepper now initialize with Runge-Kutta timesteppers of corresponding order
  - Multigrid nested iteration algorithm now allows accuracies for each stage separately (which can give a significant speed-up)
  - `dg::inverse( bc)` function is now a free-standing function to invert a boundary condition
- - `dg::Elliptic` classes now have `jump_weighting` and `multiply_sigma` member functions
+ - `dg::Elliptic` classes now have `jump_weighting` member function
  - `dg::CG` operator now has a `test-frequency` parameter to control the number of times the error condition is evaluated
  - `dg::Extrapolation` class now has a `derive` member function to interpolate the derivative of the interpolating polynomial
  - Adapt all src and diag projects to changed file and json utilities and the moved variation member
@@ -91,6 +92,7 @@ doxygen documentation, READMEs or tex writeups.
  - remove dg::MemoryTraits and associated dimensionality and memory_category traits in favor of direct host_vector and host_grid typedefs in topology classes
  - old txt input files
  - `dg::geo::DeltaFunction` and `dg::geo::Alpha` for the computation of flux-surface averages no longer needed
+ - dg::blas1::transfer (previously marked deprecated)
 ### Fixed
  - Fix bug: race condition in `dg::blas1::dot` and `dg::blas2::dot` on GPUs that led to hard to reproduce and seemingly unreasonable crashes
  - Fix bug: std namespace in diag/probes.h
@@ -104,6 +106,7 @@ doxygen documentation, READMEs or tex writeups.
  - Fix bug: Fpsi safety-factor works up to the O-point
  - Fix bug: `dg::pushForwardPerp` on functors computed wrong result (only affects `dg::geo::Hector`)
  - Fix bug(s): several bugs in `dg::geo::Hector` which computed wrong grid (happened probably when we changed the grid design to polymorphic)
+ - Fix bug: in perpendicular grid of MPI Curvlinear grid
 
 ## [v5.1]
 ### Added
diff --git a/diag/feltorSHdiag.cu b/diag/feltorSHdiag.cu
index eaf66d719..c4b3125ac 100644
--- a/diag/feltorSHdiag.cu
+++ b/diag/feltorSHdiag.cu
@@ -253,7 +253,7 @@ int main( int argc, char* argv[])
         dg::IDMatrix interpne(dg::create::interpolation(xcoo,y0coone, g2d)) ;
         
         dg::blas2::gemv(interpne,npe[0],helper1d); 
-        dg::blas1::transfer( helper1d, transfer1d);
+        dg::assign( helper1d, transfer1d);
         err_out = nc_put_vara_double( ncid_out, names1dID[0], start1d, count1d, transfer1d.data());    
         
        //get max position and value(x,y_max) of electron temperature
@@ -268,7 +268,7 @@ int main( int argc, char* argv[])
         dg::IDMatrix interpte(dg::create::interpolation(xcoo,y0coote, g2d)) ;
         
         dg::blas2::gemv(interpte,tpe[0],helper1d); 
-        dg::blas1::transfer( helper1d, transfer1d);
+        dg::assign( helper1d, transfer1d);
         err_out = nc_put_vara_double( ncid_out, names1dID[1], start1d, count1d, transfer1d.data());    
         
         
@@ -342,7 +342,7 @@ int main( int argc, char* argv[])
         dg::blas2::symv(polti,phi,helper); 
         dg::blas1::pointwiseDot(helper,helper2,helper);
         dg::blas1::axpby(1.0,helper3,p.mu[1],helper,helper3);
-        dg::blas1::transfer( helper3, transfer2d);
+        dg::assign( helper3, transfer2d);
         err_out = nc_put_vara_double( ncid_out, names2dID[3], start2d, count2d, transfer2d.data());
 
   /*
@@ -361,7 +361,7 @@ int main( int argc, char* argv[])
         dg::blas2::gemv(interpti, helper2,helper1d); 
         transfer1d=helper1d;
         err_out = nc_put_vara_double( ncid_out, names1dID[2], start1d, count1d, transfer1d.data());      */
-        dg::blas1::transfer( xcoo, transfer1d);
+        dg::assign( xcoo, transfer1d);
         err_out = nc_put_vara_double( ncid_out, names1dID[3],   start1d, count1d,transfer1d.data());       
         
         dg::blas1::transform(npe[0], npe[0], dg::PLUS<double>(-p.bgprofamp - p.nprofileamp));
diff --git a/diag/impRdiag.cu b/diag/impRdiag.cu
index fdd40911b..30b5cb6e7 100644
--- a/diag/impRdiag.cu
+++ b/diag/impRdiag.cu
@@ -263,7 +263,7 @@ int main( int argc, char* argv[])
         pol.set_chi(chi);
         pol.symv(field[0], nphys[j]);
         dg::blas1::axpby( 1., gamma_n, -1., nphys[j]);
-        dg::blas1::transfer(nphys[j], transfer2d);
+        dg::assign(nphys[j], transfer2d);
         err_out = nc_put_vara_double(ncid_out, species_wphys_id[j], start2d, count2d, transfer2d.data());
       }
       //charge number
diff --git a/diag/probes.h b/diag/probes.h
index 954a05e5e..f727b393d 100644
--- a/diag/probes.h
+++ b/diag/probes.h
@@ -64,8 +64,8 @@ probes<IMatrix, Matrix, container> :: probes (container x_c, container y_c, cons
     pol_avg(g, dg::coo2d::y)
 { 
     thrust::host_vector<double> t1, t2;
-    dg::blas1::transfer( x_c, t1);
-    dg::blas1::transfer( y_c, t2);
+    dg::assign( x_c, t1);
+    dg::assign( y_c, t2);
     dg::blas2::transfer( dg::create::interpolation( t1, t2, g, dg::NEU), probe_interp);
     assert(x_coords.size () == y_coords.size());
     std::ofstream of;
diff --git a/diag/toeflEPdiag.cu b/diag/toeflEPdiag.cu
index b03a6e9e1..d892d4d08 100644
--- a/diag/toeflEPdiag.cu
+++ b/diag/toeflEPdiag.cu
@@ -260,7 +260,7 @@ int main( int argc, char* argv[])
         //dg::IDMatrix interpne(dg::create::interpolation(xcoo,y0coone, g2d)) ;
         //
         //dg::blas2::gemv(interpne,npe[0],helper1d); 
-        //dg::blas1::transfer( helper1d, transfer1d);
+        //dg::assign( helper1d, transfer1d);
         //err_out = nc_put_vara_double( ncid_out, names1dID[0], start1d, count1d, transfer1d.data());    
         
         
diff --git a/diag/toeflRdiag.cu b/diag/toeflRdiag.cu
index 0dca27fff..b23824ddb 100644
--- a/diag/toeflRdiag.cu
+++ b/diag/toeflRdiag.cu
@@ -255,7 +255,7 @@ int main( int argc, char* argv[])
         //dg::IDMatrix interpne(dg::create::interpolation(xcoo,y0coone, g2d)) ;
         //
         //dg::blas2::gemv(interpne,npe[0],helper1d); 
-        //dg::blas1::transfer( helper1d, transfer1d);
+        //dg::assign( helper1d, transfer1d);
         //err_out = nc_put_vara_double( ncid_out, names1dID[0], start1d, count1d, transfer1d.data());    
         
         
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 25b8d345b..75d9e73f9 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -132,7 +132,7 @@ inline get_value_type<ContainerType> reduce( const ContainerType& x, get_value_t
  * @copydoc hide_iterations
  * @param source vector to copy
  * @param target (write-only) destination
- * @note in contrast to the (deprecated) \c blas1::transfer functions the \c copy function uses
+ * @note in contrast to the \c dg::assign functions the \c copy function uses
  * the execution policy to determine the implementation and thus works
  * only on types with same execution policy
  * @note catches self-assignment
@@ -611,21 +611,6 @@ inline void subroutine( Subroutine f, ContainerType&& x, ContainerTypes&&... xs)
     dg::blas1::detail::doSubroutine(tensor_category(), f, std::forward<ContainerType>(x), std::forward<ContainerTypes>(xs)...);
 }
 
-///@brief Deprecated: Use \c dg::construct<ContainerType>() instead
-///@attention This function is deprecated! Please replace with \c dg::construct (data transfer between different devices)
-template<class ContainerType, class from_ContainerType>
-inline ContainerType transfer( const from_ContainerType& from)
-{
-    return dg::construct<ContainerType>( from);
-}
-
-///@brief Deprecated: Use \c dg::assign<ContainerType>() instead
-///@attention This function is deprecated! Please replace with either \c dg::blas1::copy (parallel copy between compatible types) or \c dg::assign (data transfer between devices)"
-template<class from_ContainerType, class ContainerType>
-inline void transfer( const from_ContainerType& from, ContainerType& to)
-{
-    dg::assign<from_ContainerType, ContainerType>( from, to);
-}
 ///@}
 }//namespace blas1
 
diff --git a/inc/geometries/geometryX_elliptic_b.cu b/inc/geometries/geometryX_elliptic_b.cu
index d0d338a6f..07957b4b7 100644
--- a/inc/geometries/geometryX_elliptic_b.cu
+++ b/inc/geometries/geometryX_elliptic_b.cu
@@ -149,14 +149,14 @@ int main(int argc, char**argv)
     std::cout << hyX << "\t";
     std::cout<<t.diff()/(double)number<<"s"<<std::endl;
 
-    dg::blas1::transfer( error, X);
+    dg::assign( error, X);
     ncerr = nc_put_var_double( ncid, psiID, X.data());
-    dg::blas1::transfer( x, X);
+    dg::assign( x, X);
     ncerr = nc_put_var_double( ncid, functionID, X.data());
-    dg::blas1::transfer( solution, Y);
+    dg::assign( solution, Y);
     //dg::blas1::axpby( 1., X., -1, Y);
     ncerr = nc_put_var_double( ncid, function2ID, Y.data());
-    dg::blas1::transfer( chi, X);
+    dg::assign( chi, X);
     ncerr = nc_put_var_double( ncid, divBID, X.data());
     ncerr = nc_close( ncid);
 
diff --git a/inc/geometries/geometryX_refined_elliptic_b.cu b/inc/geometries/geometryX_refined_elliptic_b.cu
index 0905a6590..218ae459a 100644
--- a/inc/geometries/geometryX_refined_elliptic_b.cu
+++ b/inc/geometries/geometryX_refined_elliptic_b.cu
@@ -212,11 +212,11 @@ int main(int argc, char**argv)
     std::cout << hyX << "\t";
     std::cout<<t.diff()/(double)number_di<<"s"<<std::endl;
 
-    dg::blas1::transfer( error_direct, X);
+    dg::assign( error_direct, X);
     ncerr = nc_put_var_double( ncid, psiID, X.data());
-    dg::blas1::transfer( x_fine_di, X);
+    dg::assign( x_fine_di, X);
     ncerr = nc_put_var_double( ncid, functionID, X.data());
-    dg::blas1::transfer( solutionFINE, Y);
+    dg::assign( solutionFINE, Y);
     //dg::blas1::axpby( 1., X., -1, Y);
     ncerr = nc_put_var_double( ncid, function2ID, Y.data());
     ncerr = nc_close( ncid);
diff --git a/inc/geometries/geometry_advection_b.cu b/inc/geometries/geometry_advection_b.cu
index e56d1ea4c..76b6c3aee 100644
--- a/inc/geometries/geometry_advection_b.cu
+++ b/inc/geometries/geometry_advection_b.cu
@@ -176,8 +176,8 @@ int main(int argc, char** argv)
     dg::DVec curvX, curvY;
     dg::HVec tempX, tempY;
     dg::pushForwardPerp(dg::geo::CurvatureNablaBR(mag,+1), dg::geo::CurvatureNablaBZ(mag,+1), tempX, tempY, grid);
-    dg::blas1::transfer(  tempX, curvX);
-    dg::blas1::transfer(  tempY, curvY);
+    dg::assign(  tempX, curvX);
+    dg::assign(  tempY, curvY);
     dg::DMatrix dx, dy;
     dg::blas2::transfer( dg::create::dx(grid), dx);
     dg::blas2::transfer( dg::create::dy(grid), dy);
@@ -190,7 +190,7 @@ int main(int argc, char** argv)
 
     CurvatureDirPer curv(mag, psi_0, psi_1);
     dg::DVec curvature;
-    dg::blas1::transfer( dg::pullback(curv, grid), curvature);
+    dg::assign( dg::pullback(curv, grid), curvature);
 
     dg::blas1::axpby( 1., tempx, -1., curvature, tempx);
     result = dg::blas2::dot( vol, tempx);
diff --git a/inc/geometries/geometry_advection_mpib.cu b/inc/geometries/geometry_advection_mpib.cu
index ced13fae0..d63915b57 100644
--- a/inc/geometries/geometry_advection_mpib.cu
+++ b/inc/geometries/geometry_advection_mpib.cu
@@ -178,8 +178,8 @@ int main(int argc, char** argv)
     dg::MDVec curvX, curvY;
     dg::MHVec tempX, tempY;
     dg::pushForwardPerp(dg::geo::CurvatureNablaBR(mag,+1), dg::geo::CurvatureNablaBZ(mag,+1), tempX, tempY, grid);
-    dg::blas1::transfer(  tempX, curvX);
-    dg::blas1::transfer(  tempY, curvY);
+    dg::assign(  tempX, curvX);
+    dg::assign(  tempY, curvY);
     dg::MDMatrix dx, dy;
     dg::blas2::transfer( dg::create::dx(grid), dx);
     dg::blas2::transfer( dg::create::dy(grid), dy);
@@ -192,7 +192,7 @@ int main(int argc, char** argv)
 
     CurvatureDirPer curv(mag, psi_0, psi_1);
     dg::MDVec curvature;
-    dg::blas1::transfer( dg::pullback(curv, grid), curvature);
+    dg::assign( dg::pullback(curv, grid), curvature);
 
     dg::blas1::axpby( 1., tempx, -1., curvature, tempx);
     result = dg::blas2::dot( vol, tempx);
diff --git a/inc/geometries/geometry_elliptic_b.cu b/inc/geometries/geometry_elliptic_b.cu
index 4f9b4951b..303b60b12 100644
--- a/inc/geometries/geometry_elliptic_b.cu
+++ b/inc/geometries/geometry_elliptic_b.cu
@@ -104,11 +104,11 @@ int main(int argc, char**argv)
     double result = dg::blas2::dot( x, vol3d, x);
     std::cout << "               distance to solution "<<sqrt( result)<<std::endl; //don't forget sqrt when comuting errors
 
-    dg::blas1::transfer( error, X );
+    dg::assign( error, X );
     ncerr = nc_put_var_double( ncid, psiID, X.data());
-    dg::blas1::transfer( x, X );
+    dg::assign( x, X );
     ncerr = nc_put_var_double( ncid, functionID, X.data());
-    dg::blas1::transfer( solution, Y );
+    dg::assign( solution, Y );
     //dg::blas1::axpby( 1., X., -1, Y);
     ncerr = nc_put_var_double( ncid, function2ID, Y.data());
     ncerr = nc_close( ncid);
diff --git a/inc/geometries/hector.h b/inc/geometries/hector.h
index b16ffd3e2..e3cf946d0 100644
--- a/inc/geometries/hector.h
+++ b/inc/geometries/hector.h
@@ -190,8 +190,8 @@ void transform(
 {
     u_x.resize( u_zeta.size()), u_y.resize( u_zeta.size());
     thrust::host_vector<double> uh_zeta, uh_eta;
-    dg::blas1::transfer( u_zeta, uh_zeta);
-    dg::blas1::transfer( u_eta, uh_eta);
+    dg::assign( u_zeta, uh_zeta);
+    dg::assign( u_eta, uh_eta);
     dg::SparseTensor<thrust::host_vector<double> > jac = g2d.jacobian();
     dg::tensor::multiply2d( jac.transpose(), uh_zeta, uh_eta, u_x, u_y);
 }
@@ -239,11 +239,11 @@ struct Hector : public aGenerator2d
         m_conformal=m_orthogonal=true;
         ////we actually don't need m_u but it makes a good testcase
         //container psi__;
-        //dg::blas1::transfer(dg::pullback( psi, m_g2d), psi__);
+        //dg::assign(dg::pullback( psi, m_g2d), psi__);
         //dg::blas1::axpby( +1., psi__, 1.,  u); //u = c0(\tilde u + \psi-\psi_0)
         //dg::blas1::plus( u,-psi0);
         //dg::blas1::scal( u, m_c0);
-        //dg::blas1::transfer( u, m_u);
+        //dg::assign( u, m_u);
     }
 
     /**
@@ -272,11 +272,11 @@ struct Hector : public aGenerator2d
         m_conformal=false;
         ////we actually don't need m_u but it makes a good testcase
         //container psi__;
-        //dg::blas1::transfer(dg::pullback( psi, m_g2d), psi__);
+        //dg::assign(dg::pullback( psi, m_g2d), psi__);
         //dg::blas1::axpby( +1., psi__, 1.,  u); //u = c0(\tilde u + \psi-\psi_0)
         //dg::blas1::plus( u,-psi0);
         //dg::blas1::scal( u, m_c0);
-        //dg::blas1::transfer( u, m_u);
+        //dg::assign( u, m_u);
     }
 
     /**
@@ -305,11 +305,11 @@ struct Hector : public aGenerator2d
         m_orthogonal=m_conformal=false;
         ////we actually don't need m_u but it makes a good testcase
         //container psi__;
-        //dg::blas1::transfer(dg::pullback( psi, m_g2d), psi__);
+        //dg::assign(dg::pullback( psi, m_g2d), psi__);
         //dg::blas1::axpby( +1., psi__, 1.,  u); //u = c0(\tilde u + \psi-\psi_0)
         //dg::blas1::plus( u,-psi0);
         //dg::blas1::scal( u, m_c0);
-        //dg::blas1::transfer( u, m_u);
+        //dg::assign( u, m_u);
     }
 
 
@@ -487,7 +487,7 @@ struct Hector : public aGenerator2d
         dg::blas1::pointwiseDot( 1., u_zeta, perp_vol, chi.value(0,0), 0., etaVinv);
         dg::blas1::transform( etaVinv, etaV, dg::INVERT<double>());
         thrust::host_vector<double> etaVinv_h;
-        dg::blas1::transfer( etaVinv, etaVinv_h);
+        dg::assign( etaVinv, etaVinv_h);
         //now compute v_zeta and v_eta
         container v_zeta(u), v_eta(u);
         dg::blas1::axpby( -1., temp_eta, 0.,v_zeta);
diff --git a/inc/geometries/hector_t.cu b/inc/geometries/hector_t.cu
index 702d13a4b..e2c0af046 100644
--- a/inc/geometries/hector_t.cu
+++ b/inc/geometries/hector_t.cu
@@ -158,7 +158,7 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
-    dg::blas1::transfer( temp0, X);
+    dg::assign( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
     dg::HVec vol = dg::tensor::volume(metric);
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
diff --git a/inc/geometries/ribeiroX_t.cu b/inc/geometries/ribeiroX_t.cu
index 6da0e4c95..2503465be 100644
--- a/inc/geometries/ribeiroX_t.cu
+++ b/inc/geometries/ribeiroX_t.cu
@@ -155,11 +155,11 @@ int main( int argc, char* argv[])
 
     dg::blas1::pointwiseDivide( g_yy, g_xx, temp0);
     dg::blas1::axpby( 1., ones, -1., temp0, temp0);
-    dg::blas1::transfer( temp0, X);
+    dg::assign( temp0, X);
     err = nc_put_var_double( ncid, defID, periodify(X, g3d_periodic).data());
     //err = nc_put_var_double( ncid, defID, X.data());
-    dg::blas1::transfer( vol, X);
-    dg::blas1::transfer( g_yy,Y);
+    dg::assign( vol, X);
+    dg::assign( g_yy,Y);
     dg::blas1::pointwiseDot( Y, X, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g3d_periodic).data());
     //err = nc_put_var_double( ncid, volID, X.data());
@@ -170,7 +170,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
     dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
-    dg::blas1::transfer( g_xx,  temp1);
+    dg::assign( g_xx,  temp1);
     dg::blas1::pointwiseDot( temp1, temp1, temp1);
     dg::blas1::axpby( 1., temp1, -1., temp0, temp0);
     double error = sqrt( dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( temp1, w2d, temp1));
@@ -182,14 +182,14 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
-    dg::blas1::transfer( temp0, X);
+    dg::assign( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g3d_periodic).data());
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     //compare g^xx to volume form
-    dg::blas1::transfer( g_xx, temp0);
+    dg::assign( g_xx, temp0);
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(vol, w2d, vol));
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 24f82242d..15a18d160 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -99,7 +99,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
     dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
-    dg::blas1::transfer( g_xx,  temp1);
+    dg::assign( g_xx,  temp1);
     dg::blas1::pointwiseDot( temp1, temp1, temp1);
     dg::blas1::axpby( 1., temp1, -1., temp0, temp0);
     double error = sqrt( dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( temp1, w2d, temp1));
@@ -117,7 +117,7 @@ int main( int argc, char* argv[])
     if(rank==0)std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     //compare g^xx to volume form
-    dg::blas1::transfer( g_xx, temp0);
+    dg::assign( g_xx, temp0);
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(vol, w2d, vol));
diff --git a/inc/geometries/ribeiro_t.cu b/inc/geometries/ribeiro_t.cu
index 8a16f4646..abb1a5cc7 100644
--- a/inc/geometries/ribeiro_t.cu
+++ b/inc/geometries/ribeiro_t.cu
@@ -118,7 +118,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
     dg::blas1::pointwiseDot( g_xy, g_xy, temp1);
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
-    dg::blas1::transfer( g_xx,  temp1);
+    dg::assign( g_xx,  temp1);
     dg::blas1::pointwiseDot( temp1, temp1, temp1);
     dg::blas1::axpby( 1., temp1, -1., temp0, temp0);
     double error = sqrt( dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( temp1, w2d, temp1));
@@ -130,14 +130,14 @@ int main( int argc, char* argv[])
     dg::blas1::axpby( 1., temp0, -1., temp1, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
-    dg::blas1::transfer( temp0, X);
+    dg::assign( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
     std::cout << "Rel Consistency  of volume is "<<error<<"\n";
 
     //compare g^xx to volume form
-    dg::blas1::transfer( g_xx, temp0);
+    dg::assign( g_xx, temp0);
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     error=sqrt(dg::blas2::dot( temp0, w2d, temp0))/sqrt( dg::blas2::dot(vol, w2d, vol));
diff --git a/inc/geometries/simple_orthogonal_t.cu b/inc/geometries/simple_orthogonal_t.cu
index 946ffd126..41ef4c430 100644
--- a/inc/geometries/simple_orthogonal_t.cu
+++ b/inc/geometries/simple_orthogonal_t.cu
@@ -124,7 +124,7 @@ int main( int argc, char* argv[])
     dg::blas1::pointwiseDot( g_xx, g_yy, temp0);
     dg::blas1::transform( temp0, temp0, dg::SQRT<double>());
     dg::blas1::pointwiseDivide( ones, temp0, temp0);
-    dg::blas1::transfer( temp0, X);
+    dg::assign( temp0, X);
     err = nc_put_var_double( ncid, volID, periodify(X, g2d_periodic).data());
     dg::blas1::axpby( 1., temp0, -1., vol, temp0);
     double error = sqrt(dg::blas2::dot( temp0, w2d, temp0)/dg::blas2::dot( vol, w2d, vol));
diff --git a/src/ep/toeflR.cu b/src/ep/toeflR.cu
index 179712301..7b39b7c0b 100644
--- a/src/ep/toeflR.cu
+++ b/src/ep/toeflR.cu
@@ -73,7 +73,7 @@ int main( int argc, char* argv[])
         //transform field to an equidistant grid
         dvisual = y0[0];
 
-        dg::blas1::transfer( dvisual, hvisual);
+        dg::assign( dvisual, hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
@@ -85,7 +85,7 @@ int main( int argc, char* argv[])
         //transform phi
         dvisual = exp.potential();
         dg::blas2::gemv( exp.laplacianM(), dvisual, y1[1]);
-        dg::blas1::transfer( y1[1], hvisual);
+        dg::assign( y1[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
diff --git a/src/ep/toeflR.cuh b/src/ep/toeflR.cuh
index 81e40bad4..3465701df 100644
--- a/src/ep/toeflR.cuh
+++ b/src/ep/toeflR.cuh
@@ -183,7 +183,7 @@ const container& ToeflR<G, M, container>::polarisation( const std::vector<contai
     dg::blas1::plus( chi, debye_); 
     for( unsigned i=0; i<2; i++)
     {
-        dg::blas1::transfer( y[i], omega);
+        dg::assign( y[i], omega);
         dg::blas1::plus( omega, 1.); 
         dg::blas1::scal( omega, z[i]*mu[i]);
         dg::blas1::pointwiseDot( binv, omega, omega); 
diff --git a/src/ep/toefl_mpi.cu b/src/ep/toefl_mpi.cu
index 23b948936..9f25dbc72 100644
--- a/src/ep/toefl_mpi.cu
+++ b/src/ep/toefl_mpi.cu
@@ -122,19 +122,19 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<2; i++)
     {
         dg::blas2::gemv( interpolate, y0[i], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         dg::file::put_vara_double( ncid, dataIDs[i], 0, grid_out, transferH);
     }
     //pot
     transfer = test.potential();
     dg::blas2::gemv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     dg::file::put_vara_double( ncid, dataIDs[2], 0, grid_out, transferH );
     //Vor
     transfer = test.potential();
     dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);
     dg::blas2::gemv( interpolate,y1[1], transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     dg::file::put_vara_double( ncid, dataIDs[3], 0, grid_out, transferH );
     if(rank==0)err = nc_put_vara_double( ncid, tvarID, start, count, &time);
     if(rank==0)err = nc_close(ncid);
@@ -191,17 +191,17 @@ int main( int argc, char* argv[])
         for( unsigned j=0; j<2; j++)
         {
             dg::blas2::gemv( interpolate, y0[j], transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             dg::file::put_vara_double( ncid, dataIDs[j], i, grid, transferH);
         }
         transfer = test.potential();
         dg::blas2::gemv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         dg::file::put_vara_double( ncid, dataIDs[2], i, grid, transferH );
         transfer = test.potential();
         dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);        //correct?    
         dg::blas2::gemv( interpolate,y1[1], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         dg::file::put_vara_double( ncid, dataIDs[3], i, grid, transferH );
         if(rank==0)err = nc_put_vara_double( ncid, tvarID, start, count, &time);
         if(rank==0)err = nc_close(ncid);
diff --git a/src/feltorSH/feltor.cu b/src/feltorSH/feltor.cu
index 943bde82d..c14a690a7 100644
--- a/src/feltorSH/feltor.cu
+++ b/src/feltorSH/feltor.cu
@@ -114,7 +114,7 @@ int main( int argc, char* argv[])
     while ( !glfwWindowShouldClose( w ))
     {
         //draw Ne-1
-        dg::blas1::transfer( y0[0], hvisual);
+        dg::assign( y0[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(), (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax()  ,thrust::minimum<double>() );
@@ -124,7 +124,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
 
         //draw Ni-1
-        dg::blas1::transfer( y0[1], hvisual);
+        dg::assign( y0[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax()  ,thrust::minimum<double>() );
@@ -135,7 +135,7 @@ int main( int argc, char* argv[])
 
         
         //draw potential
-        dg::blas1::transfer( feltor.potential()[0], hvisual);
+        dg::assign( feltor.potential()[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax() ,thrust::minimum<double>() );
@@ -144,7 +144,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         
         //draw Te-1
-        dg::blas1::transfer( y0[2], hvisual);
+        dg::assign( y0[2], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(), (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax()  ,thrust::minimum<double>() );
@@ -154,7 +154,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
 
         //draw Ti-1
-        dg::blas1::transfer( y0[3], hvisual);
+        dg::assign( y0[3], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax()  ,thrust::minimum<double>() );
@@ -167,7 +167,7 @@ int main( int argc, char* argv[])
         //transform to Vor
         dvisual=feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
-        dg::blas1::transfer( y1[1], hvisual);
+        dg::assign( y1[1], hvisual);
          //hvisual = feltor.potential()[0];
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
diff --git a/src/feltorSH/feltor_hpc.cu b/src/feltorSH/feltor_hpc.cu
index 16b6177fb..4ad16607c 100644
--- a/src/feltorSH/feltor_hpc.cu
+++ b/src/feltorSH/feltor_hpc.cu
@@ -127,20 +127,20 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<4; i++)
     {
         dg::blas2::gemv( interpolate, y0[i], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //pot
     transfer = feltor.potential()[0];
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
 
     err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
     //Vor
     transfer = feltor.potential()[0];
     dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);            
     dg::blas2::symv( interpolate,y1[1], transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
     double time = 0;
 
@@ -226,17 +226,17 @@ int main( int argc, char* argv[])
         for( unsigned j=0; j<4; j++)
         {
             dg::blas2::symv( interpolate, y0[j], transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
         }
         transfer = feltor.potential()[0];
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
         transfer = feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);            
         dg::blas2::symv( interpolate,y1[1], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
 
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
diff --git a/src/feltorSH/feltor_mpi.cu b/src/feltorSH/feltor_mpi.cu
index 299de47e2..fd04817b4 100644
--- a/src/feltorSH/feltor_mpi.cu
+++ b/src/feltorSH/feltor_mpi.cu
@@ -181,19 +181,19 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<4; i++)
     {
         dg::blas2::gemv( interpolate, y0[i].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //pot
     transfer = feltor.potential()[0];
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
     //Vor
     transfer = feltor.potential()[0];
     dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);        
     dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
     double time = 0;
 
@@ -278,17 +278,17 @@ int main( int argc, char* argv[])
         for( unsigned j=0; j<4; j++)
         {
             dg::blas2::gemv( interpolate, y0[j].data(), transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
         }
         transfer = feltor.potential()[0];
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
         transfer = feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);        //correct?    
         dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
 //         err = nc_close(ncid);
diff --git a/src/feltorSHp/Makefile b/src/feltorSHp/Makefile
index 5490b8339..6c2463c61 100644
--- a/src/feltorSHp/Makefile
+++ b/src/feltorSHp/Makefile
@@ -8,14 +8,11 @@ include ../../config/devices/devices.mk
 INCLUDE+= -I../         # other src libraries
 INCLUDE+= -I../../inc   # other project libraries
 
-all: feltor feltor_hpc feltor_log 
+all: feltor feltor_hpc
 
 feltor: feltor.cu feltor.cuh 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB)-DDG_BENCHMARK 
 
-feltor_log: feltor_log.cu feltor_log.cuh 
-	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(GLFLAGS) $(JSONLIB)-DDG_BENCHMARK 	
-	
 feltor_hpc: feltor_hpc.cu feltor.cuh 
 	$(CC) $(OPT) $(CFLAGS) $< -o $@ $(INCLUDE) $(LIBS) $(JSONLIB) -DDG_BENCHMARK
 
diff --git a/src/feltorSHp/feltor.cu b/src/feltorSHp/feltor.cu
index 91ce80bb6..ffe68238b 100644
--- a/src/feltorSHp/feltor.cu
+++ b/src/feltorSHp/feltor.cu
@@ -105,7 +105,7 @@ int main( int argc, char* argv[])
     while ( !glfwWindowShouldClose( w ))
     {
         //draw Ne-1
-        dg::blas1::transfer( y0[0], hvisual);
+        dg::assign( y0[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(), (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax()  ,thrust::minimum<double>() );
@@ -115,7 +115,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
 
         //draw Ni-1
-        dg::blas1::transfer( y0[1], hvisual);
+        dg::assign( y0[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax()  ,thrust::minimum<double>() );
@@ -126,7 +126,7 @@ int main( int argc, char* argv[])
 
         
         //draw potential
-        dg::blas1::transfer( feltor.potential()[0], hvisual);
+        dg::assign( feltor.potential()[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax() ,thrust::minimum<double>() );
@@ -135,7 +135,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         
         //draw Te-1
-        dg::blas1::transfer( feltor.temptilde()[0], hvisual);
+        dg::assign( feltor.temptilde()[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(), (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax()  ,thrust::minimum<double>() );
@@ -145,7 +145,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
 
         //draw Ti-1
-        dg::blas1::transfer( feltor.temptilde()[1], hvisual);
+        dg::assign( feltor.temptilde()[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
         colors.scalemin() =  (double)thrust::reduce( visual.begin(), visual.end(), colors.scalemax()  ,thrust::minimum<double>() );
@@ -158,7 +158,7 @@ int main( int argc, char* argv[])
         //transform to Vor
         dvisual=feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
-        dg::blas1::transfer( y1[1], hvisual);
+        dg::assign( y1[1], hvisual);
          //hvisual = feltor.potential()[0];
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
diff --git a/src/feltorSHp/feltor_hpc.cu b/src/feltorSHp/feltor_hpc.cu
index f69ab0e81..a525e23eb 100644
--- a/src/feltorSHp/feltor_hpc.cu
+++ b/src/feltorSHp/feltor_hpc.cu
@@ -122,29 +122,29 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<2; i++)
     {
         dg::blas2::gemv( interpolate, y0[i], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //te
     transfer = feltor.temptilde()[0];
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
     //Ti
     transfer = feltor.temptilde()[1];
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
     //pot
     transfer = feltor.potential()[0];
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
     //Vor
     transfer = feltor.potential()[0];
     dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);            
     dg::blas2::symv( interpolate,y1[1], transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
     double time = 0;
 
@@ -230,29 +230,29 @@ int main( int argc, char* argv[])
         for( unsigned j=0; j<2; j++)
         {
             dg::blas2::symv( interpolate, y0[j], transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
         }
         //te
         transfer = feltor.temptilde()[0];
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
         //Ti
         transfer = feltor.temptilde()[1];
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
         //pot
         transfer = feltor.potential()[0];
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
         //vor
         transfer = feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);            
         dg::blas2::symv( interpolate,y1[1], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
 
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
diff --git a/src/feltorSHp/feltor_mpi.cu b/src/feltorSHp/feltor_mpi.cu
index d4cf2aa0c..1c62f5aae 100644
--- a/src/feltorSHp/feltor_mpi.cu
+++ b/src/feltorSHp/feltor_mpi.cu
@@ -174,29 +174,29 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<2; i++)
     {
         dg::blas2::gemv( interpolate, y0[i].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //te
     transfer = feltor.temptilde()[0];
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
     //Ti
     transfer = feltor.temptilde()[1];
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
     //pot
     transfer = feltor.potential()[0];
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
     //Vor
     transfer = feltor.potential()[0];
     dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);        
     dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
     double time = 0;
 
@@ -281,29 +281,29 @@ int main( int argc, char* argv[])
         for( unsigned j=0; j<2; j++)
         {
             dg::blas2::gemv( interpolate, y0[j].data(), transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
         }
         //te
         transfer = feltor.temptilde()[0];
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
         //Ti
         transfer = feltor.temptilde()[1];
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
         //pot
         transfer = feltor.potential()[0];
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
         //vor
         transfer = feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);        //correct?    
         dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
 //         err = nc_close(ncid);
diff --git a/src/feltorSesol/feltor.cu b/src/feltorSesol/feltor.cu
index 0ffc72674..1b04adf00 100644
--- a/src/feltorSesol/feltor.cu
+++ b/src/feltorSesol/feltor.cu
@@ -106,7 +106,7 @@ int main( int argc, char* argv[])
     while ( !glfwWindowShouldClose( w ))
     {
 
-        dg::blas1::transfer( y0[0], hvisual);
+        dg::assign( y0[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(), (double)-1e14, thrust::maximum<double>() );
 //         colors.scalemin() = -colors.scalemax();        
@@ -121,7 +121,7 @@ int main( int argc, char* argv[])
 
         //draw ions
         //thrust::transform( y1[1].begin(), y1[1].end(), dvisual.begin(), dg::PLUS<double>(-0.));//ne-1
-        dg::blas1::transfer( y0[1], hvisual);
+        dg::assign( y0[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
         //colors.scalemin() = 1.0;        
@@ -140,7 +140,7 @@ int main( int argc, char* argv[])
 //        dvisual=feltor.potential()[0];
 //        dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
 //        hvisual = y1[1];
-         dg::blas1::transfer( feltor.potential()[0], hvisual);
+         dg::assign( feltor.potential()[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
 
@@ -157,7 +157,7 @@ int main( int argc, char* argv[])
         //transform to Vor
         dvisual=feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
-        dg::blas1::transfer( y1[1], hvisual);
+        dg::assign( y1[1], hvisual);
          //hvisual = feltor.potential()[0];
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
diff --git a/src/feltorSesol/feltor_hpc.cu b/src/feltorSesol/feltor_hpc.cu
index b035e2c21..c29420566 100644
--- a/src/feltorSesol/feltor_hpc.cu
+++ b/src/feltorSesol/feltor_hpc.cu
@@ -114,11 +114,11 @@ int main( int argc, char* argv[])
         dg::IHMatrix interpolateIN = dg::create::interpolation( grid,grid_IN); 
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[0], start2dIN, count2dIN, transferINH.data());
         dg::blas2::gemv( interpolateIN, transferINH,temp);
-        dg::blas1::transfer(temp,y0[0]);
+        dg::assign(temp,y0[0]);
         errIN = nc_inq_varid(ncidIN, namesIN[1].data(), &dataIDsIN[1]);
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[1], start2dIN, count2dIN, transferINH.data());
         dg::blas2::gemv( interpolateIN, transferINH,temp);
-        dg::blas1::transfer(temp,y0[1]);      
+        dg::assign(temp,y0[1]);      
         errIN = nc_close(ncidIN);
 
     }
@@ -187,20 +187,20 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<2; i++)
     {
         dg::blas2::gemv( interpolate, y0[i], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //pot
     transfer = feltor.potential()[0];
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
 
     err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
     //Vor
     transfer = feltor.potential()[0];
     dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);            
     dg::blas2::symv( interpolate,y1[1], transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
 
 
@@ -285,17 +285,17 @@ int main( int argc, char* argv[])
         for(unsigned j = 0; j < 2; j++)
         {
             dg::blas2::symv( interpolate, y0[j], transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
         }
         transfer = feltor.potential()[0];
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
         transfer = feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);            
         dg::blas2::symv( interpolate,y1[1], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data());
 
         err = nc_put_vara_double(ncid, tvarID_field, start, count, &time);
diff --git a/src/feltorSesol/feltor_mpi.cu b/src/feltorSesol/feltor_mpi.cu
index 1808ecccd..0187f10b3 100644
--- a/src/feltorSesol/feltor_mpi.cu
+++ b/src/feltorSesol/feltor_mpi.cu
@@ -141,11 +141,11 @@ int main( int argc, char* argv[])
         if(rank==0) std::cout << "timein= "<< timeIN <<  std::endl;
         time=timeIN;
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[0], start2dIN, count2dIN,transferIN.data());
-        dg::blas1::transfer(transferIN,transferIND);
+        dg::assign(transferIN,transferIND);
         dg::blas2::gemv( interpolateIN, transferIND,y0[0].data());
         errIN = nc_inq_varid(ncidIN, namesIN[1].data(), &dataIDsIN[1]);
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[1], start2dIN, count2dIN, transferIN.data());
-        dg::blas1::transfer(transferIN,transferIND);
+        dg::assign(transferIN,transferIND);
         dg::blas2::gemv( interpolateIN, transferIND,y0[1].data());
         errIN = nc_close(ncidIN);
     }
@@ -221,19 +221,19 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<2; i++)
     {
         dg::blas2::gemv( interpolate, y0[i].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //pot
     transfer = feltor.potential()[0];
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
     //Vor
     transfer = feltor.potential()[0];
     dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);        
     dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
 
     err = nc_put_vara_double( ncid, tvarID, start, count, &time);
@@ -332,17 +332,17 @@ int main( int argc, char* argv[])
         for( unsigned j=0; j<2; j++)
         {
             dg::blas2::gemv( interpolate, y0[j].data(), transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
         }
         transfer = feltor.potential()[0];
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
         transfer = feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);        //correct?    
         dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
 //         err = nc_close(ncid);
diff --git a/src/feltorShw/feltor.cu b/src/feltorShw/feltor.cu
index dbcbcbdb5..234d026a7 100644
--- a/src/feltorShw/feltor.cu
+++ b/src/feltorShw/feltor.cu
@@ -134,7 +134,7 @@ int main( int argc, char* argv[])
     while ( !glfwWindowShouldClose( w ))
     {
 
-        dg::blas1::transfer(y0[0], hvisual);
+        dg::assign(y0[0], hvisual);
 //         if 
 //         dg::blas1::axpby(1.0,hvisual,
         dg::blas2::gemv( equi, hvisual, visual);
@@ -151,7 +151,7 @@ int main( int argc, char* argv[])
 
         //draw ions
         //thrust::transform( y1[1].begin(), y1[1].end(), dvisual.begin(), dg::PLUS<double>(-0.));//ne-1
-        dg::blas1::transfer(y0[1], hvisual);
+        dg::assign(y0[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
         //colors.scalemin() = 1.0;        
@@ -170,7 +170,7 @@ int main( int argc, char* argv[])
 //        dvisual=feltor.potential()[0];
 //        dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
 //        hvisual = y1[1];
-        dg::blas1::transfer(feltor.potential()[0], hvisual);
+        dg::assign(feltor.potential()[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
 
@@ -187,7 +187,7 @@ int main( int argc, char* argv[])
         //transform to Vor
         dvisual=feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
-        dg::blas1::transfer(y1[1], hvisual);
+        dg::assign(y1[1], hvisual);
          //hvisual = feltor.potential()[0];
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(),  (double)-1e14, thrust::maximum<double>() );
diff --git a/src/feltorShw/feltor_hpc.cu b/src/feltorShw/feltor_hpc.cu
index a7f4304bf..e1e0bd91e 100644
--- a/src/feltorShw/feltor_hpc.cu
+++ b/src/feltorShw/feltor_hpc.cu
@@ -138,11 +138,11 @@ int main( int argc, char* argv[])
         dg::IHMatrix interpolateIN = dg::create::interpolation( grid,grid_IN); 
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[0], start2dIN, count2dIN, transferINH.data());
         dg::blas2::gemv( interpolateIN, transferINH,temp);
-        dg::blas1::transfer(temp, y0[0]);
+        dg::assign(temp, y0[0]);
         errIN = nc_inq_varid(ncidIN, namesIN[1].data(), &dataIDsIN[1]);
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[1], start2dIN, count2dIN, transferINH.data());
         dg::blas2::gemv( interpolateIN, transferINH,temp);
-        dg::blas1::transfer(temp,y0[1]);      
+        dg::assign(temp,y0[1]);      
         errIN = nc_close(ncidIN);
     }
 
@@ -211,20 +211,20 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<2; i++)
     {
         dg::blas2::gemv( interpolate, y0[i], transferD);
-        dg::blas1::transfer( transferD, transferH); //transfer to host
+        dg::assign( transferD, transferH); //transfer to host
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //pot
     transfer = feltor.potential()[0];
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH); //transfer to host
+    dg::assign( transferD, transferH); //transfer to host
 
     err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
     //Vor
     transfer = feltor.potential()[0];
     dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);            
     dg::blas2::symv( interpolate,y1[1], transferD);
-    dg::blas1::transfer( transferD, transferH); //transfer to host
+    dg::assign( transferD, transferH); //transfer to host
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
 
 
@@ -313,17 +313,17 @@ int main( int argc, char* argv[])
         for(unsigned j = 0; j < 2; j++)
         {
             dg::blas2::symv( interpolate, y0[j], transferD);
-            dg::blas1::transfer( transferD, transferH); //transfer to host
+            dg::assign( transferD, transferH); //transfer to host
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
         }
         transfer = feltor.potential()[0];
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH); //transfer to host
+        dg::assign( transferD, transferH); //transfer to host
         err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
         transfer = feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);            
         dg::blas2::symv( interpolate,y1[1], transferD);
-        dg::blas1::transfer( transferD, transferH); //transfer to host
+        dg::assign( transferD, transferH); //transfer to host
         err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data());
 
         err = nc_put_vara_double(ncid, tvarID_field, start, count, &time);
diff --git a/src/feltorShw/feltor_mpi.cu b/src/feltorShw/feltor_mpi.cu
index 744327fc6..fc92762a4 100644
--- a/src/feltorShw/feltor_mpi.cu
+++ b/src/feltorShw/feltor_mpi.cu
@@ -189,11 +189,11 @@ int main( int argc, char* argv[])
         if(rank==0) std::cout << "timein= "<< timeIN <<  std::endl;
         time=timeIN;
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[0], start2dIN, count2dIN,transferIN.data());
-        dg::blas1::transfer(transferIN,transferIND);
+        dg::assign(transferIN,transferIND);
         dg::blas2::gemv( interpolateIN, transferIND,y0[0].data());
         errIN = nc_inq_varid(ncidIN, namesIN[1].data(), &dataIDsIN[1]);
         errIN = nc_get_vara_double( ncidIN, dataIDsIN[1], start2dIN, count2dIN, transferIN.data());
-        dg::blas1::transfer(transferIN,transferIND);
+        dg::assign(transferIN,transferIND);
         dg::blas2::gemv( interpolateIN, transferIND,y0[1].data());
         errIN = nc_close(ncidIN);
     }
@@ -271,19 +271,19 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<2; i++)
     {
         dg::blas2::gemv( interpolate, y0[i].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //pot
     transfer = feltor.potential()[0];
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
     //Vor
     transfer = feltor.potential()[0];
     dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);        
     dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
 
     err = nc_put_vara_double( ncid, tvarID, start, count, &time);
@@ -379,17 +379,17 @@ int main( int argc, char* argv[])
         for( unsigned j=0; j<2; j++)
         {
             dg::blas2::gemv( interpolate, y0[j].data(), transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data() );
         }
         transfer = feltor.potential()[0];
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
         transfer = feltor.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), transfer, y1[1]);        
         dg::blas2::gemv( interpolate,y1[1].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
 //         err = nc_close(ncid);
diff --git a/src/hasegawa/hw.cu b/src/hasegawa/hw.cu
index b7ab354b1..4c25aa1ff 100644
--- a/src/hasegawa/hw.cu
+++ b/src/hasegawa/hw.cu
@@ -79,7 +79,7 @@ int main( int argc, char* argv[])
         //transform field to an equidistant grid
         dvisual = y0[0];
 
-        dg::blas1::transfer( dvisual, hvisual);
+        dg::assign( dvisual, hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
@@ -91,7 +91,7 @@ int main( int argc, char* argv[])
         //transform phi
 
         dg::blas2::gemv( laplacianM, test.potential(), y1[1]);
-        dg::blas1::transfer( y1[1], hvisual);
+        dg::assign( y1[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
diff --git a/src/hasegawa/mima.cu b/src/hasegawa/mima.cu
index 691eafb3c..2aa675988 100644
--- a/src/hasegawa/mima.cu
+++ b/src/hasegawa/mima.cu
@@ -77,7 +77,7 @@ int main( int argc, char* argv[])
         //transform field to an equidistant grid
         dvisual = mima.potential();
 
-        dg::blas1::transfer( dvisual, hvisual);
+        dg::assign( dvisual, hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
@@ -88,7 +88,7 @@ int main( int argc, char* argv[])
 
         //transform phi
         dg::blas2::gemv( laplaceM, mima.potential(), y1);
-        dg::blas1::transfer( y1, hvisual);
+        dg::assign( y1, hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
diff --git a/src/impurities/toeflI.cu b/src/impurities/toeflI.cu
index 9bc6bab57..486396aa2 100644
--- a/src/impurities/toeflI.cu
+++ b/src/impurities/toeflI.cu
@@ -131,7 +131,7 @@ int main( int argc, char* argv[])
         title << std::setprecision(2) << std::scientific;
         for( unsigned i=0; i<y0.size(); i++)
         {
-            dg::blas1::transfer( y0[i], hvisual);
+            dg::assign( y0[i], hvisual);
             dg::blas2::gemv( equi, hvisual, visual);
             //compute the color scale
             colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
@@ -143,7 +143,7 @@ int main( int argc, char* argv[])
         }
         //transform phi
         dg::blas2::gemv( laplacianM, toeflI.potential()[0], y0[1]);
-        dg::blas1::transfer( y0[1], hvisual);
+        dg::assign( y0[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
diff --git a/src/impurities/toefl_hpc.cu b/src/impurities/toefl_hpc.cu
index 1e1d2fc7a..e6e235e0d 100644
--- a/src/impurities/toefl_hpc.cu
+++ b/src/impurities/toefl_hpc.cu
@@ -132,18 +132,18 @@ int main( int argc, char* argv[])
     dg::IDMatrix interpolate = dg::create::interpolation( grid_out, grid);
     for( unsigned i=0; i<3; i++)
     {   dg::blas2::gemv( interpolate, y0[i], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //Potential
     transfer = toeflI.polarization( y0);
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
     //Vorticity
     dg::blas2::gemv( diffusion.laplacianM(), transfer, y0[1]);
     dg::blas2::symv( interpolate, y0[1], transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
     err = nc_put_vara_double( ncid, tvarID, start, count, &time);
     err = nc_put_vara_double( ncid, EtimevarID, start, count, &time);
@@ -201,17 +201,17 @@ int main( int argc, char* argv[])
             err = nc_open(argv[2], NC_WRITE, &ncid);
             for( unsigned j=0; j<3; j++)
             {   dg::blas2::symv( interpolate, y0[j], transferD);
-                dg::blas1::transfer( transferD, transferH);
+                dg::assign( transferD, transferH);
                 err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
             }
             transfer = toeflI.potential()[0];
             dg::blas2::symv( interpolate, transfer, transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
             transfer = toeflI.potential()[0];
             dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);        //correct?
             dg::blas2::symv( interpolate, y1[1], transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
             err = nc_put_vara_double( ncid, tvarID, start, count, &time);
             err = nc_close(ncid);
diff --git a/src/impurities/toefl_mpi.cu b/src/impurities/toefl_mpi.cu
index 0d6cffdb7..70b329d77 100644
--- a/src/impurities/toefl_mpi.cu
+++ b/src/impurities/toefl_mpi.cu
@@ -171,18 +171,18 @@ int main( int argc, char* argv[])
     dg::IDMatrix interpolate = dg::create::interpolation( grid_out.local(), grid.local()); //create local interpolation matrix
     for( unsigned i=0; i<3; i++)
     {   dg::blas2::gemv( interpolate, y0[i].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     //Potential
     transfer = toeflI.polarization( y0);
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
     //Vorticity
     dg::blas2::gemv( diffusion.laplacianM(), transfer, y0[1]);
     dg::blas2::gemv( interpolate, y0[1].data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
     err = nc_put_vara_double( ncid, tvarID,     start, count, &time);
     err = nc_put_vara_double( ncid, EtimevarID, start, count, &time);
@@ -246,17 +246,17 @@ int main( int argc, char* argv[])
             start[0] = i;
             for( unsigned j=0; j<3; j++)
             {   dg::blas2::gemv( interpolate, y0[j].data(), transferD);
-                dg::blas1::transfer( transferD, transferH);
+                dg::assign( transferD, transferH);
                 err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
             }
             transfer = toeflI.potential()[0];
             dg::blas2::gemv( interpolate, transfer.data(), transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
             transfer = toeflI.potential()[0];
             dg::blas2::gemv( diffusion.laplacianM(), transfer, y1[1]);        //correct?
             dg::blas2::gemv( interpolate, y1[1].data(), transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
             err = nc_put_vara_double( ncid, tvarID, start, count, &time);
 #ifdef DG_BENCHMARK
diff --git a/src/lamb_dipole/shu_b.cu b/src/lamb_dipole/shu_b.cu
index 36c823eb6..33e528419 100644
--- a/src/lamb_dipole/shu_b.cu
+++ b/src/lamb_dipole/shu_b.cu
@@ -137,7 +137,7 @@ int main( int argc, char* argv[])
             dg::blas2::symv( equidistant, y0, visual);
             colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), -1., dg::AbsMax<double>() );
             //draw and swap buffers
-            dg::blas1::transfer( visual, hvisual);
+            dg::assign( visual, hvisual);
             render.renderQuad( hvisual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
             title << "Time "<<time<< " \ttook "<<t.diff()/(double)itstp<<"\t per step";
             glfwSetWindowTitle(w, title.str().c_str());
diff --git a/src/polar/polar.cu b/src/polar/polar.cu
index 4fbe5c7a9..c33c6a9ac 100644
--- a/src/polar/polar.cu
+++ b/src/polar/polar.cu
@@ -201,7 +201,7 @@ int main(int argc, char* argv[])
         dg::blas2::symv( equidistant, y0, visual);
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), -1., dg::AbsMax<double>() );
         //draw and swap buffers
-        dg::blas1::transfer( visual, hvisual);
+        dg::assign( visual, hvisual);
         render.renderQuad( hvisual, p.n*p.Nx, p.n*p.Ny, colors);
         title << "Time "<<time<< " \ttook "<<t.diff()/(double)p.itstp<<"\t per step";
         glfwSetWindowTitle(w, title.str().c_str());
diff --git a/src/reco2D/reconnection.cu b/src/reco2D/reconnection.cu
index 6857d2371..47e7d490d 100644
--- a/src/reco2D/reconnection.cu
+++ b/src/reco2D/reconnection.cu
@@ -107,7 +107,7 @@ int main( int argc, char* argv[])
     while ( !glfwWindowShouldClose( w ))
     {
         //plot electrons
-        dg::blas1::transfer( y0[0], hvisual);
+        dg::assign( y0[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(), 0., thrust::maximum<double>() );
         colors.scalemin() = -colors.scalemax();   
@@ -116,7 +116,7 @@ int main( int argc, char* argv[])
 
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         //draw ions
-        dg::blas1::transfer( y0[1], hvisual);
+        dg::assign( y0[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(), 0., thrust::maximum<double>() );
         colors.scalemin() = -colors.scalemax();   
@@ -125,7 +125,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
 
         //draw Potential
-        dg::blas1::transfer( asela.potential()[0], hvisual);
+        dg::assign( asela.potential()[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //transform to Vor
         //dvisual=asela.potential()[0];
@@ -141,7 +141,7 @@ int main( int argc, char* argv[])
         //draw Vor
         dvisual=asela.potential()[0];
         dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
-        dg::blas1::transfer( y1[1], hvisual);
+        dg::assign( y1[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         dg::blas1::scal(visual,-1.0);
         colors.scalemax() = (double)thrust::reduce( visual.begin(), visual.end(), 0.,thrust::maximum<double>()  );
@@ -151,7 +151,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
         
         //draw U_e
-        dg::blas1::transfer( asela.uparallel()[0], hvisual);
+        dg::assign( asela.uparallel()[0], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (float)thrust::reduce( visual.begin(), visual.end(), 0., thrust::maximum<double>() );
         colors.scalemin() = -colors.scalemax();   
@@ -159,7 +159,7 @@ int main( int argc, char* argv[])
          render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
 
         //draw U_i
-        dg::blas1::transfer( asela.uparallel()[1], hvisual);
+        dg::assign( asela.uparallel()[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (float)thrust::reduce( visual.begin(), visual.end(), 0., thrust::maximum<double>() );
         colors.scalemin() = -colors.scalemax();   
@@ -167,7 +167,7 @@ int main( int argc, char* argv[])
         render.renderQuad( visual, grid.n()*grid.Nx(), grid.n()*grid.Ny(), colors);
 
         //draw a parallel
-        dg::blas1::transfer(asela.aparallel(), hvisual);
+        dg::assign(asela.aparallel(), hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         colors.scalemax() = (float)thrust::reduce( visual.begin(),visual.end(), 0., thrust::maximum<double>()  );
         colors.scalemin() = - colors.scalemax();
@@ -177,7 +177,7 @@ int main( int argc, char* argv[])
          //draw j_par
         dvisual=asela.aparallel();
         dg::blas2::gemv( rolkar.laplacianM(), dvisual, y1[1]);
-        dg::blas1::transfer( y1[1], hvisual);
+        dg::assign( y1[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         
         colors.scalemax() = (float)thrust::reduce( visual.begin(),visual.end(), 0., thrust::maximum<double>()  );
diff --git a/src/reco2D/reconnection.cuh b/src/reco2D/reconnection.cuh
index 95019bac9..383c33245 100644
--- a/src/reco2D/reconnection.cuh
+++ b/src/reco2D/reconnection.cuh
@@ -38,7 +38,7 @@ struct Implicit
         p(p),
         LaplacianM_perp  ( g, g.bcx(), g.bcy(), dg::normed, dg::centered)
     {
-        dg::blas1::transfer( dg::evaluate( dg::zero, g), temp);
+        dg::assign( dg::evaluate( dg::zero, g), temp);
     }
         /**
      * @brief Return implicit terms
@@ -231,10 +231,10 @@ Asela<Grid, IMatrix, Matrix, container>::Asela( const Grid& g, Parameters p):
     p(p),  evec(6)
 { 
     ////////////////////////////init temporaries///////////////////
-    dg::blas1::transfer( dg::evaluate( dg::zero, g), chi ); 
-    dg::blas1::transfer( dg::evaluate( dg::zero, g), omega ); 
-    dg::blas1::transfer( dg::evaluate( dg::zero, g), lambda ); 
-    dg::blas1::transfer( dg::evaluate( dg::one,  g), one); 
+    dg::assign( dg::evaluate( dg::zero, g), chi ); 
+    dg::assign( dg::evaluate( dg::zero, g), omega ); 
+    dg::assign( dg::evaluate( dg::zero, g), lambda ); 
+    dg::assign( dg::evaluate( dg::one,  g), one); 
     phi.resize(2);apar.resize(2); phi[0] = phi[1] = apar[0]=apar[1] =  chi;
     npe = logn = u = u2 = un =  phi;
     arakawan = arakawau =  phi;
@@ -255,8 +255,8 @@ Asela<Grid, IMatrix, Matrix, container>::Asela( const Grid& g, Parameters p):
         multi_invgamma[u].construct(   multigrid.grid(u), g.bcx(), g.bcy(), -0.5*p.tau[1]*p.mu[1], dg::centered);
     }
     //////////////////////////init weights////////////////////////////
-    dg::blas1::transfer( dg::create::volume(g),     w2d);
-    dg::blas1::transfer( dg::create::inv_volume(g), v2d);
+    dg::assign( dg::create::volume(g),     w2d);
+    dg::assign( dg::create::inv_volume(g), v2d);
 }
 
 
diff --git a/src/reco2D/reconnection_hpc.cu b/src/reco2D/reconnection_hpc.cu
index e7e7306a4..e770b875b 100644
--- a/src/reco2D/reconnection_hpc.cu
+++ b/src/reco2D/reconnection_hpc.cu
@@ -126,35 +126,35 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<2; i++)
     {
         dg::blas2::symv( interpolate, y0[i], transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     transfer = asela.uparallel()[0];
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
     transfer = asela.uparallel()[1];
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
     transfer = asela.potential()[0];
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
     transfer = asela.aparallel();
     dg::blas2::symv( interpolate, transfer, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
     transfer = asela.potential()[0];
     dg::blas2::symv( interpolate, transfer, helperD);
     dg::blas2::symv( rolkar.laplacianM(), helperD, transferD);
     dg::blas1::scal(transferD,-1.0);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[6], start, count, transferH.data() );
     transfer = asela.aparallel();
     dg::blas2::symv( interpolate, transfer, helperD);
     dg::blas2::symv( rolkar.laplacianM(), helperD, transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[7], start, count, transferH.data() );
     double time = 0;
     err = nc_put_vara_double( ncid, tvarID, start, count, &time);
@@ -233,35 +233,35 @@ int main( int argc, char* argv[])
         for( unsigned j=0; j<2; j++)
         {
             dg::blas2::symv( interpolate, y0[j], transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
         }
         transfer = asela.uparallel()[0];
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
         transfer = asela.uparallel()[1];
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
         transfer = asela.potential()[0];
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
         transfer = asela.aparallel();
         dg::blas2::symv( interpolate, transfer, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
         transfer = asela.potential()[0];
         dg::blas2::symv( interpolate, transfer, helperD);
         dg::blas2::symv( rolkar.laplacianM(), helperD, transferD);
         dg::blas1::scal(transferD,-1.0);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[6], start, count, transferH.data() );
         transfer = asela.aparallel();
         dg::blas2::symv( interpolate, transfer, helperD);
         dg::blas2::symv( rolkar.laplacianM(), helperD, transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[7], start, count, transferH.data() );
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
         err = nc_close(ncid);
diff --git a/src/reco2D/reconnection_mpi.cu b/src/reco2D/reconnection_mpi.cu
index b057ad9e1..5acf1348a 100644
--- a/src/reco2D/reconnection_mpi.cu
+++ b/src/reco2D/reconnection_mpi.cu
@@ -171,33 +171,33 @@ int main( int argc, char* argv[])
     for( unsigned i=0; i<2; i++)
     {
         dg::blas2::gemv( interpolate, y0[i].data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[i], start, count, transferH.data() );
     }
     transfer = asela.uparallel()[0];
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
     transfer = asela.uparallel()[1];
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
     transfer = asela.potential()[0];
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
     transfer = asela.aparallel();
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
     dg::blas2::gemv( rolkar.laplacianM(), asela.potential()[0], transfer);
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
     dg::blas1::scal(transferD,-1.0);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[6], start, count, transferH.data() );
     dg::blas2::gemv( rolkar.laplacianM(), asela.aparallel(), transfer);
     dg::blas2::gemv( interpolate, transfer.data(), transferD);
-    dg::blas1::transfer( transferD, transferH);
+    dg::assign( transferD, transferH);
     err = nc_put_vara_double( ncid, dataIDs[7], start, count, transferH.data() );
     double time = 0;
     err = nc_put_vara_double( ncid, tvarID, start, count, &time);
@@ -273,34 +273,34 @@ int main( int argc, char* argv[])
         for( unsigned j=0; j<2; j++)
         {
             dg::blas2::gemv( interpolate, y0[j].data(), transferD);
-            dg::blas1::transfer( transferD, transferH);
+            dg::assign( transferD, transferH);
             err = nc_put_vara_double( ncid, dataIDs[j], start, count, transferH.data());
         }
         transfer = asela.uparallel()[0];
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[2], start, count, transferH.data() );
         transfer = asela.uparallel()[1];
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[3], start, count, transferH.data() );
         transfer = asela.potential()[0];
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[4], start, count, transferH.data() );
         transfer = asela.aparallel();
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[5], start, count, transferH.data() );
         err = nc_put_vara_double( ncid, tvarID, start, count, &time);
         dg::blas2::gemv( rolkar.laplacianM(), asela.potential()[0], transfer);
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
         dg::blas1::scal(transferD,-1.0);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[6], start, count, transferH.data() );
         dg::blas2::gemv( rolkar.laplacianM(), asela.aparallel(), transfer);
         dg::blas2::gemv( interpolate, transfer.data(), transferD);
-        dg::blas1::transfer( transferD, transferH);
+        dg::assign( transferD, transferH);
         err = nc_put_vara_double( ncid, dataIDs[7], start, count, transferH.data() );
         //err = nc_close(ncid); DONT DO IT!
 #ifdef DG_BENCHMARK
diff --git a/src/toefl/toeflR.cu b/src/toefl/toeflR.cu
index 10714cf8e..f9442a163 100644
--- a/src/toefl/toeflR.cu
+++ b/src/toefl/toeflR.cu
@@ -81,7 +81,7 @@ int main( int argc, char* argv[])
         //transform field to an equidistant grid
         dvisual = y0[0];
 
-        dg::blas1::transfer( dvisual, hvisual);
+        dg::assign( dvisual, hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
@@ -93,7 +93,7 @@ int main( int argc, char* argv[])
         //transform phi
         dvisual = ex.potential()[0];
         dg::blas2::gemv( ex.laplacianM(), dvisual, y1[1]);
-        dg::blas1::transfer( y1[1], hvisual);
+        dg::assign( y1[1], hvisual);
         dg::blas2::gemv( equi, hvisual, visual);
         //compute the color scale
         colors.scale() =  (float)thrust::reduce( visual.begin(), visual.end(), 0., dg::AbsMax<double>() );
diff --git a/src/toefl/toeflR.cuh b/src/toefl/toeflR.cuh
index 33369d18f..7cbe7468b 100644
--- a/src/toefl/toeflR.cuh
+++ b/src/toefl/toeflR.cuh
@@ -208,7 +208,7 @@ const container& Explicit<G, M, container>::polarisation( double t, const std::v
     //compute chi
     if(equations == "global" )
     {
-        dg::blas1::transfer( y[1], chi);
+        dg::assign( y[1], chi);
         dg::blas1::plus( chi, 1.);
         dg::blas1::pointwiseDot( binv, chi, chi); //\chi = n_i
         dg::blas1::pointwiseDot( binv, chi, chi); //\chi *= binv^2
@@ -222,7 +222,7 @@ const container& Explicit<G, M, container>::polarisation( double t, const std::v
     }
     else if(equations == "gravity_global" )
     {
-        dg::blas1::transfer( y[0], chi);
+        dg::assign( y[0], chi);
         dg::blas1::plus( chi, 1.);
         if( !boussinesq)
         {
@@ -234,7 +234,7 @@ const container& Explicit<G, M, container>::polarisation( double t, const std::v
     }
     else if( equations == "drift_global" )
     {
-        dg::blas1::transfer( y[0], chi);
+        dg::assign( y[0], chi);
         dg::blas1::plus( chi, 1.);
         dg::blas1::pointwiseDot( binv, chi, chi); //\chi = n_e
         dg::blas1::pointwiseDot( binv, chi, chi); //\chi *= binv^2
diff --git a/src/toefl/toefl_hpc.cu b/src/toefl/toefl_hpc.cu
index e2b963360..03a8b1cc1 100644
--- a/src/toefl/toefl_hpc.cu
+++ b/src/toefl/toefl_hpc.cu
@@ -146,7 +146,7 @@ int main( int argc, char* argv[])
     dg::blas2::symv( interpolate, transfer, transferD[3]);
     for( int k=0;k<4; k++)
     {
-        dg::blas1::transfer( transferD[k], transferH);
+        dg::assign( transferD[k], transferH);
         dg::file::put_vara_double( ncid, dataIDs[k], start, grid_out, transferH);
     }
     MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
@@ -203,7 +203,7 @@ int main( int argc, char* argv[])
         MPI_OUT err = nc_open(argv[2], NC_WRITE, &ncid);
         for( int k=0;k<4; k++)
         {
-            dg::blas1::transfer( transferD[k], transferH);
+            dg::assign( transferD[k], transferH);
             dg::file::put_vara_double( ncid, dataIDs[k], start, grid_out, transferH);
         }
         MPI_OUT err = nc_put_vara_double( ncid, tvarID, &start, &count, &time);
-- 
GitLab


From dcdcc36ffbea2ca03340610fc0614c7bce4789a0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Thu, 18 Feb 2021 12:52:13 +0100
Subject: [PATCH 525/540] Add some thoughts on (pre-)releases into CHANGELOG

---
 CHANGELOG.md | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 237171146..16be54ffd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,10 +1,21 @@
 # Changelog
 All notable changes to this project will be documented in this file.
-
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
-Only changes in code are reported here, we do not track changes in the
-doxygen documentation, READMEs or tex writeups.
-## [v5.2]
+
+We agreed to updating the master branch on feltor-dev/feltor very seldomnly and only when significant changes merit the effort to brush everything up, document and make
+everyone update their codes as well. We mark those updates with a (pre-)release tag on
+github.
+> Creating a release on github will create an associated archived version
+  on zenodo, so **we reserve releases for when we publish an article**.
+
+We kind of make up our own version numbers right now. A new major version number is often associated with a journal publication, but other than that there is no defined mapping from the amount or kind of change to a version number. 
+[Semantiv versioning](https://semver.org/) might serve as a guideline  but we are
+far away from strictly following it really.
+
+> Only changes in code are reported here, we do not track changes in the
+> doxygen documentation, READMEs or tex writeups.
+
+## [v5.2] More Multistep
 ### Added
  - M100 config file
  - json utility functions `dg::file::get, dg::file::get_idx` in `dg/file/json_utilities.h` which adds a small abstraction layer that gives a user more control over what happens if a variable is not found
@@ -108,7 +119,7 @@ doxygen documentation, READMEs or tex writeups.
  - Fix bug(s): several bugs in `dg::geo::Hector` which computed wrong grid (happened probably when we changed the grid design to polymorphic)
  - Fix bug: in perpendicular grid of MPI Curvlinear grid
 
-## [v5.1]
+## [v5.1] Adaptive Timesteppers
 ### Added
 - dg::Elliptic3d: a three-dimensional version of dg::Elliptic
 - Add 4 parameter symv member to dg::Elliptic class
-- 
GitLab


From 15d830e3917dd175f997fb5537f0114ffdb74fc9 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 15:01:54 +0100
Subject: [PATCH 526/540] Remove unnecessary typedefs in src header files

---
 src/feltorSH/feltor.cuh    | 4 ----
 src/feltorSHp/feltor.cuh   | 4 ----
 src/feltorSesol/feltor.cuh | 4 ----
 src/feltorShw/feltor.cuh   | 4 ----
 src/hasegawa/hw.cuh        | 2 --
 src/hasegawa/mima.cuh      | 2 --
 6 files changed, 20 deletions(-)

diff --git a/src/feltorSH/feltor.cuh b/src/feltorSH/feltor.cuh
index ea64c8dc8..ac52a32fe 100644
--- a/src/feltorSH/feltor.cuh
+++ b/src/feltorSH/feltor.cuh
@@ -60,11 +60,7 @@ struct Implicit
 template< class Geometry, class Matrix, class container>
 struct Explicit
 {
-    //typedef std::vector<container> Vector;
     using value_type = dg::get_value_type<container>;
-    //typedef typename thrust::iterator_system<typename container::iterator>::type MemorySpace;
-    //typedef cusp::ell_matrix<int, value_type, MemorySpace> Matrix;
-    //typedef dg::DMatrix Matrix; //fastest device Matrix (does this conflict with 
 
     Explicit( const Geometry& g, eule::Parameters p);
 
diff --git a/src/feltorSHp/feltor.cuh b/src/feltorSHp/feltor.cuh
index 642b82a42..bdcc2353d 100644
--- a/src/feltorSHp/feltor.cuh
+++ b/src/feltorSHp/feltor.cuh
@@ -65,11 +65,7 @@ struct Implicit
 template< class Geometry, class Matrix, class container>
 struct Explicit
 {
-    //typedef std::vector<container> Vector;
     using value_type = dg::get_value_type<container>;
-    //typedef typename thrust::iterator_system<typename container::iterator>::type MemorySpace;
-    //typedef cusp::ell_matrix<int, value_type, MemorySpace> Matrix;
-    //typedef dg::DMatrix Matrix; //fastest device Matrix (does this conflict with 
 
     Explicit( const Geometry& g, eule::Parameters p);
 
diff --git a/src/feltorSesol/feltor.cuh b/src/feltorSesol/feltor.cuh
index f9a14d25d..42f2f2ed2 100644
--- a/src/feltorSesol/feltor.cuh
+++ b/src/feltorSesol/feltor.cuh
@@ -56,11 +56,7 @@ struct Implicit
 template< class Geometry, class Matrix, class container>
 struct Explicit
 {
-    //typedef std::vector<container> Vector;
     using value_type = dg::get_value_type<container>;
-    //typedef typename thrust::iterator_system<typename container::iterator>::type MemorySpace;
-    //typedef cusp::ell_matrix<int, value_type, MemorySpace> Matrix;
-    //typedef dg::DMatrix Matrix; //fastest device Matrix (does this conflict with 
 
     Explicit( const Geometry& g, eule::Parameters p);
 
diff --git a/src/feltorShw/feltor.cuh b/src/feltorShw/feltor.cuh
index 436d9a0b5..9170c69b4 100644
--- a/src/feltorShw/feltor.cuh
+++ b/src/feltorShw/feltor.cuh
@@ -58,11 +58,7 @@ struct Implicit
 template< class Geometry, class Matrix, class container >
 struct Explicit
 {
-    //typedef std::vector<container> Vector;
     using value_type = dg::get_value_type<container>;
-    //typedef typename thrust::iterator_system<typename container::iterator>::type MemorySpace;
-    //typedef cusp::ell_matrix<int, value_type, MemorySpace> Matrix;
-    //typedef dg::DMatrix Matrix; //fastest device Matrix (does this conflict with 
 
     Explicit( const Geometry& g, eule::Parameters p);
 
diff --git a/src/hasegawa/hw.cuh b/src/hasegawa/hw.cuh
index 685a454e7..766ac0010 100644
--- a/src/hasegawa/hw.cuh
+++ b/src/hasegawa/hw.cuh
@@ -43,8 +43,6 @@ struct HW
 {
     typedef std::vector<container> Vector;
     typedef typename container::value_type value_type;
-    typedef typename thrust::iterator_system<typename container::iterator>::type MemorySpace;
-    //typedef cusp::ell_matrix<int, value_type, MemorySpace> Matrix;
 
     /**
      * @brief Construct a HW solver object
diff --git a/src/hasegawa/mima.cuh b/src/hasegawa/mima.cuh
index 6c1527a7a..7a7cb540c 100644
--- a/src/hasegawa/mima.cuh
+++ b/src/hasegawa/mima.cuh
@@ -36,8 +36,6 @@ struct Mima
 {
     typedef std::vector<container> Vector;
     typedef typename container::value_type value_type;
-    typedef typename thrust::iterator_system<typename container::iterator>::type MemorySpace;
-    //typedef cusp::ell_matrix<int, value_type, MemorySpace> Matrix;
 
     /**
      * @brief Construct a Mima solver object
-- 
GitLab


From 054b32c5cbc63c25cd2e49b403c713ab8c89cafe Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 15:05:24 +0100
Subject: [PATCH 527/540] Add entry to .gitignore

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 54914461d..660113d9e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,4 @@ src/feltor/doc/
 *.log
 *.png
 *.pvsm
+*.bak
-- 
GitLab


From 6245e33f6dc79952364304d5c23621fed0916a50 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 15:07:25 +0100
Subject: [PATCH 528/540] Add example in tensor documentation

---
 inc/dg/topology/tensor.h         | 16 ++++++++++++++++
 inc/dg/topology/transform.h      |  1 +
 inc/geometries/curvilinear.h     |  3 ++-
 inc/geometries/mpi_curvilinear.h |  3 ++-
 4 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/inc/dg/topology/tensor.h b/inc/dg/topology/tensor.h
index 32ea0eae6..9a1d95d3e 100644
--- a/inc/dg/topology/tensor.h
+++ b/inc/dg/topology/tensor.h
@@ -23,6 +23,22 @@ namespace dg
 * The integers represent a gather index into a stored array of containers.
 * In this way duplicate entries are stored only once, which helps to
 * avoid unnecessary memory accesses.
+* For example an orthogonal metric is represented as follows
+* \f[
+* \begin{pmatrix}
+* g^{xx} & 0 & 0\\
+* 0 & g^{yy} & 0 \\
+* 0 & 0 & g^{zz}
+* \end{pmatrix}
+* \quad\rightarrow\quad
+* \text{idx} = \begin{pmatrix}
+* 1 & 0 & 0 \\
+* 0 & 2 & 0 \\
+* 0 & 0 & 3
+* \end{pmatrix} \quad \text{values} = \begin{pmatrix}
+* 0 & g^{xx} & g^{yy} & g^{zz}
+* \end{pmatrix}
+* \f]
 * @tparam container must be default constructible and copyable.
 * @ingroup sparsematrix
 */
diff --git a/inc/dg/topology/transform.h b/inc/dg/topology/transform.h
index 1d4a13a38..25f63f1d9 100644
--- a/inc/dg/topology/transform.h
+++ b/inc/dg/topology/transform.h
@@ -210,6 +210,7 @@ namespace create{
 ///@{
 
 
+//Note that for the volume function to work properly all 2d grids must set the g_22 element to 1!!
 
 /**
  * @brief Create the volume element on the grid (including weights!!)
diff --git a/inc/geometries/curvilinear.h b/inc/geometries/curvilinear.h
index b27ac0d39..2080eee5e 100644
--- a/inc/geometries/curvilinear.h
+++ b/inc/geometries/curvilinear.h
@@ -222,7 +222,8 @@ RealCurvilinearGrid2d<real_type>::RealCurvilinearGrid2d( RealCurvilinearProductG
     m_jac=g.jacobian();
     m_metric=g.metric();
     // we rely on the fact that the 3d grid uses square to compute its metric
-    // so the (2,2) entry is value 3 that we need to set to 1
+    // so the (2,2) entry is value 3 that we need to set to 1 (for the
+    // create::volume function to work properly)
     dg::blas1::copy( 1., m_metric.values()[3]);
     m_map.pop_back();
 }
diff --git a/inc/geometries/mpi_curvilinear.h b/inc/geometries/mpi_curvilinear.h
index ab4945e89..658110f64 100644
--- a/inc/geometries/mpi_curvilinear.h
+++ b/inc/geometries/mpi_curvilinear.h
@@ -207,7 +207,8 @@ RealCurvilinearMPIGrid2d<real_type>::RealCurvilinearMPIGrid2d( const RealCurvili
         m_metric.values()[i].set_communicator( comm, comm_mod, comm_mod_reduce);
     }
     // we rely on the fact that the 3d grid uses square to compute its metric
-    // so the (2,2) entry is value 3 that we need to set to 1
+    // so the (2,2) entry is value 3 that we need to set to 1 (for the
+    // create::volume function to work properly)
     dg::blas1::copy( 1., m_metric.values()[3]);
     for( unsigned i=0; i<m_map.size(); i++)
     {
-- 
GitLab


From f7d5797cf2c6d8800c41c0f17b67755ef490a7bf Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 15:09:00 +0100
Subject: [PATCH 529/540] Fix findOpoint in separatrix_orthogonal_t.cu

---
 inc/geometries/separatrix_orthogonal_t.cu | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/inc/geometries/separatrix_orthogonal_t.cu b/inc/geometries/separatrix_orthogonal_t.cu
index 8cc3b79f3..099ecbaf9 100644
--- a/inc/geometries/separatrix_orthogonal_t.cu
+++ b/inc/geometries/separatrix_orthogonal_t.cu
@@ -94,7 +94,7 @@ int main( int argc, char* argv[])
     dg::geo::solovev::Parameters gp(js);
     dg::geo::TokamakMagneticField mag = dg::geo::createSolovevField(gp);
     double R_O = gp.R_0, Z_O = 0.;
-    dg::geo::findXpoint( mag.get_psip(), R_O, Z_O);
+    dg::geo::findOpoint( mag.get_psip(), R_O, Z_O);
     const double psipmin = mag.psip()(R_O, Z_O);
     std::cout << "Psi min "<<psipmin<<"\n";
     dg::Timer t;
@@ -102,7 +102,7 @@ int main( int argc, char* argv[])
     double psi_0 = -20;
     std::cin >> psi_0;
     std::cout << "Typed "<<psi_0<<"\n";
-    std::cout << "Type fx and fy ( fx*Nx and fy*Ny must be integer) 1/4, 1/22 \n";
+    std::cout << "Type fx and fy ( fx*Nx and fy*Ny must be integer) 0.25 0.04545454545454545 \n";
     double fx_0=1./4., fy_0=1./22.;
     std::cin >> fx_0>> fy_0;
     std::cout << "Typed "<<fx_0<<" "<<fy_0<<"\n";
-- 
GitLab


From 9c9dcbda029209e7a0afe0fd1f4c1146eecb0e62 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 15:12:21 +0100
Subject: [PATCH 530/540] Fix MPI bug if host mpi code is compiled with nvcc

- we don't need to involve cuda calls for host mpi
- this is practically never encountered in practice, why would you
compile host code with nvcc?
---
 inc/dg/backend/mpi_collective.h   |  6 ++++--
 inc/dg/backend/mpi_vector.h       | 22 ++++++++++++++--------
 inc/geometries/mpi_fieldaligned.h |  6 ++++--
 3 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/inc/dg/backend/mpi_collective.h b/inc/dg/backend/mpi_collective.h
index 14fc9b83f..4439731ee 100644
--- a/inc/dg/backend/mpi_collective.h
+++ b/inc/dg/backend/mpi_collective.h
@@ -108,7 +108,8 @@ void Collective<Index, Device>::scatter( const Device& values, Device& store) co
 {
     //assert( store.size() == store_size() );
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
-    cudaDeviceSynchronize(); //needs to be called
+    if( std::is_same< get_execution_policy<Device>, CudaTag>::value ) //could be serial tag
+        cudaDeviceSynchronize(); //needs to be called
 #endif //THRUST_DEVICE_SYSTEM
 #ifdef _DG_CUDA_UNAWARE_MPI
     m_values.data() = values;
@@ -138,7 +139,8 @@ void Collective<Index, Device>::gather( const Device& gatherFrom, Device& values
     //assert( gatherFrom.size() == store_size() );
     values.resize( values_size() );
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
-    cudaDeviceSynchronize(); //needs to be called
+    if( std::is_same< get_execution_policy<Device>, CudaTag>::value ) //could be serial tag
+        cudaDeviceSynchronize(); //needs to be called
 #endif //THRUST_DEVICE_SYSTEM
 #ifdef _DG_CUDA_UNAWARE_MPI
     m_store.data() = gatherFrom;
diff --git a/inc/dg/backend/mpi_vector.h b/inc/dg/backend/mpi_vector.h
index 730ab58c9..76a7a5ed1 100644
--- a/inc/dg/backend/mpi_vector.h
+++ b/inc/dg/backend/mpi_vector.h
@@ -323,6 +323,8 @@ struct NearestNeighborComm
     {
         MPI_Waitall( 4, rqst, MPI_STATUSES_IGNORE );
 #ifdef _DG_CUDA_UNAWARE_MPI
+    if( std::is_same< get_execution_policy<Vector>, CudaTag>::value ) //could be serial tag
+    {
         unsigned size = buffer_size();
         cudaMemcpy( thrust::raw_pointer_cast(&m_internal_buffer.data()[0*size]), //dst
                     thrust::raw_pointer_cast(&m_internal_host_buffer.data()[0*size]), //src
@@ -331,6 +333,7 @@ struct NearestNeighborComm
         cudaMemcpy( thrust::raw_pointer_cast(&m_internal_buffer.data()[5*size]), //dst
                     thrust::raw_pointer_cast(&m_internal_host_buffer.data()[5*size]), //src
                     size*sizeof(get_value_type<Vector>), cudaMemcpyHostToDevice);
+    }
 #endif
     }
     private:
@@ -487,14 +490,17 @@ void NearestNeighborComm<I,B,V>::sendrecv( const_pointer_type sb1_ptr, const_poi
 {
     unsigned size = buffer_size();
 #ifdef _DG_CUDA_UNAWARE_MPI
-    cudaMemcpy( thrust::raw_pointer_cast(&m_internal_host_buffer.data()[1*size]),//dst
-        sb1_ptr, size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost); //src
-    cudaMemcpy( thrust::raw_pointer_cast(&m_internal_host_buffer.data()[4*size]),  //dst
-        sb2_ptr, size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost); //src
-    sb1_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[1*size]);
-    sb2_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[4*size]);
-    rb1_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[0*size]);
-    rb2_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[5*size]);
+    if( std::is_same< get_execution_policy<V>, CudaTag>::value ) //could be serial tag
+    {
+        cudaMemcpy( thrust::raw_pointer_cast(&m_internal_host_buffer.data()[1*size]),//dst
+            sb1_ptr, size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost); //src
+        cudaMemcpy( thrust::raw_pointer_cast(&m_internal_host_buffer.data()[4*size]),  //dst
+            sb2_ptr, size*sizeof(get_value_type<V>), cudaMemcpyDeviceToHost); //src
+        sb1_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[1*size]);
+        sb2_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[4*size]);
+        rb1_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[0*size]);
+        rb2_ptr = thrust::raw_pointer_cast(&m_internal_host_buffer.data()[5*size]);
+    }
 //This is a mistake if called with a host_vector
 #endif
     MPI_Isend( sb1_ptr, size,
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 581367063..fc9640662 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -29,7 +29,8 @@ void sendForward( const thrust_vector0& in, thrust_vector1& out, MPI_Comm comm)
     MPI_Status status;
     MPI_Cart_shift( comm, 2, +1, &source, &dest);
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
-    cudaDeviceSynchronize();//wait until device functions are finished before sending data
+    if( std::is_same< get_execution_policy<thrust_vector0>, CudaTag>::value) //could be serial tag
+        cudaDeviceSynchronize();//wait until device functions are finished before sending data
 #endif //THRUST_DEVICE_SYSTEM
     unsigned size = in.size();
     MPI_Sendrecv(   thrust::raw_pointer_cast(in.data()), size, MPI_DOUBLE,  //sender
@@ -46,7 +47,8 @@ void sendBackward( const thrust_vector0& in, thrust_vector1& out, MPI_Comm comm)
     MPI_Status status;
     MPI_Cart_shift( comm, 2, -1, &source, &dest);
 #if THRUST_DEVICE_SYSTEM==THRUST_DEVICE_SYSTEM_CUDA
-    cudaDeviceSynchronize();//wait until device functions are finished before sending data
+    if( std::is_same< get_execution_policy<thrust_vector0>, CudaTag>::value) //could be serial tag
+        cudaDeviceSynchronize();//wait until device functions are finished before sending data
 #endif //THRUST_DEVICE_SYSTEM
     unsigned size = in.size();
     MPI_Sendrecv(   thrust::raw_pointer_cast(in.data()), size, MPI_DOUBLE,  //sender
-- 
GitLab


From cd3d1b5a74819690d7fa94a6cb809dc815070554 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 15:15:25 +0100
Subject: [PATCH 531/540] Fix bug in output in elliptic2d_b.cu

---
 inc/dg/arakawa_t.cu    | 2 +-
 inc/dg/elliptic.h      | 4 ++++
 inc/dg/elliptic2d_b.cu | 4 ++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/inc/dg/arakawa_t.cu b/inc/dg/arakawa_t.cu
index cd8d09b26..362b296fe 100644
--- a/inc/dg/arakawa_t.cu
+++ b/inc/dg/arakawa_t.cu
@@ -103,6 +103,6 @@ int main()
     //n = 5 -> p = 5    |
     // quantities are all conserved to 1e-15 for periodic bc
     // for dirichlet bc these are not better conserved than normal jacobian
-    std::cout << "\nContinue with gradient_t.cu !\n\n";
+    std::cout << "\nContinue with topology/average_t.cu !\n\n";
     return 0;
 }
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index 1d00274ef..bdd14eaf3 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -17,6 +17,10 @@
   */
 namespace dg
 {
+// Note that there are many tests for this file : elliptic2d_b,
+// elliptic2d_mpib, elliptic_b, elliptic_mpib, ellipticX2d_b
+// And don't forget inc/geometries/elliptic3d_t (testing alignment and
+// projection tensors as Chi)
 
 /**
  * @brief A 2d negative elliptic differential operator \f$ -\nabla \cdot ( \mathbf{\chi}\cdot \nabla ) \f$
diff --git a/inc/dg/elliptic2d_b.cu b/inc/dg/elliptic2d_b.cu
index cc1511f35..55f44b2c7 100644
--- a/inc/dg/elliptic2d_b.cu
+++ b/inc/dg/elliptic2d_b.cu
@@ -157,8 +157,8 @@ int main()
     pol_forward.variation( 1., chi, x, 0., error);
     dg::blas1::axpby( 1., variatio, -1., error);
     err = dg::blas2::dot( w2d, error);
-    norm = dg::blas2::dot( w2d, variatio);
-    std::cout << " "<<sqrt( err/norm) << "\n";
+    const double norm_var = dg::blas2::dot( w2d, variatio);
+    std::cout << " "<<sqrt( err/norm_var) << "\n";
     }
 
     {
-- 
GitLab


From b6f0fcf703bc79fb9996632033e9b3a66adf7dae Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 15:16:11 +0100
Subject: [PATCH 532/540] Use C++-14 helper typedefs in predicate

- makes our meta-programming a bit easier to read
---
 inc/dg/backend/blas1_dispatch_mpi.h    |  2 +-
 inc/dg/backend/blas1_dispatch_vector.h |  2 +-
 inc/dg/backend/blas2_cusp.h            |  8 +++----
 inc/dg/backend/exblas/config.h         |  2 +-
 inc/dg/backend/predicate.h             | 33 +++++++++++++-------------
 inc/dg/backend/tensor_traits.h         | 24 +++++++++----------
 inc/dg/backend/tensor_traits_cusp.h    |  4 ++--
 inc/dg/backend/tensor_traits_scalar.h  |  2 +-
 inc/dg/backend/tensor_traits_std.h     |  4 ++--
 inc/dg/backend/tensor_traits_thrust.h  |  6 ++---
 inc/dg/backend/view.h                  | 12 +++++-----
 inc/dg/backend/view_t.cu               |  4 ++--
 inc/dg/blas1.h                         |  2 +-
 inc/dg/topology/base_geometryX.h       |  4 ++--
 inc/dg/topology/operator.h             |  2 +-
 inc/dg/topology/split_and_join.h       |  6 ++---
 16 files changed, 58 insertions(+), 59 deletions(-)

diff --git a/inc/dg/backend/blas1_dispatch_mpi.h b/inc/dg/backend/blas1_dispatch_mpi.h
index 5945d8df4..de1d23fb0 100644
--- a/inc/dg/backend/blas1_dispatch_mpi.h
+++ b/inc/dg/backend/blas1_dispatch_mpi.h
@@ -25,7 +25,7 @@ Vector1 doConstruct( const Vector2& in, MPIVectorTag, MPIVectorTag, Params&& ...
 {
     Vector1 out;
     out.set_communicator(in.communicator(), in.communicator_mod(), in.communicator_mod_reduce());
-    using container1 = typename std::decay<Vector1>::type::container_type;
+    using container1 = typename std::decay_t<Vector1>::container_type;
     out.data() = dg::construct<container1>( in.data(), std::forward<Params>(ps)...);
     return out;
 }
diff --git a/inc/dg/backend/blas1_dispatch_vector.h b/inc/dg/backend/blas1_dispatch_vector.h
index 31b58f928..f39d64078 100644
--- a/inc/dg/backend/blas1_dispatch_vector.h
+++ b/inc/dg/backend/blas1_dispatch_vector.h
@@ -161,7 +161,7 @@ inline std::vector<int64_t> doDot_superacc( const Vector1& x1, const Vector2& x2
 template< class size_type, class Subroutine, class container, class ...Containers>
 inline void doSubroutine_dispatch( RecursiveVectorTag, OmpTag, size_type size, Subroutine f, container&& x, Containers&&... xs)
 {
-    //using inner_container = typename std::decay<container>::type::value_type;
+    //using inner_container = typename std::decay_t<container>::value_type;
     if( !omp_in_parallel())//to catch recursive calls
     {
         #pragma omp parallel
diff --git a/inc/dg/backend/blas2_cusp.h b/inc/dg/backend/blas2_cusp.h
index d41ff46e1..fc0d11a54 100644
--- a/inc/dg/backend/blas2_cusp.h
+++ b/inc/dg/backend/blas2_cusp.h
@@ -36,7 +36,7 @@ inline void doSymv_cusp_dispatch( Matrix&& m,
                     cusp::csr_format,
                     OmpTag)
 {
-    typedef typename std::decay<Matrix>::type::index_type index_type;
+    typedef typename std::decay_t<Matrix>::index_type index_type;
     using value_type = get_value_type<Container1>;
     const value_type* RESTRICT val_ptr = thrust::raw_pointer_cast( &m.values[0]);
     const index_type* RESTRICT row_ptr = thrust::raw_pointer_cast( &m.row_offsets[0]);
@@ -81,7 +81,7 @@ inline void doSymv( Matrix&& m,
     static_assert( std::is_base_of<SharedVectorTag, get_tensor_category<Vector2>>::value,
         "All data layouts must derive from the same vector category (SharedVectorTag in this case)!");
     static_assert( std::is_same< get_execution_policy<Vector1>, get_execution_policy<Vector2> >::value, "Execution policies must be equal!");
-    typedef typename std::decay<Matrix>::type::value_type value_type;
+    typedef typename std::decay_t<Matrix>::value_type value_type;
     static_assert( std::is_same< get_value_type<Vector1>, value_type >::value,
         "Value types must be equal"
     );
@@ -94,7 +94,7 @@ inline void doSymv( Matrix&& m,
     assert( m.num_cols == x.size() );
 #endif //DG_DEBUG
     doSymv_cusp_dispatch( std::forward<Matrix>(m),x,y,
-            typename std::decay<Matrix>::type::format(),
+            typename std::decay_t<Matrix>::format(),
             get_execution_policy<Vector1>());
 }
 template< class Matrix, class Vector1, class Vector2>
@@ -110,7 +110,7 @@ inline void doSymv( Matrix&& m,
     assert( m.num_rows == y.size() );
     assert( m.num_cols == x.size() );
 #endif //DG_DEBUG
-    using inner_container = typename std::decay<Vector1>::type::value_type;
+    using inner_container = typename std::decay_t<Vector1>::value_type;
     for ( unsigned i=0; i<x.size(); i++)
         doSymv( std::forward<Matrix>(m), x[i], y[i], CuspMatrixTag(), get_tensor_category<inner_container>());
 }
diff --git a/inc/dg/backend/exblas/config.h b/inc/dg/backend/exblas/config.h
index 4eb2cc204..17102ffa9 100644
--- a/inc/dg/backend/exblas/config.h
+++ b/inc/dg/backend/exblas/config.h
@@ -104,7 +104,7 @@ struct ValueTraits<T*>
     using value_type = T;
 };
 template<class U>
-using has_floating_value = typename std::conditional< std::is_floating_point<typename ValueTraits<U>::value_type>::value, std::true_type, std::false_type>::type;
+using has_floating_value = std::conditional_t< std::is_floating_point<typename ValueTraits<U>::value_type>::value, std::true_type, std::false_type>;
 ///@endcond
 
 }//namespace exblas
diff --git a/inc/dg/backend/predicate.h b/inc/dg/backend/predicate.h
index 650ddbccf..cd01fe047 100644
--- a/inc/dg/backend/predicate.h
+++ b/inc/dg/backend/predicate.h
@@ -14,14 +14,14 @@ struct find_if_impl;
 template<template <typename> class Predicate, unsigned n, class Default, class T>
 struct find_if_impl<Predicate, n, Default, T>
 {
-    using type = typename std::conditional< Predicate<T>::value, T, Default>::type;
+    using type = std::conditional_t< Predicate<T>::value, T, Default>;
     static constexpr unsigned value = Predicate<T>::value ? n : n+1;
 };
 
 template<template <typename> class Predicate, unsigned n, class Default, class T, class... Ts>
 struct find_if_impl<Predicate, n, Default, T, Ts...>
 {
-    using type = typename std::conditional< Predicate<T>::value, T, typename find_if_impl<Predicate, n+1, Default, Ts...>::type>::type;
+    using type = std::conditional_t< Predicate<T>::value, T, typename find_if_impl<Predicate, n+1, Default, Ts...>::type>;
     static constexpr unsigned value = Predicate<T>::value ? n : find_if_impl<Predicate, n+1, Default, Ts...>::value;
 };
 }//namespace detail
@@ -29,14 +29,14 @@ struct find_if_impl<Predicate, n, Default, T, Ts...>
 //access the element at position index
 //we name it get_idx and not get so we do not get a conflict with std::get
 template<size_t index, typename T, typename... Ts>
-inline typename std::enable_if<index==0, T>::type
+inline std::enable_if_t<index==0, T>
 get_idx(T&& t, Ts&&... ts) {
     return std::forward<T>(t);
 }
 
 template<size_t index, typename T, typename... Ts>
-inline typename std::enable_if<(index > 0) && index <= sizeof...(Ts),
-          typename std::tuple_element<index, std::tuple<T, Ts...>>::type>::type
+inline std::enable_if_t<(index > 0) && index <= sizeof...(Ts),
+          std::tuple_element_t<index, std::tuple<T, Ts...>>>
 get_idx(T&& t, Ts&&... ts) {
     return get_idx<index-1>(std::forward<Ts>(ts)...);
 }
@@ -51,39 +51,38 @@ using find_if_v = std::integral_constant<unsigned, detail::find_if_impl<Predicat
 /////////////////////////////////////////////////////////////////////////////////
 //is scalar
 template< class T>
-using is_scalar = typename std::conditional< std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, std::true_type, std::false_type>::type;
+using is_scalar = std::conditional_t< std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, std::true_type, std::false_type>;
 template< class T>
-using is_not_scalar = typename std::conditional< !std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, std::true_type, std::false_type>::type;
+using is_not_scalar = std::conditional_t< !std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, std::true_type, std::false_type>;
 //is vector (or scalar)
 template< class T>
-using is_vector = typename std::conditional< std::is_base_of<AnyVectorTag, get_tensor_category<T>>::value, std::true_type, std::false_type>::type;
+using is_vector = std::conditional_t< std::is_base_of<AnyVectorTag, get_tensor_category<T>>::value, std::true_type, std::false_type>;
 //is matrix (or vector or scalar)
 template< class T>
-using is_matrix = typename std::conditional< std::is_base_of<AnyMatrixTag, get_tensor_category<T>>::value, std::true_type, std::false_type>::type;
+using is_matrix = std::conditional_t< std::is_base_of<AnyMatrixTag, get_tensor_category<T>>::value, std::true_type, std::false_type>;
 
 namespace detail
 {
 template<class Category>
-using find_base_category = typename
-    std::conditional< std::is_base_of<SharedVectorTag, Category>::value, SharedVectorTag,
-    typename std::conditional< std::is_base_of<RecursiveVectorTag, Category>::value, RecursiveVectorTag, MPIVectorTag>::type>::type;
+using find_base_category = std::conditional_t< std::is_base_of<SharedVectorTag, Category>::value, SharedVectorTag,
+        std::conditional_t< std::is_base_of<RecursiveVectorTag, Category>::value, RecursiveVectorTag, MPIVectorTag>>;
 }//namesapce detail
 //is scalar or same tensor category
 template<class T, class Category>
-using is_scalar_or_same_base_category = typename std::conditional< std::is_base_of<detail::find_base_category<Category>, get_tensor_category<T>>::value || is_scalar<T>::value , std::true_type, std::false_type>::type;
+using is_scalar_or_same_base_category = std::conditional_t< std::is_base_of<detail::find_base_category<Category>, get_tensor_category<T>>::value || is_scalar<T>::value , std::true_type, std::false_type>;
 
 
 //has trivial policy
 template< class T>
-using has_any_policy = typename std::conditional< std::is_same<AnyPolicyTag, get_execution_policy<T>>::value, std::true_type, std::false_type>::type;
+using has_any_policy = std::conditional_t< std::is_same<AnyPolicyTag, get_execution_policy<T>>::value, std::true_type, std::false_type>;
 template< class T>
-using has_not_any_policy = typename std::conditional< !std::is_same<AnyPolicyTag, get_execution_policy<T>>::value, std::true_type, std::false_type>::type;
+using has_not_any_policy = std::conditional_t< !std::is_same<AnyPolicyTag, get_execution_policy<T>>::value, std::true_type, std::false_type>;
 //has any or same policy tag
 template<class U, class Policy>
-using has_any_or_same_policy = typename std::conditional< std::is_same<get_execution_policy<U>, Policy>::value || has_any_policy<U>::value, std::true_type, std::false_type>::type;
+using has_any_or_same_policy = std::conditional_t< std::is_same<get_execution_policy<U>, Policy>::value || has_any_policy<U>::value, std::true_type, std::false_type>;
 //is not scalar and has a nontrivial policy
 template< class T>
-using is_not_scalar_has_not_any_policy = typename std::conditional< !is_scalar<T>::value && !has_any_policy<T>::value, std::true_type, std::false_type>::type;
+using is_not_scalar_has_not_any_policy = std::conditional_t< !is_scalar<T>::value && !has_any_policy<T>::value, std::true_type, std::false_type>;
 
 /////////////////////////////////////////////////////////////////////////////////
 //from stackoverflow implement Columbo's bool pack trick to check parameter packs
diff --git a/inc/dg/backend/tensor_traits.h b/inc/dg/backend/tensor_traits.h
index 7ce505364..bbc196149 100644
--- a/inc/dg/backend/tensor_traits.h
+++ b/inc/dg/backend/tensor_traits.h
@@ -28,27 +28,27 @@ template< class Vector, class Enable=void>
 struct TensorTraits;
 
 template<class Vector>
-using get_value_type = typename TensorTraits<typename std::decay<Vector>::type>::value_type;
+using get_value_type = typename TensorTraits<std::decay_t<Vector>>::value_type;
 template<class Vector>
-using get_tensor_category = typename TensorTraits< typename std::decay<Vector>::type >::tensor_category;
+using get_tensor_category = typename TensorTraits< std::decay_t<Vector>>::tensor_category;
 template<class Vector>
-using get_execution_policy = typename TensorTraits<typename std::decay<Vector>::type>::execution_policy;
+using get_execution_policy = typename TensorTraits<std::decay_t<Vector>>::execution_policy;
 
 ///@}
 
 ///@cond
 ////////////get element, pointer and data
 template<class T> //T = SharedVector
-using get_pointer_type = typename std::conditional< std::is_const< typename std::remove_reference<T>::type >::value,
-    const get_value_type<T>*, get_value_type<T>* >::type;
+using get_pointer_type = std::conditional_t< std::is_const< std::remove_reference_t<T> >::value,
+    const get_value_type<T>*, get_value_type<T>* >;
 
 template<class T> //T = RecursiveVector
-using get_element_type = typename std::conditional< std::is_const< typename std::remove_reference<T>::type >::value,
-    const typename std::decay<T>::type::value_type&, typename std::decay<T>::type::value_type& >::type;
+using get_element_type = std::conditional_t< std::is_const< std::remove_reference_t<T> >::value,
+    const typename std::decay_t<T>::value_type&, typename std::decay_t<T>::value_type& >;
 
 template<class T>//T = MPIVector
-using get_data_type = typename std::conditional< std::is_const< typename std::remove_reference<T>::type >::value,
-    const typename std::decay<T>::type::container_type&, typename std::decay<T>::type::container_type& >::type;
+using get_data_type = std::conditional_t< std::is_const< std::remove_reference_t<T> >::value,
+    const typename std::decay_t<T>::container_type&, typename std::decay_t<T>::container_type& >;
 
 template<class T>
 inline get_element_type<T> do_get_vector_element( T&& v, unsigned i, RecursiveVectorTag)//-> decltype(v[i]){
@@ -83,19 +83,19 @@ inline T&& do_get_pointer_or_reference( T&& v, AnyScalarTag){
 ///@endcond
 
 //template<class T>
-//inline typename std::conditional<std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, T&&, get_element_type<T> >::type get_vector_element( T&& v, unsigned i )// -> decltype( do_get_vector_element( std::forward<T>(v), i, get_tensor_category<T>()) )
+//inline std::conditional_t<std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, T&&, get_element_type<T> > get_vector_element( T&& v, unsigned i )// -> decltype( do_get_vector_element( std::forward<T>(v), i, get_tensor_category<T>()) )
 //{
 //    return do_get_vector_element( std::forward<T>(v), i, get_tensor_category<T>());
 //}
 //
 //template<class T>
-//inline typename std::conditional<std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, T, get_data_type<T> >::type get_data( T&& v)//-> decltype(do_get_data( std::forward<T>(v), get_tensor_category<T>() ))
+//inline std::conditional_t<std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, T, get_data_type<T> > get_data( T&& v)//-> decltype(do_get_data( std::forward<T>(v), get_tensor_category<T>() ))
 //{
 //    return do_get_data( std::forward<T>(v), get_tensor_category<T>());
 //}
 //
 //template<class T>
-//typename std::conditional<std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, T, get_pointer_type<T> >::type get_pointer_or_reference( T&& v )// -> decltype( do_get_pointer_or_reference( std::forward<T>(v), get_tensor_category<T>()))
+//std::conditional<std::is_base_of<AnyScalarTag, get_tensor_category<T>>::value, T, get_pointer_type<T> > get_pointer_or_reference( T&& v )// -> decltype( do_get_pointer_or_reference( std::forward<T>(v), get_tensor_category<T>()))
 //{
 //    return do_get_pointer_or_reference( std::forward<T>(v), get_tensor_category<T>());
 //}
diff --git a/inc/dg/backend/tensor_traits_cusp.h b/inc/dg/backend/tensor_traits_cusp.h
index 8f108d1b2..32b5db25b 100644
--- a/inc/dg/backend/tensor_traits_cusp.h
+++ b/inc/dg/backend/tensor_traits_cusp.h
@@ -20,7 +20,7 @@ namespace dg
 ///@{
 template<class T>
 struct TensorTraits<cusp::array1d<T,cusp::host_memory>,
-    typename std::enable_if< std::is_arithmetic<T>::value>::type>
+    std::enable_if_t< std::is_arithmetic<T>::value>>
 {
     using value_type        = T;
     using tensor_category   = CuspVectorTag;
@@ -28,7 +28,7 @@ struct TensorTraits<cusp::array1d<T,cusp::host_memory>,
 };
 template<class T>
 struct TensorTraits<cusp::array1d<T,cusp::device_memory>,
-    typename std::enable_if< std::is_arithmetic<T>::value>::type>
+    std::enable_if_t< std::is_arithmetic<T>::value>>
 {
     using value_type        = T;
     using tensor_category   = CuspVectorTag;
diff --git a/inc/dg/backend/tensor_traits_scalar.h b/inc/dg/backend/tensor_traits_scalar.h
index ed5a39204..82d3cc440 100644
--- a/inc/dg/backend/tensor_traits_scalar.h
+++ b/inc/dg/backend/tensor_traits_scalar.h
@@ -10,7 +10,7 @@ namespace dg
 
 ///@brief Recognize arithmetic types as scalars
 template<class T>
-struct TensorTraits<T, typename std::enable_if< std::is_arithmetic<T>::value>::type>
+struct TensorTraits<T, std::enable_if_t< std::is_arithmetic<T>::value>>
 {
     using value_type        = T;
     using tensor_category   = ScalarTag;
diff --git a/inc/dg/backend/tensor_traits_std.h b/inc/dg/backend/tensor_traits_std.h
index 2ff9dbfd3..586db2486 100644
--- a/inc/dg/backend/tensor_traits_std.h
+++ b/inc/dg/backend/tensor_traits_std.h
@@ -27,14 +27,14 @@ struct TensorTraits<std::vector<T>>
  */
 //template<class T, std::size_t N>
 //struct TensorTraits<std::array<T, N>,
-//    typename std::enable_if< std::is_arithmetic<T>::value>::type>
+//    std::enable_if_t< std::is_arithmetic<T>::value>>
 //{
 //    using value_type        = T;
 //    using tensor_category   = StdArrayTag;
 //    using execution_policy  = SerialTag;
 //};
 template<class T, std::size_t N>
-struct TensorTraits<std::array<T, N>>//, typename std::enable_if< !std::is_arithmetic<T>::value>::type>
+struct TensorTraits<std::array<T, N>>//, std::enable_if_t< !std::is_arithmetic<T>::value>>
 {
     using value_type        = get_value_type<T>;
     using tensor_category   = ArrayVectorTag;
diff --git a/inc/dg/backend/tensor_traits_thrust.h b/inc/dg/backend/tensor_traits_thrust.h
index d651b0a47..d088149a7 100644
--- a/inc/dg/backend/tensor_traits_thrust.h
+++ b/inc/dg/backend/tensor_traits_thrust.h
@@ -12,7 +12,7 @@ namespace dg
 
 ///@brief prototypical Shared Vector with Serial Tag
 template<class T>
-struct TensorTraits<thrust::host_vector<T> >//, typename std::enable_if< std::is_arithmetic<T>::value>::type>
+struct TensorTraits<thrust::host_vector<T> >//, std::enable_if_t< std::is_arithmetic<T>::value>>
 {
     using value_type        = T;
     using tensor_category   = ThrustVectorTag;
@@ -20,7 +20,7 @@ struct TensorTraits<thrust::host_vector<T> >//, typename std::enable_if< std::is
 };
 //template<class T>
 //struct TensorTraits<thrust::host_vector<T>,
-//    typename std::enable_if< !std::is_arithmetic<T>::value>::type>
+//    std::enable_if_t< !std::is_arithmetic<T>::value>>
 //{
 //    using value_type        = get_value_type<T>;
 //    using tensor_category   = RecursiveVectorTag;
@@ -29,7 +29,7 @@ struct TensorTraits<thrust::host_vector<T> >//, typename std::enable_if< std::is
 
 ///@brief prototypical Shared Vector with Cuda or Omp Tag
 template<class T>
-struct TensorTraits<thrust::device_vector<T> >//, typename std::enable_if<std::is_arithmetic<T>::value>::type>
+struct TensorTraits<thrust::device_vector<T> >//, std::enable_if_t<std::is_arithmetic<T>::value>>
 {
     using value_type        = T;
     using tensor_category   = ThrustVectorTag;
diff --git a/inc/dg/backend/view.h b/inc/dg/backend/view.h
index 51c84acd4..e70790ddc 100644
--- a/inc/dg/backend/view.h
+++ b/inc/dg/backend/view.h
@@ -8,7 +8,7 @@ namespace dg
 
 
 /**
- * @brief A vector view class, usable in \c dg::blas1 functions
+ * @brief A vector view class, usable in \c dg functions
  *
  * @ingroup view
  * The view class holds a pointer and a size. It does not own the pointer.
@@ -23,7 +23,7 @@ namespace dg
  * you want to use the \c dg::blas1 functions without specializing \c TensorTraits
  * for your own vector class or deep copying data, like the following example demonstrates:
  * @code
-SomeVectorClass vector( 1e6, 20.); //vector of size 1e6, all elements equal 20
+SomeDeviceVectorClass vector( 1e6, 20.); //vector of size 1e6, all elements equal 20
 
 //create a view of a device vector to enable parallel execution
 dg::View<dg::DVec> view( vector.data(), vector.size());
@@ -41,13 +41,13 @@ dg::blas1::copy( 7., view); //elements of vector now equal 7 instead of 20
 template<class ThrustVector >
 struct View
 {
-    using iterator = typename std::conditional<std::is_const<ThrustVector>::value,
+    using iterator = std::conditional_t<std::is_const<ThrustVector>::value,
           typename ThrustVector::const_iterator,
-          typename ThrustVector::iterator>::type;
+          typename ThrustVector::iterator>;
     using const_iterator = typename ThrustVector::const_iterator;
-    using pointer = typename std::conditional<std::is_const<ThrustVector>::value,
+    using pointer = std::conditional_t<std::is_const<ThrustVector>::value,
           typename ThrustVector::const_pointer,
-          typename ThrustVector::pointer>::type;
+          typename ThrustVector::pointer>;
     using const_pointer = typename ThrustVector::const_pointer;
     ///@brief Initialize empty view
     View( void): m_ptr(), m_size(0){}
diff --git a/inc/dg/backend/view_t.cu b/inc/dg/backend/view_t.cu
index 0660d7ff4..a042c7fce 100644
--- a/inc/dg/backend/view_t.cu
+++ b/inc/dg/backend/view_t.cu
@@ -10,14 +10,14 @@ int main()
 
     std::vector<double> test( 100, 3.);
 
-    dg::View<dg::DVec> view( test.data(), test.size());
+    dg::View<thrust::host_vector<double>> view( test.data(), test.size());
     dg::blas1::copy( 7., view);
     std::cout << "The original now has "<<test[0]<<" (7)\n";
     view.construct( &test[50], 50);
     dg::blas1::copy( 3., view);
     std::cout << "The original now has "<<test[0]<<" (7) and "<<test[50]<<" (3)\n";
     const std::vector<double> const_test( 100, 42);
-    const dg::View<const dg::DVec> const_view( const_test.data(), 50);
+    const dg::View<const thrust::host_vector<double>> const_view( const_test.data(), 50);
     dg::blas1::copy( const_view, view);
     std::cout << "The original now has "<<test[50]<<" (42)\n";
 
diff --git a/inc/dg/blas1.h b/inc/dg/blas1.h
index 75d9e73f9..cef17eeb4 100644
--- a/inc/dg/blas1.h
+++ b/inc/dg/blas1.h
@@ -607,7 +607,7 @@ inline void subroutine( Subroutine f, ContainerType&& x, ContainerTypes&&... xs)
             dg::is_scalar_or_same_base_category<ContainerTypes, tensor_category>::value...
             >::value,
         "All container types must be either Scalar or have compatible Vector categories (AnyVector or Same base class)!");
-    //using basic_tag_type  = typename std::conditional< all_true< is_scalar<ContainerType>::value, is_scalar<ContainerTypes>::value... >::value, AnyScalarTag , AnyVectorTag >::type;
+    //using basic_tag_type  = std::conditional_t< all_true< is_scalar<ContainerType>::value, is_scalar<ContainerTypes>::value... >::value, AnyScalarTag , AnyVectorTag >;
     dg::blas1::detail::doSubroutine(tensor_category(), f, std::forward<ContainerType>(x), std::forward<ContainerTypes>(xs)...);
 }
 
diff --git a/inc/dg/topology/base_geometryX.h b/inc/dg/topology/base_geometryX.h
index 955570c4d..be5aba6f1 100644
--- a/inc/dg/topology/base_geometryX.h
+++ b/inc/dg/topology/base_geometryX.h
@@ -40,10 +40,10 @@ struct aRealGeometryX2d : public aRealTopologyX2d<real_type>
     aRealGeometryX2d& operator=( const aRealGeometryX2d& src) = default;
     private:
     virtual SparseTensor<thrust::host_vector<real_type> > do_compute_metric()const {
-        return SparseTensor<thrust::host_vector<real_type> >();
+        return SparseTensor<thrust::host_vector<real_type> >(*this);
     }
     virtual SparseTensor<thrust::host_vector<real_type> > do_compute_jacobian()const {
-        return SparseTensor<thrust::host_vector<real_type> >();
+        return SparseTensor<thrust::host_vector<real_type> >(*this);
     }
     virtual std::vector<thrust::host_vector<real_type> > do_compute_map()const{
         std::vector<thrust::host_vector<real_type> > map(2);
diff --git a/inc/dg/topology/operator.h b/inc/dg/topology/operator.h
index f30442e1e..fa789a58b 100644
--- a/inc/dg/topology/operator.h
+++ b/inc/dg/topology/operator.h
@@ -50,7 +50,7 @@ class Operator
      * @param last
      */
     template< class InputIterator>
-    Operator( InputIterator first, InputIterator last, typename std::enable_if<!std::is_integral<InputIterator>::value>::type* = 0): data_(first, last)
+    Operator( InputIterator first, InputIterator last, std::enable_if_t<!std::is_integral<InputIterator>::value>* = 0): data_(first, last)
     {
         unsigned n = std::distance( first, last);
         n_ = (unsigned)sqrt( (value_type)n);
diff --git a/inc/dg/topology/split_and_join.h b/inc/dg/topology/split_and_join.h
index 6923162dd..671fb769b 100644
--- a/inc/dg/topology/split_and_join.h
+++ b/inc/dg/topology/split_and_join.h
@@ -55,10 +55,10 @@ std::vector<View<SharedContainer>> split( SharedContainer& in, const aRealTopolo
 #ifdef MPI_VERSION
 
 template<class MPIContainer>
-using get_mpi_view_type = typename
-    std::conditional< std::is_const<MPIContainer>::value,
+using get_mpi_view_type =
+    std::conditional_t< std::is_const<MPIContainer>::value,
     MPI_Vector<View<const typename MPIContainer::container_type>>,
-    MPI_Vector<View<typename MPIContainer::container_type>> >::type;
+    MPI_Vector<View<typename MPIContainer::container_type>> >;
 
 /** @brief MPI Version of split (fast version)
  *
-- 
GitLab


From b57bb7d8d3fb19d69281567cc20410d5de247821 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 16:02:24 +0100
Subject: [PATCH 533/540] Fix bug in mpi_fieldaligned.set_boundaries

---
 inc/dg/topology/tensor.h           | 1 +
 inc/geometries/README              | 6 +++++-
 inc/geometries/ds_straight_mpit.cu | 4 ++--
 inc/geometries/mpi_fieldaligned.h  | 9 ++++-----
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/inc/dg/topology/tensor.h b/inc/dg/topology/tensor.h
index 9a1d95d3e..6d556f982 100644
--- a/inc/dg/topology/tensor.h
+++ b/inc/dg/topology/tensor.h
@@ -3,6 +3,7 @@
 #include "grid.h"
 #include "operator.h"
 #include "evaluation.h"
+#include "evaluationX.h"
 #include "dg/functors.h"
 #include "dg/blas1.h"
 
diff --git a/inc/geometries/README b/inc/geometries/README
index 33c41eb83..9882c54c9 100644
--- a/inc/geometries/README
+++ b/inc/geometries/README
@@ -22,17 +22,21 @@
         utilitiesX.h
 
 #magnetic fields
+    make_field.h
     solovev.h
     solovev_parameters.h
+    polynomial.h
+    polynomial_parameters.h
     taylor.h
     guenther.h
     toroidal.h
+    modified.h
 
     #utilities
         testfunctors.h
         fluxfunctions.h
+        adaption.h
     magnetic_field.h
-    adaption.h
 
 #miscellaneous
     ds.h
diff --git a/inc/geometries/ds_straight_mpit.cu b/inc/geometries/ds_straight_mpit.cu
index 18f03f2b1..a786e0834 100644
--- a/inc/geometries/ds_straight_mpit.cu
+++ b/inc/geometries/ds_straight_mpit.cu
@@ -31,7 +31,7 @@ int main(int argc, char **argv)
     if(rank==0)std::cout << "# Test straight field lines and boundaries in z.\n";
     dg::mpi_init3d( dg::DIR, dg::DIR, dg::NEU, n, Nx, Ny, Nz, comm);
     dg::CartesianMPIGrid3d g3d( -1, 1, -1, 1, 0.1, M_PI+0.1, n, Nx, Ny, Nz, dg::DIR, dg::DIR, dg::NEU, comm);
-    dg::CartesianMPIGrid2d perp_grid( -1, 1, -1, 1, n, Nx, Ny, dg::DIR, dg::DIR, comm);
+    dg::CartesianMPIGrid2d perp_grid( -1, 1, -1, 1, n, Nx, Ny, dg::DIR, dg::DIR, g3d.get_perp_comm());
     if(rank==0)std::cout <<"# You typed\n"
               <<"n:  "<<n<<"\n"
               <<"Nx: "<<Nx<<"\n"
@@ -54,7 +54,7 @@ int main(int argc, char **argv)
     ds.set_boundaries( dg::DIR, sin(g3d.z0()),sin(g3d.z1()));
     ds( constfunc, derivative);
     t.toc();
-    std::cout << "Straight:\n";
+    if(rank==0)std::cout << "Straight:\n";
     if(rank==0)std::cout << "# Application of parallel Derivative took  "<<t.diff()<<"s\n";
     dg::blas1::axpby( 1., constsolution, -1., derivative);
     double norm = dg::blas2::dot( constsolution, w3d, constsolution);
diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index fc9640662..57204daa7 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -102,9 +102,8 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
     void set_boundaries( dg::bc bcz, double left, double right)
     {
         m_bcz = bcz;
-        const dg::MPIGrid2d g2d( 0., 1., 0., 1., m_g->global().n(), m_g->global().Nx(), m_g->global().Ny(), m_g->get_perp_comm() );
-        m_left  = dg::evaluate( dg::CONSTANT(left), g2d);
-        m_right = dg::evaluate( dg::CONSTANT(right),g2d);
+        dg::blas1::copy( left, m_left);
+        dg::blas1::copy( right, m_right);
     }
 
     void set_boundaries( dg::bc bcz, const MPI_Vector<LocalContainer>& left, const MPI_Vector<LocalContainer>& right)
@@ -117,8 +116,8 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
     void set_boundaries( dg::bc bcz, const MPI_Vector<LocalContainer>& global, double scal_left, double scal_right)
     {
         dg::split( global, m_f, *m_g);
-        dg::blas1::axpby( scal_left, m_f[0],               0., m_left);
-        dg::blas1::axpby( scal_right, m_f[m_g->local().Nz()], 0., m_left);
+        dg::blas1::axpby( scal_left,  m_f[0],                   0., m_left);
+        dg::blas1::axpby( scal_right, m_f[m_g->local().Nz()-1], 0., m_left);
         m_bcz = bcz;
     }
 
-- 
GitLab


From 15803f9892f545601b578de102d53ce05b458029 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 18:24:30 +0100
Subject: [PATCH 534/540] Fix initialization bug in DS

---
 inc/geometries/ds.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inc/geometries/ds.h b/inc/geometries/ds.h
index 936a84a24..17cb3ca63 100644
--- a/inc/geometries/ds.h
+++ b/inc/geometries/ds.h
@@ -570,7 +570,7 @@ struct DS
 ////////////////////////////////////DEFINITIONS////////////////////////////////////////
 
 template<class Geometry, class I, class M, class container>
-DS<Geometry, I, M,container>::DS( Fieldaligned<Geometry, I, container> fa, dg::direction dir): m_fa(fa)
+DS<Geometry, I, M,container>::DS( Fieldaligned<Geometry, I, container> fa, dg::direction dir): m_fa(fa), m_dir(dir)
 {
     dg::assign( dg::create::volume(     fa.grid()), m_vol3d);
     dg::assign( dg::create::weights(    fa.grid()), m_weights_wo_vol);
-- 
GitLab


From 02c0c7e48093c04f7d804173b9daee3a243b54ee Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 18:25:06 +0100
Subject: [PATCH 535/540] Fix parallel boundary condition in mpi_fieldaligned

---
 inc/geometries/mpi_fieldaligned.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/inc/geometries/mpi_fieldaligned.h b/inc/geometries/mpi_fieldaligned.h
index 57204daa7..a363568c6 100644
--- a/inc/geometries/mpi_fieldaligned.h
+++ b/inc/geometries/mpi_fieldaligned.h
@@ -117,7 +117,7 @@ struct Fieldaligned< ProductMPIGeometry, MPIDistMat<LocalIMatrix, CommunicatorXY
     {
         dg::split( global, m_f, *m_g);
         dg::blas1::axpby( scal_left,  m_f[0],                   0., m_left);
-        dg::blas1::axpby( scal_right, m_f[m_g->local().Nz()-1], 0., m_left);
+        dg::blas1::axpby( scal_right, m_f[m_g->local().Nz()-1], 0., m_right);
         m_bcz = bcz;
     }
 
@@ -419,7 +419,7 @@ void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::ePlus( enum whichM
 
     //3. apply right boundary conditions in last plane
     unsigned i0=m_Nz-1;
-    if( m_bcz != dg::PER && m_g->z1() == m_g->global().z1())
+    if( m_bcz != dg::PER && m_g->local().z1() == m_g->global().z1())
     {
         if( m_bcz == dg::DIR || m_bcz == dg::NEU_DIR)
             dg::blas1::axpby( 2, m_right, -1., m_f[i0], m_ghostP);
@@ -465,7 +465,7 @@ void Fieldaligned<G,MPIDistMat<M,C>, MPI_Vector<container> >::eMinus( enum which
 
     //3. apply left boundary conditions in first plane
     unsigned i0=0;
-    if( m_bcz != dg::PER && m_g->z0() == m_g->global().z0())
+    if( m_bcz != dg::PER && m_g->local().z0() == m_g->global().z0())
     {
         if( m_bcz == dg::DIR || m_bcz == dg::DIR_NEU)
             dg::blas1::axpby( 2., m_left,  -1., m_f[i0], m_ghostM);
-- 
GitLab


From 454cb78db205bad7b5d66645c497ae60063d57ff Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Fri, 19 Feb 2021 18:50:59 +0100
Subject: [PATCH 536/540] Update CHANGELOG

---
 CHANGELOG.md                            | 2 ++
 inc/geometries/conformalX_elliptic_b.cu | 7 ++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 16be54ffd..dbddd94c8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -118,6 +118,8 @@ far away from strictly following it really.
  - Fix bug: `dg::pushForwardPerp` on functors computed wrong result (only affects `dg::geo::Hector`)
  - Fix bug(s): several bugs in `dg::geo::Hector` which computed wrong grid (happened probably when we changed the grid design to polymorphic)
  - Fix bug: in perpendicular grid of MPI Curvlinear grid
+ - Fix bug: missing direction initialization in mpi fieldaligned class
+ - Fix bug: non-zero parallel boundary condition in mpi fieldaligned
 
 ## [v5.1] Adaptive Timesteppers
 ### Added
diff --git a/inc/geometries/conformalX_elliptic_b.cu b/inc/geometries/conformalX_elliptic_b.cu
index cca39771f..f7dcf7b98 100644
--- a/inc/geometries/conformalX_elliptic_b.cu
+++ b/inc/geometries/conformalX_elliptic_b.cu
@@ -15,7 +15,7 @@ void compute_error_elliptic( const dg::geo::TokamakMagneticField& c, const dg::g
 {
     dg::Elliptic<dg::geo::CurvilinearGridX2d, dg::Composite<dg::DMatrix>, dg::DVec> pol( g2d, dg::not_normed, dg::forward);
     ////////////////////////blob solution////////////////////////////////////////
-    const dg::DVec b        = dg::pullback( dg::geo::EllipticBlobDirNeuM(c,psi_0, psi_1, 480, -300, 70.,1.), g2d);
+    dg::DVec b        = dg::pullback( dg::geo::EllipticBlobDirNeuM(c,psi_0, psi_1, 480, -300, 70.,1.), g2d);
     const dg::DVec chi      = dg::pullback( dg::ONE(), g2d);
     const dg::DVec solution = dg::pullback( dg::geo::FuncDirNeu(c, psi_0, psi_1, 480, -300, 70., 1. ), g2d);
     //////////////////////////blob solution on X-point/////////////////////////////
@@ -43,9 +43,10 @@ void compute_error_elliptic( const dg::geo::TokamakMagneticField& c, const dg::g
     std::cout << eps<<"\t";
     dg::Timer t;
     t.tic();
-    dg::Invert<dg::DVec > invert( x, g2d.size(), eps);
+    dg::CG<dg::DVec > invert( x, g2d.size());
+    dg::blas2::symv( pol.weights(), b, b);
     //unsigned number = invert(pol, x,b, vol2d, inv_vol2d );
-    unsigned number = invert(pol, x,b, vol2d, v2d, v2d ); //inv weights are better preconditioners
+    unsigned number = invert(pol, x,b, v2d, v2d, eps ); //inv weights are better preconditioners
     std::cout <<number<<"\t";
     t.toc();
     dg::blas1::axpby( 1.,x,-1., solution, error);
-- 
GitLab


From 0ac1a079ab0aa881c54a2b4551b7776984fea70c Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 22 Feb 2021 16:15:26 +0100
Subject: [PATCH 537/540] Update documentation on sparseblockmat

---
 inc/dg/backend/sparseblockmat.cuh |  4 ++++
 inc/dg/backend/sparseblockmat.h   | 33 ++++++++++++++++++++++++++++---
 2 files changed, 34 insertions(+), 3 deletions(-)

diff --git a/inc/dg/backend/sparseblockmat.cuh b/inc/dg/backend/sparseblockmat.cuh
index 13d704277..ff6adae84 100644
--- a/inc/dg/backend/sparseblockmat.cuh
+++ b/inc/dg/backend/sparseblockmat.cuh
@@ -14,6 +14,8 @@ namespace dg
 * This class holds a copy of a EllSparseBlockMat on the device, which may
 be gpu or omp depending on the THRUST_DEVICE_SYSTEM macro. It can be applied
 to device vectors and does the same thing as the host version
+
+@copydetails EllSparseBlockMat
 */
 template<class value_type>
 struct EllSparseBlockMatDevice
@@ -77,6 +79,8 @@ struct EllSparseBlockMatDevice
 * @ingroup sparsematrix
 * This class holds a copy of a CooSparseBlockMat on the device, which may
 be gpu or omp depending on the THRUST_DEVICE_SYSTEM macro. It does the same thing as the host version with the difference that it applies to device vectors.
+
+@copydetails CooSparseBlockMat
 */
 template<class value_type>
 struct CooSparseBlockMatDevice
diff --git a/inc/dg/backend/sparseblockmat.h b/inc/dg/backend/sparseblockmat.h
index 8427ce1df..36e6cd577 100644
--- a/inc/dg/backend/sparseblockmat.h
+++ b/inc/dg/backend/sparseblockmat.h
@@ -23,7 +23,20 @@ of nonrecurrent blocks is small.
 The indices and blocks are those of a one-dimensional problem. When we want
 to apply the matrix to a multidimensional vector we can multiply it by
 Kronecker deltas of the form
-\f[  1\otimes M \otimes 1\f]
+\f[ M = \begin{pmatrix}
+B & C &   &   &   & \\
+A & B & C &   &   & \\
+  & A & B & C &   & \\
+  &   & A & B & C & \\
+  &   &   &...&   &
+  \end{pmatrix}\rightarrow
+\text{data} = ( A, B, C, 0)\quad \text{cols_idx} = ( 0,0,1,0,1,2,1,2,3,2,3,4,...)
+\quad \text{data_idx} = ( 3,1,2,0,1,2,0,1,2,0,1,2,...)\f]
+where \f$A,\ B,\ C,\ 0\f$ are \f$n\times n\f$ block matrices. The 0 is used
+for padding in order to keep the number of elements per line constant as 3
+(in this example \c blocks_per_line=3, \c num_different_blocks=4).
+The matrix M has \c num_rows rows and \c num_cols columns of blocks.
+\f[  1_\mathrm{left}\otimes M \otimes 1_\mathrm{right}\f]
 where \f$ 1\f$ are diagonal matrices of variable size and \f$ M\f$ is our
 one-dimensional matrix.
 */
@@ -90,7 +103,7 @@ struct EllSparseBlockMat
     thrust::host_vector<value_type> data;//!< The data array is of size n*n*num_different_blocks and contains the blocks. The first block is contained in the first n*n elements, then comes the next block, etc.
     thrust::host_vector<int> cols_idx; //!< is of size num_rows*num_blocks_per_line and contains the column indices % n into the vector
     thrust::host_vector<int> data_idx; //!< has the same size as cols_idx and contains indices into the data array, i.e. the block number
-    thrust::host_vector<int> right_range; //!< range
+    thrust::host_vector<int> right_range; //!< range, [0,right_size] per default (can be used to apply the matrix to only part of the right rows
     int num_rows; //!< number of block rows, each row contains blocks
     int num_cols; //!< number of block columns
     int blocks_per_line; //!< number of blocks in each line
@@ -112,10 +125,24 @@ struct EllSparseBlockMat
 * The clue is that instead of a values array we use an index array with
 indices into a data array that contains the actual blocks. This safes storage if the number
 of nonrecurrent blocks is small.
+\f[
+M = \begin{pmatrix}
+A &   &  B&  & & \\
+  & C &   &  & & \\
+  &   &   &  & & \\
+  & A &   &  & B & C
+\end{pmatrix}
+\rightarrow
+\text{data}=(A,B,C)\quad \text{rows_idx} = ( 0,0,1,3,3,3)
+\quad\text{cols_idx} = (0,2,1,1,4,5)
+\f]
+where \f$A,\ B,\ C,\ 0\f$ are \f$n\times n\f$ block matrices.
+The matrix M in this example has \c num_rows=4, \c num_cols=6, \c num_entries=6.
+
 The indices and blocks are those of a one-dimensional problem. When we want
 to apply the matrix to a multidimensional vector we can multiply it by
 Kronecker deltas of the form
-\f[  1\otimes M \otimes 1\f]
+\f[  1_\mathrm{left}\otimes M \otimes 1_\mathrm{right}\f]
 where \f$ 1\f$ are diagonal matrices of variable size and \f$ M\f$ is our
 one-dimensional matrix.
 @note This matrix type is used for the computation of boundary points in
-- 
GitLab


From 6d4650e202805d79366058de9e493802af8a16c0 Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 22 Feb 2021 16:16:53 +0100
Subject: [PATCH 538/540] Fix GPU bug for derivatives on X-point grids

- don't know how this one was introduced
---
 inc/dg/backend/sparseblockmat_gpu_kernels.cuh | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/inc/dg/backend/sparseblockmat_gpu_kernels.cuh b/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
index 7be76d55c..49523cc95 100644
--- a/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
+++ b/inc/dg/backend/sparseblockmat_gpu_kernels.cuh
@@ -26,7 +26,9 @@ template<class value_type>
             i = (rrn)%num_rows,
             k = (rr)%n,
             j=right_range[0]+row%right_;
-        y[row]*= beta;
+        int idx = ((s*num_rows+i)*n+k)*right_size+j;
+        //idx != row ( if right_range[0] != 0)
+        y[idx]*= beta;
         for( int d=0; d<blocks_per_line; d++)
         {
             value_type temp=0;
@@ -34,7 +36,7 @@ template<class value_type>
             int J = (s*num_cols+cols_idx[i*blocks_per_line+d])*n;
             for( int q=0; q<n; q++) //multiplication-loop
                 temp =fma( data[ B+q], x[(J+q)*right_size+j], temp);
-            y[row] = fma( alpha, temp, y[row]);
+            y[idx]=fma( alpha, temp, y[idx]);
         }
     }
 
@@ -88,9 +90,11 @@ template<class value_type, size_t n, size_t blocks_per_line>
                 for( int q=0; q<n; q++) //multiplication-loop
                     temp[d] = fma( data[ B+q], x[(J+q)*right_size+j], temp[d]);
             }
-            y[row]*= beta;
+            int idx = ((s*num_rows+i)*n+k)*right_size+j;
+            //idx != row ( if right_range[0] != 0)
+            y[idx]*= beta;
             for( int d=0; d<blocks_per_line; d++)
-                y[row] = fma( alpha, temp[d], y[row]);
+                y[idx] = fma( alpha, temp[d], y[idx]);
         }
     }
 }
-- 
GitLab


From 393695045f5db5dd46e4b1f29d105d58909e708f Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 22 Feb 2021 20:01:58 +0100
Subject: [PATCH 539/540] Slight corrections to a few test programs

---
 inc/dg/blas_t.cu                         |  1 +
 inc/dg/elliptic.h                        |  3 ++-
 inc/dg/topology/evaluation_mpit.cu       |  2 +-
 inc/dg/topology/evaluation_t.cu          |  2 +-
 inc/dg/topology/multiply_t.cu            |  2 +-
 inc/dg/topology/operator_t.cu            |  2 +-
 inc/geometries/geometry_elliptic_mpib.cu |  9 +++++++++
 inc/geometries/ribeiro_mpit.cu           | 11 ++++++++++-
 8 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/inc/dg/blas_t.cu b/inc/dg/blas_t.cu
index 923acc78d..4fb06284e 100644
--- a/inc/dg/blas_t.cu
+++ b/inc/dg/blas_t.cu
@@ -41,6 +41,7 @@ int main()
     std::cout << "Recursive Scalar/Vetor addition   "<< (arrdvec1[0][0] == 26 && arrdvec1[1][0]==46.)<<std::endl;
     // test the examples in the documentation
     // dg::blas1::subroutine( []__host__ __device__(double& v){ v+=1.;}, dvec1);
+    dg::blas1::plus( dvec1, 1);
     std::array<dg::DVec, 3> array_v{ dvec1, dvec1, dvec1}, array_w(array_v);
     std::array<double, 3> array_p{ 1,2,3};
     dg::blas1::subroutine( Expression(), dvec1, array_w[2], 3);
diff --git a/inc/dg/elliptic.h b/inc/dg/elliptic.h
index bdd14eaf3..e6ed9017c 100644
--- a/inc/dg/elliptic.h
+++ b/inc/dg/elliptic.h
@@ -20,7 +20,8 @@ namespace dg
 // Note that there are many tests for this file : elliptic2d_b,
 // elliptic2d_mpib, elliptic_b, elliptic_mpib, ellipticX2d_b
 // And don't forget inc/geometries/elliptic3d_t (testing alignment and
-// projection tensors as Chi)
+// projection tensors as Chi) geometry_elliptic_b, geometry_elliptic_mpib,
+// and geometryX_elliptic_b and geometryX_refined_elliptic_b
 
 /**
  * @brief A 2d negative elliptic differential operator \f$ -\nabla \cdot ( \mathbf{\chi}\cdot \nabla ) \f$
diff --git a/inc/dg/topology/evaluation_mpit.cu b/inc/dg/topology/evaluation_mpit.cu
index 1572b24b6..5c9b362c6 100644
--- a/inc/dg/topology/evaluation_mpit.cu
+++ b/inc/dg/topology/evaluation_mpit.cu
@@ -59,7 +59,7 @@ int main(int argc, char** argv)
     if(rank==0)std::cout << "Correct integral is       "<<std::setw(6)<<sol2d<<std::endl;
     if(rank==0)std::cout << "2d error is               "<<(integral2d-sol2d)<<"\n\n";
     float integralf2d = dg::blas1::dot( wf2d, funcf2d); res.d = integralf2d;
-    if(rank==0)std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\t" << res.i - 4525606114229747712<< "\n";
+    if(rank==0)std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\n";
     float solf2d = 0.;
     if(rank==0)std::cout << "Correct integral is       "<<std::setw(6)<<solf2d<<std::endl;
     if(rank==0)std::cout << "2d error (float)          "<<(integralf2d-solf2d)<<"\n\n";
diff --git a/inc/dg/topology/evaluation_t.cu b/inc/dg/topology/evaluation_t.cu
index d60d473f8..266f6f1af 100644
--- a/inc/dg/topology/evaluation_t.cu
+++ b/inc/dg/topology/evaluation_t.cu
@@ -76,7 +76,7 @@ int main()
     std::cout << "2d error is               "<<(integral2d-sol2d)<<"\n\n";
 
     float integralf2d = dg::blas1::dot( wf2d, funcf2d); res.d = integralf2d;
-    std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\t" << res.i - 4525606114229747712<< "\n";
+    std::cout << "2D integral (float)       "<<std::setw(6)<<integralf2d <<"\n";
     float solf2d = 0;
     std::cout << "Correct integral is       "<<std::setw(6)<<solf2d<<std::endl;
     std::cout << "2d error (float)          "<<(integralf2d-solf2d)<<"\n\n";
diff --git a/inc/dg/topology/multiply_t.cu b/inc/dg/topology/multiply_t.cu
index 01f1d14ed..2632ffce7 100644
--- a/inc/dg/topology/multiply_t.cu
+++ b/inc/dg/topology/multiply_t.cu
@@ -31,7 +31,7 @@ int main()
     std::cout << "Begin T\n"; print(t);
     dg::tensor::scal(t,mu);
     std::cout<< "Scale with 5 \n";print(t);
-    dg::tensor::scal(t,1);
+    dg::tensor::scal(t,1.);
     std::cout << "Scale with empty element \n";print(t);
     dg::tensor::scal(t,1./5.);
     std::cout << "Scale with 1/5 \n";print(t);
diff --git a/inc/dg/topology/operator_t.cu b/inc/dg/topology/operator_t.cu
index 83b38d5f6..e709de8d1 100644
--- a/inc/dg/topology/operator_t.cu
+++ b/inc/dg/topology/operator_t.cu
@@ -27,7 +27,7 @@ int main()
     std::cout << "Inverse Operator\n"<<inv_op<<"\n";
     std::cout << "Multiplication\n"<<inv_op*op<<"\n";
 
-    op.zero();
+    //op.zero();
     op(0,2) = op(1,1) = op(2,0) = 0;// op(3,3)= 1;
     std::cout << "Operator\n"<<op<<"\n";
     inv_op = dg::create::invert(op);
diff --git a/inc/geometries/geometry_elliptic_mpib.cu b/inc/geometries/geometry_elliptic_mpib.cu
index 09b648e0d..0af889412 100644
--- a/inc/geometries/geometry_elliptic_mpib.cu
+++ b/inc/geometries/geometry_elliptic_mpib.cu
@@ -24,7 +24,16 @@ int main(int argc, char**argv)
     unsigned n, Nx, Ny, Nz;
     MPI_Comm comm;
     dg::mpi_init3d( dg::DIR, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
+    int dims[3], periods[3], coords[3];
+    MPI_Cart_get( comm, 3, dims, periods, coords);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    if( dims[2] != 1)
+    {
+        // because of netcdf output
+        if(rank==0) std::cout << "Please do not parallelize in z!\n";
+        MPI_Finalize();
+        return 0;
+    }
     Json::Value js;
     if( argc==1)
     {
diff --git a/inc/geometries/ribeiro_mpit.cu b/inc/geometries/ribeiro_mpit.cu
index 15a18d160..bddc7997f 100644
--- a/inc/geometries/ribeiro_mpit.cu
+++ b/inc/geometries/ribeiro_mpit.cu
@@ -31,7 +31,16 @@ int main( int argc, char* argv[])
     unsigned n, Nx, Ny, Nz;
     MPI_Comm comm;
     dg::mpi_init3d( dg::DIR, dg::PER, dg::PER, n, Nx, Ny, Nz, comm);
+    int dims[3], periods[3], coords[3];
+    MPI_Cart_get( comm, 3, dims, periods, coords);
     MPI_Comm_rank( MPI_COMM_WORLD, &rank);
+    if( dims[2] != 1)
+    {
+        // because of netcdf output
+        if(rank==0) std::cout << "Please do not parallelize in z!\n";
+        MPI_Finalize();
+        return 0;
+    }
     Json::Value js;
     if( argc==1)
     {
@@ -46,7 +55,7 @@ int main( int argc, char* argv[])
     dg::geo::solovev::Parameters gp(js);
     dg::geo::CylindricalFunctorsLvl2 psip = dg::geo::solovev::createPsip( gp);
     if(rank==0)std::cout << "Psi min "<<psip.f()(gp.R_0, 0)<<"\n";
-    if(rank==0)std::cout << "Type psi_0 and psi_1\n";
+    if(rank==0)std::cout << "Type psi_0 (-20) and psi_1 (-4)\n";
     double psi_0, psi_1;
     if(rank==0)std::cin >> psi_0>> psi_1;
     MPI_Bcast( &psi_0, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD);
-- 
GitLab


From 0d1df1ac59358f2eba2a1c88355ff941cc94cadd Mon Sep 17 00:00:00 2001
From: Matthias <mattwi@fysik.dtu.dk>
Date: Mon, 22 Feb 2021 20:08:49 +0100
Subject: [PATCH 540/540] Update CHANGELOG

---
 CHANGELOG.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index dbddd94c8..b6329c0e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,7 @@ github.
 > Creating a release on github will create an associated archived version
   on zenodo, so **we reserve releases for when we publish an article**.
 
-We kind of make up our own version numbers right now. A new major version number is often associated with a journal publication, but other than that there is no defined mapping from the amount or kind of change to a version number. 
+We kind of make up our own version numbers right now. A new major version number is often associated with a journal publication, but other than that there is no defined mapping from the amount or kind of change to a version number.
 [Semantiv versioning](https://semver.org/) might serve as a guideline  but we are
 far away from strictly following it really.
 
@@ -107,10 +107,10 @@ far away from strictly following it really.
 ### Fixed
  - Fix bug: race condition in `dg::blas1::dot` and `dg::blas2::dot` on GPUs that led to hard to reproduce and seemingly unreasonable crashes
  - Fix bug: std namespace in diag/probes.h
- - Fix bug: const in `exblas::cpu::get_element` 
+ - Fix bug: const in `exblas::cpu::get_element`
  - Fix bug: correct  indices in `exblas::cpu::make_vcl_vec8d`
  - Fix bug: infinite creation of MPI communicators in `exblas::mpi_reduce_communicator` . Lead to MPI crashes due to memory overflow.
- - correct capture of cuda-aware mpi, 
+ - Fix bug: correct capture of cuda-aware mpi in configuration
  - Fix bug: test for no-communication in mpi_communicator.h (indicated false positives)
  - Fix bug: coefficient and initialization in `dg::Extrpolate`
  - Fix bug: Fpsi safety-factor in case nan is encountered still works
@@ -119,7 +119,9 @@ far away from strictly following it really.
  - Fix bug(s): several bugs in `dg::geo::Hector` which computed wrong grid (happened probably when we changed the grid design to polymorphic)
  - Fix bug: in perpendicular grid of MPI Curvlinear grid
  - Fix bug: missing direction initialization in mpi fieldaligned class
+ - Fix bug: host mpi code compiled with nvcc
  - Fix bug: non-zero parallel boundary condition in mpi fieldaligned
+ - Fix bug: GPU symv on X-point grids
 
 ## [v5.1] Adaptive Timesteppers
 ### Added
@@ -160,7 +162,7 @@ far away from strictly following it really.
 - default constructor of MPI\_Vector constructs empty communicator instead of MPI\_COMM\_WORLD
 - set\_communicator in MPI\_Vector takes three arguments now to avoid group
   creation
-- Configure cuda-aware mpi test a warning instead of an error
+- cuda-aware mpi no longer a requirement, fall-back to traditional mpi implemented
 - rewrite feltordiag.cu merging ncdiag and filamentdiag
 - Remove container argument from dg::geo::SafetyFactor constructor (Since it always takes Alpha)
 - More general interface for geometries/init.h functors including Nprofile and ZonalFlow (Old Psi functors are now regular functors)
-- 
GitLab